efaimo 0.1.1 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +128 -5
- package/README.md +69 -7
- package/SKILL.md +16 -9
- package/dist/check/check.js +8 -0
- package/dist/cli.js +179 -27
- package/dist/clients/introspect.js +10 -2
- package/dist/core/engine.d.ts +2 -1
- package/dist/core/engine.js +4 -0
- package/dist/core/types.d.ts +121 -0
- package/dist/find/bm25.d.ts +77 -0
- package/dist/find/bm25.js +107 -0
- package/dist/find/find.d.ts +120 -0
- package/dist/find/find.js +403 -0
- package/dist/find/tokenize.d.ts +64 -0
- package/dist/find/tokenize.js +116 -0
- package/dist/index.d.ts +6 -3
- package/dist/index.js +13 -3
- package/dist/reporters/json.d.ts +19 -1
- package/dist/reporters/json.js +13 -2
- package/dist/reporters/markdown.d.ts +2 -1
- package/dist/reporters/markdown.js +109 -11
- package/dist/reporters/pretty.d.ts +12 -1
- package/dist/reporters/pretty.js +177 -10
- package/dist/rules/find/index.d.ts +4 -0
- package/dist/rules/find/index.js +262 -0
- package/dist/rules/injection.d.ts +1 -2
- package/dist/rules/injection.js +0 -11
- package/dist/rules/mcp/index.js +21 -1
- package/dist/rules/skill/index.js +5 -3
- package/dist/rules/version.d.ts +26 -0
- package/dist/rules/version.js +26 -0
- package/dist/targets/repoScan.js +0 -0
- package/dist/testing/anthropicRunner.d.ts +30 -0
- package/dist/testing/anthropicRunner.js +43 -8
- package/dist/testing/harness.d.ts +33 -1
- package/dist/testing/harness.js +66 -7
- package/dist/testing/openaiRunner.js +15 -8
- package/dist/testing/stats.d.ts +16 -2
- package/dist/testing/stats.js +26 -2
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/weigh/anthropicExact.d.ts +12 -1
- package/dist/weigh/anthropicExact.js +9 -1
- package/dist/weigh/weigh.js +10 -3
- package/docs/INTEGRATIONS.md +15 -1
- package/docs/METHODOLOGY.md +111 -7
- package/docs/RULES.md +42 -5
- package/examples/scenario.crm.yaml +8 -0
- package/examples/scenario.example.yaml +8 -0
- package/package.json +2 -2
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The tokenizer the findability simulation runs on.
|
|
3
|
+
*
|
|
4
|
+
* Tool search (both variants) matches against a tool's name, description,
|
|
5
|
+
* argument names and argument descriptions. Anthropic does not publish the
|
|
6
|
+
* analyzer it uses, so this is a stated approximation rather than a
|
|
7
|
+
* reproduction, and docs/METHODOLOGY.md says exactly that. What matters is
|
|
8
|
+
* that it is deterministic and written down: the same catalog always produces
|
|
9
|
+
* the same terms, on every machine.
|
|
10
|
+
*
|
|
11
|
+
* Splitting rules, in order:
|
|
12
|
+
* 1. camelCase and PascalCase boundaries become separators (`getWeather` ->
|
|
13
|
+
* `get weather`), because a regex like `.*weather.*` has to reach the
|
|
14
|
+
* second half of a camelCase name and a naive split on `[^a-z0-9]` never
|
|
15
|
+
* would.
|
|
16
|
+
* 2. Runs of digits after letters split too (`s3Bucket` keeps `s3`, but
|
|
17
|
+
* `getV2Config` yields `get v2 config`).
|
|
18
|
+
* 3. Everything that is not a letter or a digit is a separator, so
|
|
19
|
+
* `get_weather`, `get-weather` and `get.weather` all land on the same
|
|
20
|
+
* two terms.
|
|
21
|
+
* 4. Lowercase, and drop single characters, which carry no retrieval signal
|
|
22
|
+
* and only distort the length normalization.
|
|
23
|
+
*/
|
|
24
|
+
/**
|
|
25
|
+
* Longest field this will tokenize.
|
|
26
|
+
*
|
|
27
|
+
* Every string here comes from the audited server, which may be hostile and is
|
|
28
|
+
* in any case not obliged to be sensible. The split below is linear, but linear
|
|
29
|
+
* on ten megabytes is still ten megabytes, done twice per description, and the
|
|
30
|
+
* connect timeout bounds the RPC rather than what happens after it. A tool
|
|
31
|
+
* definition that needs more than this to describe itself has a finding of its
|
|
32
|
+
* own waiting in `check`.
|
|
33
|
+
*/
|
|
34
|
+
export declare const MAX_FIELD_CHARS = 200000;
|
|
35
|
+
export declare function tokenize(text: string): string[];
|
|
36
|
+
/**
|
|
37
|
+
* English function words, excluded from QUERY SELECTION only.
|
|
38
|
+
*
|
|
39
|
+
* They stay in the index on purpose. BM25's IDF already discounts a term that
|
|
40
|
+
* appears in most documents, and removing them from the index would also
|
|
41
|
+
* change every document's length, which changes every score. They are excluded
|
|
42
|
+
* from query selection because that step ranks a tool's own terms by TF-IDF,
|
|
43
|
+
* and in a small catalog a function word repeated three times in one
|
|
44
|
+
* description can outrank the domain noun that actually identifies the tool.
|
|
45
|
+
*
|
|
46
|
+
* Deliberately short and boring: this list is part of the published method, so
|
|
47
|
+
* every word in it has to be defensible as a word nobody searches for.
|
|
48
|
+
*/
|
|
49
|
+
export declare const QUERY_STOPWORDS: Set<string>;
|
|
50
|
+
/**
|
|
51
|
+
* Name tokens that identify nothing on their own.
|
|
52
|
+
*
|
|
53
|
+
* Used by E144 only. Anthropic's own debugging note for tool search says
|
|
54
|
+
* Claude writes broad patterns such as `.*weather.*`, and that adding common
|
|
55
|
+
* keywords to descriptions improves discoverability. A name assembled purely
|
|
56
|
+
* from these words carries no such keyword, so the only thing that can reach
|
|
57
|
+
* the tool is its description.
|
|
58
|
+
*
|
|
59
|
+
* A word being here does NOT make a name bad: `get_weather` has `weather`.
|
|
60
|
+
* The rule fires only when EVERY token of a name is in this set.
|
|
61
|
+
*/
|
|
62
|
+
export declare const GENERIC_NAME_TOKENS: Set<string>;
|
|
63
|
+
/** True when a tool name is built only from words that identify nothing. */
|
|
64
|
+
export declare function nameIsGeneric(name: string): boolean;
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The tokenizer the findability simulation runs on.
|
|
3
|
+
*
|
|
4
|
+
* Tool search (both variants) matches against a tool's name, description,
|
|
5
|
+
* argument names and argument descriptions. Anthropic does not publish the
|
|
6
|
+
* analyzer it uses, so this is a stated approximation rather than a
|
|
7
|
+
* reproduction, and docs/METHODOLOGY.md says exactly that. What matters is
|
|
8
|
+
* that it is deterministic and written down: the same catalog always produces
|
|
9
|
+
* the same terms, on every machine.
|
|
10
|
+
*
|
|
11
|
+
* Splitting rules, in order:
|
|
12
|
+
* 1. camelCase and PascalCase boundaries become separators (`getWeather` ->
|
|
13
|
+
* `get weather`), because a regex like `.*weather.*` has to reach the
|
|
14
|
+
* second half of a camelCase name and a naive split on `[^a-z0-9]` never
|
|
15
|
+
* would.
|
|
16
|
+
* 2. Runs of digits after letters split too (`s3Bucket` keeps `s3`, but
|
|
17
|
+
* `getV2Config` yields `get v2 config`).
|
|
18
|
+
* 3. Everything that is not a letter or a digit is a separator, so
|
|
19
|
+
* `get_weather`, `get-weather` and `get.weather` all land on the same
|
|
20
|
+
* two terms.
|
|
21
|
+
* 4. Lowercase, and drop single characters, which carry no retrieval signal
|
|
22
|
+
* and only distort the length normalization.
|
|
23
|
+
*/
|
|
24
|
+
/**
|
|
25
|
+
* Longest field this will tokenize.
|
|
26
|
+
*
|
|
27
|
+
* Every string here comes from the audited server, which may be hostile and is
|
|
28
|
+
* in any case not obliged to be sensible. The split below is linear, but linear
|
|
29
|
+
* on ten megabytes is still ten megabytes, done twice per description, and the
|
|
30
|
+
* connect timeout bounds the RPC rather than what happens after it. A tool
|
|
31
|
+
* definition that needs more than this to describe itself has a finding of its
|
|
32
|
+
* own waiting in `check`.
|
|
33
|
+
*/
|
|
34
|
+
export const MAX_FIELD_CHARS = 200_000;
|
|
35
|
+
export function tokenize(text) {
|
|
36
|
+
if (!text)
|
|
37
|
+
return [];
|
|
38
|
+
return (text.length > MAX_FIELD_CHARS ? text.slice(0, MAX_FIELD_CHARS) : text)
|
|
39
|
+
// ACRONYMFollowed -> ACRONYM Followed, before the general camel split, so
|
|
40
|
+
// `HTTPServer` does not become `httpserver` or `h t t p server`.
|
|
41
|
+
//
|
|
42
|
+
// One capital, not `[A-Z]+`. The quantified form is quadratic on a run of
|
|
43
|
+
// capitals with no lowercase after it: every start position backtracks the
|
|
44
|
+
// whole run. Measured on the old expression, `'A'.repeat(n)` took 43ms at
|
|
45
|
+
// 10k, 7.8s at 100k and 147s at 400k, so one 300KB shouted description
|
|
46
|
+
// pinned a core for minutes while `weigh` and `check` on the same server
|
|
47
|
+
// answered in under a second. `([A-Z])([A-Z][a-z])` matches the same
|
|
48
|
+
// boundaries in linear time: on `HTTPServer` it still splits at `P|Se`.
|
|
49
|
+
.replace(/([A-Z])([A-Z][a-z])/g, "$1 $2")
|
|
50
|
+
.replace(/([a-z0-9])([A-Z])/g, "$1 $2")
|
|
51
|
+
.replace(/([A-Za-z])(\d)/g, "$1 $2")
|
|
52
|
+
.split(/[^A-Za-z0-9]+/)
|
|
53
|
+
.map((s) => s.toLowerCase())
|
|
54
|
+
.filter((s) => s.length > 1);
|
|
55
|
+
}
|
|
56
|
+
/**
|
|
57
|
+
* English function words, excluded from QUERY SELECTION only.
|
|
58
|
+
*
|
|
59
|
+
* They stay in the index on purpose. BM25's IDF already discounts a term that
|
|
60
|
+
* appears in most documents, and removing them from the index would also
|
|
61
|
+
* change every document's length, which changes every score. They are excluded
|
|
62
|
+
* from query selection because that step ranks a tool's own terms by TF-IDF,
|
|
63
|
+
* and in a small catalog a function word repeated three times in one
|
|
64
|
+
* description can outrank the domain noun that actually identifies the tool.
|
|
65
|
+
*
|
|
66
|
+
* Deliberately short and boring: this list is part of the published method, so
|
|
67
|
+
* every word in it has to be defensible as a word nobody searches for.
|
|
68
|
+
*/
|
|
69
|
+
export const QUERY_STOPWORDS = new Set([
|
|
70
|
+
// Two and three letter function words. The tokenizer drops single
|
|
71
|
+
// characters, so without these a description like "Delete all records in the
|
|
72
|
+
// datastore" probes with `in`, and a term that appears in half the catalog
|
|
73
|
+
// spends one of the four query slots saying nothing.
|
|
74
|
+
"in", "on", "at", "to", "of", "or", "as", "by", "is", "be", "an", "if",
|
|
75
|
+
"do", "so", "up", "we", "us", "am", "he", "she", "him", "her", "his", "me",
|
|
76
|
+
"my", "no", "yes",
|
|
77
|
+
"the", "and", "for", "with", "that", "this", "from", "into", "onto", "than",
|
|
78
|
+
"then", "they", "them", "their", "there", "these", "those", "will", "would",
|
|
79
|
+
"shall", "should", "can", "could", "may", "might", "must", "not", "but",
|
|
80
|
+
"are", "was", "were", "been", "being", "have", "has", "had", "does", "did",
|
|
81
|
+
"you", "your", "yours", "its", "it", "our", "ours", "any", "all", "each",
|
|
82
|
+
"such", "only", "also", "more", "most", "some", "other", "when", "where",
|
|
83
|
+
"which", "while", "what", "who", "whom", "how", "why", "use", "used",
|
|
84
|
+
"using", "via", "per", "out", "off", "over", "under", "about", "above",
|
|
85
|
+
"below", "between", "before", "after", "here", "one", "two", "yes", "no",
|
|
86
|
+
]);
|
|
87
|
+
/**
|
|
88
|
+
* Name tokens that identify nothing on their own.
|
|
89
|
+
*
|
|
90
|
+
* Used by E144 only. Anthropic's own debugging note for tool search says
|
|
91
|
+
* Claude writes broad patterns such as `.*weather.*`, and that adding common
|
|
92
|
+
* keywords to descriptions improves discoverability. A name assembled purely
|
|
93
|
+
* from these words carries no such keyword, so the only thing that can reach
|
|
94
|
+
* the tool is its description.
|
|
95
|
+
*
|
|
96
|
+
* A word being here does NOT make a name bad: `get_weather` has `weather`.
|
|
97
|
+
* The rule fires only when EVERY token of a name is in this set.
|
|
98
|
+
*/
|
|
99
|
+
export const GENERIC_NAME_TOKENS = new Set([
|
|
100
|
+
"get", "set", "put", "post", "patch", "add", "new", "create", "make",
|
|
101
|
+
"update", "edit", "modify", "change", "delete", "remove", "drop",
|
|
102
|
+
"list", "read", "write", "fetch", "load", "save", "send", "call",
|
|
103
|
+
"run", "exec", "execute", "invoke", "do", "handle", "process", "perform",
|
|
104
|
+
"query", "search", "find", "lookup", "check", "test", "start", "stop",
|
|
105
|
+
"open", "close", "begin", "end", "init", "setup", "config", "configure",
|
|
106
|
+
"tool", "api", "data", "item", "items", "object", "thing", "value",
|
|
107
|
+
"values", "result", "results", "action", "request", "response", "info",
|
|
108
|
+
"helper", "util", "utils", "main", "default", "generic", "custom", "my",
|
|
109
|
+
]);
|
|
110
|
+
/** True when a tool name is built only from words that identify nothing. */
|
|
111
|
+
export function nameIsGeneric(name) {
|
|
112
|
+
const terms = tokenize(name);
|
|
113
|
+
if (!terms.length)
|
|
114
|
+
return true;
|
|
115
|
+
return terms.every((t) => GENERIC_NAME_TOKENS.has(t));
|
|
116
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -7,10 +7,13 @@ export { weighServer, weighSkills } from "./weigh/weigh.js";
|
|
|
7
7
|
export { diffServerWeigh, type WeighDiff } from "./weigh/diff.js";
|
|
8
8
|
export { countTokens } from "./weigh/tokens.js";
|
|
9
9
|
export { findSkills, parseSkillFile } from "./skills/parse.js";
|
|
10
|
+
export { analyzeFind, indexedTerms, taskQuery, DEFAULT_TOP_K, DEFAULT_QUERY_TERMS, DEFER_RECOMMENDED_TOKENS, DEFER_RECOMMENDED_TOOLS, type FindOptions } from "./find/find.js";
|
|
11
|
+
export { runFindRules } from "./core/engine.js";
|
|
12
|
+
export { RULES_VERSION } from "./rules/version.js";
|
|
10
13
|
export { checkMcpTarget, checkMcpRepoOnly, checkSkillSet, type SkillReport, type CheckSkillResult } from "./check/check.js";
|
|
11
14
|
export { parseScenario, runScenario, armSystems, type Scenario, type TestReport, type Runner, } from "./testing/harness.js";
|
|
12
|
-
export { anthropicRunner } from "./testing/anthropicRunner.js";
|
|
15
|
+
export { anthropicRunner, acceptsSamplingParams, anthropicRequestBody } from "./testing/anthropicRunner.js";
|
|
13
16
|
export { openaiRunner, providerForModel } from "./testing/openaiRunner.js";
|
|
14
|
-
export { renderCheckPretty, renderSkillSetPretty, renderServerWeighPretty, renderSkillWeighPretty, renderDiffPretty, renderScenarioPlan, renderTestReportPretty, } from "./reporters/pretty.js";
|
|
15
|
-
export { renderCheckMarkdown, renderSkillSetMarkdown, renderWeighMarkdown, renderDiffMarkdown, } from "./reporters/markdown.js";
|
|
17
|
+
export { renderCheckPretty, renderSkillSetPretty, renderServerWeighPretty, renderSkillWeighPretty, renderDiffPretty, renderScenarioPlan, renderTestReportPretty, renderFindPretty, renderFindFindingsPretty, } from "./reporters/pretty.js";
|
|
18
|
+
export { renderCheckMarkdown, renderSkillSetMarkdown, renderWeighMarkdown, renderDiffMarkdown, renderFindMarkdown, } from "./reporters/markdown.js";
|
|
16
19
|
export { toJsonEnvelope } from "./reporters/json.js";
|
package/dist/index.js
CHANGED
|
@@ -14,13 +14,23 @@ export { diffServerWeigh } from "./weigh/diff.js";
|
|
|
14
14
|
export { countTokens } from "./weigh/tokens.js";
|
|
15
15
|
// Skills
|
|
16
16
|
export { findSkills, parseSkillFile } from "./skills/parse.js";
|
|
17
|
+
// Find (findability under deferred tool loading)
|
|
18
|
+
//
|
|
19
|
+
// Exported because docs/INTEGRATIONS.md promises "every capability is also a
|
|
20
|
+
// typed function", and the blanket re-export of core/types.js was already
|
|
21
|
+
// publishing FindResult and ToolFindEntry: a public type with nothing public
|
|
22
|
+
// that produces one. runFindRules is here for the same reason its output is,
|
|
23
|
+
// even though the rule TABLE stays private like the other two.
|
|
24
|
+
export { analyzeFind, indexedTerms, taskQuery, DEFAULT_TOP_K, DEFAULT_QUERY_TERMS, DEFER_RECOMMENDED_TOKENS, DEFER_RECOMMENDED_TOOLS } from "./find/find.js";
|
|
25
|
+
export { runFindRules } from "./core/engine.js";
|
|
26
|
+
export { RULES_VERSION } from "./rules/version.js";
|
|
17
27
|
// Check (audit)
|
|
18
28
|
export { checkMcpTarget, checkMcpRepoOnly, checkSkillSet } from "./check/check.js";
|
|
19
29
|
// Test (skill A/B outcome harness)
|
|
20
30
|
export { parseScenario, runScenario, armSystems, } from "./testing/harness.js";
|
|
21
|
-
export { anthropicRunner } from "./testing/anthropicRunner.js";
|
|
31
|
+
export { anthropicRunner, acceptsSamplingParams, anthropicRequestBody } from "./testing/anthropicRunner.js";
|
|
22
32
|
export { openaiRunner, providerForModel } from "./testing/openaiRunner.js";
|
|
23
33
|
// Reporters (report -> string; stable value-in/string-out contract)
|
|
24
|
-
export { renderCheckPretty, renderSkillSetPretty, renderServerWeighPretty, renderSkillWeighPretty, renderDiffPretty, renderScenarioPlan, renderTestReportPretty, } from "./reporters/pretty.js";
|
|
25
|
-
export { renderCheckMarkdown, renderSkillSetMarkdown, renderWeighMarkdown, renderDiffMarkdown, } from "./reporters/markdown.js";
|
|
34
|
+
export { renderCheckPretty, renderSkillSetPretty, renderServerWeighPretty, renderSkillWeighPretty, renderDiffPretty, renderScenarioPlan, renderTestReportPretty, renderFindPretty, renderFindFindingsPretty, } from "./reporters/pretty.js";
|
|
35
|
+
export { renderCheckMarkdown, renderSkillSetMarkdown, renderWeighMarkdown, renderDiffMarkdown, renderFindMarkdown, } from "./reporters/markdown.js";
|
|
26
36
|
export { toJsonEnvelope } from "./reporters/json.js";
|
package/dist/reporters/json.d.ts
CHANGED
|
@@ -1 +1,19 @@
|
|
|
1
|
-
export
|
|
1
|
+
export type EnvelopeKind = "check" | "weigh" | "test" | "find";
|
|
2
|
+
export interface EnvelopeOptions {
|
|
3
|
+
/**
|
|
4
|
+
* Include `generatedAt`. Default true for a report a human reads.
|
|
5
|
+
*
|
|
6
|
+
* Off for anything meant to be committed, diffed or compared. A wall-clock
|
|
7
|
+
* stamp inside the payload makes every regeneration a diff even when not one
|
|
8
|
+
* measured number moved, which is how a re-run that changed nothing still
|
|
9
|
+
* produces a commit that has to be reviewed as though it might have. This
|
|
10
|
+
* project has shipped several releases whose only data change was described
|
|
11
|
+
* in its own notes as "stamp-only".
|
|
12
|
+
*
|
|
13
|
+
* `--out` baselines always drop it: a baseline exists to be compared against,
|
|
14
|
+
* and a field that differs on every write is the one field a comparison must
|
|
15
|
+
* ignore.
|
|
16
|
+
*/
|
|
17
|
+
timestamp?: boolean;
|
|
18
|
+
}
|
|
19
|
+
export declare function toJsonEnvelope(kind: EnvelopeKind, data: unknown, opts?: EnvelopeOptions): string;
|
package/dist/reporters/json.js
CHANGED
|
@@ -1,10 +1,21 @@
|
|
|
1
1
|
import { VERSION } from "../version.js";
|
|
2
|
-
|
|
2
|
+
import { RULES_VERSION } from "../rules/version.js";
|
|
3
|
+
export function toJsonEnvelope(kind, data, opts = {}) {
|
|
3
4
|
return JSON.stringify({
|
|
4
5
|
tool: "efaimo",
|
|
5
6
|
version: VERSION,
|
|
7
|
+
// Which ruleset produced this. The tool version alone does not say it: a
|
|
8
|
+
// patch release can change what a rule fires on, and every published
|
|
9
|
+
// grade is only reproducible if the reader can tell which rules were
|
|
10
|
+
// being run. See src/rules/version.ts.
|
|
11
|
+
//
|
|
12
|
+
// Only where rules actually ran. `weigh` counts tokens and `test` runs
|
|
13
|
+
// trials; neither evaluates a rule, and stamping a ruleset onto their
|
|
14
|
+
// output would be a report asserting which rules produced it when none
|
|
15
|
+
// did.
|
|
16
|
+
...(kind === "check" || kind === "find" ? { rulesVersion: RULES_VERSION } : {}),
|
|
6
17
|
kind,
|
|
7
|
-
generatedAt: new Date().toISOString(),
|
|
18
|
+
...(opts.timestamp === false ? {} : { generatedAt: new Date().toISOString() }),
|
|
8
19
|
data,
|
|
9
20
|
}, null, 2);
|
|
10
21
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { CheckReport, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
|
|
1
|
+
import type { CheckReport, Finding, FindResult, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
|
|
2
2
|
import type { CheckSkillResult } from "../check/check.js";
|
|
3
3
|
import type { WeighDiff } from "../weigh/diff.js";
|
|
4
4
|
export declare function renderCheckMarkdown(report: CheckReport): string;
|
|
5
5
|
export declare function renderSkillSetMarkdown(res: CheckSkillResult): string;
|
|
6
6
|
export declare function renderWeighMarkdown(w: ServerWeighResult | SkillSetWeighResult): string;
|
|
7
|
+
export declare function renderFindMarkdown(f: FindResult, findings: Finding[]): string;
|
|
7
8
|
export declare function renderDiffMarkdown(d: WeighDiff): string;
|
|
@@ -3,23 +3,57 @@ import { safeText } from "../util/safeText.js";
|
|
|
3
3
|
const SEV_LABEL = { error: "🔴 error", warn: "🟡 warn", info: "🔵 info" };
|
|
4
4
|
// Everything through here came from the audited target, and
|
|
5
5
|
// docs/INTEGRATIONS.md recommends piping --md into $GITHUB_STEP_SUMMARY, so a
|
|
6
|
-
// hostile tool name or description renders into a page the reader trusts.
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
|
|
6
|
+
// hostile tool name or description renders into a page the reader trusts. An
|
|
7
|
+
// auditor's verdict must not be rewritable by the thing being audited.
|
|
8
|
+
//
|
|
9
|
+
// Two escapes that were here did not work, both found by adversarial review:
|
|
10
|
+
//
|
|
11
|
+
// `\|` was applied without escaping the backslash first. A name containing
|
|
12
|
+
// backslash-pipe became `\\|`, which GFM reads as an escaped backslash
|
|
13
|
+
// followed by a LIVE cell delimiter. Extra cells past the header count
|
|
14
|
+
// are dropped, so a hostile server could push the real verdict column
|
|
15
|
+
// off the end of the row and supply its own.
|
|
16
|
+
//
|
|
17
|
+
// `` ` `` was neutralised by putting a zero-width space BEFORE it, which
|
|
18
|
+
// changes nothing: a code span ends at the next backtick run whatever
|
|
19
|
+
// precedes it. The rest of the name then rendered as live markup, and a
|
|
20
|
+
// `[click](url)` in a tool name became a link in the summary. A
|
|
21
|
+
// backtick cannot be escaped inside a code span at all, so it is
|
|
22
|
+
// substituted rather than escaped; a tool whose name contains one is
|
|
23
|
+
// still recognisable and can no longer close the span.
|
|
24
|
+
function escCode(s) {
|
|
13
25
|
return safeText(s)
|
|
26
|
+
.replace(/[\r\n]+/g, " ")
|
|
27
|
+
.replace(/\\/g, "\\\\")
|
|
14
28
|
.replace(/\|/g, "\\|")
|
|
15
|
-
.replace(/`/g, "
|
|
16
|
-
.replace(/\n/g, " ");
|
|
29
|
+
.replace(/`/g, "'");
|
|
17
30
|
}
|
|
31
|
+
// For a cell that is NOT wrapped in a code span, where markdown and inline
|
|
32
|
+
// HTML are both live. Every value that reaches one of these should really be
|
|
33
|
+
// in a code span; this is the belt for the places that are not, such as rule
|
|
34
|
+
// messages, which embed a tool name mid-sentence.
|
|
35
|
+
function escText(s) {
|
|
36
|
+
return escCode(s)
|
|
37
|
+
.replace(/&/g, "&")
|
|
38
|
+
.replace(/</g, "<")
|
|
39
|
+
.replace(/>/g, ">")
|
|
40
|
+
.replace(/([[\]*_~])/g, "\\$1");
|
|
41
|
+
}
|
|
42
|
+
/** Backwards-compatible alias for the code-span form, which is what most call sites want. */
|
|
43
|
+
const esc = escCode;
|
|
18
44
|
function findingsTable(lines, findings) {
|
|
19
45
|
lines.push("| severity | rule | finding |");
|
|
20
46
|
lines.push("|---|---|---|");
|
|
21
47
|
for (const f of findings) {
|
|
22
|
-
|
|
48
|
+
// escText, not escCode: a message is prose, not a code span, so markdown
|
|
49
|
+
// and inline HTML are both live in this cell. The message embeds a tool
|
|
50
|
+
// name the target chose.
|
|
51
|
+
//
|
|
52
|
+
// detail and fixHint were dropped here, so E141's "every word it has also
|
|
53
|
+
// appears on: ..." and E142's "outranked by: ..." existed in the terminal
|
|
54
|
+
// and in JSON and silently vanished from the surface the CI docs recommend.
|
|
55
|
+
const extra = [f.detail, f.fixHint && `fix: ${f.fixHint}`].filter(Boolean).join(" ");
|
|
56
|
+
lines.push(`| ${SEV_LABEL[f.severity]} | ${f.ruleId} | ${escText(f.message)}${extra ? ` <br>${escText(extra)}` : ""} |`);
|
|
23
57
|
}
|
|
24
58
|
}
|
|
25
59
|
export function renderCheckMarkdown(report) {
|
|
@@ -85,7 +119,10 @@ export function renderWeighMarkdown(w) {
|
|
|
85
119
|
lines.push(`| Claude-style | ${w.totals.claudeStyle.toLocaleString("en-US")} |`);
|
|
86
120
|
lines.push(`| OpenAI tools | ${w.totals.openaiTools.toLocaleString("en-US")} |`);
|
|
87
121
|
if (w.anthropicExactTotal !== undefined) {
|
|
88
|
-
|
|
122
|
+
// Names the model, like the terminal does. Without it the same figure
|
|
123
|
+
// was reproducible from --json and from pretty output but not from --md,
|
|
124
|
+
// which is the surface the CI docs recommend piping into a summary.
|
|
125
|
+
lines.push(`| anthropic exact (count_tokens, ${escCode(w.anthropicExactModel ?? "model not recorded")}) | ${w.anthropicExactTotal.toLocaleString("en-US")} |`);
|
|
89
126
|
}
|
|
90
127
|
lines.push("");
|
|
91
128
|
if (w.perTool.length) {
|
|
@@ -115,6 +152,67 @@ export function renderWeighMarkdown(w) {
|
|
|
115
152
|
lines.push(`> ${esc(note)}`);
|
|
116
153
|
return lines.join("\n");
|
|
117
154
|
}
|
|
155
|
+
export function renderFindMarkdown(f, findings) {
|
|
156
|
+
const lines = [];
|
|
157
|
+
lines.push(`## efaimo find: ${f.distinct.count}/${f.distinct.total} tools own a distinct word`);
|
|
158
|
+
lines.push("");
|
|
159
|
+
lines.push(`target: \`${esc(f.label)}\``);
|
|
160
|
+
lines.push("");
|
|
161
|
+
lines.push(`**distinct ${f.distinct.count}/${f.distinct.total} (${f.distinct.pct}%)** tools that own a word no other tool has. ` +
|
|
162
|
+
`Probe ${f.probe.returned}/${f.probe.total} (${f.probe.pct}%) returned by a simulated search for their own description.`);
|
|
163
|
+
lines.push("");
|
|
164
|
+
if (f.distinctVacuous) {
|
|
165
|
+
lines.push(`> **Distinct is vacuous here.** A one-tool catalog has nothing to be distinct from.`);
|
|
166
|
+
lines.push("");
|
|
167
|
+
}
|
|
168
|
+
if (f.windowCoversCatalog) {
|
|
169
|
+
lines.push(`> **The probe is vacuous here.** ${f.toolCount} tools and a result window of ${f.method.topK}, so no tool can fall outside it.`);
|
|
170
|
+
lines.push("");
|
|
171
|
+
}
|
|
172
|
+
lines.push("| words | tool | vocabulary no other tool has | probe |");
|
|
173
|
+
lines.push("|---|---|---|---|");
|
|
174
|
+
const byWorst = [...f.perTool].sort((a, b) => {
|
|
175
|
+
const ar = a.rank ?? Number.POSITIVE_INFINITY;
|
|
176
|
+
const br = b.rank ?? Number.POSITIVE_INFINITY;
|
|
177
|
+
return a.ownTermCount - b.ownTermCount || br - ar || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
|
|
178
|
+
});
|
|
179
|
+
// Capped, like every other table here. Uncapped, a 3,000-tool catalog wrote
|
|
180
|
+
// 74 KB of rows and a large aggregate would blow GitHub's 1 MiB step-summary
|
|
181
|
+
// limit, failing the step it was meant to annotate.
|
|
182
|
+
const MD_ROWS = 50;
|
|
183
|
+
for (const t of byWorst.slice(0, MD_ROWS)) {
|
|
184
|
+
const own = t.ownTermCount === 0
|
|
185
|
+
? t.sharedWith.length
|
|
186
|
+
? `none, every word also on ${t.sharedWith.map((s) => `\`${escCode(s)}\``).join(", ")}`
|
|
187
|
+
: "none"
|
|
188
|
+
: `\`${escCode(t.ownTerms.join(", "))}\`` +
|
|
189
|
+
(t.ownTermCount > t.ownTerms.length ? ` (+${t.ownTermCount - t.ownTerms.length})` : "");
|
|
190
|
+
// Blank when it has nothing to say, matching the terminal. A column of
|
|
191
|
+
// "ok" down every row is a value that never varies presented as a
|
|
192
|
+
// measurement, which is the thing renderFindPretty refuses to do.
|
|
193
|
+
const probe = t.rank === undefined ? "no description" : t.reachable ? "" : `rank ${t.rank}`;
|
|
194
|
+
lines.push(`| ${t.ownTermCount} | \`${escCode(t.name)}\` | ${own} | ${probe} |`);
|
|
195
|
+
}
|
|
196
|
+
if (byWorst.length > MD_ROWS) {
|
|
197
|
+
lines.push(`| | *(+${byWorst.length - MD_ROWS} more, see \`--json\`)* | | |`);
|
|
198
|
+
}
|
|
199
|
+
lines.push("");
|
|
200
|
+
lines.push("### findings");
|
|
201
|
+
lines.push("");
|
|
202
|
+
if (!findings.length)
|
|
203
|
+
lines.push("no findability findings.");
|
|
204
|
+
else
|
|
205
|
+
findingsTable(lines, findings);
|
|
206
|
+
lines.push("");
|
|
207
|
+
// A blank line between blockquotes, or CommonMark lazily continues them into
|
|
208
|
+
// one paragraph and five separate notes render as one wall of prose.
|
|
209
|
+
for (const note of f.notes) {
|
|
210
|
+
lines.push(`> ${escText(note)}`);
|
|
211
|
+
lines.push("");
|
|
212
|
+
}
|
|
213
|
+
lines.push(`<sub>efaimo v${VERSION}</sub>`);
|
|
214
|
+
return lines.join("\n");
|
|
215
|
+
}
|
|
118
216
|
export function renderDiffMarkdown(d) {
|
|
119
217
|
const sign = d.delta >= 0 ? "+" : "";
|
|
120
218
|
const lines = [];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CheckReport, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
|
|
1
|
+
import type { CheckReport, Finding, FindResult, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
|
|
2
2
|
import type { CheckSkillResult } from "../check/check.js";
|
|
3
3
|
import type { Scenario, TestReport } from "../testing/harness.js";
|
|
4
4
|
import type { WeighDiff } from "../weigh/diff.js";
|
|
@@ -9,6 +9,17 @@ export declare function renderTestReportPretty(r: TestReport): string;
|
|
|
9
9
|
export declare function renderSkillSetPretty(res: CheckSkillResult): string;
|
|
10
10
|
export declare function renderServerWeighPretty(w: ServerWeighResult): string;
|
|
11
11
|
export declare function renderSkillWeighPretty(w: SkillSetWeighResult): string;
|
|
12
|
+
/**
|
|
13
|
+
* `efaimo find`.
|
|
14
|
+
*
|
|
15
|
+
* Two numbers, both as fractions with their denominators printed, neither a
|
|
16
|
+
* letter. The first is a property of the catalog; the second is a simulation
|
|
17
|
+
* and is labelled as one on the line itself, because a saturated 100% next to
|
|
18
|
+
* a real measurement reads like a good score unless the page says otherwise.
|
|
19
|
+
*/
|
|
20
|
+
export declare function renderFindPretty(f: FindResult): string;
|
|
21
|
+
/** The findings half of `efaimo find`, printed under the table. */
|
|
22
|
+
export declare function renderFindFindingsPretty(f: FindResult, findings: Finding[]): string;
|
|
12
23
|
export declare function renderDiffPretty(d: WeighDiff, opts: {
|
|
13
24
|
maxTokens?: number;
|
|
14
25
|
allowIncreasePct?: number;
|