efaimo 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -1
- package/README.md +68 -6
- package/SKILL.md +14 -7
- package/dist/cli.js +162 -22
- package/dist/core/engine.d.ts +2 -1
- package/dist/core/engine.js +4 -0
- package/dist/core/types.d.ts +121 -0
- package/dist/find/bm25.d.ts +77 -0
- package/dist/find/bm25.js +107 -0
- package/dist/find/find.d.ts +120 -0
- package/dist/find/find.js +403 -0
- package/dist/find/tokenize.d.ts +64 -0
- package/dist/find/tokenize.js +116 -0
- package/dist/index.d.ts +6 -3
- package/dist/index.js +13 -3
- package/dist/reporters/json.d.ts +19 -1
- package/dist/reporters/json.js +13 -2
- package/dist/reporters/markdown.d.ts +2 -1
- package/dist/reporters/markdown.js +109 -11
- package/dist/reporters/pretty.d.ts +12 -1
- package/dist/reporters/pretty.js +174 -9
- package/dist/rules/find/index.d.ts +4 -0
- package/dist/rules/find/index.js +262 -0
- package/dist/rules/version.d.ts +26 -0
- package/dist/rules/version.js +26 -0
- package/dist/testing/anthropicRunner.d.ts +30 -0
- package/dist/testing/anthropicRunner.js +43 -8
- package/dist/testing/harness.d.ts +32 -1
- package/dist/testing/harness.js +28 -3
- package/dist/testing/stats.d.ts +16 -0
- package/dist/testing/stats.js +26 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/weigh/anthropicExact.d.ts +12 -1
- package/dist/weigh/anthropicExact.js +9 -1
- package/dist/weigh/weigh.js +10 -3
- package/docs/INTEGRATIONS.md +15 -1
- package/docs/METHODOLOGY.md +110 -6
- package/docs/RULES.md +40 -3
- package/examples/scenario.crm.yaml +8 -0
- package/examples/scenario.example.yaml +8 -0
- package/package.json +1 -1
package/dist/reporters/json.js
CHANGED
|
@@ -1,10 +1,21 @@
|
|
|
1
1
|
import { VERSION } from "../version.js";
|
|
2
|
-
|
|
2
|
+
import { RULES_VERSION } from "../rules/version.js";
|
|
3
|
+
export function toJsonEnvelope(kind, data, opts = {}) {
|
|
3
4
|
return JSON.stringify({
|
|
4
5
|
tool: "efaimo",
|
|
5
6
|
version: VERSION,
|
|
7
|
+
// Which ruleset produced this. The tool version alone does not say it: a
|
|
8
|
+
// patch release can change what a rule fires on, and every published
|
|
9
|
+
// grade is only reproducible if the reader can tell which rules were
|
|
10
|
+
// being run. See src/rules/version.ts.
|
|
11
|
+
//
|
|
12
|
+
// Only where rules actually ran. `weigh` counts tokens and `test` runs
|
|
13
|
+
// trials; neither evaluates a rule, and stamping a ruleset onto their
|
|
14
|
+
// output would be a report asserting which rules produced it when none
|
|
15
|
+
// did.
|
|
16
|
+
...(kind === "check" || kind === "find" ? { rulesVersion: RULES_VERSION } : {}),
|
|
6
17
|
kind,
|
|
7
|
-
generatedAt: new Date().toISOString(),
|
|
18
|
+
...(opts.timestamp === false ? {} : { generatedAt: new Date().toISOString() }),
|
|
8
19
|
data,
|
|
9
20
|
}, null, 2);
|
|
10
21
|
}
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
-
import type { CheckReport, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
|
|
1
|
+
import type { CheckReport, Finding, FindResult, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
|
|
2
2
|
import type { CheckSkillResult } from "../check/check.js";
|
|
3
3
|
import type { WeighDiff } from "../weigh/diff.js";
|
|
4
4
|
export declare function renderCheckMarkdown(report: CheckReport): string;
|
|
5
5
|
export declare function renderSkillSetMarkdown(res: CheckSkillResult): string;
|
|
6
6
|
export declare function renderWeighMarkdown(w: ServerWeighResult | SkillSetWeighResult): string;
|
|
7
|
+
export declare function renderFindMarkdown(f: FindResult, findings: Finding[]): string;
|
|
7
8
|
export declare function renderDiffMarkdown(d: WeighDiff): string;
|
|
@@ -3,23 +3,57 @@ import { safeText } from "../util/safeText.js";
|
|
|
3
3
|
const SEV_LABEL = { error: "🔴 error", warn: "🟡 warn", info: "🔵 info" };
|
|
4
4
|
// Everything through here came from the audited target, and
|
|
5
5
|
// docs/INTEGRATIONS.md recommends piping --md into $GITHUB_STEP_SUMMARY, so a
|
|
6
|
-
// hostile tool name or description renders into a page the reader trusts.
|
|
7
|
-
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
|
|
6
|
+
// hostile tool name or description renders into a page the reader trusts. An
|
|
7
|
+
// auditor's verdict must not be rewritable by the thing being audited.
|
|
8
|
+
//
|
|
9
|
+
// Two escapes that were here did not work, both found by adversarial review:
|
|
10
|
+
//
|
|
11
|
+
// `\|` was applied without escaping the backslash first. A name containing
|
|
12
|
+
// backslash-pipe became `\\|`, which GFM reads as an escaped backslash
|
|
13
|
+
// followed by a LIVE cell delimiter. Extra cells past the header count
|
|
14
|
+
// are dropped, so a hostile server could push the real verdict column
|
|
15
|
+
// off the end of the row and supply its own.
|
|
16
|
+
//
|
|
17
|
+
// `` ` `` was neutralised by putting a zero-width space BEFORE it, which
|
|
18
|
+
// changes nothing: a code span ends at the next backtick run whatever
|
|
19
|
+
// precedes it. The rest of the name then rendered as live markup, and a
|
|
20
|
+
// `[click](url)` in a tool name became a link in the summary. A
|
|
21
|
+
// backtick cannot be escaped inside a code span at all, so it is
|
|
22
|
+
// substituted rather than escaped; a tool whose name contains one is
|
|
23
|
+
// still recognisable and can no longer close the span.
|
|
24
|
+
function escCode(s) {
|
|
13
25
|
return safeText(s)
|
|
26
|
+
.replace(/[\r\n]+/g, " ")
|
|
27
|
+
.replace(/\\/g, "\\\\")
|
|
14
28
|
.replace(/\|/g, "\\|")
|
|
15
|
-
.replace(/`/g, "
|
|
16
|
-
.replace(/\n/g, " ");
|
|
29
|
+
.replace(/`/g, "'");
|
|
17
30
|
}
|
|
31
|
+
// For a cell that is NOT wrapped in a code span, where markdown and inline
|
|
32
|
+
// HTML are both live. Every value that reaches one of these should really be
|
|
33
|
+
// in a code span; this is the belt for the places that are not, such as rule
|
|
34
|
+
// messages, which embed a tool name mid-sentence.
|
|
35
|
+
function escText(s) {
|
|
36
|
+
return escCode(s)
|
|
37
|
+
.replace(/&/g, "&")
|
|
38
|
+
.replace(/</g, "<")
|
|
39
|
+
.replace(/>/g, ">")
|
|
40
|
+
.replace(/([[\]*_~])/g, "\\$1");
|
|
41
|
+
}
|
|
42
|
+
/** Backwards-compatible alias for the code-span form, which is what most call sites want. */
|
|
43
|
+
const esc = escCode;
|
|
18
44
|
function findingsTable(lines, findings) {
|
|
19
45
|
lines.push("| severity | rule | finding |");
|
|
20
46
|
lines.push("|---|---|---|");
|
|
21
47
|
for (const f of findings) {
|
|
22
|
-
|
|
48
|
+
// escText, not escCode: a message is prose, not a code span, so markdown
|
|
49
|
+
// and inline HTML are both live in this cell. The message embeds a tool
|
|
50
|
+
// name the target chose.
|
|
51
|
+
//
|
|
52
|
+
// detail and fixHint were dropped here, so E141's "every word it has also
|
|
53
|
+
// appears on: ..." and E142's "outranked by: ..." existed in the terminal
|
|
54
|
+
// and in JSON and silently vanished from the surface the CI docs recommend.
|
|
55
|
+
const extra = [f.detail, f.fixHint && `fix: ${f.fixHint}`].filter(Boolean).join(" ");
|
|
56
|
+
lines.push(`| ${SEV_LABEL[f.severity]} | ${f.ruleId} | ${escText(f.message)}${extra ? ` <br>${escText(extra)}` : ""} |`);
|
|
23
57
|
}
|
|
24
58
|
}
|
|
25
59
|
export function renderCheckMarkdown(report) {
|
|
@@ -85,7 +119,10 @@ export function renderWeighMarkdown(w) {
|
|
|
85
119
|
lines.push(`| Claude-style | ${w.totals.claudeStyle.toLocaleString("en-US")} |`);
|
|
86
120
|
lines.push(`| OpenAI tools | ${w.totals.openaiTools.toLocaleString("en-US")} |`);
|
|
87
121
|
if (w.anthropicExactTotal !== undefined) {
|
|
88
|
-
|
|
122
|
+
// Names the model, like the terminal does. Without it the same figure
|
|
123
|
+
// was reproducible from --json and from pretty output but not from --md,
|
|
124
|
+
// which is the surface the CI docs recommend piping into a summary.
|
|
125
|
+
lines.push(`| anthropic exact (count_tokens, ${escCode(w.anthropicExactModel ?? "model not recorded")}) | ${w.anthropicExactTotal.toLocaleString("en-US")} |`);
|
|
89
126
|
}
|
|
90
127
|
lines.push("");
|
|
91
128
|
if (w.perTool.length) {
|
|
@@ -115,6 +152,67 @@ export function renderWeighMarkdown(w) {
|
|
|
115
152
|
lines.push(`> ${esc(note)}`);
|
|
116
153
|
return lines.join("\n");
|
|
117
154
|
}
|
|
155
|
+
export function renderFindMarkdown(f, findings) {
|
|
156
|
+
const lines = [];
|
|
157
|
+
lines.push(`## efaimo find: ${f.distinct.count}/${f.distinct.total} tools own a distinct word`);
|
|
158
|
+
lines.push("");
|
|
159
|
+
lines.push(`target: \`${esc(f.label)}\``);
|
|
160
|
+
lines.push("");
|
|
161
|
+
lines.push(`**distinct ${f.distinct.count}/${f.distinct.total} (${f.distinct.pct}%)** tools that own a word no other tool has. ` +
|
|
162
|
+
`Probe ${f.probe.returned}/${f.probe.total} (${f.probe.pct}%) returned by a simulated search for their own description.`);
|
|
163
|
+
lines.push("");
|
|
164
|
+
if (f.distinctVacuous) {
|
|
165
|
+
lines.push(`> **Distinct is vacuous here.** A one-tool catalog has nothing to be distinct from.`);
|
|
166
|
+
lines.push("");
|
|
167
|
+
}
|
|
168
|
+
if (f.windowCoversCatalog) {
|
|
169
|
+
lines.push(`> **The probe is vacuous here.** ${f.toolCount} tools and a result window of ${f.method.topK}, so no tool can fall outside it.`);
|
|
170
|
+
lines.push("");
|
|
171
|
+
}
|
|
172
|
+
lines.push("| words | tool | vocabulary no other tool has | probe |");
|
|
173
|
+
lines.push("|---|---|---|---|");
|
|
174
|
+
const byWorst = [...f.perTool].sort((a, b) => {
|
|
175
|
+
const ar = a.rank ?? Number.POSITIVE_INFINITY;
|
|
176
|
+
const br = b.rank ?? Number.POSITIVE_INFINITY;
|
|
177
|
+
return a.ownTermCount - b.ownTermCount || br - ar || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
|
|
178
|
+
});
|
|
179
|
+
// Capped, like every other table here. Uncapped, a 3,000-tool catalog wrote
|
|
180
|
+
// 74 KB of rows and a large aggregate would blow GitHub's 1 MiB step-summary
|
|
181
|
+
// limit, failing the step it was meant to annotate.
|
|
182
|
+
const MD_ROWS = 50;
|
|
183
|
+
for (const t of byWorst.slice(0, MD_ROWS)) {
|
|
184
|
+
const own = t.ownTermCount === 0
|
|
185
|
+
? t.sharedWith.length
|
|
186
|
+
? `none, every word also on ${t.sharedWith.map((s) => `\`${escCode(s)}\``).join(", ")}`
|
|
187
|
+
: "none"
|
|
188
|
+
: `\`${escCode(t.ownTerms.join(", "))}\`` +
|
|
189
|
+
(t.ownTermCount > t.ownTerms.length ? ` (+${t.ownTermCount - t.ownTerms.length})` : "");
|
|
190
|
+
// Blank when it has nothing to say, matching the terminal. A column of
|
|
191
|
+
// "ok" down every row is a value that never varies presented as a
|
|
192
|
+
// measurement, which is the thing renderFindPretty refuses to do.
|
|
193
|
+
const probe = t.rank === undefined ? "no description" : t.reachable ? "" : `rank ${t.rank}`;
|
|
194
|
+
lines.push(`| ${t.ownTermCount} | \`${escCode(t.name)}\` | ${own} | ${probe} |`);
|
|
195
|
+
}
|
|
196
|
+
if (byWorst.length > MD_ROWS) {
|
|
197
|
+
lines.push(`| | *(+${byWorst.length - MD_ROWS} more, see \`--json\`)* | | |`);
|
|
198
|
+
}
|
|
199
|
+
lines.push("");
|
|
200
|
+
lines.push("### findings");
|
|
201
|
+
lines.push("");
|
|
202
|
+
if (!findings.length)
|
|
203
|
+
lines.push("no findability findings.");
|
|
204
|
+
else
|
|
205
|
+
findingsTable(lines, findings);
|
|
206
|
+
lines.push("");
|
|
207
|
+
// A blank line between blockquotes, or CommonMark lazily continues them into
|
|
208
|
+
// one paragraph and five separate notes render as one wall of prose.
|
|
209
|
+
for (const note of f.notes) {
|
|
210
|
+
lines.push(`> ${escText(note)}`);
|
|
211
|
+
lines.push("");
|
|
212
|
+
}
|
|
213
|
+
lines.push(`<sub>efaimo v${VERSION}</sub>`);
|
|
214
|
+
return lines.join("\n");
|
|
215
|
+
}
|
|
118
216
|
export function renderDiffMarkdown(d) {
|
|
119
217
|
const sign = d.delta >= 0 ? "+" : "";
|
|
120
218
|
const lines = [];
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { CheckReport, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
|
|
1
|
+
import type { CheckReport, Finding, FindResult, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
|
|
2
2
|
import type { CheckSkillResult } from "../check/check.js";
|
|
3
3
|
import type { Scenario, TestReport } from "../testing/harness.js";
|
|
4
4
|
import type { WeighDiff } from "../weigh/diff.js";
|
|
@@ -9,6 +9,17 @@ export declare function renderTestReportPretty(r: TestReport): string;
|
|
|
9
9
|
export declare function renderSkillSetPretty(res: CheckSkillResult): string;
|
|
10
10
|
export declare function renderServerWeighPretty(w: ServerWeighResult): string;
|
|
11
11
|
export declare function renderSkillWeighPretty(w: SkillSetWeighResult): string;
|
|
12
|
+
/**
|
|
13
|
+
* `efaimo find`.
|
|
14
|
+
*
|
|
15
|
+
* Two numbers, both as fractions with their denominators printed, neither a
|
|
16
|
+
* letter. The first is a property of the catalog; the second is a simulation
|
|
17
|
+
* and is labelled as one on the line itself, because a saturated 100% next to
|
|
18
|
+
* a real measurement reads like a good score unless the page says otherwise.
|
|
19
|
+
*/
|
|
20
|
+
export declare function renderFindPretty(f: FindResult): string;
|
|
21
|
+
/** The findings half of `efaimo find`, printed under the table. */
|
|
22
|
+
export declare function renderFindFindingsPretty(f: FindResult, findings: Finding[]): string;
|
|
12
23
|
export declare function renderDiffPretty(d: WeighDiff, opts: {
|
|
13
24
|
maxTokens?: number;
|
|
14
25
|
allowIncreasePct?: number;
|
package/dist/reporters/pretty.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import pc from "picocolors";
|
|
2
2
|
import { formatWindowShare } from "../weigh/window.js";
|
|
3
3
|
import { sortFindings } from "../core/grade.js";
|
|
4
|
+
import { minimumDetectableDelta } from "../testing/stats.js";
|
|
4
5
|
import { VERSION } from "../version.js";
|
|
5
6
|
import { safeText } from "../util/safeText.js";
|
|
6
7
|
const DOCS_RULES = "https://github.com/efaimo-ai/efaimo/blob/main/docs/RULES.md";
|
|
@@ -32,8 +33,23 @@ function gradeColor(letter, text) {
|
|
|
32
33
|
function n(x) {
|
|
33
34
|
return x.toLocaleString("en-US");
|
|
34
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* A value that has to stay inside one line of a table or one line of a
|
|
38
|
+
* finding.
|
|
39
|
+
*
|
|
40
|
+
* `safeText` strips control characters but deliberately keeps newlines,
|
|
41
|
+
* because a detail block is meant to be multi-line. That is the wrong default
|
|
42
|
+
* for a single cell: a tool named
|
|
43
|
+
* `nl_tool\n 9 forged_row weather, forecast` renders as a SECOND
|
|
44
|
+
* row byte-identical in shape to a real one, and under `--no-color`, which is
|
|
45
|
+
* how CI logs read, there is no tell at all. The audited thing must not be
|
|
46
|
+
* able to write rows in the auditor's table.
|
|
47
|
+
*/
|
|
48
|
+
function cell(s) {
|
|
49
|
+
return safeText(s).replace(/[\r\n]+/g, " ");
|
|
50
|
+
}
|
|
35
51
|
function renderFinding(lines, f) {
|
|
36
|
-
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${
|
|
52
|
+
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${cell(f.message)}`);
|
|
37
53
|
if (f.detail) {
|
|
38
54
|
for (const d of safeText(f.detail).split("\n"))
|
|
39
55
|
lines.push(paint(pc.dim, ` ${d}`));
|
|
@@ -80,13 +96,23 @@ export function renderCheckPretty(report) {
|
|
|
80
96
|
export function renderScenarioPlan(s) {
|
|
81
97
|
const calls = s.trials * 2 * 2;
|
|
82
98
|
const firstLine = s.task.split("\n")[0].slice(0, 60);
|
|
99
|
+
const mde = minimumDetectableDelta(s.trials);
|
|
83
100
|
return [
|
|
84
101
|
paint(pc.dim, `efaimo v${VERSION}`),
|
|
85
102
|
`test (dry run) ${paint(pc.bold, s.name)}`,
|
|
86
103
|
"",
|
|
87
104
|
` skill ${s.skillName}`,
|
|
88
105
|
` model ${s.model}`,
|
|
106
|
+
` judge ${s.judgeModel}${s.judgeModel === s.model
|
|
107
|
+
? paint(pc.yellow, " (same model grades its own answers; set judge_model or --judge-model to separate them)")
|
|
108
|
+
: ""}`,
|
|
89
109
|
` plan ${s.trials} trials x 2 arms x (task + judge) = ${paint(pc.bold, String(calls))} API calls`,
|
|
110
|
+
// What this plan cannot detect, before any tokens are spent. A run that
|
|
111
|
+
// cannot reach significance produces a green that means nothing, and the
|
|
112
|
+
// only useful moment to learn that is now.
|
|
113
|
+
` power ${Number.isFinite(mde)
|
|
114
|
+
? `the smallest gap this size can call significant is ${paint(mde > 40 ? pc.yellow : pc.dim, `${mde.toFixed(0)} points`)}${mde > 40 ? paint(pc.yellow, " (raise trials)") : ""}`
|
|
115
|
+
: paint(pc.red, "no outcome at this size can reach p < 0.05; raise trials")}`,
|
|
90
116
|
` task ${paint(pc.dim, firstLine + (s.task.length > 60 ? "..." : ""))}`,
|
|
91
117
|
"",
|
|
92
118
|
paint(pc.yellow, "dry run: no API calls made. add --live to run it (Claude models need ANTHROPIC_API_KEY, GPT models need OPENAI_API_KEY)."),
|
|
@@ -98,7 +124,7 @@ export function renderTestReportPretty(r) {
|
|
|
98
124
|
const lines = [
|
|
99
125
|
paint(pc.dim, `efaimo v${VERSION}`),
|
|
100
126
|
`test ${paint(pc.bold, r.scenario)}`,
|
|
101
|
-
`skill ${r.skill} model ${r.model}`,
|
|
127
|
+
`skill ${r.skill} model ${r.model} judge ${r.judgeModel}${r.judgeModel === r.model ? paint(pc.dim, " (same model)") : ""}`,
|
|
102
128
|
"",
|
|
103
129
|
` with skill ${r.withSkill.passes}/${r.withSkill.scored} pass (${r.withSkill.passRate.toFixed(0)}%)`,
|
|
104
130
|
` without skill ${r.withoutSkill.passes}/${r.withoutSkill.scored} pass (${r.withoutSkill.passRate.toFixed(0)}%)`,
|
|
@@ -130,21 +156,21 @@ export function renderSkillSetPretty(res) {
|
|
|
130
156
|
for (const s of [...res.perSkill].sort((a, b) => a.report.grade.score - b.report.grade.score)) {
|
|
131
157
|
const g = s.report.grade;
|
|
132
158
|
const c = s.report.counts;
|
|
133
|
-
lines.push(` ${gradeColor(g.letter, `${g.letter} (${String(g.score).padStart(3)})`)} ${
|
|
159
|
+
lines.push(` ${gradeColor(g.letter, `${g.letter} (${String(g.score).padStart(3)})`)} ${cell(s.name).padEnd(nameW)} ` +
|
|
134
160
|
`${paint(pc.red, String(c.error))}e ${paint(pc.yellow, String(c.warn))}w ${paint(pc.cyan, String(c.info))}i`);
|
|
135
161
|
}
|
|
136
162
|
if (res.setFindings.length) {
|
|
137
163
|
lines.push("");
|
|
138
164
|
lines.push(paint(pc.bold, "across the set"));
|
|
139
165
|
for (const f of sortFindings(res.setFindings)) {
|
|
140
|
-
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${
|
|
166
|
+
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${cell(f.message)}`);
|
|
141
167
|
}
|
|
142
168
|
}
|
|
143
169
|
const flagged = res.perSkill.filter((s) => s.report.findings.length);
|
|
144
170
|
if (flagged.length) {
|
|
145
171
|
lines.push("");
|
|
146
172
|
for (const s of flagged) {
|
|
147
|
-
lines.push(paint(pc.bold, `${
|
|
173
|
+
lines.push(paint(pc.bold, `${cell(s.name)} ${gradeColor(s.report.grade.letter, s.report.grade.letter)}`));
|
|
148
174
|
for (const f of s.report.findings) {
|
|
149
175
|
lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
|
|
150
176
|
if (f.fixHint)
|
|
@@ -170,7 +196,7 @@ export function renderServerWeighPretty(w) {
|
|
|
170
196
|
if (w.instructionsTokens > 0)
|
|
171
197
|
lines.push(` server instructions ${n(w.instructionsTokens).padStart(4)}`);
|
|
172
198
|
if (w.anthropicExactTotal !== undefined) {
|
|
173
|
-
lines.push(` ${paint(pc.green, "anthropic exact")} ${n(w.anthropicExactTotal).padStart(8)} (count_tokens
|
|
199
|
+
lines.push(` ${paint(pc.green, "anthropic exact")} ${n(w.anthropicExactTotal).padStart(8)} (count_tokens, ${safeText(w.anthropicExactModel ?? "model not recorded")})`);
|
|
174
200
|
}
|
|
175
201
|
if (w.perTool.length) {
|
|
176
202
|
lines.push("");
|
|
@@ -183,9 +209,9 @@ export function renderServerWeighPretty(w) {
|
|
|
183
209
|
// column in its own flagship demo. Capped so one pathological name cannot
|
|
184
210
|
// push the numbers off a terminal.
|
|
185
211
|
const shown = w.perTool.slice(0, 8);
|
|
186
|
-
const toolW = Math.min(38, Math.max(20, ...shown.map((t) =>
|
|
212
|
+
const toolW = Math.min(38, Math.max(20, ...shown.map((t) => cell(t.name).length)));
|
|
187
213
|
for (const [i, t] of shown.entries()) {
|
|
188
|
-
lines.push(` ${String(i + 1).padStart(2)}. ${
|
|
214
|
+
lines.push(` ${String(i + 1).padStart(2)}. ${cell(t.name).padEnd(toolW)} ${n(t.tokens.claudeStyle).padStart(7)} desc ${n(t.descriptionTokens)} | schema ${n(t.schemaTokens)}`);
|
|
189
215
|
}
|
|
190
216
|
if (w.perTool.length > 8)
|
|
191
217
|
lines.push(paint(pc.dim, ` (+${w.perTool.length - 8} more)`));
|
|
@@ -205,7 +231,7 @@ export function renderSkillWeighPretty(w) {
|
|
|
205
231
|
lines.push("");
|
|
206
232
|
lines.push(` ${"skill".padEnd(28)} ${"metadata".padStart(8)} ${"body".padStart(9)} ${"lines".padStart(6)} refs`);
|
|
207
233
|
for (const s of w.perSkill) {
|
|
208
|
-
lines.push(` ${
|
|
234
|
+
lines.push(` ${cell(s.name).padEnd(28)} ${n(s.metadataTokens).padStart(8)} ${n(s.bodyTokens).padStart(9)} ${String(s.bodyLines).padStart(6)} ${s.refFileCount ? `${s.refFileCount} files ${n(s.refFileTokens)}` : "-"}`);
|
|
209
235
|
}
|
|
210
236
|
lines.push("");
|
|
211
237
|
lines.push(`totals: metadata ${paint(pc.bold, n(w.totals.metadata))} (always loaded) | body ${n(w.totals.body)} (on trigger) | referenced ${n(w.totals.refFiles)} (on demand)`);
|
|
@@ -214,6 +240,145 @@ export function renderSkillWeighPretty(w) {
|
|
|
214
240
|
lines.push(paint(pc.dim, `note: ${safeText(note)}`));
|
|
215
241
|
return lines.join("\n");
|
|
216
242
|
}
|
|
243
|
+
/**
|
|
244
|
+
* `efaimo find`.
|
|
245
|
+
*
|
|
246
|
+
* Two numbers, both as fractions with their denominators printed, neither a
|
|
247
|
+
* letter. The first is a property of the catalog; the second is a simulation
|
|
248
|
+
* and is labelled as one on the line itself, because a saturated 100% next to
|
|
249
|
+
* a real measurement reads like a good score unless the page says otherwise.
|
|
250
|
+
*/
|
|
251
|
+
export function renderFindPretty(f) {
|
|
252
|
+
const lines = [];
|
|
253
|
+
lines.push(paint(pc.dim, `efaimo v${VERSION}`));
|
|
254
|
+
lines.push(`find mcp ${paint(pc.bold, safeText(f.label))}`);
|
|
255
|
+
lines.push(paint(pc.dim, `tools ${f.toolCount} result window ${f.method.topK} BM25 k1=${f.method.bm25.k1} b=${f.method.bm25.b}, ${f.method.queryTerms} query terms`));
|
|
256
|
+
lines.push("");
|
|
257
|
+
// Coloured on the rule condition, not on invented cutoffs. An earlier
|
|
258
|
+
// version banded this at 95 and 80 percent, two numbers that appear in no
|
|
259
|
+
// rule, no doc and no ADR, inside the one command whose whole design note
|
|
260
|
+
// says it does not grade. Red means E141 fired; that is a threshold the
|
|
261
|
+
// ruleset already owns.
|
|
262
|
+
const dpct = f.distinct.pct;
|
|
263
|
+
const anyOwnsNothing = f.distinct.count < f.distinct.total;
|
|
264
|
+
const dcolor = f.distinctVacuous ? pc.dim : anyOwnsNothing ? pc.red : pc.green;
|
|
265
|
+
const dFrac = `${f.distinct.count}/${f.distinct.total} (${dpct}%)`;
|
|
266
|
+
const pFrac = `${f.probe.returned}/${f.probe.total} (${f.probe.pct}%)`;
|
|
267
|
+
// Pad on the plain text, never on the painted string: colour codes are bytes
|
|
268
|
+
// that padEnd counts and a terminal does not, which is how a coloured column
|
|
269
|
+
// ends up crooked in exactly the demo everyone screenshots.
|
|
270
|
+
const fracW = Math.max(dFrac.length, pFrac.length);
|
|
271
|
+
lines.push(`distinct ${paint(dcolor, dFrac)}${" ".repeat(fracW - dFrac.length)}` +
|
|
272
|
+
` ${paint(pc.dim, "tools that own a word no other tool has")}`);
|
|
273
|
+
if (f.distinctVacuous) {
|
|
274
|
+
lines.push(paint(pc.yellow, ` vacuous: a one-tool catalog has nothing to be distinct from, so this is 100% for any tool`));
|
|
275
|
+
}
|
|
276
|
+
lines.push(`probe ${paint(pc.dim, pFrac)}${" ".repeat(fracW - pFrac.length)}` +
|
|
277
|
+
` ${paint(pc.dim, "returned by a simulated search for their own description")}`);
|
|
278
|
+
if (f.windowCoversCatalog) {
|
|
279
|
+
lines.push(paint(pc.yellow, ` vacuous: ${f.toolCount} tools fit inside a window of ${f.method.topK}, so none can fall outside it`));
|
|
280
|
+
}
|
|
281
|
+
lines.push("");
|
|
282
|
+
// Worst first. A reader scanning the top of the table should be looking at
|
|
283
|
+
// the tools that cannot be told apart, not at an alphabetical list.
|
|
284
|
+
const byWorst = [...f.perTool].sort((a, b) => {
|
|
285
|
+
const ar = a.rank ?? Number.POSITIVE_INFINITY;
|
|
286
|
+
const br = b.rank ?? Number.POSITIVE_INFINITY;
|
|
287
|
+
return (a.ownTermCount - b.ownTermCount ||
|
|
288
|
+
br - ar ||
|
|
289
|
+
(a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
290
|
+
});
|
|
291
|
+
const shown = byWorst.slice(0, 15);
|
|
292
|
+
const nameW = Math.min(34, Math.max(12, ...shown.map((t) => cell(t.name).length)));
|
|
293
|
+
lines.push(paint(pc.dim, ` words ${"tool".padEnd(nameW)} vocabulary no other tool has`));
|
|
294
|
+
for (const t of shown) {
|
|
295
|
+
const count = t.ownTermCount === 0
|
|
296
|
+
? paint(pc.red, " 0")
|
|
297
|
+
: t.ownTermCount === 1
|
|
298
|
+
? paint(pc.yellow, " 1")
|
|
299
|
+
: paint(pc.green, String(t.ownTermCount).padStart(5));
|
|
300
|
+
let tail;
|
|
301
|
+
if (t.ownTermCount === 0) {
|
|
302
|
+
tail = paint(pc.red, t.sharedWith.length ? `none: every word also on ${cell(t.sharedWith.join(", "))}` : "none");
|
|
303
|
+
}
|
|
304
|
+
else {
|
|
305
|
+
const more = t.ownTermCount - t.ownTerms.length;
|
|
306
|
+
tail = paint(pc.dim, cell(t.ownTerms.join(", ")) + (more > 0 ? ` (+${more})` : ""));
|
|
307
|
+
}
|
|
308
|
+
// The probe only ever appears as a flag on a row it failed. A column of
|
|
309
|
+
// "1" down the whole table would be a number that never varies presented
|
|
310
|
+
// as if it were a measurement.
|
|
311
|
+
const probe = t.rank === undefined
|
|
312
|
+
? paint(pc.red, " [probe: no description]")
|
|
313
|
+
: t.reachable
|
|
314
|
+
? ""
|
|
315
|
+
: paint(pc.red, ` [probe: rank ${t.rank}]`);
|
|
316
|
+
lines.push(` ${count} ${cell(t.name).padEnd(nameW)} ${tail}${probe}`);
|
|
317
|
+
}
|
|
318
|
+
// Describe the hidden rows from the hidden rows, never from an assumption.
|
|
319
|
+
// This line used to read "(+N more, each owning at least one word)" without
|
|
320
|
+
// checking: on a catalog where all 18 tools owned nothing it printed that
|
|
321
|
+
// three lines under a `0/18` headline saying the opposite.
|
|
322
|
+
const hiddenRows = byWorst.slice(shown.length);
|
|
323
|
+
if (hiddenRows.length) {
|
|
324
|
+
const hiddenAtZero = hiddenRows.filter((t) => t.ownTermCount === 0).length;
|
|
325
|
+
lines.push(paint(pc.dim, ` (+${hiddenRows.length} more` +
|
|
326
|
+
(hiddenAtZero ? `, ${hiddenAtZero} of them owning no word` : ", each owning at least one word") +
|
|
327
|
+
`; --json for all of them)`));
|
|
328
|
+
}
|
|
329
|
+
return lines.join("\n");
|
|
330
|
+
}
|
|
331
|
+
/** The findings half of `efaimo find`, printed under the table. */
|
|
332
|
+
export function renderFindFindingsPretty(f, findings) {
|
|
333
|
+
const lines = [""];
|
|
334
|
+
if (!findings.length) {
|
|
335
|
+
lines.push(paint(pc.green, " no findability findings."));
|
|
336
|
+
}
|
|
337
|
+
else {
|
|
338
|
+
for (const x of sortFindings(findings))
|
|
339
|
+
renderFinding(lines, x);
|
|
340
|
+
}
|
|
341
|
+
lines.push("");
|
|
342
|
+
for (const note of f.notes) {
|
|
343
|
+
// The two vacuity notes are already printed inline, in yellow, on the row
|
|
344
|
+
// the reader is looking at. Repeating them here as prose was the same
|
|
345
|
+
// warning twice in one screen; the data still carries them for --json and
|
|
346
|
+
// --md, where there is no inline row to carry it.
|
|
347
|
+
if (note.startsWith("VACUOUS"))
|
|
348
|
+
continue;
|
|
349
|
+
for (const [i, line] of wrapNote(safeText(note)).entries()) {
|
|
350
|
+
lines.push(paint(pc.dim, i === 0 ? `note: ${line}` : ` ${line}`));
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
// What to do next. A clean run used to end on "no findability findings" and
|
|
354
|
+
// a rules URL, which tells a first-time reader nothing about how to keep it
|
|
355
|
+
// that way. `distinct` is named because it is the number that can fail at
|
|
356
|
+
// any catalog size; the probe cannot.
|
|
357
|
+
if (!f.distinctVacuous) {
|
|
358
|
+
lines.push(paint(pc.dim, `gate it: efaimo find "<server>" --min-distinct ${Math.floor(f.distinct.pct)}` +
|
|
359
|
+
(f.distinct.pct < 100 ? " (raise it as you fix the tools above)" : "")));
|
|
360
|
+
}
|
|
361
|
+
lines.push(paint(pc.dim, `rules: ${DOCS_RULES}`));
|
|
362
|
+
return lines.join("\n");
|
|
363
|
+
}
|
|
364
|
+
/** Soft-wrap a long note at ~92 columns on word boundaries, so terminals do not ragged-wrap mid-number. */
|
|
365
|
+
function wrapNote(s, width = 92) {
|
|
366
|
+
const out = [];
|
|
367
|
+
for (const paragraph of s.split("\n")) {
|
|
368
|
+
let line = "";
|
|
369
|
+
for (const word of paragraph.split(/\s+/)) {
|
|
370
|
+
if (line && line.length + 1 + word.length > width) {
|
|
371
|
+
out.push(line);
|
|
372
|
+
line = word;
|
|
373
|
+
}
|
|
374
|
+
else {
|
|
375
|
+
line = line ? `${line} ${word}` : word;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
out.push(line);
|
|
379
|
+
}
|
|
380
|
+
return out;
|
|
381
|
+
}
|
|
217
382
|
export function renderDiffPretty(d, opts) {
|
|
218
383
|
const lines = [];
|
|
219
384
|
const sign = d.delta >= 0 ? "+" : "";
|