efaimo 0.1.2 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/CHANGELOG.md +63 -1
  2. package/README.md +68 -6
  3. package/SKILL.md +14 -7
  4. package/dist/cli.js +162 -22
  5. package/dist/core/engine.d.ts +2 -1
  6. package/dist/core/engine.js +4 -0
  7. package/dist/core/types.d.ts +121 -0
  8. package/dist/find/bm25.d.ts +77 -0
  9. package/dist/find/bm25.js +107 -0
  10. package/dist/find/find.d.ts +120 -0
  11. package/dist/find/find.js +403 -0
  12. package/dist/find/tokenize.d.ts +64 -0
  13. package/dist/find/tokenize.js +116 -0
  14. package/dist/index.d.ts +6 -3
  15. package/dist/index.js +13 -3
  16. package/dist/reporters/json.d.ts +19 -1
  17. package/dist/reporters/json.js +13 -2
  18. package/dist/reporters/markdown.d.ts +2 -1
  19. package/dist/reporters/markdown.js +109 -11
  20. package/dist/reporters/pretty.d.ts +12 -1
  21. package/dist/reporters/pretty.js +174 -9
  22. package/dist/rules/find/index.d.ts +4 -0
  23. package/dist/rules/find/index.js +262 -0
  24. package/dist/rules/version.d.ts +26 -0
  25. package/dist/rules/version.js +26 -0
  26. package/dist/testing/anthropicRunner.d.ts +30 -0
  27. package/dist/testing/anthropicRunner.js +43 -8
  28. package/dist/testing/harness.d.ts +32 -1
  29. package/dist/testing/harness.js +28 -3
  30. package/dist/testing/stats.d.ts +16 -0
  31. package/dist/testing/stats.js +26 -0
  32. package/dist/version.d.ts +1 -1
  33. package/dist/version.js +1 -1
  34. package/dist/weigh/anthropicExact.d.ts +12 -1
  35. package/dist/weigh/anthropicExact.js +9 -1
  36. package/dist/weigh/weigh.js +10 -3
  37. package/docs/INTEGRATIONS.md +15 -1
  38. package/docs/METHODOLOGY.md +110 -6
  39. package/docs/RULES.md +40 -3
  40. package/examples/scenario.crm.yaml +8 -0
  41. package/examples/scenario.example.yaml +8 -0
  42. package/package.json +1 -1
@@ -1,10 +1,21 @@
1
1
  import { VERSION } from "../version.js";
2
- export function toJsonEnvelope(kind, data) {
2
+ import { RULES_VERSION } from "../rules/version.js";
3
+ export function toJsonEnvelope(kind, data, opts = {}) {
3
4
  return JSON.stringify({
4
5
  tool: "efaimo",
5
6
  version: VERSION,
7
+ // Which ruleset produced this. The tool version alone does not say it: a
8
+ // patch release can change what a rule fires on, and every published
9
+ // grade is only reproducible if the reader can tell which rules were
10
+ // being run. See src/rules/version.ts.
11
+ //
12
+ // Only where rules actually ran. `weigh` counts tokens and `test` runs
13
+ // trials; neither evaluates a rule, and stamping a ruleset onto their
14
+ // output would be a report asserting which rules produced it when none
15
+ // did.
16
+ ...(kind === "check" || kind === "find" ? { rulesVersion: RULES_VERSION } : {}),
6
17
  kind,
7
- generatedAt: new Date().toISOString(),
18
+ ...(opts.timestamp === false ? {} : { generatedAt: new Date().toISOString() }),
8
19
  data,
9
20
  }, null, 2);
10
21
  }
@@ -1,7 +1,8 @@
1
- import type { CheckReport, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
1
+ import type { CheckReport, Finding, FindResult, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
2
2
  import type { CheckSkillResult } from "../check/check.js";
3
3
  import type { WeighDiff } from "../weigh/diff.js";
4
4
  export declare function renderCheckMarkdown(report: CheckReport): string;
5
5
  export declare function renderSkillSetMarkdown(res: CheckSkillResult): string;
6
6
  export declare function renderWeighMarkdown(w: ServerWeighResult | SkillSetWeighResult): string;
7
+ export declare function renderFindMarkdown(f: FindResult, findings: Finding[]): string;
7
8
  export declare function renderDiffMarkdown(d: WeighDiff): string;
@@ -3,23 +3,57 @@ import { safeText } from "../util/safeText.js";
3
3
  const SEV_LABEL = { error: "🔴 error", warn: "🟡 warn", info: "🔵 info" };
4
4
  // Everything through here came from the audited target, and
5
5
  // docs/INTEGRATIONS.md recommends piping --md into $GITHUB_STEP_SUMMARY, so a
6
- // hostile tool name or description renders into a page the reader trusts.
7
- // Two things were missing: control characters (raw ESC bytes were written
8
- // through literally) and backticks (most call sites wrap the value in a code
9
- // span, so one backtick closes it and the rest of the name becomes markup).
10
- // The backtick is neutralised with a zero-width space rather than a backslash,
11
- // because a backslash is not an escape inside a code span.
12
- function esc(s) {
6
+ // hostile tool name or description renders into a page the reader trusts. An
7
+ // auditor's verdict must not be rewritable by the thing being audited.
8
+ //
9
+ // Two escapes that were here did not work, both found by adversarial review:
10
+ //
11
+ // `\|` was applied without escaping the backslash first. A name containing
12
+ // backslash-pipe became `\\|`, which GFM reads as an escaped backslash
13
+ // followed by a LIVE cell delimiter. Extra cells past the header count
14
+ // are dropped, so a hostile server could push the real verdict column
15
+ // off the end of the row and supply its own.
16
+ //
17
+ // `` ` `` was neutralised by putting a zero-width space BEFORE it, which
18
+ // changes nothing: a code span ends at the next backtick run whatever
19
+ // precedes it. The rest of the name then rendered as live markup, and a
20
+ // `[click](url)` in a tool name became a link in the summary. A
21
+ // backtick cannot be escaped inside a code span at all, so it is
22
+ // substituted rather than escaped; a tool whose name contains one is
23
+ // still recognisable and can no longer close the span.
24
+ function escCode(s) {
13
25
  return safeText(s)
26
+ .replace(/[\r\n]+/g, " ")
27
+ .replace(/\\/g, "\\\\")
14
28
  .replace(/\|/g, "\\|")
15
- .replace(/`/g, "​`")
16
- .replace(/\n/g, " ");
29
+ .replace(/`/g, "'");
17
30
  }
31
+ // For a cell that is NOT wrapped in a code span, where markdown and inline
32
+ // HTML are both live. Every value that reaches one of these should really be
33
+ // in a code span; this is the belt for the places that are not, such as rule
34
+ // messages, which embed a tool name mid-sentence.
35
+ function escText(s) {
36
+ return escCode(s)
37
+ .replace(/&/g, "&")
38
+ .replace(/</g, "&lt;")
39
+ .replace(/>/g, "&gt;")
40
+ .replace(/([[\]*_~])/g, "\\$1");
41
+ }
42
+ /** Backwards-compatible alias for the code-span form, which is what most call sites want. */
43
+ const esc = escCode;
18
44
  function findingsTable(lines, findings) {
19
45
  lines.push("| severity | rule | finding |");
20
46
  lines.push("|---|---|---|");
21
47
  for (const f of findings) {
22
- lines.push(`| ${SEV_LABEL[f.severity]} | ${f.ruleId} | ${esc(f.message)} |`);
48
+ // escText, not escCode: a message is prose, not a code span, so markdown
49
+ // and inline HTML are both live in this cell. The message embeds a tool
50
+ // name the target chose.
51
+ //
52
+ // detail and fixHint were dropped here, so E141's "every word it has also
53
+ // appears on: ..." and E142's "outranked by: ..." existed in the terminal
54
+ // and in JSON and silently vanished from the surface the CI docs recommend.
55
+ const extra = [f.detail, f.fixHint && `fix: ${f.fixHint}`].filter(Boolean).join(" ");
56
+ lines.push(`| ${SEV_LABEL[f.severity]} | ${f.ruleId} | ${escText(f.message)}${extra ? ` <br>${escText(extra)}` : ""} |`);
23
57
  }
24
58
  }
25
59
  export function renderCheckMarkdown(report) {
@@ -85,7 +119,10 @@ export function renderWeighMarkdown(w) {
85
119
  lines.push(`| Claude-style | ${w.totals.claudeStyle.toLocaleString("en-US")} |`);
86
120
  lines.push(`| OpenAI tools | ${w.totals.openaiTools.toLocaleString("en-US")} |`);
87
121
  if (w.anthropicExactTotal !== undefined) {
88
- lines.push(`| anthropic exact (API) | ${w.anthropicExactTotal.toLocaleString("en-US")} |`);
122
+ // Names the model, like the terminal does. Without it the same figure
123
+ // was reproducible from --json and from pretty output but not from --md,
124
+ // which is the surface the CI docs recommend piping into a summary.
125
+ lines.push(`| anthropic exact (count_tokens, ${escCode(w.anthropicExactModel ?? "model not recorded")}) | ${w.anthropicExactTotal.toLocaleString("en-US")} |`);
89
126
  }
90
127
  lines.push("");
91
128
  if (w.perTool.length) {
@@ -115,6 +152,67 @@ export function renderWeighMarkdown(w) {
115
152
  lines.push(`> ${esc(note)}`);
116
153
  return lines.join("\n");
117
154
  }
155
+ export function renderFindMarkdown(f, findings) {
156
+ const lines = [];
157
+ lines.push(`## efaimo find: ${f.distinct.count}/${f.distinct.total} tools own a distinct word`);
158
+ lines.push("");
159
+ lines.push(`target: \`${esc(f.label)}\``);
160
+ lines.push("");
161
+ lines.push(`**distinct ${f.distinct.count}/${f.distinct.total} (${f.distinct.pct}%)** tools that own a word no other tool has. ` +
162
+ `Probe ${f.probe.returned}/${f.probe.total} (${f.probe.pct}%) returned by a simulated search for their own description.`);
163
+ lines.push("");
164
+ if (f.distinctVacuous) {
165
+ lines.push(`> **Distinct is vacuous here.** A one-tool catalog has nothing to be distinct from.`);
166
+ lines.push("");
167
+ }
168
+ if (f.windowCoversCatalog) {
169
+ lines.push(`> **The probe is vacuous here.** ${f.toolCount} tools and a result window of ${f.method.topK}, so no tool can fall outside it.`);
170
+ lines.push("");
171
+ }
172
+ lines.push("| words | tool | vocabulary no other tool has | probe |");
173
+ lines.push("|---|---|---|---|");
174
+ const byWorst = [...f.perTool].sort((a, b) => {
175
+ const ar = a.rank ?? Number.POSITIVE_INFINITY;
176
+ const br = b.rank ?? Number.POSITIVE_INFINITY;
177
+ return a.ownTermCount - b.ownTermCount || br - ar || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0);
178
+ });
179
+ // Capped, like every other table here. Uncapped, a 3,000-tool catalog wrote
180
+ // 74 KB of rows and a large aggregate would blow GitHub's 1 MiB step-summary
181
+ // limit, failing the step it was meant to annotate.
182
+ const MD_ROWS = 50;
183
+ for (const t of byWorst.slice(0, MD_ROWS)) {
184
+ const own = t.ownTermCount === 0
185
+ ? t.sharedWith.length
186
+ ? `none, every word also on ${t.sharedWith.map((s) => `\`${escCode(s)}\``).join(", ")}`
187
+ : "none"
188
+ : `\`${escCode(t.ownTerms.join(", "))}\`` +
189
+ (t.ownTermCount > t.ownTerms.length ? ` (+${t.ownTermCount - t.ownTerms.length})` : "");
190
+ // Blank when it has nothing to say, matching the terminal. A column of
191
+ // "ok" down every row is a value that never varies presented as a
192
+ // measurement, which is the thing renderFindPretty refuses to do.
193
+ const probe = t.rank === undefined ? "no description" : t.reachable ? "" : `rank ${t.rank}`;
194
+ lines.push(`| ${t.ownTermCount} | \`${escCode(t.name)}\` | ${own} | ${probe} |`);
195
+ }
196
+ if (byWorst.length > MD_ROWS) {
197
+ lines.push(`| | *(+${byWorst.length - MD_ROWS} more, see \`--json\`)* | | |`);
198
+ }
199
+ lines.push("");
200
+ lines.push("### findings");
201
+ lines.push("");
202
+ if (!findings.length)
203
+ lines.push("no findability findings.");
204
+ else
205
+ findingsTable(lines, findings);
206
+ lines.push("");
207
+ // A blank line between blockquotes, or CommonMark lazily continues them into
208
+ // one paragraph and five separate notes render as one wall of prose.
209
+ for (const note of f.notes) {
210
+ lines.push(`> ${escText(note)}`);
211
+ lines.push("");
212
+ }
213
+ lines.push(`<sub>efaimo v${VERSION}</sub>`);
214
+ return lines.join("\n");
215
+ }
118
216
  export function renderDiffMarkdown(d) {
119
217
  const sign = d.delta >= 0 ? "+" : "";
120
218
  const lines = [];
@@ -1,4 +1,4 @@
1
- import type { CheckReport, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
1
+ import type { CheckReport, Finding, FindResult, ServerWeighResult, SkillSetWeighResult } from "../core/types.js";
2
2
  import type { CheckSkillResult } from "../check/check.js";
3
3
  import type { Scenario, TestReport } from "../testing/harness.js";
4
4
  import type { WeighDiff } from "../weigh/diff.js";
@@ -9,6 +9,17 @@ export declare function renderTestReportPretty(r: TestReport): string;
9
9
  export declare function renderSkillSetPretty(res: CheckSkillResult): string;
10
10
  export declare function renderServerWeighPretty(w: ServerWeighResult): string;
11
11
  export declare function renderSkillWeighPretty(w: SkillSetWeighResult): string;
12
+ /**
13
+ * `efaimo find`.
14
+ *
15
+ * Two numbers, both as fractions with their denominators printed, neither a
16
+ * letter. The first is a property of the catalog; the second is a simulation
17
+ * and is labelled as one on the line itself, because a saturated 100% next to
18
+ * a real measurement reads like a good score unless the page says otherwise.
19
+ */
20
+ export declare function renderFindPretty(f: FindResult): string;
21
+ /** The findings half of `efaimo find`, printed under the table. */
22
+ export declare function renderFindFindingsPretty(f: FindResult, findings: Finding[]): string;
12
23
  export declare function renderDiffPretty(d: WeighDiff, opts: {
13
24
  maxTokens?: number;
14
25
  allowIncreasePct?: number;
@@ -1,6 +1,7 @@
1
1
  import pc from "picocolors";
2
2
  import { formatWindowShare } from "../weigh/window.js";
3
3
  import { sortFindings } from "../core/grade.js";
4
+ import { minimumDetectableDelta } from "../testing/stats.js";
4
5
  import { VERSION } from "../version.js";
5
6
  import { safeText } from "../util/safeText.js";
6
7
  const DOCS_RULES = "https://github.com/efaimo-ai/efaimo/blob/main/docs/RULES.md";
@@ -32,8 +33,23 @@ function gradeColor(letter, text) {
32
33
  function n(x) {
33
34
  return x.toLocaleString("en-US");
34
35
  }
36
+ /**
37
+ * A value that has to stay inside one line of a table or one line of a
38
+ * finding.
39
+ *
40
+ * `safeText` strips control characters but deliberately keeps newlines,
41
+ * because a detail block is meant to be multi-line. That is the wrong default
42
+ * for a single cell: a tool named
43
+ * `nl_tool\n 9 forged_row weather, forecast` renders as a SECOND
44
+ * row byte-identical in shape to a real one, and under `--no-color`, which is
45
+ * how CI logs read, there is no tell at all. The audited thing must not be
46
+ * able to write rows in the auditor's table.
47
+ */
48
+ function cell(s) {
49
+ return safeText(s).replace(/[\r\n]+/g, " ");
50
+ }
35
51
  function renderFinding(lines, f) {
36
- lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
52
+ lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${cell(f.message)}`);
37
53
  if (f.detail) {
38
54
  for (const d of safeText(f.detail).split("\n"))
39
55
  lines.push(paint(pc.dim, ` ${d}`));
@@ -80,13 +96,23 @@ export function renderCheckPretty(report) {
80
96
  export function renderScenarioPlan(s) {
81
97
  const calls = s.trials * 2 * 2;
82
98
  const firstLine = s.task.split("\n")[0].slice(0, 60);
99
+ const mde = minimumDetectableDelta(s.trials);
83
100
  return [
84
101
  paint(pc.dim, `efaimo v${VERSION}`),
85
102
  `test (dry run) ${paint(pc.bold, s.name)}`,
86
103
  "",
87
104
  ` skill ${s.skillName}`,
88
105
  ` model ${s.model}`,
106
+ ` judge ${s.judgeModel}${s.judgeModel === s.model
107
+ ? paint(pc.yellow, " (same model grades its own answers; set judge_model or --judge-model to separate them)")
108
+ : ""}`,
89
109
  ` plan ${s.trials} trials x 2 arms x (task + judge) = ${paint(pc.bold, String(calls))} API calls`,
110
+ // What this plan cannot detect, before any tokens are spent. A run that
111
+ // cannot reach significance produces a green that means nothing, and the
112
+ // only useful moment to learn that is now.
113
+ ` power ${Number.isFinite(mde)
114
+ ? `the smallest gap this size can call significant is ${paint(mde > 40 ? pc.yellow : pc.dim, `${mde.toFixed(0)} points`)}${mde > 40 ? paint(pc.yellow, " (raise trials)") : ""}`
115
+ : paint(pc.red, "no outcome at this size can reach p < 0.05; raise trials")}`,
90
116
  ` task ${paint(pc.dim, firstLine + (s.task.length > 60 ? "..." : ""))}`,
91
117
  "",
92
118
  paint(pc.yellow, "dry run: no API calls made. add --live to run it (Claude models need ANTHROPIC_API_KEY, GPT models need OPENAI_API_KEY)."),
@@ -98,7 +124,7 @@ export function renderTestReportPretty(r) {
98
124
  const lines = [
99
125
  paint(pc.dim, `efaimo v${VERSION}`),
100
126
  `test ${paint(pc.bold, r.scenario)}`,
101
- `skill ${r.skill} model ${r.model}`,
127
+ `skill ${r.skill} model ${r.model} judge ${r.judgeModel}${r.judgeModel === r.model ? paint(pc.dim, " (same model)") : ""}`,
102
128
  "",
103
129
  ` with skill ${r.withSkill.passes}/${r.withSkill.scored} pass (${r.withSkill.passRate.toFixed(0)}%)`,
104
130
  ` without skill ${r.withoutSkill.passes}/${r.withoutSkill.scored} pass (${r.withoutSkill.passRate.toFixed(0)}%)`,
@@ -130,21 +156,21 @@ export function renderSkillSetPretty(res) {
130
156
  for (const s of [...res.perSkill].sort((a, b) => a.report.grade.score - b.report.grade.score)) {
131
157
  const g = s.report.grade;
132
158
  const c = s.report.counts;
133
- lines.push(` ${gradeColor(g.letter, `${g.letter} (${String(g.score).padStart(3)})`)} ${safeText(s.name).padEnd(nameW)} ` +
159
+ lines.push(` ${gradeColor(g.letter, `${g.letter} (${String(g.score).padStart(3)})`)} ${cell(s.name).padEnd(nameW)} ` +
134
160
  `${paint(pc.red, String(c.error))}e ${paint(pc.yellow, String(c.warn))}w ${paint(pc.cyan, String(c.info))}i`);
135
161
  }
136
162
  if (res.setFindings.length) {
137
163
  lines.push("");
138
164
  lines.push(paint(pc.bold, "across the set"));
139
165
  for (const f of sortFindings(res.setFindings)) {
140
- lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
166
+ lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${cell(f.message)}`);
141
167
  }
142
168
  }
143
169
  const flagged = res.perSkill.filter((s) => s.report.findings.length);
144
170
  if (flagged.length) {
145
171
  lines.push("");
146
172
  for (const s of flagged) {
147
- lines.push(paint(pc.bold, `${safeText(s.name)} ${gradeColor(s.report.grade.letter, s.report.grade.letter)}`));
173
+ lines.push(paint(pc.bold, `${cell(s.name)} ${gradeColor(s.report.grade.letter, s.report.grade.letter)}`));
148
174
  for (const f of s.report.findings) {
149
175
  lines.push(` ${sevGlyph(f)} ${paint(pc.bold, f.ruleId)} ${safeText(f.message)}`);
150
176
  if (f.fixHint)
@@ -170,7 +196,7 @@ export function renderServerWeighPretty(w) {
170
196
  if (w.instructionsTokens > 0)
171
197
  lines.push(` server instructions ${n(w.instructionsTokens).padStart(4)}`);
172
198
  if (w.anthropicExactTotal !== undefined) {
173
- lines.push(` ${paint(pc.green, "anthropic exact")} ${n(w.anthropicExactTotal).padStart(8)} (count_tokens API)`);
199
+ lines.push(` ${paint(pc.green, "anthropic exact")} ${n(w.anthropicExactTotal).padStart(8)} (count_tokens, ${safeText(w.anthropicExactModel ?? "model not recorded")})`);
174
200
  }
175
201
  if (w.perTool.length) {
176
202
  lines.push("");
@@ -183,9 +209,9 @@ export function renderServerWeighPretty(w) {
183
209
  // column in its own flagship demo. Capped so one pathological name cannot
184
210
  // push the numbers off a terminal.
185
211
  const shown = w.perTool.slice(0, 8);
186
- const toolW = Math.min(38, Math.max(20, ...shown.map((t) => safeText(t.name).length)));
212
+ const toolW = Math.min(38, Math.max(20, ...shown.map((t) => cell(t.name).length)));
187
213
  for (const [i, t] of shown.entries()) {
188
- lines.push(` ${String(i + 1).padStart(2)}. ${safeText(t.name).padEnd(toolW)} ${n(t.tokens.claudeStyle).padStart(7)} desc ${n(t.descriptionTokens)} | schema ${n(t.schemaTokens)}`);
214
+ lines.push(` ${String(i + 1).padStart(2)}. ${cell(t.name).padEnd(toolW)} ${n(t.tokens.claudeStyle).padStart(7)} desc ${n(t.descriptionTokens)} | schema ${n(t.schemaTokens)}`);
189
215
  }
190
216
  if (w.perTool.length > 8)
191
217
  lines.push(paint(pc.dim, ` (+${w.perTool.length - 8} more)`));
@@ -205,7 +231,7 @@ export function renderSkillWeighPretty(w) {
205
231
  lines.push("");
206
232
  lines.push(` ${"skill".padEnd(28)} ${"metadata".padStart(8)} ${"body".padStart(9)} ${"lines".padStart(6)} refs`);
207
233
  for (const s of w.perSkill) {
208
- lines.push(` ${safeText(s.name).padEnd(28)} ${n(s.metadataTokens).padStart(8)} ${n(s.bodyTokens).padStart(9)} ${String(s.bodyLines).padStart(6)} ${s.refFileCount ? `${s.refFileCount} files ${n(s.refFileTokens)}` : "-"}`);
234
+ lines.push(` ${cell(s.name).padEnd(28)} ${n(s.metadataTokens).padStart(8)} ${n(s.bodyTokens).padStart(9)} ${String(s.bodyLines).padStart(6)} ${s.refFileCount ? `${s.refFileCount} files ${n(s.refFileTokens)}` : "-"}`);
209
235
  }
210
236
  lines.push("");
211
237
  lines.push(`totals: metadata ${paint(pc.bold, n(w.totals.metadata))} (always loaded) | body ${n(w.totals.body)} (on trigger) | referenced ${n(w.totals.refFiles)} (on demand)`);
@@ -214,6 +240,145 @@ export function renderSkillWeighPretty(w) {
214
240
  lines.push(paint(pc.dim, `note: ${safeText(note)}`));
215
241
  return lines.join("\n");
216
242
  }
243
+ /**
244
+ * `efaimo find`.
245
+ *
246
+ * Two numbers, both as fractions with their denominators printed, neither a
247
+ * letter. The first is a property of the catalog; the second is a simulation
248
+ * and is labelled as one on the line itself, because a saturated 100% next to
249
+ * a real measurement reads like a good score unless the page says otherwise.
250
+ */
251
+ export function renderFindPretty(f) {
252
+ const lines = [];
253
+ lines.push(paint(pc.dim, `efaimo v${VERSION}`));
254
+ lines.push(`find mcp ${paint(pc.bold, safeText(f.label))}`);
255
+ lines.push(paint(pc.dim, `tools ${f.toolCount} result window ${f.method.topK} BM25 k1=${f.method.bm25.k1} b=${f.method.bm25.b}, ${f.method.queryTerms} query terms`));
256
+ lines.push("");
257
+ // Coloured on the rule condition, not on invented cutoffs. An earlier
258
+ // version banded this at 95 and 80 percent, two numbers that appear in no
259
+ // rule, no doc and no ADR, inside the one command whose whole design note
260
+ // says it does not grade. Red means E141 fired; that is a threshold the
261
+ // ruleset already owns.
262
+ const dpct = f.distinct.pct;
263
+ const anyOwnsNothing = f.distinct.count < f.distinct.total;
264
+ const dcolor = f.distinctVacuous ? pc.dim : anyOwnsNothing ? pc.red : pc.green;
265
+ const dFrac = `${f.distinct.count}/${f.distinct.total} (${dpct}%)`;
266
+ const pFrac = `${f.probe.returned}/${f.probe.total} (${f.probe.pct}%)`;
267
+ // Pad on the plain text, never on the painted string: colour codes are bytes
268
+ // that padEnd counts and a terminal does not, which is how a coloured column
269
+ // ends up crooked in exactly the demo everyone screenshots.
270
+ const fracW = Math.max(dFrac.length, pFrac.length);
271
+ lines.push(`distinct ${paint(dcolor, dFrac)}${" ".repeat(fracW - dFrac.length)}` +
272
+ ` ${paint(pc.dim, "tools that own a word no other tool has")}`);
273
+ if (f.distinctVacuous) {
274
+ lines.push(paint(pc.yellow, ` vacuous: a one-tool catalog has nothing to be distinct from, so this is 100% for any tool`));
275
+ }
276
+ lines.push(`probe ${paint(pc.dim, pFrac)}${" ".repeat(fracW - pFrac.length)}` +
277
+ ` ${paint(pc.dim, "returned by a simulated search for their own description")}`);
278
+ if (f.windowCoversCatalog) {
279
+ lines.push(paint(pc.yellow, ` vacuous: ${f.toolCount} tools fit inside a window of ${f.method.topK}, so none can fall outside it`));
280
+ }
281
+ lines.push("");
282
+ // Worst first. A reader scanning the top of the table should be looking at
283
+ // the tools that cannot be told apart, not at an alphabetical list.
284
+ const byWorst = [...f.perTool].sort((a, b) => {
285
+ const ar = a.rank ?? Number.POSITIVE_INFINITY;
286
+ const br = b.rank ?? Number.POSITIVE_INFINITY;
287
+ return (a.ownTermCount - b.ownTermCount ||
288
+ br - ar ||
289
+ (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
290
+ });
291
+ const shown = byWorst.slice(0, 15);
292
+ const nameW = Math.min(34, Math.max(12, ...shown.map((t) => cell(t.name).length)));
293
+ lines.push(paint(pc.dim, ` words ${"tool".padEnd(nameW)} vocabulary no other tool has`));
294
+ for (const t of shown) {
295
+ const count = t.ownTermCount === 0
296
+ ? paint(pc.red, " 0")
297
+ : t.ownTermCount === 1
298
+ ? paint(pc.yellow, " 1")
299
+ : paint(pc.green, String(t.ownTermCount).padStart(5));
300
+ let tail;
301
+ if (t.ownTermCount === 0) {
302
+ tail = paint(pc.red, t.sharedWith.length ? `none: every word also on ${cell(t.sharedWith.join(", "))}` : "none");
303
+ }
304
+ else {
305
+ const more = t.ownTermCount - t.ownTerms.length;
306
+ tail = paint(pc.dim, cell(t.ownTerms.join(", ")) + (more > 0 ? ` (+${more})` : ""));
307
+ }
308
+ // The probe only ever appears as a flag on a row it failed. A column of
309
+ // "1" down the whole table would be a number that never varies presented
310
+ // as if it were a measurement.
311
+ const probe = t.rank === undefined
312
+ ? paint(pc.red, " [probe: no description]")
313
+ : t.reachable
314
+ ? ""
315
+ : paint(pc.red, ` [probe: rank ${t.rank}]`);
316
+ lines.push(` ${count} ${cell(t.name).padEnd(nameW)} ${tail}${probe}`);
317
+ }
318
+ // Describe the hidden rows from the hidden rows, never from an assumption.
319
+ // This line used to read "(+N more, each owning at least one word)" without
320
+ // checking: on a catalog where all 18 tools owned nothing it printed that
321
+ // three lines under a `0/18` headline saying the opposite.
322
+ const hiddenRows = byWorst.slice(shown.length);
323
+ if (hiddenRows.length) {
324
+ const hiddenAtZero = hiddenRows.filter((t) => t.ownTermCount === 0).length;
325
+ lines.push(paint(pc.dim, ` (+${hiddenRows.length} more` +
326
+ (hiddenAtZero ? `, ${hiddenAtZero} of them owning no word` : ", each owning at least one word") +
327
+ `; --json for all of them)`));
328
+ }
329
+ return lines.join("\n");
330
+ }
331
+ /** The findings half of `efaimo find`, printed under the table. */
332
+ export function renderFindFindingsPretty(f, findings) {
333
+ const lines = [""];
334
+ if (!findings.length) {
335
+ lines.push(paint(pc.green, " no findability findings."));
336
+ }
337
+ else {
338
+ for (const x of sortFindings(findings))
339
+ renderFinding(lines, x);
340
+ }
341
+ lines.push("");
342
+ for (const note of f.notes) {
343
+ // The two vacuity notes are already printed inline, in yellow, on the row
344
+ // the reader is looking at. Repeating them here as prose was the same
345
+ // warning twice in one screen; the data still carries them for --json and
346
+ // --md, where there is no inline row to carry it.
347
+ if (note.startsWith("VACUOUS"))
348
+ continue;
349
+ for (const [i, line] of wrapNote(safeText(note)).entries()) {
350
+ lines.push(paint(pc.dim, i === 0 ? `note: ${line}` : ` ${line}`));
351
+ }
352
+ }
353
+ // What to do next. A clean run used to end on "no findability findings" and
354
+ // a rules URL, which tells a first-time reader nothing about how to keep it
355
+ // that way. `distinct` is named because it is the number that can fail at
356
+ // any catalog size; the probe cannot.
357
+ if (!f.distinctVacuous) {
358
+ lines.push(paint(pc.dim, `gate it: efaimo find "<server>" --min-distinct ${Math.floor(f.distinct.pct)}` +
359
+ (f.distinct.pct < 100 ? " (raise it as you fix the tools above)" : "")));
360
+ }
361
+ lines.push(paint(pc.dim, `rules: ${DOCS_RULES}`));
362
+ return lines.join("\n");
363
+ }
364
+ /** Soft-wrap a long note at ~92 columns on word boundaries, so terminals do not ragged-wrap mid-number. */
365
+ function wrapNote(s, width = 92) {
366
+ const out = [];
367
+ for (const paragraph of s.split("\n")) {
368
+ let line = "";
369
+ for (const word of paragraph.split(/\s+/)) {
370
+ if (line && line.length + 1 + word.length > width) {
371
+ out.push(line);
372
+ line = word;
373
+ }
374
+ else {
375
+ line = line ? `${line} ${word}` : word;
376
+ }
377
+ }
378
+ out.push(line);
379
+ }
380
+ return out;
381
+ }
217
382
  export function renderDiffPretty(d, opts) {
218
383
  const lines = [];
219
384
  const sign = d.delta >= 0 ? "+" : "";
@@ -0,0 +1,4 @@
1
+ import type { FindRule } from "../../core/types.js";
2
+ export declare const FIND_RULES: FindRule[];
3
+ /** E141-E146 are findability rules: reported by `efaimo find`, never part of a `check` grade. */
4
+ export declare function isFindabilityRuleId(id: string): boolean;