@davesheffer/hunch 1.36.0 → 1.37.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/cli/index.js CHANGED
@@ -66,7 +66,8 @@ import { scaffoldProviders, regenerateGrounding, refreshExistingGrounding, refre
66
66
  import { healClaudeConfigCaseSplit } from "../integrations/claudeConfig.js";
67
67
  import { formatSearchHit, formatStructure } from "../core/format.js";
68
68
  import { isStateKind, renderStateLine, stateSupplements } from "../core/stateDelivery.js";
69
- import { taskSupplements } from "../core/taskDelivery.js";
69
+ import { taskSelectionSupplements } from "../core/taskDelivery.js";
70
+ import { buildTaskRankingQuery } from "../core/taskQuery.js";
70
71
  import { diagnoseIssueCorrectionStage, formatCorrectionStageDiagnostic } from "../core/correctionStage.js";
71
72
  import { compileVerifiedEvidenceMap, formatVerifiedEvidenceMap } from "../core/evidenceMap.js";
72
73
  import { collectCorrectionStageSources } from "../extractors/correctionSources.js";
@@ -3885,7 +3886,7 @@ program
3885
3886
  }
3886
3887
  else {
3887
3888
  console.log("");
3888
- renderPolicyEvaluations(policyResults).forEach((line) => console.log(line));
3889
+ renderPolicyEvaluations(policyResults, { compact: true }).forEach((line) => console.log(line));
3889
3890
  }
3890
3891
  }
3891
3892
  if (sarif) {
@@ -4211,7 +4212,7 @@ program
4211
4212
  decisionCorpus: store.recs("decisions"),
4212
4213
  historical: !!asOf,
4213
4214
  profile: opts.profile,
4214
- supplements: [...stateGrounding, ...(asOf ? [] : taskSupplements(store.tasksFor(target, 3), target))],
4215
+ supplements: [...stateGrounding, ...(asOf ? [] : taskSelectionSupplements(store.selectTasksFor(target, buildTaskRankingQuery(root, opts.task ?? null, target)), target))],
4215
4216
  });
4216
4217
  process.stdout.write(envelope.text);
4217
4218
  if (opts.task) {
@@ -4881,7 +4882,7 @@ program
4881
4882
  // from this file. No diff exists yet, so this is context — "don't re-add X" —
4882
4883
  // not a block; the commit-time `hunch check` does the actual gating.
4883
4884
  const retired = store.retiredForFile(target).filter((r) => r.symbols.length || r.deps.length);
4884
- const recentTasks = taskSupplements(store.tasksFor(target, 3), target);
4885
+ const recentTasks = taskSelectionSupplements(store.selectTasksFor(target, buildTaskRankingQuery(root, hookReportTaskId(root, provider, evt), target)), target);
4885
4886
  const hasContent = ctx.constraints.length ||
4886
4887
  ctx.decisions.length ||
4887
4888
  ctx.bugs.length ||
@@ -4918,19 +4919,27 @@ program
4918
4919
  // Delivery receipts (dec_925f4bcaad): the ledger of what actually reached
4919
4920
  // an agent. A full injection is a serve; a delta one-liner attests the
4920
4921
  // earlier serve is still standing. Never throws, never blocks.
4921
- const receipts = (event) => recordServed(root, envelope.delivered.map((item) => ({
4922
- event,
4923
- kind: item.kind,
4924
- record_id: item.record_id,
4925
- target,
4926
- session_id: evt.session_id,
4927
- rank: item.rank,
4928
- delivery_reason: item.delivery_reason,
4929
- provenance_status: item.provenance_status,
4930
- token_cost: item.token_cost,
4931
- delivery_profile: envelope.profile,
4932
- ranking_policy: envelope.ranking_policy,
4933
- })));
4922
+ const receipts = (event) => recordServed(root, [
4923
+ ...envelope.delivered.map((item) => ({
4924
+ event,
4925
+ kind: item.kind,
4926
+ record_id: item.record_id,
4927
+ target,
4928
+ session_id: evt.session_id,
4929
+ rank: item.rank,
4930
+ delivery_reason: item.delivery_reason,
4931
+ provenance_status: item.provenance_status,
4932
+ token_cost: item.token_cost,
4933
+ delivery_profile: envelope.profile,
4934
+ ranking_policy: envelope.ranking_policy,
4935
+ })),
4936
+ // Delivered task lines are receipts too: they feed access-based recency.
4937
+ ...envelope.supplements.filter((s) => s.kind === "recent-task" && s.delivered).map((s) => ({
4938
+ event, kind: "tasks", record_id: s.id, target, session_id: evt.session_id,
4939
+ rank: s.rank, delivery_reason: "supplemental", token_cost: s.token_cost,
4940
+ delivery_profile: envelope.profile, ranking_policy: envelope.ranking_policy,
4941
+ })),
4942
+ ]);
4934
4943
  const reportTaskId = hookReportTaskId(root, provider, evt);
4935
4944
  // A new authoritative prompt gets its own full delivery. An earlier
4936
4945
  // prompt's session-level delta cannot establish this task's receipt.
@@ -8,6 +8,8 @@ import { renderTaskReport, writeTaskReportHtml } from "../core/taskReportRender.
8
8
  import { assertReportPath } from "../core/taskReportPaths.js";
9
9
  import { publicTaskReport } from "../core/taskReportPublic.js";
10
10
  import { mergeDurableTaskSummaries, persistTaskRecord } from "../core/taskRecord.js";
11
+ import { evaluateTaskRanking, renderRankEval } from "../core/taskRankEval.js";
12
+ import { taskRecordStats } from "../core/taskRecordStats.js";
11
13
  export function registerTaskReportCommands(program, openStore) {
12
14
  const task = program.command("task").description("Record an explicit task lifecycle for Hunch contribution reports");
13
15
  task.command("start <title>").option("--id <id>", "retry an exact existing task identity")
@@ -106,6 +108,37 @@ export function registerTaskReportCommands(program, openStore) {
106
108
  console.log(` memory saved ${stats.with_save} ${pct(stats.with_save)}`);
107
109
  console.log(` edit denied ${stats.with_refusal} ${pct(stats.with_refusal)}`);
108
110
  console.log(` nothing observed ${stats.empty} ${pct(stats.empty)}`);
111
+ // Graph-record proxies (dec_66925aa0ee): do agents redo verified work, or repeat a violation?
112
+ try {
113
+ const opened = openStore();
114
+ try {
115
+ const rs = taskRecordStats(opened.store.recs("tasks"));
116
+ const rate = (r, n) => r === null ? "–" : `${Math.round(r * 100)}% of ${n}`;
117
+ console.log(`Graph task records: ${rs.records}`);
118
+ console.log(` re-verified an earlier check (24h) ${rate(rs.reverification_rate, rs.reverify_candidates)}`);
119
+ console.log(` repeated an earlier violation ${rate(rs.repeat_violation_rate, rs.violation_candidates)}`);
120
+ }
121
+ finally {
122
+ opened.store.close();
123
+ }
124
+ }
125
+ catch { /* no store: ledger stats only */ }
126
+ });
127
+ task.command("rank-eval").description("Offline leave-one-out check of task-record ranking against 'latest 3 on the file' (Hit@5, MRR, paired bootstrap CI); the pre-registered metric behind dec_66925aa0ee")
128
+ .option("--since <days>", "only task records finished in the last N days", "365")
129
+ .option("--split <fraction>", "evaluate the newest fraction of cases (temporal split)", "0.3")
130
+ .option("--json", "machine-readable report")
131
+ .action((opts) => {
132
+ const { store } = openStore();
133
+ try {
134
+ const cutoff = Date.now() - (Number(opts.since) || 365) * 86_400_000;
135
+ const records = store.recs("tasks").filter((r) => (Date.parse(r.finished_at) || 0) >= cutoff);
136
+ const report = evaluateTaskRanking(records, { split: Math.min(1, Math.max(0.05, Number(opts.split) || 0.3)) });
137
+ console.log(opts.json ? JSON.stringify(report, null, 2) : renderRankEval(report));
138
+ }
139
+ finally {
140
+ store.close();
141
+ }
109
142
  });
110
143
  task.command("status").description("One line for a terminal status line: the current prompt's task when Claude Code's status-line JSON arrives on stdin, otherwise the most recent task here")
111
144
  .option("--json", "machine-readable summary")
@@ -5,4 +5,10 @@
5
5
  * environmental reason read as one actionable block instead of ten
6
6
  * (fnd_b421b3f7ab); satisfied and violated policies always stay one per line. */
7
7
  import type { PolicyEvaluationSet } from "./service.js";
8
- export declare function renderPolicyEvaluations(results: PolicyEvaluationSet[]): string[];
8
+ export interface RenderPolicyOptions {
9
+ /** Commit-time rendering: a grouped non-evaluation is one line and satisfied
10
+ * receipts collapse to one line. Violations, blocks and gate errors always
11
+ * render in full. `hunch policy evaluate` keeps the complete form. */
12
+ compact?: boolean;
13
+ }
14
+ export declare function renderPolicyEvaluations(results: PolicyEvaluationSet[], options?: RenderPolicyOptions): string[];
@@ -7,10 +7,25 @@ function groupKey(r) {
7
7
  return `one ${r.policy.id}`;
8
8
  return `${r.policy.state} ${result} ${r.evaluation.explanation}`;
9
9
  }
10
- export function renderPolicyEvaluations(results) {
10
+ /** The cause of a grouped non-evaluation in a few words: the explanation up to
11
+ * its first parenthesis, semicolon or dash, so "no dependency snapshot cache
12
+ * exists on this machine (.hunch-cache/behavior-deps); executable behavior …"
13
+ * reads as its first clause. */
14
+ function shortCause(explanation) {
15
+ const cut = explanation.split(/[(;]| — /)[0].trim();
16
+ return cut.length > 110 ? `${cut.slice(0, 109).trimEnd()}…` : cut;
17
+ }
18
+ export function renderPolicyEvaluations(results, options = {}) {
11
19
  if (!results.length)
12
20
  return ["No Constitution policies matched."];
13
21
  const out = [`Constitution policy evaluation: ${results.length} canonical receipt(s)`];
22
+ if (options.compact) {
23
+ const satisfied = results.filter((r) => r.evaluation.result === "satisfied" && !r.blocks && !r.gate_error);
24
+ if (satisfied.length > 1) {
25
+ out.push(` ✅ ${satisfied.length} policies satisfied: ${satisfied.map((r) => r.policy.id).join(", ")}`);
26
+ results = results.filter((r) => !satisfied.includes(r));
27
+ }
28
+ }
14
29
  const groups = new Map();
15
30
  for (const r of results) {
16
31
  const key = groupKey(r);
@@ -29,6 +44,11 @@ export function renderPolicyEvaluations(results) {
29
44
  rendered.add(member);
30
45
  const ids = members.map((m) => m.policy.id);
31
46
  const receipts = members.map((m) => `${m.policy.id}=${m.evaluation.deterministic_hash.slice(0, 17)}`);
47
+ if (options.compact) {
48
+ // One line per cause at commit time; the ids and receipts are one command away.
49
+ out.push(` ${icon} ${members.length} policies [${r.policy.state}] ${r.evaluation.result} — ${shortCause(r.evaluation.explanation)} · hunch policy evaluate for ids and receipts`);
50
+ continue;
51
+ }
32
52
  out.push(` ${icon} ${members.length} policies [${r.policy.state}] ${r.evaluation.result} — same cause`);
33
53
  out.push(` ${r.evaluation.explanation}`);
34
54
  out.push(` policies: ${ids.join(", ")}`);
@@ -0,0 +1,15 @@
1
+ import type { CochangeStrength } from "./taskRanking.js";
2
+ export interface CochangeOptions {
3
+ /** Commits touching the target to inspect (newest first). */
4
+ maxCommits?: number;
5
+ /** Commits touching more files than this are bulk moves, not coupling. */
6
+ maxFilesPerCommit?: number;
7
+ timeoutMs?: number;
8
+ /** Set false to bypass the .hunch-cache read/write (tests). */
9
+ cache?: boolean;
10
+ }
11
+ /** Parse `git log --format=%H --name-only` output into per-commit file lists. */
12
+ export declare function parseNameOnlyLog(output: string): string[][];
13
+ /** Co-change strength for every file that changed with `file`. */
14
+ export declare function computeCochange(commits: readonly (readonly string[])[], file: string, maxFilesPerCommit: number): Map<string, CochangeStrength>;
15
+ export declare function cochangeFor(root: string, file: string, options?: CochangeOptions): Map<string, CochangeStrength>;
@@ -0,0 +1,113 @@
1
+ /** Co-change (evolutionary coupling) for one file, from git history.
2
+ *
3
+ * Files that changed together with the target in past commits are a
4
+ * deterministic proximity signal (Zimmermann et al. ROSE; Ying et al.): a
5
+ * task that touched such a file is about the same place even when the paths
6
+ * differ. Bounded on purpose: the last N commits touching the target, bulk
7
+ * commits ignored, a hard time limit, and a per-HEAD cache under
8
+ * .hunch-cache so the pre-edit hook never pays twice. Any failure yields an
9
+ * empty map — the ranking term goes to zero, nothing throws. */
10
+ import { execFileSync } from "node:child_process";
11
+ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
12
+ import { createHash } from "node:crypto";
13
+ import { join } from "node:path";
14
+ function gitEnv() {
15
+ const env = {};
16
+ for (const [k, v] of Object.entries(process.env))
17
+ if (!k.startsWith("GIT_"))
18
+ env[k] = v;
19
+ env.GIT_OPTIONAL_LOCKS = "0";
20
+ env.GIT_TERMINAL_PROMPT = "0";
21
+ return env;
22
+ }
23
+ function headSha(root, env, timeoutMs) {
24
+ try {
25
+ return execFileSync("git", ["-C", root, "rev-parse", "HEAD"], { env, timeout: timeoutMs, encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] }).trim() || null;
26
+ }
27
+ catch {
28
+ return null;
29
+ }
30
+ }
31
+ /** Parse `git log --format=%H --name-only` output into per-commit file lists. */
32
+ export function parseNameOnlyLog(output) {
33
+ const commits = [];
34
+ let current = null;
35
+ for (const raw of output.split("\n")) {
36
+ const line = raw.trim();
37
+ if (!line)
38
+ continue;
39
+ if (/^[0-9a-f]{40}$/.test(line)) {
40
+ current = [];
41
+ commits.push(current);
42
+ continue;
43
+ }
44
+ current?.push(line.replace(/\\/g, "/"));
45
+ }
46
+ return commits;
47
+ }
48
+ /** Co-change strength for every file that changed with `file`. */
49
+ export function computeCochange(commits, file, maxFilesPerCommit) {
50
+ const target = file.replace(/\\/g, "/");
51
+ const counts = new Map();
52
+ let touching = 0;
53
+ for (const files of commits) {
54
+ if (!files.includes(target) || files.length > maxFilesPerCommit)
55
+ continue;
56
+ touching++;
57
+ for (const f of files)
58
+ if (f !== target)
59
+ counts.set(f, (counts.get(f) ?? 0) + 1);
60
+ }
61
+ const out = new Map();
62
+ if (!touching)
63
+ return out;
64
+ for (const [f, count] of counts)
65
+ out.set(f, { count, strength: count / touching });
66
+ return out;
67
+ }
68
+ export function cochangeFor(root, file, options = {}) {
69
+ const maxCommits = options.maxCommits ?? 500;
70
+ const maxFiles = options.maxFilesPerCommit ?? 30;
71
+ const timeoutMs = options.timeoutMs ?? 2_000;
72
+ const target = file.replace(/\\/g, "/");
73
+ const env = gitEnv();
74
+ const head = headSha(root, env, timeoutMs);
75
+ if (!head)
76
+ return new Map();
77
+ const cacheDir = join(root, ".hunch-cache", "cochange");
78
+ const cacheFile = join(cacheDir, `${head.slice(0, 12)}-${createHash("sha256").update(`${target}\n${maxCommits}\n${maxFiles}`).digest("hex").slice(0, 16)}.json`);
79
+ if (options.cache !== false && existsSync(cacheFile)) {
80
+ try {
81
+ const parsed = JSON.parse(readFileSync(cacheFile, "utf8"));
82
+ return new Map(Object.entries(parsed));
83
+ }
84
+ catch { /* recompute */ }
85
+ }
86
+ // `git log -- <file> --name-only` lists only files matching the pathspec, so
87
+ // co-changed files would never appear. Two steps: the commits that touched the
88
+ // target, then each commit's complete file list.
89
+ let output;
90
+ try {
91
+ const shas = execFileSync("git", ["-C", root, "log", "--format=%H", "-n", String(maxCommits), "--", target], {
92
+ env, timeout: timeoutMs, encoding: "utf8", maxBuffer: 4_000_000, stdio: ["ignore", "pipe", "ignore"],
93
+ }).split(/\r?\n/).filter((line) => /^[0-9a-f]{40}$/.test(line));
94
+ if (!shas.length)
95
+ return new Map();
96
+ output = execFileSync("git", ["-C", root, "show", "--format=%H", "--name-only", "--no-renames", ...shas], {
97
+ env, timeout: timeoutMs, encoding: "utf8", maxBuffer: 16_000_000, stdio: ["ignore", "pipe", "ignore"],
98
+ });
99
+ }
100
+ catch {
101
+ return new Map();
102
+ }
103
+ const result = computeCochange(parseNameOnlyLog(output), target, maxFiles);
104
+ if (options.cache !== false) {
105
+ try {
106
+ mkdirSync(cacheDir, { recursive: true });
107
+ writeFileSync(cacheFile, JSON.stringify(Object.fromEntries(result)));
108
+ }
109
+ catch { /* cache is a convenience */ }
110
+ }
111
+ return result;
112
+ }
113
+ //# sourceMappingURL=cochange.js.map
@@ -7,9 +7,13 @@
7
7
  * share the brief's budget and are advisory: a task line is history, never a
8
8
  * rule, and never an instruction to repeat or skip anything. */
9
9
  import type { DeliverySupplement } from "./delivery.js";
10
+ import type { TaskSelection } from "./taskRanking.js";
10
11
  import type { TaskRecord } from "./types.js";
11
12
  export declare const TASK_SUPPLEMENT_LIMIT = 3;
12
13
  /** One bounded line for a task: identity, when, what reached it, what it did. */
13
14
  export declare function describeTaskRecord(t: TaskRecord): string;
15
+ /** Render a ranked, slotted selection (dec_66925aa0ee): one line per pick with
16
+ * its slot and the two strongest factual reasons. Empty selection → nothing. */
17
+ export declare function taskSelectionSupplements(selection: TaskSelection, target: string): DeliverySupplement[];
14
18
  /** Newest first, bounded. Empty input yields no supplement at all (no header noise). */
15
19
  export declare function taskSupplements(tasks: readonly TaskRecord[], target: string, limit?: number): DeliverySupplement[];
@@ -19,6 +19,36 @@ export function describeTaskRecord(t) {
19
19
  const files = t.files.length ? `files ${t.files.slice(0, 4).join(", ")}${t.files.length > 4 ? "…" : ""}` : null;
20
20
  return `${t.id} · ${when} · ${t.state} · "${clip(t.title, 80)}" — ${[lessons, applied, saved, check, violated, denied, files].filter(Boolean).join(" · ")}`;
21
21
  }
22
+ function summarizeRecord(t) {
23
+ const lessons = t.lessons.length ? `${t.lessons.length} lesson(s)` : "no memory delivered";
24
+ const applied = t.applied.length ? `applied ${t.applied.length}` : null;
25
+ const saved = t.saved.length ? `saved ${t.saved.length}` : null;
26
+ const last = t.checks.at(-1);
27
+ const check = last ? `check ${last.state}` : null;
28
+ return [lessons, applied, saved, check].filter(Boolean).join(", ");
29
+ }
30
+ const SLOT_LABEL = { latest: "latest ", violation: "problem ", relevant: "relevant" };
31
+ /** Render a ranked, slotted selection (dec_66925aa0ee): one line per pick with
32
+ * its slot and the two strongest factual reasons. Empty selection → nothing. */
33
+ export function taskSelectionSupplements(selection, target) {
34
+ if (!selection.picks.length)
35
+ return [];
36
+ const counts = { latest: 0, violation: 0, relevant: 0 };
37
+ for (const p of selection.picks)
38
+ counts[p.slot]++;
39
+ const parts = [counts.latest ? "latest" : null, counts.violation ? "problem" : null, counts.relevant ? `relevant ${counts.relevant}` : null].filter(Boolean).join(" · ");
40
+ return [
41
+ {
42
+ id: "recent-tasks", kind: "recent-tasks", priority: 415,
43
+ text: `RECENT TASKS on ${target} — ${parts} — earlier agent work here, from graph memory (advisory history, not rules): build on what was verified instead of redoing it blind.${selection.more > 0 ? ` ${selection.more} more: hunch task list ${target}.` : ""}`,
44
+ },
45
+ ...selection.picks.map((p, i) => {
46
+ const t = p.ranked.record;
47
+ const reasons = p.ranked.reasons.slice(0, 2).join(" · ");
48
+ return { id: t.id, kind: "recent-task", priority: 414 - i, text: `${SLOT_LABEL[p.slot]} ${t.id} · ${t.finished_at.slice(0, 10)} · "${clip(t.title, 80)}" — ${reasons} · ${summarizeRecord(t)}` };
49
+ }),
50
+ ];
51
+ }
22
52
  /** Newest first, bounded. Empty input yields no supplement at all (no header noise). */
23
53
  export function taskSupplements(tasks, target, limit = TASK_SUPPLEMENT_LIMIT) {
24
54
  const recent = [...tasks]
@@ -0,0 +1,7 @@
1
+ import { type RankingQuery } from "./taskRanking.js";
2
+ export interface TaskQueryOptions {
3
+ /** A phrase the caller has (hunch_context's target when it is not a path). */
4
+ phrase?: string | null;
5
+ now?: number;
6
+ }
7
+ export declare function buildTaskRankingQuery(root: string, taskId: string | null | undefined, target: string, options?: TaskQueryOptions): RankingQuery;
@@ -0,0 +1,44 @@
1
+ /** The agent's own context as a ranking query.
2
+ *
3
+ * What a task has already done is the best statement of what it is doing:
4
+ * the files context was delivered for, the rules and decisions it received,
5
+ * applied or saved, and its title when the repository opted into prompt
6
+ * titles. All of it is already in the local ledger; nothing new is stored and
7
+ * no prompt text is read. Without a task id the query is the target alone. */
8
+ import { readTaskReport } from "./taskReport.js";
9
+ import { targetLooksLikePath } from "./taskRecord.js";
10
+ import { normalizePath } from "./taskRanking.js";
11
+ const GENERIC_TITLES = new Set(["Assistant task", "Claude task"]);
12
+ export function buildTaskRankingQuery(root, taskId, target, options = {}) {
13
+ const now = options.now ?? Date.now();
14
+ const files = new Set();
15
+ const recordIds = new Set();
16
+ let phrase = options.phrase && !targetLooksLikePath(options.phrase) ? options.phrase : null;
17
+ if (targetLooksLikePath(target))
18
+ files.add(normalizePath(target));
19
+ else if (!phrase)
20
+ phrase = target;
21
+ if (taskId) {
22
+ try {
23
+ const report = readTaskReport(root, taskId);
24
+ for (const d of report.deliveries) {
25
+ if (d.target && targetLooksLikePath(d.target))
26
+ files.add(normalizePath(d.target));
27
+ for (const r of d.records)
28
+ recordIds.add(r.record_id);
29
+ }
30
+ for (const c of report.conformance)
31
+ for (const f of c.files)
32
+ files.add(normalizePath(f));
33
+ for (const c of report.claims)
34
+ recordIds.add(c.record_id);
35
+ for (const s of report.saves)
36
+ recordIds.add(s.record.record_id);
37
+ if (!GENERIC_TITLES.has(report.task.title))
38
+ phrase = phrase ?? report.task.title;
39
+ }
40
+ catch { /* an unreadable ledger degrades to a target-only query */ }
41
+ }
42
+ return { target: normalizePath(target), files, recordIds, phrase, now };
43
+ }
44
+ //# sourceMappingURL=taskQuery.js.map
@@ -0,0 +1,70 @@
1
+ /** Offline evaluation of task-record ranking (dec_66925aa0ee, PR B).
2
+ *
3
+ * Leave-one-out over finished task records: for each task T, rank the tasks
4
+ * that finished before it with a query built from T's own record, and ask
5
+ * whether the records T evidently needed appear in the five slots. Compared
6
+ * against the previous behaviour ("latest 3 on the file") with a paired
7
+ * bootstrap confidence interval. Deterministic: seeded resampling, no clock.
8
+ *
9
+ * Ground truth is what the record itself proves T used: an older task that
10
+ * received or saved a record T applied, that ran the same check T ran on an
11
+ * overlapping file, or whose violated rule T received. This is a proxy, not a
12
+ * label; the metric is pre-registered so the ranker cannot be tuned to it and
13
+ * then evaluated on the same window. */
14
+ import { type RankingContext, type RankingQuery, type RankingWeights } from "./taskRanking.js";
15
+ import type { TaskRecord } from "./types.js";
16
+ export declare const TASK_RANK_EVAL_SCHEMA: "hunch.task-rank-eval/1";
17
+ export interface EvalCase {
18
+ task: TaskRecord;
19
+ file: string;
20
+ older: TaskRecord[];
21
+ truth: Set<string>;
22
+ }
23
+ export interface RankerScore {
24
+ name: string;
25
+ hit5: number;
26
+ mrr: number;
27
+ hits: number[];
28
+ }
29
+ export interface RankEvalReport {
30
+ schema: typeof TASK_RANK_EVAL_SCHEMA;
31
+ tasks: number;
32
+ evaluable: number;
33
+ split: {
34
+ fraction: number;
35
+ evaluated: number;
36
+ note: string | null;
37
+ };
38
+ rankers: Array<Omit<RankerScore, "hits">>;
39
+ /** Paired difference ranked − baseline on Hit@5. */
40
+ delta_hit5: {
41
+ mean: number;
42
+ ci95: [number, number];
43
+ resamples: number;
44
+ };
45
+ verdict: "ranked-better" | "baseline-better" | "inconclusive" | "insufficient-data";
46
+ weights: RankingWeights;
47
+ }
48
+ /** What T evidently used from older tasks. */
49
+ export declare function groundTruth(task: TaskRecord, older: readonly TaskRecord[]): Set<string>;
50
+ /** T's own record as the query it would have had before editing. */
51
+ export declare function evalQuery(task: TaskRecord): RankingQuery;
52
+ /** A store-free context: IDF over the older set, lexical by token overlap. */
53
+ export declare function pureContext(older: readonly TaskRecord[], query: RankingQuery): RankingContext;
54
+ export declare function buildCases(records: readonly TaskRecord[]): EvalCase[];
55
+ export type Ranker = (c: EvalCase) => string[];
56
+ export declare function rankedRanker(weights?: Readonly<RankingWeights>): Ranker;
57
+ export declare const latest3Ranker: Ranker;
58
+ export declare function scoreRanker(name: string, ranker: Ranker, cases: readonly EvalCase[]): RankerScore;
59
+ export declare function pairedBootstrap(a: readonly number[], b: readonly number[], resamples?: number, seed?: number): {
60
+ mean: number;
61
+ ci95: [number, number];
62
+ };
63
+ export interface RankEvalOptions {
64
+ split?: number;
65
+ minEvaluated?: number;
66
+ resamples?: number;
67
+ weights?: Readonly<RankingWeights>;
68
+ }
69
+ export declare function evaluateTaskRanking(records: readonly TaskRecord[], options?: RankEvalOptions): RankEvalReport;
70
+ export declare function renderRankEval(report: RankEvalReport): string;