@davesheffer/hunch 1.36.0 → 1.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +26 -17
- package/dist/cli/taskReport.js +33 -0
- package/dist/constitution/renderEvaluations.d.ts +7 -1
- package/dist/constitution/renderEvaluations.js +21 -1
- package/dist/core/cochange.d.ts +15 -0
- package/dist/core/cochange.js +113 -0
- package/dist/core/taskDelivery.d.ts +4 -0
- package/dist/core/taskDelivery.js +30 -0
- package/dist/core/taskQuery.d.ts +7 -0
- package/dist/core/taskQuery.js +44 -0
- package/dist/core/taskRankEval.d.ts +70 -0
- package/dist/core/taskRankEval.js +195 -0
- package/dist/core/taskRanking.d.ts +113 -0
- package/dist/core/taskRanking.js +202 -0
- package/dist/core/taskRecord.d.ts +5 -0
- package/dist/core/taskRecord.js +30 -2
- package/dist/core/taskRecordStats.d.ts +14 -0
- package/dist/core/taskRecordStats.js +45 -0
- package/dist/core/types.d.ts +2 -0
- package/dist/core/types.js +1 -0
- package/dist/mcp/server.js +24 -15
- package/dist/store/hunchStore.d.ts +20 -0
- package/dist/store/hunchStore.js +110 -0
- package/package.json +1 -1
- package/server.json +2 -2
|
@@ -0,0 +1,195 @@
|
|
|
1
|
+
/** Offline evaluation of task-record ranking (dec_66925aa0ee, PR B).
|
|
2
|
+
*
|
|
3
|
+
* Leave-one-out over finished task records: for each task T, rank the tasks
|
|
4
|
+
* that finished before it with a query built from T's own record, and ask
|
|
5
|
+
* whether the records T evidently needed appear in the five slots. Compared
|
|
6
|
+
* against the previous behaviour ("latest 3 on the file") with a paired
|
|
7
|
+
* bootstrap confidence interval. Deterministic: seeded resampling, no clock.
|
|
8
|
+
*
|
|
9
|
+
* Ground truth is what the record itself proves T used: an older task that
|
|
10
|
+
* received or saved a record T applied, that ran the same check T ran on an
|
|
11
|
+
* overlapping file, or whose violated rule T received. This is a proxy, not a
|
|
12
|
+
* label; the metric is pre-registered so the ranker cannot be tuned to it and
|
|
13
|
+
* then evaluated on the same window. */
|
|
14
|
+
import { DEFAULT_WEIGHTS, normalizePath, rankTaskRecords, recordIdsOf, selectTaskSlots } from "./taskRanking.js";
|
|
15
|
+
export const TASK_RANK_EVAL_SCHEMA = "hunch.task-rank-eval/1";
|
|
16
|
+
const GENERIC_TITLES = new Set(["Assistant task", "Claude task"]);
|
|
17
|
+
function tokens(text) {
|
|
18
|
+
return new Set(text.toLowerCase().split(/[^a-z0-9_]+/).filter((t) => t.length > 2));
|
|
19
|
+
}
|
|
20
|
+
/** What T evidently used from older tasks. */
|
|
21
|
+
export function groundTruth(task, older) {
|
|
22
|
+
const applied = new Set(task.applied.map((a) => a.record_id));
|
|
23
|
+
const received = new Set(task.lessons.map((l) => l.record_id));
|
|
24
|
+
const labels = new Set(task.checks.map((c) => c.label));
|
|
25
|
+
const files = new Set(task.files.map(normalizePath));
|
|
26
|
+
const truth = new Set();
|
|
27
|
+
for (const o of older) {
|
|
28
|
+
const ids = recordIdsOf(o);
|
|
29
|
+
const overlap = o.files.some((f) => files.has(normalizePath(f)));
|
|
30
|
+
if ([...applied].some((id) => ids.has(id))) {
|
|
31
|
+
truth.add(o.id);
|
|
32
|
+
continue;
|
|
33
|
+
}
|
|
34
|
+
if (overlap && o.checks.some((c) => labels.has(c.label))) {
|
|
35
|
+
truth.add(o.id);
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
if (o.conformance.some((c) => c.outcome === "violated" && received.has(c.record_id))) {
|
|
39
|
+
truth.add(o.id);
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
return truth;
|
|
44
|
+
}
|
|
45
|
+
/** T's own record as the query it would have had before editing. */
|
|
46
|
+
export function evalQuery(task) {
|
|
47
|
+
const files = new Set(task.files.map(normalizePath));
|
|
48
|
+
return {
|
|
49
|
+
target: normalizePath(task.files[0] ?? task.title),
|
|
50
|
+
files,
|
|
51
|
+
recordIds: new Set(task.lessons.map((l) => l.record_id)),
|
|
52
|
+
phrase: GENERIC_TITLES.has(task.title) ? null : task.title,
|
|
53
|
+
now: Date.parse(task.finished_at) || 0,
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
/** A store-free context: IDF over the older set, lexical by token overlap. */
|
|
57
|
+
export function pureContext(older, query) {
|
|
58
|
+
const df = new Map();
|
|
59
|
+
for (const r of older)
|
|
60
|
+
for (const id of recordIdsOf(r))
|
|
61
|
+
df.set(id, (df.get(id) ?? 0) + 1);
|
|
62
|
+
const n = Math.max(1, older.length);
|
|
63
|
+
const lexical = new Map();
|
|
64
|
+
if (query.phrase) {
|
|
65
|
+
const q = tokens(query.phrase);
|
|
66
|
+
let best = 0;
|
|
67
|
+
const raw = new Map();
|
|
68
|
+
for (const r of older) {
|
|
69
|
+
const doc = tokens(`${r.title} ${r.lessons.map((l) => l.title).join(" ")}`);
|
|
70
|
+
let inter = 0;
|
|
71
|
+
for (const t of q)
|
|
72
|
+
if (doc.has(t))
|
|
73
|
+
inter++;
|
|
74
|
+
const score = q.size ? inter / q.size : 0;
|
|
75
|
+
if (score > 0) {
|
|
76
|
+
raw.set(r.id, score);
|
|
77
|
+
best = Math.max(best, score);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
for (const [id, s] of raw)
|
|
81
|
+
lexical.set(id, s / best);
|
|
82
|
+
}
|
|
83
|
+
return {
|
|
84
|
+
dependents: new Set(), cochange: new Map(), lexical, anchorsAlive: () => 1,
|
|
85
|
+
ruleStats: (id) => { const d = df.get(id) ?? 0; return { df: d, idf: Math.log((n + 1) / (d + 1)) + 1e-6 }; },
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
export function buildCases(records) {
|
|
89
|
+
const sorted = [...records].filter((r) => r.state === "completed").sort((a, b) => a.finished_at.localeCompare(b.finished_at) || a.id.localeCompare(b.id));
|
|
90
|
+
const cases = [];
|
|
91
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
92
|
+
const task = sorted[i];
|
|
93
|
+
if (!task.files.length)
|
|
94
|
+
continue;
|
|
95
|
+
const older = sorted.slice(0, i);
|
|
96
|
+
if (!older.length)
|
|
97
|
+
continue;
|
|
98
|
+
const truth = groundTruth(task, older);
|
|
99
|
+
if (!truth.size)
|
|
100
|
+
continue;
|
|
101
|
+
cases.push({ task, file: normalizePath(task.files[0]), older, truth });
|
|
102
|
+
}
|
|
103
|
+
return cases;
|
|
104
|
+
}
|
|
105
|
+
export function rankedRanker(weights = DEFAULT_WEIGHTS) {
|
|
106
|
+
return (c) => {
|
|
107
|
+
const q = evalQuery(c.task);
|
|
108
|
+
const ranked = rankTaskRecords(c.older, q, pureContext(c.older, q), weights);
|
|
109
|
+
return selectTaskSlots(ranked).picks.map((p) => p.ranked.record.id);
|
|
110
|
+
};
|
|
111
|
+
}
|
|
112
|
+
export const latest3Ranker = (c) => c.older.filter((r) => r.files.map(normalizePath).includes(c.file))
|
|
113
|
+
.sort((a, b) => b.finished_at.localeCompare(a.finished_at) || a.id.localeCompare(b.id))
|
|
114
|
+
.slice(0, 3).map((r) => r.id);
|
|
115
|
+
export function scoreRanker(name, ranker, cases) {
|
|
116
|
+
const hits = [];
|
|
117
|
+
let rr = 0;
|
|
118
|
+
for (const c of cases) {
|
|
119
|
+
const picks = ranker(c).slice(0, 5);
|
|
120
|
+
const rank = picks.findIndex((id) => c.truth.has(id));
|
|
121
|
+
hits.push(rank >= 0 ? 1 : 0);
|
|
122
|
+
if (rank >= 0)
|
|
123
|
+
rr += 1 / (rank + 1);
|
|
124
|
+
}
|
|
125
|
+
const n = cases.length || 1;
|
|
126
|
+
return { name, hit5: hits.reduce((s, h) => s + h, 0) / n, mrr: rr / n, hits };
|
|
127
|
+
}
|
|
128
|
+
/** xorshift32: deterministic resampling so two runs agree byte for byte. */
|
|
129
|
+
function prng(seed) {
|
|
130
|
+
let x = seed >>> 0 || 0x9e3779b9;
|
|
131
|
+
return () => { x ^= x << 13; x >>>= 0; x ^= x >>> 17; x ^= x << 5; x >>>= 0; return x / 0x1_0000_0000; };
|
|
132
|
+
}
|
|
133
|
+
export function pairedBootstrap(a, b, resamples = 1000, seed = 42) {
|
|
134
|
+
const n = a.length;
|
|
135
|
+
if (!n)
|
|
136
|
+
return { mean: 0, ci95: [0, 0] };
|
|
137
|
+
const diffs = a.map((x, i) => x - (b[i] ?? 0));
|
|
138
|
+
const mean = diffs.reduce((s, d) => s + d, 0) / n;
|
|
139
|
+
const rnd = prng(seed);
|
|
140
|
+
const means = [];
|
|
141
|
+
for (let r = 0; r < resamples; r++) {
|
|
142
|
+
let s = 0;
|
|
143
|
+
for (let i = 0; i < n; i++)
|
|
144
|
+
s += diffs[Math.floor(rnd() * n)];
|
|
145
|
+
means.push(s / n);
|
|
146
|
+
}
|
|
147
|
+
means.sort((x, y) => x - y);
|
|
148
|
+
const at = (p) => means[Math.min(means.length - 1, Math.max(0, Math.floor(p * (means.length - 1))))];
|
|
149
|
+
return { mean, ci95: [at(0.025), at(0.975)] };
|
|
150
|
+
}
|
|
151
|
+
export function evaluateTaskRanking(records, options = {}) {
|
|
152
|
+
const split = options.split ?? 0.3;
|
|
153
|
+
const minEvaluated = options.minEvaluated ?? 5;
|
|
154
|
+
const all = buildCases(records);
|
|
155
|
+
let cases = all.slice(Math.floor(all.length * (1 - split)));
|
|
156
|
+
let note = null;
|
|
157
|
+
if (cases.length < minEvaluated) {
|
|
158
|
+
cases = all;
|
|
159
|
+
note = `fewer than ${minEvaluated} cases in the newest ${Math.round(split * 100)}%; evaluated every case`;
|
|
160
|
+
}
|
|
161
|
+
const weights = { ...(options.weights ?? DEFAULT_WEIGHTS) };
|
|
162
|
+
const ranked = scoreRanker("ranked", rankedRanker(weights), cases);
|
|
163
|
+
const baseline = scoreRanker("latest3", latest3Ranker, cases);
|
|
164
|
+
const resamples = options.resamples ?? 1000;
|
|
165
|
+
const delta = pairedBootstrap(ranked.hits, baseline.hits, resamples);
|
|
166
|
+
let verdict = "inconclusive";
|
|
167
|
+
if (cases.length < minEvaluated)
|
|
168
|
+
verdict = "insufficient-data";
|
|
169
|
+
else if (delta.ci95[0] > 0)
|
|
170
|
+
verdict = "ranked-better";
|
|
171
|
+
else if (delta.ci95[1] < 0)
|
|
172
|
+
verdict = "baseline-better";
|
|
173
|
+
return {
|
|
174
|
+
schema: TASK_RANK_EVAL_SCHEMA,
|
|
175
|
+
tasks: records.length,
|
|
176
|
+
evaluable: all.length,
|
|
177
|
+
split: { fraction: split, evaluated: cases.length, note },
|
|
178
|
+
rankers: [ranked, baseline].map(({ hits: _h, ...rest }) => rest),
|
|
179
|
+
delta_hit5: { ...delta, resamples },
|
|
180
|
+
verdict,
|
|
181
|
+
weights,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
export function renderRankEval(report) {
|
|
185
|
+
const pct = (x) => `${(x * 100).toFixed(0)}%`;
|
|
186
|
+
const lines = [
|
|
187
|
+
`Task ranking evaluation — ${report.evaluable} evaluable of ${report.tasks} task record(s); evaluated ${report.split.evaluated}${report.split.note ? ` (${report.split.note})` : ` (newest ${Math.round(report.split.fraction * 100)}%)`}`,
|
|
188
|
+
];
|
|
189
|
+
for (const r of report.rankers)
|
|
190
|
+
lines.push(` ${r.name.padEnd(8)} Hit@5 ${pct(r.hit5).padStart(4)} MRR ${r.mrr.toFixed(2)}`);
|
|
191
|
+
lines.push(` Δ Hit@5 ranked − latest3: ${(report.delta_hit5.mean * 100).toFixed(0)} pts, 95% CI [${(report.delta_hit5.ci95[0] * 100).toFixed(0)}, ${(report.delta_hit5.ci95[1] * 100).toFixed(0)}] (${report.delta_hit5.resamples} resamples)`);
|
|
192
|
+
lines.push(` verdict: ${report.verdict}${report.verdict === "insufficient-data" ? " — keep collecting; the kill rule needs a CI that excludes zero" : ""}`);
|
|
193
|
+
return lines.join("\n");
|
|
194
|
+
}
|
|
195
|
+
//# sourceMappingURL=taskRankEval.js.map
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
/** Ranking of task records for pre-edit delivery (dec_66925aa0ee).
|
|
2
|
+
*
|
|
3
|
+
* Pure functions over plain data: no I/O, no clock, no model. The store gathers
|
|
4
|
+
* candidates and computes the corpus-level inputs (dependents, co-change, IDF,
|
|
5
|
+
* lexical scores, anchor liveness); this module gates, scores, slots and
|
|
6
|
+
* explains. Same inputs always give the same picks, so a selection is
|
|
7
|
+
* replayable and every line carries the reasons that produced it.
|
|
8
|
+
*
|
|
9
|
+
* Design, from the research synthesis: gate before rank (unfiltered injection
|
|
10
|
+
* is what lost in every negative result), structure over similarity, verified
|
|
11
|
+
* outcome as the importance proxy, recency as a decayed term with a floor and
|
|
12
|
+
* never a cutoff, convex weighted sum (not RRF, which degenerates on short
|
|
13
|
+
* lists), IDF-weighted overlap instead of Jaccard on tiny id sets, slots plus
|
|
14
|
+
* MMR instead of a flat top-k, one factual reason per pick. */
|
|
15
|
+
import type { TaskRecord } from "./types.js";
|
|
16
|
+
export interface RankingQuery {
|
|
17
|
+
/** File (posix path) or phrase the agent is working on. */
|
|
18
|
+
target: string;
|
|
19
|
+
/** Files this task has already touched (from its ledger). */
|
|
20
|
+
files: ReadonlySet<string>;
|
|
21
|
+
/** Record ids delivered, applied or saved in this task so far. */
|
|
22
|
+
recordIds: ReadonlySet<string>;
|
|
23
|
+
/** Task title (opt-in) or the hunch_context phrase; null when unknown. */
|
|
24
|
+
phrase: string | null;
|
|
25
|
+
/** Milliseconds since epoch; passed in so ranking is replayable. */
|
|
26
|
+
now: number;
|
|
27
|
+
}
|
|
28
|
+
export interface CochangeStrength {
|
|
29
|
+
/** Commits in which the file and the target changed together. */
|
|
30
|
+
count: number;
|
|
31
|
+
/** count / max(commits touching either), in [0, 1]. */
|
|
32
|
+
strength: number;
|
|
33
|
+
}
|
|
34
|
+
export interface RankingContext {
|
|
35
|
+
/** Files that depend on the target or share its component (posix paths). */
|
|
36
|
+
dependents: ReadonlySet<string>;
|
|
37
|
+
/** Co-change strength per file, for files that changed with the target. */
|
|
38
|
+
cochange: ReadonlyMap<string, CochangeStrength>;
|
|
39
|
+
/** Corpus statistics for a record id across all task records. */
|
|
40
|
+
ruleStats: (recordId: string) => {
|
|
41
|
+
idf: number;
|
|
42
|
+
df: number;
|
|
43
|
+
};
|
|
44
|
+
/** bm25 of the query phrase over task title + lesson titles, normalized to the top hit. */
|
|
45
|
+
lexical: ReadonlyMap<string, number>;
|
|
46
|
+
/** Fraction of a record's files that still exist; 1 when it names no files. */
|
|
47
|
+
anchorsAlive: (record: TaskRecord) => number;
|
|
48
|
+
/** Task ids some later record supersedes; never delivered. */
|
|
49
|
+
superseded?: ReadonlySet<string>;
|
|
50
|
+
/** Last time a task line was delivered (ms since epoch), when receipts know. */
|
|
51
|
+
lastDelivered?: (taskId: string) => number | null;
|
|
52
|
+
}
|
|
53
|
+
export interface RankingWeights {
|
|
54
|
+
file: number;
|
|
55
|
+
rules: number;
|
|
56
|
+
outcome: number;
|
|
57
|
+
lexical: number;
|
|
58
|
+
recency: number;
|
|
59
|
+
workingSet: number;
|
|
60
|
+
}
|
|
61
|
+
/** A prior, not a measurement. Changed only through `hunch task rank-eval`. */
|
|
62
|
+
export declare const DEFAULT_WEIGHTS: Readonly<RankingWeights>;
|
|
63
|
+
export declare const RECENCY_HALF_LIFE_DAYS = 30;
|
|
64
|
+
export declare const RECENCY_FLOOR = 0.1;
|
|
65
|
+
export declare const COCHANGE_MIN_COUNT = 2;
|
|
66
|
+
/** A shared record admits a candidate only when it is informative: present in
|
|
67
|
+
* at most half of all task records (idf ≥ ln 2). A rule every task receives
|
|
68
|
+
* says nothing about relatedness; it still contributes to the score, weakly. */
|
|
69
|
+
export declare const RULE_GATE_MIN_IDF: number;
|
|
70
|
+
export type RankingTerm = keyof RankingWeights;
|
|
71
|
+
export interface RankedTask {
|
|
72
|
+
record: TaskRecord;
|
|
73
|
+
score: number;
|
|
74
|
+
terms: Record<RankingTerm, number>;
|
|
75
|
+
/** Factual reasons, strongest first (weighted term contribution). */
|
|
76
|
+
reasons: string[];
|
|
77
|
+
anchorsAlive: number;
|
|
78
|
+
/** True when the record names the exact target file. */
|
|
79
|
+
exactFile: boolean;
|
|
80
|
+
/** True when a delivered rule was violated or a check failed in the record. */
|
|
81
|
+
problem: boolean;
|
|
82
|
+
}
|
|
83
|
+
export type SlotName = "latest" | "violation" | "relevant";
|
|
84
|
+
export interface SlotPick {
|
|
85
|
+
slot: SlotName;
|
|
86
|
+
ranked: RankedTask;
|
|
87
|
+
}
|
|
88
|
+
export interface TaskSelection {
|
|
89
|
+
picks: SlotPick[];
|
|
90
|
+
/** Ranked candidates not shown. */
|
|
91
|
+
more: number;
|
|
92
|
+
candidates: number;
|
|
93
|
+
}
|
|
94
|
+
export interface SlotOptions {
|
|
95
|
+
limit?: number;
|
|
96
|
+
relevant?: number;
|
|
97
|
+
lambda?: number;
|
|
98
|
+
}
|
|
99
|
+
export declare function normalizePath(path: string): string;
|
|
100
|
+
/** Every record id a task record references: lessons received, applications, saves. */
|
|
101
|
+
export declare function recordIdsOf(record: TaskRecord): Set<string>;
|
|
102
|
+
export declare function recencyTerm(record: TaskRecord, now: number, lastDelivered?: number | null): number;
|
|
103
|
+
export declare function outcomeTerm(record: TaskRecord): {
|
|
104
|
+
value: number;
|
|
105
|
+
reason: string | null;
|
|
106
|
+
};
|
|
107
|
+
/** Gate, score and explain one record. Null when the gate rejects it. */
|
|
108
|
+
export declare function rankTaskRecord(record: TaskRecord, query: RankingQuery, ctx: RankingContext, weights?: Readonly<RankingWeights>): RankedTask | null;
|
|
109
|
+
export declare function rankTaskRecords(records: readonly TaskRecord[], query: RankingQuery, ctx: RankingContext, weights?: Readonly<RankingWeights>): RankedTask[];
|
|
110
|
+
/** Jaccard over files ∪ record ids: the similarity MMR penalizes. */
|
|
111
|
+
export declare function taskSimilarity(a: TaskRecord, b: TaskRecord): number;
|
|
112
|
+
/** Slots: latest on the exact file, most recent problem, then relevant by MMR. */
|
|
113
|
+
export declare function selectTaskSlots(ranked: readonly RankedTask[], options?: SlotOptions): TaskSelection;
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
/** A prior, not a measurement. Changed only through `hunch task rank-eval`. */
|
|
2
|
+
export const DEFAULT_WEIGHTS = Object.freeze({
|
|
3
|
+
file: 0.30, rules: 0.25, outcome: 0.15, lexical: 0.10, recency: 0.10, workingSet: 0.10,
|
|
4
|
+
});
|
|
5
|
+
export const RECENCY_HALF_LIFE_DAYS = 30;
|
|
6
|
+
export const RECENCY_FLOOR = 0.1;
|
|
7
|
+
export const COCHANGE_MIN_COUNT = 2;
|
|
8
|
+
/** A shared record admits a candidate only when it is informative: present in
|
|
9
|
+
* at most half of all task records (idf ≥ ln 2). A rule every task receives
|
|
10
|
+
* says nothing about relatedness; it still contributes to the score, weakly. */
|
|
11
|
+
export const RULE_GATE_MIN_IDF = Math.log(2);
|
|
12
|
+
const DAY_MS = 86_400_000;
|
|
13
|
+
export function normalizePath(path) {
|
|
14
|
+
return path.trim().replace(/\\/g, "/").replace(/^\.\//, "");
|
|
15
|
+
}
|
|
16
|
+
/** Every record id a task record references: lessons received, applications, saves. */
|
|
17
|
+
export function recordIdsOf(record) {
|
|
18
|
+
const ids = new Set();
|
|
19
|
+
for (const l of record.lessons)
|
|
20
|
+
ids.add(l.record_id);
|
|
21
|
+
for (const a of record.applied)
|
|
22
|
+
ids.add(a.record_id);
|
|
23
|
+
for (const s of record.saved)
|
|
24
|
+
ids.add(s.record_id);
|
|
25
|
+
return ids;
|
|
26
|
+
}
|
|
27
|
+
function recordFiles(record) {
|
|
28
|
+
return record.files.map(normalizePath);
|
|
29
|
+
}
|
|
30
|
+
/** Age from the later of finishing and the last delivery: a record that keeps
|
|
31
|
+
* being delivered stays warm (access-based decay, as in Generative Agents). */
|
|
32
|
+
function ageDays(record, now, lastDelivered = null) {
|
|
33
|
+
const finished = Date.parse(record.finished_at);
|
|
34
|
+
const anchor = Math.max(Number.isFinite(finished) ? finished : Number.NEGATIVE_INFINITY, lastDelivered ?? Number.NEGATIVE_INFINITY);
|
|
35
|
+
if (!Number.isFinite(anchor))
|
|
36
|
+
return Number.POSITIVE_INFINITY;
|
|
37
|
+
return Math.max(0, (now - anchor) / DAY_MS);
|
|
38
|
+
}
|
|
39
|
+
export function recencyTerm(record, now, lastDelivered = null) {
|
|
40
|
+
const days = ageDays(record, now, lastDelivered);
|
|
41
|
+
if (!Number.isFinite(days))
|
|
42
|
+
return RECENCY_FLOOR;
|
|
43
|
+
return Math.max(RECENCY_FLOOR, Math.pow(0.5, days / RECENCY_HALF_LIFE_DAYS));
|
|
44
|
+
}
|
|
45
|
+
export function outcomeTerm(record) {
|
|
46
|
+
if (record.conformance.some((c) => c.outcome === "violated"))
|
|
47
|
+
return { value: 1, reason: "RULE VIOLATED" };
|
|
48
|
+
if (record.checks.some((c) => c.state === "failed" || c.state === "timed out"))
|
|
49
|
+
return { value: 0.8, reason: "check failed" };
|
|
50
|
+
if (record.saved.length)
|
|
51
|
+
return { value: 0.6, reason: `saved ${record.saved[0].record_id}${record.saved.length > 1 ? ` +${record.saved.length - 1}` : ""}` };
|
|
52
|
+
if (record.applied.some((a) => a.supported_by))
|
|
53
|
+
return { value: 0.5, reason: "applied a rule (rule-supported)" };
|
|
54
|
+
if (record.checks.some((c) => c.state === "passed"))
|
|
55
|
+
return { value: 0.3, reason: "check passed" };
|
|
56
|
+
return { value: 0, reason: null };
|
|
57
|
+
}
|
|
58
|
+
function humanAge(days) {
|
|
59
|
+
if (!Number.isFinite(days))
|
|
60
|
+
return "undated";
|
|
61
|
+
if (days < 1)
|
|
62
|
+
return "today";
|
|
63
|
+
if (days < 2)
|
|
64
|
+
return "yesterday";
|
|
65
|
+
return `${Math.floor(days)} days ago`;
|
|
66
|
+
}
|
|
67
|
+
function clip(text, max) {
|
|
68
|
+
return text.length <= max ? text : `${text.slice(0, max - 1).trimEnd()}…`;
|
|
69
|
+
}
|
|
70
|
+
/** Gate, score and explain one record. Null when the gate rejects it. */
|
|
71
|
+
export function rankTaskRecord(record, query, ctx, weights = DEFAULT_WEIGHTS) {
|
|
72
|
+
const target = normalizePath(query.target);
|
|
73
|
+
if (ctx.superseded?.has(record.id))
|
|
74
|
+
return null; // a later task verified over it
|
|
75
|
+
const files = recordFiles(record);
|
|
76
|
+
const alive = files.length ? ctx.anchorsAlive(record) : 1;
|
|
77
|
+
if (files.length && alive === 0)
|
|
78
|
+
return null; // nothing it names still exists
|
|
79
|
+
// --- file: same file 1, dependent / same component 0.5, co-change 0.3 × strength-ish
|
|
80
|
+
const exactFile = files.includes(target);
|
|
81
|
+
const dependent = !exactFile && files.some((f) => ctx.dependents.has(f));
|
|
82
|
+
let cochangeHit = null;
|
|
83
|
+
if (!exactFile && !dependent) {
|
|
84
|
+
for (const f of files) {
|
|
85
|
+
const c = ctx.cochange.get(f);
|
|
86
|
+
if (c && c.count >= COCHANGE_MIN_COUNT && (!cochangeHit || c.count > cochangeHit.count))
|
|
87
|
+
cochangeHit = { file: f, count: c.count };
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const fileValue = exactFile ? 1 : dependent ? 0.5 : cochangeHit ? 0.3 : 0;
|
|
91
|
+
// --- rules: IDF-weighted intersection over the current task's own ids
|
|
92
|
+
const ids = recordIdsOf(record);
|
|
93
|
+
const shared = [];
|
|
94
|
+
let queryMass = 0;
|
|
95
|
+
for (const id of query.recordIds) {
|
|
96
|
+
const s = ctx.ruleStats(id);
|
|
97
|
+
queryMass += s.idf;
|
|
98
|
+
if (ids.has(id))
|
|
99
|
+
shared.push({ id, ...s });
|
|
100
|
+
}
|
|
101
|
+
const rulesValue = queryMass > 0 ? Math.min(1, shared.reduce((sum, s) => sum + s.idf, 0) / queryMass) : 0;
|
|
102
|
+
// --- gate: structure or an informative shared rule; lexical/recency/outcome alone never admit
|
|
103
|
+
const informative = shared.some((s) => s.idf >= RULE_GATE_MIN_IDF);
|
|
104
|
+
if (fileValue === 0 && !informative)
|
|
105
|
+
return null;
|
|
106
|
+
const outcome = outcomeTerm(record);
|
|
107
|
+
const lexicalValue = Math.max(0, Math.min(1, ctx.lexical.get(record.id) ?? 0));
|
|
108
|
+
const lastDelivered = ctx.lastDelivered?.(record.id) ?? null;
|
|
109
|
+
const recencyValue = recencyTerm(record, query.now, lastDelivered);
|
|
110
|
+
const touched = files.filter((f) => query.files.has(f));
|
|
111
|
+
const workingSetValue = query.files.size ? Math.min(1, touched.length / query.files.size) : 0;
|
|
112
|
+
const terms = {
|
|
113
|
+
file: fileValue, rules: rulesValue, outcome: outcome.value, lexical: lexicalValue, recency: recencyValue, workingSet: workingSetValue,
|
|
114
|
+
};
|
|
115
|
+
const score = Object.keys(terms).reduce((sum, k) => sum + weights[k] * terms[k], 0);
|
|
116
|
+
// --- reasons, ordered by weighted contribution
|
|
117
|
+
const reasons = [];
|
|
118
|
+
if (exactFile)
|
|
119
|
+
reasons.push({ weight: weights.file * 1, text: "same file" });
|
|
120
|
+
else if (dependent)
|
|
121
|
+
reasons.push({ weight: weights.file * 0.5, text: `dependent of ${target}` });
|
|
122
|
+
else if (cochangeHit)
|
|
123
|
+
reasons.push({ weight: weights.file * 0.3, text: `co-changed with ${target} in ${cochangeHit.count} commits` });
|
|
124
|
+
if (shared.length) {
|
|
125
|
+
const top = [...shared].sort((a, b) => b.idf - a.idf)[0];
|
|
126
|
+
reasons.push({ weight: weights.rules * rulesValue, text: `shares ${top.id}${top.df ? ` (${top.df === 1 ? "only here" : `${top.df} tasks`})` : ""}${shared.length > 1 ? ` +${shared.length - 1}` : ""}` });
|
|
127
|
+
}
|
|
128
|
+
if (outcome.reason)
|
|
129
|
+
reasons.push({ weight: weights.outcome * outcome.value, text: outcome.reason });
|
|
130
|
+
if (lexicalValue > 0 && query.phrase)
|
|
131
|
+
reasons.push({ weight: weights.lexical * lexicalValue, text: `matches "${clip(query.phrase, 40)}"` });
|
|
132
|
+
if (touched.length)
|
|
133
|
+
reasons.push({ weight: weights.workingSet * workingSetValue, text: `also touched ${touched[0]} this task` });
|
|
134
|
+
const finishedAge = ageDays(record, query.now);
|
|
135
|
+
const effectiveAge = ageDays(record, query.now, lastDelivered);
|
|
136
|
+
reasons.push({ weight: weights.recency * recencyValue, text: effectiveAge < finishedAge ? `delivered ${humanAge(effectiveAge)}` : humanAge(finishedAge) });
|
|
137
|
+
if (alive < 1)
|
|
138
|
+
reasons.push({ weight: 0, text: "files since changed" });
|
|
139
|
+
reasons.sort((a, b) => b.weight - a.weight);
|
|
140
|
+
return {
|
|
141
|
+
record, score, terms, reasons: reasons.map((r) => r.text), anchorsAlive: alive, exactFile,
|
|
142
|
+
problem: outcome.value >= 0.8,
|
|
143
|
+
};
|
|
144
|
+
}
|
|
145
|
+
/** Newest first, then id — the tie order the research insists on. */
|
|
146
|
+
function newestFirst(a, b) {
|
|
147
|
+
return b.record.finished_at.localeCompare(a.record.finished_at) || a.record.id.localeCompare(b.record.id);
|
|
148
|
+
}
|
|
149
|
+
export function rankTaskRecords(records, query, ctx, weights = DEFAULT_WEIGHTS) {
|
|
150
|
+
const ranked = [];
|
|
151
|
+
for (const record of records) {
|
|
152
|
+
const r = rankTaskRecord(record, query, ctx, weights);
|
|
153
|
+
if (r)
|
|
154
|
+
ranked.push(r);
|
|
155
|
+
}
|
|
156
|
+
return ranked.sort((a, b) => (b.score - a.score) || newestFirst(a, b));
|
|
157
|
+
}
|
|
158
|
+
/** Jaccard over files ∪ record ids: the similarity MMR penalizes. */
|
|
159
|
+
export function taskSimilarity(a, b) {
|
|
160
|
+
const sa = new Set([...recordFiles(a), ...recordIdsOf(a)]);
|
|
161
|
+
const sb = new Set([...recordFiles(b), ...recordIdsOf(b)]);
|
|
162
|
+
if (!sa.size && !sb.size)
|
|
163
|
+
return 0;
|
|
164
|
+
let inter = 0;
|
|
165
|
+
for (const x of sa)
|
|
166
|
+
if (sb.has(x))
|
|
167
|
+
inter++;
|
|
168
|
+
return inter / (sa.size + sb.size - inter);
|
|
169
|
+
}
|
|
170
|
+
/** Slots: latest on the exact file, most recent problem, then relevant by MMR. */
|
|
171
|
+
export function selectTaskSlots(ranked, options = {}) {
|
|
172
|
+
const limit = Math.max(1, options.limit ?? 5);
|
|
173
|
+
const relevantMax = Math.max(0, options.relevant ?? 3);
|
|
174
|
+
const lambda = options.lambda ?? 0.7;
|
|
175
|
+
const picks = [];
|
|
176
|
+
const taken = new Set();
|
|
177
|
+
const take = (slot, r) => { picks.push({ slot, ranked: r }); taken.add(r.record.id); };
|
|
178
|
+
const latest = [...ranked].filter((r) => r.exactFile).sort(newestFirst)[0];
|
|
179
|
+
if (latest)
|
|
180
|
+
take("latest", latest);
|
|
181
|
+
const violation = [...ranked].filter((r) => r.problem && !taken.has(r.record.id)).sort(newestFirst)[0];
|
|
182
|
+
if (violation && picks.length < limit)
|
|
183
|
+
take("violation", violation);
|
|
184
|
+
let remaining = ranked.filter((r) => !taken.has(r.record.id));
|
|
185
|
+
while (picks.length < limit && picks.length - (latest ? 1 : 0) - (violation ? 1 : 0) < relevantMax && remaining.length) {
|
|
186
|
+
let best = null, bestValue = -Infinity;
|
|
187
|
+
for (const r of remaining) {
|
|
188
|
+
const redundancy = picks.length ? Math.max(...picks.map((p) => taskSimilarity(p.ranked.record, r.record))) : 0;
|
|
189
|
+
const value = lambda * r.score - (1 - lambda) * redundancy;
|
|
190
|
+
if (value > bestValue || (value === bestValue && best && newestFirst(r, best) < 0)) {
|
|
191
|
+
best = r;
|
|
192
|
+
bestValue = value;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
if (!best)
|
|
196
|
+
break;
|
|
197
|
+
take("relevant", best);
|
|
198
|
+
remaining = remaining.filter((r) => r !== best);
|
|
199
|
+
}
|
|
200
|
+
return { picks, more: ranked.length - picks.length, candidates: ranked.length };
|
|
201
|
+
}
|
|
202
|
+
//# sourceMappingURL=taskRanking.js.map
|
|
@@ -18,6 +18,11 @@ export declare function taskRecordFromReport(report: TaskReport): TaskRecord | n
|
|
|
18
18
|
* private save, or a delivered lesson that lives only there — must not be named
|
|
19
19
|
* in a public record; the store's own routing (unified/shared mode) wins first. */
|
|
20
20
|
export declare function taskRecordHome(store: HunchStore, record: TaskRecord): TaskRecordHome;
|
|
21
|
+
/** Older records this one verified over: same file, a shared record, and this
|
|
22
|
+
* task's last check passed with no rule violated. Superseded records stay in
|
|
23
|
+
* the graph (append-only) but are never delivered; the newer line carries the
|
|
24
|
+
* verified state. Bounded. */
|
|
25
|
+
export declare function computeSupersedes(record: TaskRecord, others: readonly TaskRecord[], limit?: number): string[];
|
|
21
26
|
export interface PersistedTaskRecord {
|
|
22
27
|
record: TaskRecord;
|
|
23
28
|
home: TaskRecordHome;
|
package/dist/core/taskRecord.js
CHANGED
|
@@ -111,6 +111,33 @@ export function taskRecordHome(store, record) {
|
|
|
111
111
|
}
|
|
112
112
|
return "public";
|
|
113
113
|
}
|
|
114
|
+
/** Older records this one verified over: same file, a shared record, and this
|
|
115
|
+
* task's last check passed with no rule violated. Superseded records stay in
|
|
116
|
+
* the graph (append-only) but are never delivered; the newer line carries the
|
|
117
|
+
* verified state. Bounded. */
|
|
118
|
+
export function computeSupersedes(record, others, limit = 20) {
|
|
119
|
+
const last = record.checks.at(-1);
|
|
120
|
+
if (!last || last.state !== "passed")
|
|
121
|
+
return [];
|
|
122
|
+
if (record.conformance.some((c) => c.outcome === "violated"))
|
|
123
|
+
return [];
|
|
124
|
+
const files = new Set(record.files.map((f) => f.replace(/\\/g, "/")));
|
|
125
|
+
const ids = new Set([...record.lessons.map((l) => l.record_id), ...record.applied.map((a) => a.record_id), ...record.saved.map((s) => s.record_id)]);
|
|
126
|
+
if (!files.size || !ids.size)
|
|
127
|
+
return [];
|
|
128
|
+
const out = [];
|
|
129
|
+
for (const o of others) {
|
|
130
|
+
if (o.id === record.id || o.finished_at >= record.finished_at)
|
|
131
|
+
continue;
|
|
132
|
+
if (!o.files.some((f) => files.has(f.replace(/\\/g, "/"))))
|
|
133
|
+
continue;
|
|
134
|
+
const oids = [...o.lessons.map((l) => l.record_id), ...o.applied.map((a) => a.record_id), ...o.saved.map((s) => s.record_id)];
|
|
135
|
+
if (!oids.some((id) => ids.has(id)))
|
|
136
|
+
continue;
|
|
137
|
+
out.push(o.id);
|
|
138
|
+
}
|
|
139
|
+
return out.sort().slice(0, limit);
|
|
140
|
+
}
|
|
114
141
|
/** Write (or refresh) the graph record for a finished task. Idempotent on the
|
|
115
142
|
* report hash. A record never changes home once written. Returns null for an
|
|
116
143
|
* open task, an empty report, or when task records are disabled locally. */
|
|
@@ -118,9 +145,10 @@ export function persistTaskRecord(root, store, taskId, options = {}) {
|
|
|
118
145
|
if (!taskRecordsEnabled(root))
|
|
119
146
|
return null;
|
|
120
147
|
const report = readTaskReport(root, taskId, reportSourceSnapshot(root).hash);
|
|
121
|
-
const
|
|
122
|
-
if (!
|
|
148
|
+
const built = taskRecordFromReport(report);
|
|
149
|
+
if (!built)
|
|
123
150
|
return null;
|
|
151
|
+
const record = { ...built, supersedes: computeSupersedes(built, store.recs("tasks")) };
|
|
124
152
|
const inPrivate = store.hasPrivate ? store.getPrivateRec("tasks", record.id) : undefined;
|
|
125
153
|
const inPublic = store.json.get("tasks", record.id);
|
|
126
154
|
const home = inPrivate ? "private" : inPublic ? "public" : taskRecordHome(store, record);
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { TaskRecord } from "./types.js";
|
|
2
|
+
export interface TaskRecordStats {
|
|
3
|
+
records: number;
|
|
4
|
+
/** Later tasks that had a chance to re-verify (an earlier overlapping task with a check within the window). */
|
|
5
|
+
reverify_candidates: number;
|
|
6
|
+
reverified: number;
|
|
7
|
+
reverification_rate: number | null;
|
|
8
|
+
/** Later tasks that received a rule an earlier overlapping task violated. */
|
|
9
|
+
violation_candidates: number;
|
|
10
|
+
repeated_violations: number;
|
|
11
|
+
repeat_violation_rate: number | null;
|
|
12
|
+
window_hours: number;
|
|
13
|
+
}
|
|
14
|
+
export declare function taskRecordStats(records: readonly TaskRecord[], windowHours?: number): TaskRecordStats;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/** Online proxies for whether delivered task history is doing its job,
|
|
2
|
+
* computed from task records alone (no agent claims):
|
|
3
|
+
* - re-verification: a later task on an overlapping file re-ran a check with
|
|
4
|
+
* the same label as an earlier task within a window;
|
|
5
|
+
* - repeat violation: a rule violated in an earlier task was violated again
|
|
6
|
+
* in a later task on an overlapping file.
|
|
7
|
+
* Both should fall if the RECENT TASKS lines are read and acted on. */
|
|
8
|
+
import { normalizePath } from "./taskRanking.js";
|
|
9
|
+
export function taskRecordStats(records, windowHours = 24) {
|
|
10
|
+
const sorted = [...records].filter((r) => r.state === "completed").sort((a, b) => a.finished_at.localeCompare(b.finished_at) || a.id.localeCompare(b.id));
|
|
11
|
+
const windowMs = windowHours * 3_600_000;
|
|
12
|
+
let reverifyCandidates = 0, reverified = 0, violationCandidates = 0, repeated = 0;
|
|
13
|
+
for (let i = 0; i < sorted.length; i++) {
|
|
14
|
+
const t = sorted[i];
|
|
15
|
+
const files = new Set(t.files.map(normalizePath));
|
|
16
|
+
const tEnd = Date.parse(t.finished_at) || 0;
|
|
17
|
+
const overlapping = sorted.slice(0, i).filter((o) => o.files.some((f) => files.has(normalizePath(f))));
|
|
18
|
+
if (!overlapping.length)
|
|
19
|
+
continue;
|
|
20
|
+
const recent = overlapping.filter((o) => tEnd - (Date.parse(o.finished_at) || 0) <= windowMs);
|
|
21
|
+
const earlierLabels = new Set(recent.flatMap((o) => o.checks.map((c) => c.label)));
|
|
22
|
+
if (earlierLabels.size) {
|
|
23
|
+
reverifyCandidates++;
|
|
24
|
+
if (t.checks.some((c) => earlierLabels.has(c.label)))
|
|
25
|
+
reverified++;
|
|
26
|
+
}
|
|
27
|
+
const violatedBefore = new Set(overlapping.flatMap((o) => o.conformance.filter((c) => c.outcome === "violated").map((c) => c.record_id)));
|
|
28
|
+
const received = new Set(t.lessons.map((l) => l.record_id));
|
|
29
|
+
const exposed = [...violatedBefore].filter((id) => received.has(id));
|
|
30
|
+
if (exposed.length) {
|
|
31
|
+
violationCandidates++;
|
|
32
|
+
if (t.conformance.some((c) => c.outcome === "violated" && violatedBefore.has(c.record_id)))
|
|
33
|
+
repeated++;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
records: sorted.length,
|
|
38
|
+
reverify_candidates: reverifyCandidates, reverified,
|
|
39
|
+
reverification_rate: reverifyCandidates ? reverified / reverifyCandidates : null,
|
|
40
|
+
violation_candidates: violationCandidates, repeated_violations: repeated,
|
|
41
|
+
repeat_violation_rate: violationCandidates ? repeated / violationCandidates : null,
|
|
42
|
+
window_hours: windowHours,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=taskRecordStats.js.map
|
package/dist/core/types.d.ts
CHANGED
|
@@ -658,6 +658,7 @@ export declare const TaskRecordSchema: z.ZodObject<{
|
|
|
658
658
|
}, z.core.$strip>>>;
|
|
659
659
|
refusals: z.ZodDefault<z.ZodNumber>;
|
|
660
660
|
files: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
661
|
+
supersedes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
661
662
|
source_snapshot: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
662
663
|
report_hash: z.ZodString;
|
|
663
664
|
provenance: z.ZodObject<{
|
|
@@ -1523,6 +1524,7 @@ export declare const SCHEMAS: {
|
|
|
1523
1524
|
}, z.core.$strip>>>;
|
|
1524
1525
|
refusals: z.ZodDefault<z.ZodNumber>;
|
|
1525
1526
|
files: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
1527
|
+
supersedes: z.ZodDefault<z.ZodArray<z.ZodString>>;
|
|
1526
1528
|
source_snapshot: z.ZodDefault<z.ZodNullable<z.ZodString>>;
|
|
1527
1529
|
report_hash: z.ZodString;
|
|
1528
1530
|
provenance: z.ZodObject<{
|
package/dist/core/types.js
CHANGED
|
@@ -502,6 +502,7 @@ export const TaskRecordSchema = z.object({
|
|
|
502
502
|
})).default([]).describe("Hunch's deterministic evaluation of each delivered rule against the changed files"),
|
|
503
503
|
refusals: z.number().int().nonnegative().default(0).describe("edits the native gate denied during the task"),
|
|
504
504
|
files: z.array(z.string()).default([]).describe("files the task touched: delivery targets, rule-checked changes, denied edits"),
|
|
505
|
+
supersedes: z.array(z.string()).default([]).describe("older task records this task verified over: same file, shared record, passing check; superseded records are not delivered"),
|
|
505
506
|
source_snapshot: z.string().nullable().default(null).describe("bounded source snapshot hash at the last check, when one ran"),
|
|
506
507
|
report_hash: z.string().describe("content hash of the full local report this record summarizes"),
|
|
507
508
|
provenance: ProvenanceSchema,
|