@inixiative/foundry-core 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/feedback/correction.ts +8 -1
- package/src/feedback/experiment-gate.ts +89 -0
- package/src/feedback/review-findings.ts +237 -0
- package/src/feedback/warden-labels.ts +5 -2
- package/src/feedback/warden-prefilter.ts +5 -3
- package/src/feedback/warden-probes.ts +383 -15
- package/src/index.ts +47 -0
- package/src/thread.ts +15 -0
- package/src/types.ts +12 -0
package/package.json
CHANGED
|
@@ -31,6 +31,8 @@ export interface CorrectionEvent {
|
|
|
31
31
|
readonly timestamp: string;
|
|
32
32
|
/** What the agent said right before the correction ('' at the start of a session). */
|
|
33
33
|
readonly before: string;
|
|
34
|
+
/** The person's previous message: what the agent was answering when it earned the correction. */
|
|
35
|
+
readonly ask?: string;
|
|
34
36
|
/** The person's message. */
|
|
35
37
|
readonly correction: string;
|
|
36
38
|
/** The strongest signal, and every signal that fired. */
|
|
@@ -167,6 +169,7 @@ export function extractCorrections(
|
|
|
167
169
|
const minScore = options.minScore ?? 0;
|
|
168
170
|
const corrections: CorrectionEvent[] = [];
|
|
169
171
|
let before = '';
|
|
172
|
+
let ask = '';
|
|
170
173
|
let pendingInterruption = false;
|
|
171
174
|
let secretsScrubbed = 0;
|
|
172
175
|
// An identical message resent within ten minutes is one correction, not a repeat.
|
|
@@ -181,6 +184,8 @@ export function extractCorrections(
|
|
|
181
184
|
if (!text.trim()) continue;
|
|
182
185
|
const detection = detectCorrection(text, before, { interrupted: pendingInterruption });
|
|
183
186
|
pendingInterruption = false;
|
|
187
|
+
const previousAsk = ask;
|
|
188
|
+
ask = text.trim();
|
|
184
189
|
const key = text.toLowerCase().replace(/\s+/g, ' ').trim();
|
|
185
190
|
const at = Date.parse(turn.timestamp);
|
|
186
191
|
const sent = recent.get(key);
|
|
@@ -190,12 +195,14 @@ export function extractCorrections(
|
|
|
190
195
|
// Scrub before truncating, so a cut can't leave part of a secret unmatched.
|
|
191
196
|
const scrubbedBefore = scrubSecrets(before);
|
|
192
197
|
const scrubbedText = scrubSecrets(text.trim());
|
|
193
|
-
|
|
198
|
+
const scrubbedAsk = scrubSecrets(previousAsk);
|
|
199
|
+
secretsScrubbed += scrubbedBefore.hits + scrubbedText.hits + scrubbedAsk.hits;
|
|
194
200
|
const event = {
|
|
195
201
|
...(turn.id ? { messageId: turn.id } : {}),
|
|
196
202
|
...(turn.sourceRef ? { sourceRef: turn.sourceRef } : {}),
|
|
197
203
|
timestamp: turn.timestamp,
|
|
198
204
|
before: scrubbedBefore.text.slice(-2000),
|
|
205
|
+
...(scrubbedAsk.text ? { ask: scrubbedAsk.text.slice(0, 4000) } : {}),
|
|
199
206
|
correction: scrubbedText.text.slice(0, 4000),
|
|
200
207
|
signal: detection.signal,
|
|
201
208
|
signals: detection.signals,
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The experiment gate (`.claude/skills/experiment/SKILL.md`), shared by Foundry's eval scripts and
|
|
3
|
+
* Oracle's replay runner: a wide run (more than TINY_MOMENTS moments, or whatever else the caller
|
|
4
|
+
* counts as wide) needs `--experiment <design.md>` whose pre-registration sections are filled and
|
|
5
|
+
* whose `<design>.review.md` says `verdict: go` for this exact design (`design-sha256:` of its
|
|
6
|
+
* bytes). A tiny test runs without one.
|
|
7
|
+
*/
|
|
8
|
+
import { createHash } from 'node:crypto';
|
|
9
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
10
|
+
import { resolve } from 'node:path';
|
|
11
|
+
|
|
12
|
+
export const TINY_MOMENTS = 3;
|
|
13
|
+
|
|
14
|
+
/** The `## ` headings of `.claude/skills/experiment/design.md`, every one required. */
|
|
15
|
+
export const DESIGN_SECTIONS = [
|
|
16
|
+
'Question',
|
|
17
|
+
'Variable',
|
|
18
|
+
'Arms',
|
|
19
|
+
'Fixtures and power',
|
|
20
|
+
'Baseline',
|
|
21
|
+
'Gates',
|
|
22
|
+
'Cost ceiling and stop rules',
|
|
23
|
+
'Provenance',
|
|
24
|
+
'What would make the result wrong',
|
|
25
|
+
] as const;
|
|
26
|
+
|
|
27
|
+
export type ExperimentRecord = { design: string; designSha256: string; review: string };
|
|
28
|
+
|
|
29
|
+
export const reviewPathOf = (design: string) => design.replace(/\.md$/, '.review.md');
|
|
30
|
+
|
|
31
|
+
const sha256 = (path: string) => createHash('sha256').update(readFileSync(path)).digest('hex');
|
|
32
|
+
|
|
33
|
+
/** Why the design may not run wide; empty when it may. */
|
|
34
|
+
export const experimentProblems = (designPath: string): string[] => {
|
|
35
|
+
const design = resolve(designPath);
|
|
36
|
+
if (!design.endsWith('.md')) return [`${design}: the design is a .md file`];
|
|
37
|
+
if (!existsSync(design)) return [`${design}: no such design`];
|
|
38
|
+
const problems: string[] = [];
|
|
39
|
+
const text = readFileSync(design, 'utf8');
|
|
40
|
+
const sections = new Map<string, string>();
|
|
41
|
+
let current: string | undefined;
|
|
42
|
+
for (const line of text.split('\n')) {
|
|
43
|
+
const heading = /^##\s+(.+?)\s*$/.exec(line);
|
|
44
|
+
if (heading) {
|
|
45
|
+
current = heading[1]?.toLowerCase();
|
|
46
|
+
if (current) sections.set(current, sections.get(current) ?? '');
|
|
47
|
+
} else if (/^#\s/.test(line)) current = undefined;
|
|
48
|
+
else if (current) sections.set(current, `${sections.get(current) ?? ''}${line}\n`);
|
|
49
|
+
}
|
|
50
|
+
for (const name of DESIGN_SECTIONS) {
|
|
51
|
+
const body = sections.get(name.toLowerCase());
|
|
52
|
+
if (body === undefined) problems.push(`missing section "## ${name}"`);
|
|
53
|
+
else if (!body.replace(/<!--[\s\S]*?-->/g, '').trim())
|
|
54
|
+
problems.push(`section "## ${name}" is empty`);
|
|
55
|
+
}
|
|
56
|
+
if (text.includes('<fill')) problems.push('unfilled <fill: …> placeholders remain');
|
|
57
|
+
const review = reviewPathOf(design);
|
|
58
|
+
if (!existsSync(review)) return [...problems, `no adversarial review at ${review}`];
|
|
59
|
+
const reviewText = readFileSync(review, 'utf8');
|
|
60
|
+
if (!/^verdict:\s*go\s*$/im.test(reviewText))
|
|
61
|
+
problems.push(`${review} has no "verdict: go" line`);
|
|
62
|
+
const reviewed = /^design-sha256:\s*([0-9a-f]{64})\s*$/im.exec(reviewText)?.[1];
|
|
63
|
+
if (!reviewed) problems.push(`${review} has no "design-sha256: <hex>" line`);
|
|
64
|
+
else if (reviewed !== sha256(design))
|
|
65
|
+
problems.push('the design changed after its review (design-sha256 differs): review it again');
|
|
66
|
+
return problems;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
/** Throws unless the run may go ahead; returns the provenance to record when a design was named. */
|
|
70
|
+
export const requireExperiment = (
|
|
71
|
+
experiment: string | undefined,
|
|
72
|
+
wide: boolean,
|
|
73
|
+
): ExperimentRecord | undefined => {
|
|
74
|
+
if (!experiment) {
|
|
75
|
+
if (!wide) return undefined;
|
|
76
|
+
throw Error(
|
|
77
|
+
`a wide run (more than ${TINY_MOMENTS} moments) needs --experiment <design.md> ` +
|
|
78
|
+
'with a reviewed pre-registration: see .claude/skills/experiment/SKILL.md',
|
|
79
|
+
);
|
|
80
|
+
}
|
|
81
|
+
const problems = experimentProblems(experiment);
|
|
82
|
+
if (problems.length)
|
|
83
|
+
throw Error(
|
|
84
|
+
`--experiment ${experiment} may not run:\n - ${problems.join('\n - ')}\n` +
|
|
85
|
+
'see .claude/skills/experiment/SKILL.md',
|
|
86
|
+
);
|
|
87
|
+
const design = resolve(experiment);
|
|
88
|
+
return { design, designSha256: sha256(design), review: reviewPathOf(design) };
|
|
89
|
+
};
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import {
|
|
2
|
+
type Authorship,
|
|
3
|
+
CONCEPT_CLUSTERS,
|
|
4
|
+
type CorrectionTarget,
|
|
5
|
+
MISS_KINDS,
|
|
6
|
+
type MissKind,
|
|
7
|
+
} from './warden-probes';
|
|
8
|
+
|
|
9
|
+
// The Prospector's reading of written review: PR reviews and ticket comments where the person said the
|
|
10
|
+
// work should have used, extended or ported something that already existed. Data and pure functions
|
|
11
|
+
// only: callers fetch the text and make the model call.
|
|
12
|
+
|
|
13
|
+
/** One piece of written review, with what it was about. */
|
|
14
|
+
export interface ReviewMoment {
|
|
15
|
+
readonly id: string;
|
|
16
|
+
/** Where it was written, e.g. `PR #2116 (CHANGES_REQUESTED review, inline)` or `ZLT-4321 comment`. */
|
|
17
|
+
readonly where: string;
|
|
18
|
+
/** What the work was: PR title and description, or the issue's title and description. */
|
|
19
|
+
readonly work: string;
|
|
20
|
+
/** The code it was written on, when inline (path, line, diff hunk). */
|
|
21
|
+
readonly code?: string;
|
|
22
|
+
/** The message it replies to, when it is a reply. */
|
|
23
|
+
readonly replyTo?: string;
|
|
24
|
+
readonly text: string;
|
|
25
|
+
/** `directive`: a description the person wrote of work still to do (a ticket, a project), not a review of done work. */
|
|
26
|
+
readonly mode?: 'review' | 'directive';
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export type FindingScope = 'architectural' | 'code';
|
|
30
|
+
|
|
31
|
+
/** How much a finding counts in scoring: generic code-correctness findings are lightly down-weighted (Aron, 2026-10-15). */
|
|
32
|
+
export const SCOPE_WEIGHT: Readonly<Record<FindingScope, number>> = {
|
|
33
|
+
architectural: 1,
|
|
34
|
+
code: 0.75,
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
/** One place the reviewer pushed toward something that already existed. */
|
|
38
|
+
export interface ReviewFinding {
|
|
39
|
+
readonly target: CorrectionTarget;
|
|
40
|
+
/** The reviewer's own words that say it, verbatim, at most 300 characters. */
|
|
41
|
+
readonly excerpt: string;
|
|
42
|
+
/** `architectural`: about which system, primitive or boundary to build on; `code`: a generic code-correctness fix. */
|
|
43
|
+
readonly scope?: FindingScope;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface ReviewReading {
|
|
47
|
+
readonly findings: readonly ReviewFinding[];
|
|
48
|
+
/** The model's read of the voice: typed by a person, or written by an AI agent. A weak signal. */
|
|
49
|
+
readonly voice?: 'human' | 'agent';
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* Words a reuse finding almost always carries. A change-request review is read whole; everything else
|
|
54
|
+
* is read only when it matches (the method of Zealot's core-primitives findings, ZLT-5638).
|
|
55
|
+
*/
|
|
56
|
+
export const REUSE_HINT =
|
|
57
|
+
/\b(?:already (?:exists?|have|has|does|handles?|in|imported|installed|there)|existing|exists|re-?use|reimplement\w*|re-implement\w*|hand-?roll\w*|hand-?written|duplicat\w*|copy|copies|copied|parallel|second (?:copy|version|implementation)|third (?:copy|type system)|instead of|port(?:ed|ing)? (?:from|it|this|the)|from (?:the )?template|template'?s|primitive|extend(?:ing)? (?:the|it)|use (?:the |our )?[`\w]|we have|there'?s (?:a|an|already)|same (?:shape|thing|as)|like \w+ does|convention|pattern|wrapper|shared|generic|lodash|helper|util|registry|consolidate|fold (?:into|it)|one place|single source)\b/i;
|
|
58
|
+
|
|
59
|
+
export const REVIEW_FINDINGS_PROMPT = `You read code review written by one reviewer, Aron, on pull requests and tickets. For each item, find every place where he says the work should have USED, EXTENDED or PORTED something that ALREADY EXISTS instead of what was built: a primitive (helper, wrapper, hook, component, class, module, package export, library already installed), a system (an existing subsystem or flow), a pattern (an established way the codebase does a thing, e.g. a route template or a module convention), or a rule (a standing convention or ruling, e.g. "no select on internal reads").
|
|
60
|
+
|
|
61
|
+
A finding needs both: the existing thing, and what was done instead. Not findings: bug reports, missing tests, style nits, new requirements, questions, praise, status updates, an author's replies about what they fixed, and anything that points at something not yet built.
|
|
62
|
+
|
|
63
|
+
For each finding:
|
|
64
|
+
- name: the existing thing as the codebase names it, at most 80 characters (e.g. "paginate() + filterLens", "Skeleton component", "lodash omit", "PolymorphismRegistry"). Name the thing, not the advice.
|
|
65
|
+
- kind: "primitive" | "system" | "pattern" | "rule"
|
|
66
|
+
- concept: its concept cluster; use one of the given clusters when one fits, else a new short kebab-case cluster.
|
|
67
|
+
- where: the file, package or doc it lives in, only when the item says so.
|
|
68
|
+
- builtInstead: what the work did instead, one line, at most 160 characters. For a directive item, what the work would otherwise have built, or "" when the item doesn't say.
|
|
69
|
+
- miss: why it was missed, judged from the item: "unaware" (didn't know it existed), "misfit" (knew it, worked around a gap instead of extending it), "wrapper" (wrapped it under a new name), "unclear".
|
|
70
|
+
- excerpt: Aron's words that say it, verbatim, at most 300 characters.
|
|
71
|
+
- scope: "architectural" when it is about which system, primitive, module boundary or pattern to build on; "code" when it is a generic code-correctness fix (a library call, a small helper, an idiom).
|
|
72
|
+
|
|
73
|
+
Items marked "Mode: directive" are descriptions Aron wrote of work still to be done (a ticket, a project). There a finding is a place where he directs the work to use, extend or port something that already exists, by name.
|
|
74
|
+
|
|
75
|
+
Also judge the voice of each item: "human" when it reads as typed by a person (terse, informal, lowercase, typos, first-person asides), "agent" when it reads as written by an AI coding agent (headed sections, bold labels, numbered findings, dense backticked citations, "Fixed in <sha>" replies, polished em-dash prose).
|
|
76
|
+
|
|
77
|
+
Respond with JSON only: { "results": { "<id>": { "voice": "human" | "agent", "findings": [ { "name": string, "kind": string, "concept": string, "where"?: string, "builtInstead": string, "miss": string, "excerpt": string, "scope": "architectural" | "code" } ] } } } with "findings": [] when there are none.`;
|
|
78
|
+
|
|
79
|
+
/** The reader's input for a batch of review moments. */
|
|
80
|
+
export function reviewFindingMessages(
|
|
81
|
+
moments: readonly ReviewMoment[],
|
|
82
|
+
): { role: 'system' | 'user'; content: string }[] {
|
|
83
|
+
const body = moments
|
|
84
|
+
.map((m) =>
|
|
85
|
+
[
|
|
86
|
+
`## ${m.id}`,
|
|
87
|
+
`Where: ${m.where}`,
|
|
88
|
+
...(m.mode === 'directive' ? ['Mode: directive'] : []),
|
|
89
|
+
`### The work\n${m.work.slice(0, 1200)}`,
|
|
90
|
+
...(m.code ? [`### Code it was written on\n${m.code.slice(-1500)}`] : []),
|
|
91
|
+
...(m.replyTo ? [`### Replying to\n${m.replyTo.slice(0, 800)}`] : []),
|
|
92
|
+
`### Aron wrote\n${m.text.slice(0, 4000)}`,
|
|
93
|
+
].join('\n'),
|
|
94
|
+
)
|
|
95
|
+
.join('\n\n');
|
|
96
|
+
return [
|
|
97
|
+
{ role: 'system', content: REVIEW_FINDINGS_PROMPT },
|
|
98
|
+
{
|
|
99
|
+
role: 'user',
|
|
100
|
+
content: `Concept clusters: ${CONCEPT_CLUSTERS.join(', ')}\n\nRead each item below.\n\n${body}`,
|
|
101
|
+
},
|
|
102
|
+
];
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const KINDS = new Set(['primitive', 'system', 'pattern', 'rule']);
|
|
106
|
+
const kebab = (text: string) =>
|
|
107
|
+
text
|
|
108
|
+
.trim()
|
|
109
|
+
.toLowerCase()
|
|
110
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
111
|
+
.replace(/^-|-$/g, '');
|
|
112
|
+
|
|
113
|
+
/** One reading per id; undefined when missing or malformed (the caller retries it alone). */
|
|
114
|
+
export function parseReviewFindings(
|
|
115
|
+
content: string,
|
|
116
|
+
ids: readonly string[],
|
|
117
|
+
): (ReviewReading | undefined)[] {
|
|
118
|
+
let results: Record<string, { voice?: unknown; findings?: unknown }> = {};
|
|
119
|
+
try {
|
|
120
|
+
results =
|
|
121
|
+
(
|
|
122
|
+
JSON.parse(content.slice(content.indexOf('{'), content.lastIndexOf('}') + 1)) as {
|
|
123
|
+
results?: typeof results;
|
|
124
|
+
}
|
|
125
|
+
).results ?? {};
|
|
126
|
+
} catch {
|
|
127
|
+
return ids.map(() => undefined);
|
|
128
|
+
}
|
|
129
|
+
return ids.map((id) => {
|
|
130
|
+
const item = results[id];
|
|
131
|
+
if (!item || !Array.isArray(item.findings)) return undefined;
|
|
132
|
+
const findings: ReviewFinding[] = [];
|
|
133
|
+
for (const raw of item.findings as Record<string, unknown>[]) {
|
|
134
|
+
if (!raw || typeof raw.name !== 'string' || !raw.name.trim()) continue;
|
|
135
|
+
if (typeof raw.concept !== 'string' || !kebab(raw.concept)) continue;
|
|
136
|
+
const miss = MISS_KINDS.includes(raw.miss as MissKind) ? (raw.miss as MissKind) : 'unclear';
|
|
137
|
+
findings.push({
|
|
138
|
+
target: {
|
|
139
|
+
name: raw.name.trim().slice(0, 120),
|
|
140
|
+
kind: (KINDS.has(String(raw.kind)) ? raw.kind : 'pattern') as CorrectionTarget['kind'],
|
|
141
|
+
concept: kebab(raw.concept),
|
|
142
|
+
...(typeof raw.where === 'string' && raw.where.trim()
|
|
143
|
+
? { where: raw.where.trim().slice(0, 160) }
|
|
144
|
+
: {}),
|
|
145
|
+
...(typeof raw.builtInstead === 'string' && raw.builtInstead.trim()
|
|
146
|
+
? { builtInstead: raw.builtInstead.trim().slice(0, 240) }
|
|
147
|
+
: {}),
|
|
148
|
+
miss,
|
|
149
|
+
},
|
|
150
|
+
excerpt: typeof raw.excerpt === 'string' ? raw.excerpt.trim().slice(0, 300) : '',
|
|
151
|
+
...(raw.scope === 'architectural' || raw.scope === 'code' ? { scope: raw.scope } : {}),
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
return {
|
|
155
|
+
findings,
|
|
156
|
+
...(item.voice === 'human' || item.voice === 'agent' ? { voice: item.voice } : {}),
|
|
157
|
+
};
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/** Markers an agent leaves in what it posts, and the trace of text typed into a web form. */
|
|
162
|
+
export const AUTHORSHIP_MARKERS = {
|
|
163
|
+
/** An agent's signature line. */
|
|
164
|
+
footer:
|
|
165
|
+
/(?:generated (?:by|with) \[?claude|🤖 generated|co-authored-by: claude|^_?generated by .{0,40}_?$)/im,
|
|
166
|
+
/** Claude Code's terminal glyphs: an agent's output pasted in by hand. */
|
|
167
|
+
pasted: /(?:^|\n)\s*[⏺⎿]/,
|
|
168
|
+
/** CRLF line endings: typed (or edited) in the GitHub web UI; gh and the API post LF. */
|
|
169
|
+
webForm: /\r\n/,
|
|
170
|
+
} as const;
|
|
171
|
+
|
|
172
|
+
/** What the person's own sessions say about a comment: an agent posted it, or the person typed it. */
|
|
173
|
+
export interface SessionEvidence {
|
|
174
|
+
readonly posted?: boolean;
|
|
175
|
+
/** The text appears in another tool call: a body file an agent drafted, or a quote. Weak. */
|
|
176
|
+
readonly drafted?: boolean;
|
|
177
|
+
readonly typed?: boolean;
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
/**
|
|
181
|
+
* Who wrote a comment posted under the person's account. Strong evidence decides; a model's read of
|
|
182
|
+
* the voice is the weak fallback, then plain style. A typed prompt outranks an agent posting it: the
|
|
183
|
+
* person dictated the words.
|
|
184
|
+
*/
|
|
185
|
+
export function authorshipOf(
|
|
186
|
+
text: string,
|
|
187
|
+
evidence: SessionEvidence = {},
|
|
188
|
+
voice?: 'human' | 'agent',
|
|
189
|
+
): Authorship {
|
|
190
|
+
const signals: string[] = [];
|
|
191
|
+
if (evidence.typed) signals.push('session:typed');
|
|
192
|
+
if (evidence.posted) signals.push('session:posted');
|
|
193
|
+
if (evidence.drafted) signals.push('session:drafted');
|
|
194
|
+
if (AUTHORSHIP_MARKERS.footer.test(text)) signals.push('marker:footer');
|
|
195
|
+
if (AUTHORSHIP_MARKERS.pasted.test(text)) signals.push('marker:pasted');
|
|
196
|
+
if (AUTHORSHIP_MARKERS.webForm.test(text)) signals.push('marker:web-form');
|
|
197
|
+
const style = styleOf(text);
|
|
198
|
+
signals.push(`style:${style}`);
|
|
199
|
+
if (voice) signals.push(`voice:${voice}`);
|
|
200
|
+
if (evidence.typed) return { by: 'human', confidence: 'strong', signals };
|
|
201
|
+
if (evidence.posted || signals.includes('marker:footer') || signals.includes('marker:pasted'))
|
|
202
|
+
return { by: 'agent', confidence: 'strong', signals };
|
|
203
|
+
if (signals.includes('marker:web-form')) return { by: 'human', confidence: 'strong', signals };
|
|
204
|
+
// A draft can also be an agent quoting the person's comment back, so it only outweighs style.
|
|
205
|
+
return { by: evidence.drafted ? 'agent' : (voice ?? style), confidence: 'weak', signals };
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/** Plain style: an agent's headings, bold labels, numbered findings and dense citations. */
|
|
209
|
+
export function styleOf(text: string): 'human' | 'agent' {
|
|
210
|
+
let agent = 0;
|
|
211
|
+
if (/^#{1,4} /m.test(text)) agent += 2;
|
|
212
|
+
if (/\*\*[^*\n]{2,60}\*\*/.test(text)) agent++;
|
|
213
|
+
if (/^\s*\d+\. \*\*/m.test(text)) agent += 2;
|
|
214
|
+
if ((text.match(/`[^`\n]+`/g) ?? []).length >= 4) agent++;
|
|
215
|
+
if ((text.match(/—/g) ?? []).length >= 2) agent++;
|
|
216
|
+
if (/^(?:Fixed in|Addressed in|Done in) [0-9a-f]{7}/im.test(text)) agent += 2;
|
|
217
|
+
if (text.length > 900) agent++;
|
|
218
|
+
let human = 0;
|
|
219
|
+
if (/^[a-z]/.test(text.trim())) human += 2;
|
|
220
|
+
if (!/[.!?:`)]\s*$/.test(text.trim())) human++;
|
|
221
|
+
if (/\b(?:im|dont|cant|doesnt|isnt|wont|u|ur|pls|thx|lol|imo|tbh)\b/.test(text)) human += 2;
|
|
222
|
+
if (text.length < 200) human++;
|
|
223
|
+
return agent > human ? 'agent' : 'human';
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
/**
|
|
227
|
+
* A distinctive run of a comment that survives JSON and shell quoting unchanged, to find the session
|
|
228
|
+
* that posted it. Undefined when the text has no such run.
|
|
229
|
+
*/
|
|
230
|
+
export function evidenceSnippet(text: string, length = 48): string | undefined {
|
|
231
|
+
let best = '';
|
|
232
|
+
for (const run of text.split(/[^A-Za-z0-9 ,.;:()/_=+-]+/))
|
|
233
|
+
if (run.trim().length > best.length) best = run.trim();
|
|
234
|
+
if (best.length < Math.min(length, 32)) return undefined;
|
|
235
|
+
const start = Math.max(0, Math.floor((best.length - length) / 2));
|
|
236
|
+
return best.slice(start, start + length).trim();
|
|
237
|
+
}
|
|
@@ -14,7 +14,7 @@ export interface WardenLabel {
|
|
|
14
14
|
readonly shouldKnow?: string;
|
|
15
15
|
}
|
|
16
16
|
|
|
17
|
-
export type WardenTriggerKind = 'user-message' | 'agent-action' | 'agent-plan';
|
|
17
|
+
export type WardenTriggerKind = 'user-message' | 'agent-action' | 'agent-plan' | 'subagent-spawn';
|
|
18
18
|
|
|
19
19
|
/**
|
|
20
20
|
* A labelled moment for the Warden prefilter: which Warden domains the trigger touches. Never commit
|
|
@@ -215,7 +215,10 @@ export function fixtureProbe(
|
|
|
215
215
|
kind: relevant.length ? 'positive' : 'negative',
|
|
216
216
|
ctx: fixture.ctx,
|
|
217
217
|
...(fixture.project ? { project: fixture.project } : {}),
|
|
218
|
-
|
|
218
|
+
// A user-message fixture is asked at the user-message trigger; the others are the agent's turn.
|
|
219
|
+
...(fixture.trigger.kind === 'user-message'
|
|
220
|
+
? { moment: '', ask: fixture.trigger.text }
|
|
221
|
+
: { moment: fixture.trigger.text }),
|
|
219
222
|
...(relevant.length
|
|
220
223
|
? {
|
|
221
224
|
gold: {
|
|
@@ -6,8 +6,9 @@
|
|
|
6
6
|
import type { WardenTriggerKind } from './warden-labels';
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
|
-
* Where a moment arrives: before (the user's message)
|
|
10
|
-
* finished turn). The labelled moments are
|
|
9
|
+
* Where a moment arrives: before (the user's message), after (the agent's actions of a turn or its
|
|
10
|
+
* plan, and its finished turn), or at a handoff to a subagent. The labelled moments are
|
|
11
|
+
* `WardenFixture`s (`warden-labels`).
|
|
11
12
|
*/
|
|
12
13
|
export type PrefilterTrigger = WardenTriggerKind | 'post-turn';
|
|
13
14
|
|
|
@@ -172,8 +173,9 @@ export function prefilterQuestion(description: string) {
|
|
|
172
173
|
export function prefilterState(trigger: PrefilterTrigger, text: string, context?: string): string {
|
|
173
174
|
const label = {
|
|
174
175
|
'user-message': "The user's message",
|
|
175
|
-
'agent-action': "The agent's
|
|
176
|
+
'agent-action': "The agent's actions this turn",
|
|
176
177
|
'agent-plan': "The agent's plan",
|
|
178
|
+
'subagent-spawn': 'The task the agent is handing to a subagent',
|
|
177
179
|
'post-turn': "The agent's finished turn",
|
|
178
180
|
}[trigger];
|
|
179
181
|
const moment = window(text.trim(), PREFILTER_MOMENT_CHARS);
|
|
@@ -2,8 +2,28 @@
|
|
|
2
2
|
// scope and right. Data and pure functions only, so Oracle can score runs on a core-only dependency;
|
|
3
3
|
// callers bring the probes, the Warden calls and the judge call.
|
|
4
4
|
//
|
|
5
|
-
// A probe is one moment of a real session. A positive probe is
|
|
6
|
-
// correction
|
|
5
|
+
// A probe is one moment of a real session. A positive probe is a moment that drew the user's sharp
|
|
6
|
+
// correction; a negative probe is one the user let pass. Each trigger point asks about its own part
|
|
7
|
+
// of the moment: `user-message` about the user's ask (advise), `agent-action` about the agent's turn
|
|
8
|
+
// (guard). A Warden anticipated a correction when it said it first, at either trigger.
|
|
9
|
+
|
|
10
|
+
import type { TokenCounts } from '../token-counts';
|
|
11
|
+
|
|
12
|
+
/** Where the Prospector found a probe. */
|
|
13
|
+
export type ProbeSource = 'session' | 'github' | 'linear';
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Whether the person wrote a correction or an agent wrote it under their account. `strong` rests on
|
|
17
|
+
* a marker (an agent footer, a posting command in a session, text typed in the web UI); `weak` on style.
|
|
18
|
+
*/
|
|
19
|
+
export interface Authorship {
|
|
20
|
+
readonly by: 'human' | 'agent';
|
|
21
|
+
readonly confidence: 'strong' | 'weak';
|
|
22
|
+
readonly signals: readonly string[];
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** The trigger points a probe is asked at (live: `user-message` advises, `agent-action` guards). */
|
|
26
|
+
export type WardenProbeTrigger = 'user-message' | 'agent-action';
|
|
7
27
|
|
|
8
28
|
/** One real moment a chorus is asked about. Never commit probes: they hold session text. */
|
|
9
29
|
export interface WardenProbe {
|
|
@@ -12,29 +32,62 @@ export interface WardenProbe {
|
|
|
12
32
|
/** Which context's Wardens may see it (UserEvidence probes only ever see the UserEvidence ladder). */
|
|
13
33
|
readonly ctx: string;
|
|
14
34
|
readonly project?: string;
|
|
15
|
-
/**
|
|
35
|
+
/** Where the Prospector found it: a session, a GitHub PR review, a Linear issue. Absent means session. */
|
|
36
|
+
readonly source?: ProbeSource;
|
|
37
|
+
/** Who wrote the correction, for sources where the person's account also posts agent-written text. */
|
|
38
|
+
readonly authorship?: Authorship;
|
|
39
|
+
/** How much the probe counts in weighted scores (default 1): generic code-correctness findings count less. */
|
|
40
|
+
readonly weight?: number;
|
|
41
|
+
/** What the agent just did or said, as the Warden sees it ('' when only the user's ask was recorded). */
|
|
16
42
|
readonly moment: string;
|
|
17
|
-
/**
|
|
18
|
-
readonly
|
|
43
|
+
/** The user's message the agent was answering: what the `user-message` trigger advises on. */
|
|
44
|
+
readonly ask?: string;
|
|
45
|
+
/** When the user answered (ISO), and the period it falls in, so a backtest trains on earlier probes only. */
|
|
46
|
+
readonly at?: string;
|
|
47
|
+
readonly period?: 'train' | 'test';
|
|
48
|
+
/** Positive probes: the Warden domains that own the rule the user enforced next (when known), and its one-line statements. */
|
|
49
|
+
readonly gold?: {
|
|
50
|
+
readonly domains: readonly string[];
|
|
51
|
+
readonly statements: readonly string[];
|
|
52
|
+
/** What the user pushed the agent toward: the existing rule, primitive, system or pattern. */
|
|
53
|
+
readonly target?: CorrectionTarget;
|
|
54
|
+
};
|
|
19
55
|
/** The user's actual next message. The judge sees it; Wardens never do. */
|
|
20
56
|
readonly hidden?: string;
|
|
57
|
+
/** The Warden's own understanding of the thread at this moment, as a live call supplies it. */
|
|
58
|
+
readonly threadKnowledge?: string;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** The trigger points that can ask about a probe: those whose input it recorded. */
|
|
62
|
+
export function probeTriggers(probe: WardenProbe): WardenProbeTrigger[] {
|
|
63
|
+
return [
|
|
64
|
+
...(probe.ask?.trim() ? (['user-message'] as const) : []),
|
|
65
|
+
...(probe.moment.trim() ? (['agent-action'] as const) : []),
|
|
66
|
+
];
|
|
21
67
|
}
|
|
22
68
|
|
|
23
69
|
/** One Warden's answer on one probe, and the judge's verdict when it spoke. */
|
|
24
70
|
export interface WardenProbeAnswer {
|
|
25
71
|
readonly probeId: string;
|
|
26
72
|
readonly domain: string;
|
|
73
|
+
/** The trigger point that asked; `agent-action` when unrecorded. */
|
|
74
|
+
readonly trigger?: WardenProbeTrigger;
|
|
27
75
|
readonly noComment: boolean;
|
|
28
76
|
/** The protocol was broken; counted apart, never as silence or speech. */
|
|
29
77
|
readonly error?: string;
|
|
30
78
|
readonly text?: string;
|
|
31
79
|
readonly elapsedMs?: number;
|
|
80
|
+
/** What the call used, as the provider reported it (cache reads show a warm, primed prefix). */
|
|
81
|
+
readonly tokens?: Omit<TokenCounts, 'providerUsage'>;
|
|
82
|
+
/** Transport failures of earlier attempts at this call, re-asked before this answer. */
|
|
83
|
+
readonly retried?: readonly string[];
|
|
32
84
|
readonly judged?: WardenJudgement;
|
|
33
85
|
}
|
|
34
86
|
|
|
35
87
|
/** One probe run as a turn: every Warden asked at once, as the live chorus is. */
|
|
36
88
|
export interface WardenProbeTurn {
|
|
37
89
|
readonly probeId: string;
|
|
90
|
+
readonly trigger?: WardenProbeTrigger;
|
|
38
91
|
/** Warden calls made for this probe. */
|
|
39
92
|
readonly calls: number;
|
|
40
93
|
/** Wall clock from the first ask to the last answer. */
|
|
@@ -65,8 +118,30 @@ export interface CharterScore {
|
|
|
65
118
|
readonly speakPrecision: number | null;
|
|
66
119
|
/** Remarks outside the speaking Warden's domain. */
|
|
67
120
|
readonly scopeViolations: number;
|
|
68
|
-
/**
|
|
121
|
+
/** Of positive probes with gold domains, those where an owning Warden named the correction before the user did. */
|
|
69
122
|
readonly recall: number | null;
|
|
123
|
+
/** The primary measure: of all positive probes, those where any Warden, at any trigger, said the correction first. */
|
|
124
|
+
readonly anticipated: number | null;
|
|
125
|
+
/** `anticipated` with each positive counted at its `weight`. */
|
|
126
|
+
readonly weightedAnticipated: number | null;
|
|
127
|
+
/** Remarks the judge did not match to a real correction (on negative probes, or saying something else). */
|
|
128
|
+
readonly wrongRemarks: number;
|
|
129
|
+
/** Wrong remarks per probe asked: how often the chorus speaks when the user would not have. */
|
|
130
|
+
readonly wrongPerProbe: number | null;
|
|
131
|
+
/** The same measures per trigger point, over the probes that trigger could ask about. */
|
|
132
|
+
readonly byTrigger: Partial<
|
|
133
|
+
Record<
|
|
134
|
+
WardenProbeTrigger,
|
|
135
|
+
{
|
|
136
|
+
readonly probes: number;
|
|
137
|
+
readonly positives: number;
|
|
138
|
+
readonly anticipated: number | null;
|
|
139
|
+
readonly remarks: number;
|
|
140
|
+
readonly speakPrecision: number | null;
|
|
141
|
+
readonly wrongRemarks: number;
|
|
142
|
+
}
|
|
143
|
+
>
|
|
144
|
+
>;
|
|
70
145
|
readonly remarks: number;
|
|
71
146
|
readonly silences: number;
|
|
72
147
|
/** Remarks the judge could not grade: counted as unmatched, reported apart. */
|
|
@@ -88,6 +163,16 @@ function percentile(sorted: readonly number[], p: number): number {
|
|
|
88
163
|
|
|
89
164
|
const ratio = (n: number, d: number) => (d ? Math.round((n / d) * 1000) / 1000 : null);
|
|
90
165
|
|
|
166
|
+
/** The 95% Wilson interval of `n` successes in `d` trials, to three places; null when nothing was tried. */
|
|
167
|
+
export function wilsonInterval(n: number, d: number): [number, number] | null {
|
|
168
|
+
if (!d) return null;
|
|
169
|
+
const z = 1.96;
|
|
170
|
+
const p = n / d;
|
|
171
|
+
const centre = (p + (z * z) / (2 * d)) / (1 + (z * z) / d);
|
|
172
|
+
const half = (z * Math.sqrt((p * (1 - p)) / d + (z * z) / (4 * d * d))) / (1 + (z * z) / d);
|
|
173
|
+
return [Math.round((centre - half) * 1000) / 1000, Math.round((centre + half) * 1000) / 1000];
|
|
174
|
+
}
|
|
175
|
+
|
|
91
176
|
/**
|
|
92
177
|
* Score one charter run. "Nothing applied" means a negative probe, or a positive probe whose gold
|
|
93
178
|
* domains do not include this Warden; a remark there that the judge matched to the real correction
|
|
@@ -109,6 +194,11 @@ export function scoreCharterRun(
|
|
|
109
194
|
silences = 0,
|
|
110
195
|
unjudged = 0;
|
|
111
196
|
const recalled = new Set<string>();
|
|
197
|
+
const anticipated = new Set<string>();
|
|
198
|
+
const perTrigger = new Map<
|
|
199
|
+
WardenProbeTrigger,
|
|
200
|
+
{ probes: Set<string>; anticipated: Set<string>; remarks: number; right: number }
|
|
201
|
+
>();
|
|
112
202
|
for (const a of answers) {
|
|
113
203
|
calls++;
|
|
114
204
|
const probe = byId.get(a.probeId);
|
|
@@ -117,16 +207,32 @@ export function scoreCharterRun(
|
|
|
117
207
|
if (a.error) {
|
|
118
208
|
// A broken answer is neither silence nor a remark; where nothing applied, it is a failed silence.
|
|
119
209
|
errors++;
|
|
210
|
+
perTrigger.get(a.trigger ?? 'agent-action')?.probes.add(probe.id);
|
|
120
211
|
if (!owned) quietCases++;
|
|
121
212
|
continue;
|
|
122
213
|
}
|
|
214
|
+
const trigger = a.trigger ?? 'agent-action';
|
|
215
|
+
const tally = perTrigger.get(trigger) ?? {
|
|
216
|
+
probes: new Set<string>(),
|
|
217
|
+
anticipated: new Set<string>(),
|
|
218
|
+
remarks: 0,
|
|
219
|
+
right: 0,
|
|
220
|
+
};
|
|
221
|
+
perTrigger.set(trigger, tally);
|
|
222
|
+
tally.probes.add(probe.id);
|
|
123
223
|
if (a.noComment) silences++;
|
|
124
224
|
else {
|
|
125
225
|
remarks++;
|
|
226
|
+
tally.remarks++;
|
|
126
227
|
if (!a.judged) unjudged++;
|
|
127
228
|
if (a.judged?.matches) {
|
|
128
229
|
remarksRight++;
|
|
230
|
+
tally.right++;
|
|
129
231
|
if (owned) recalled.add(probe.id);
|
|
232
|
+
if (probe.kind === 'positive') {
|
|
233
|
+
anticipated.add(probe.id);
|
|
234
|
+
tally.anticipated.add(probe.id);
|
|
235
|
+
}
|
|
130
236
|
}
|
|
131
237
|
if (a.judged && !a.judged.inDomain) scope++;
|
|
132
238
|
}
|
|
@@ -135,7 +241,28 @@ export function scoreCharterRun(
|
|
|
135
241
|
if (a.noComment) quietRight++;
|
|
136
242
|
}
|
|
137
243
|
}
|
|
138
|
-
const positives = probes.filter((p) => p.kind === 'positive')
|
|
244
|
+
const positives = probes.filter((p) => p.kind === 'positive');
|
|
245
|
+
const owners = positives.filter((p) => p.gold?.domains.length).length;
|
|
246
|
+
const asked = new Set(answers.map((a) => a.probeId));
|
|
247
|
+
const byTrigger: CharterScore['byTrigger'] = Object.fromEntries(
|
|
248
|
+
[...perTrigger].map(([trigger, t]) => {
|
|
249
|
+
const reached = probes.filter((p) => t.probes.has(p.id));
|
|
250
|
+
return [
|
|
251
|
+
trigger,
|
|
252
|
+
{
|
|
253
|
+
probes: reached.length,
|
|
254
|
+
positives: reached.filter((p) => p.kind === 'positive').length,
|
|
255
|
+
anticipated: ratio(
|
|
256
|
+
t.anticipated.size,
|
|
257
|
+
reached.filter((p) => p.kind === 'positive').length,
|
|
258
|
+
),
|
|
259
|
+
remarks: t.remarks,
|
|
260
|
+
speakPrecision: ratio(t.right, t.remarks),
|
|
261
|
+
wrongRemarks: t.remarks - t.right,
|
|
262
|
+
},
|
|
263
|
+
];
|
|
264
|
+
}),
|
|
265
|
+
);
|
|
139
266
|
return {
|
|
140
267
|
probes: probes.length,
|
|
141
268
|
calls,
|
|
@@ -143,7 +270,18 @@ export function scoreCharterRun(
|
|
|
143
270
|
silencePrecision: ratio(quietRight, quietCases),
|
|
144
271
|
speakPrecision: ratio(remarksRight, remarks),
|
|
145
272
|
scopeViolations: scope,
|
|
146
|
-
recall: ratio(recalled.size,
|
|
273
|
+
recall: ratio(recalled.size, owners),
|
|
274
|
+
anticipated: ratio(anticipated.size, positives.length),
|
|
275
|
+
weightedAnticipated: (() => {
|
|
276
|
+
const total = positives.reduce((n, p) => n + (p.weight ?? 1), 0);
|
|
277
|
+
const said = positives
|
|
278
|
+
.filter((p) => anticipated.has(p.id))
|
|
279
|
+
.reduce((n, p) => n + (p.weight ?? 1), 0);
|
|
280
|
+
return total ? Math.round((said / total) * 1000) / 1000 : null;
|
|
281
|
+
})(),
|
|
282
|
+
wrongRemarks: remarks - remarksRight,
|
|
283
|
+
wrongPerProbe: ratio(remarks - remarksRight, probes.filter((p) => asked.has(p.id)).length),
|
|
284
|
+
byTrigger,
|
|
147
285
|
remarks,
|
|
148
286
|
silences,
|
|
149
287
|
unjudged,
|
|
@@ -166,31 +304,205 @@ export function scoreCharterRun(
|
|
|
166
304
|
};
|
|
167
305
|
}
|
|
168
306
|
|
|
307
|
+
/** What a correction pushed the agent toward. Absent when it pushed toward nothing that already existed. */
|
|
308
|
+
export interface CorrectionTarget {
|
|
309
|
+
/** The thing itself, named as the codebase names it (e.g. "Inquiry handler autoApprove"). */
|
|
310
|
+
readonly name: string;
|
|
311
|
+
readonly kind: 'primitive' | 'system' | 'pattern' | 'rule';
|
|
312
|
+
/** The concept cluster it belongs to (kebab-case, e.g. "concurrency-locking"): the routing label. */
|
|
313
|
+
readonly concept: string;
|
|
314
|
+
/** Where it lives, when the moment says (file, package, doc). */
|
|
315
|
+
readonly where?: string;
|
|
316
|
+
/** The canonical name shared by every correction that pushed toward the same thing. */
|
|
317
|
+
readonly canonical?: string;
|
|
318
|
+
/** What was built instead of using it, when the source says (PR reviews do). */
|
|
319
|
+
readonly builtInstead?: string;
|
|
320
|
+
/**
|
|
321
|
+
* Why it was missed, from the thread: `unaware` (didn't know it existed), `misfit` (knew it and
|
|
322
|
+
* worked around a gap instead of extending it), `wrapper` (wrapped it under a new name), `unclear`.
|
|
323
|
+
*/
|
|
324
|
+
readonly miss?: MissKind;
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
export const MISS_KINDS = ['unaware', 'misfit', 'wrapper', 'unclear'] as const;
|
|
328
|
+
export type MissKind = (typeof MISS_KINDS)[number];
|
|
329
|
+
|
|
330
|
+
/** Concept clusters the labeler starts from; it may add one when none fits. */
|
|
331
|
+
export const CONCEPT_CLUSTERS = [
|
|
332
|
+
'concurrency-locking',
|
|
333
|
+
'requests-inquiry',
|
|
334
|
+
'access-signets-permissions',
|
|
335
|
+
'audit-log',
|
|
336
|
+
'errors-responses',
|
|
337
|
+
'data-schema-db',
|
|
338
|
+
'queries-db-client',
|
|
339
|
+
'jobs-queues-cron',
|
|
340
|
+
'caching',
|
|
341
|
+
'http-routes-api',
|
|
342
|
+
'validation-types',
|
|
343
|
+
'testing-fixtures-vcr',
|
|
344
|
+
'frontend-ui',
|
|
345
|
+
'email-communications',
|
|
346
|
+
'config-env-setup',
|
|
347
|
+
'tooling-ci-lint',
|
|
348
|
+
'naming',
|
|
349
|
+
'docs-conventions',
|
|
350
|
+
'shared-helpers-utils',
|
|
351
|
+
'process-merge-ship',
|
|
352
|
+
'process-communication',
|
|
353
|
+
] as const;
|
|
354
|
+
|
|
355
|
+
export const CORRECTION_TARGET_PROMPT = `You label corrections a developer gave an AI coding agent. Each item shows the developer's request, the agent's turn when recorded, and the developer's correction.
|
|
356
|
+
|
|
357
|
+
Most corrections push the agent toward something that ALREADY EXISTS: a primitive (a helper, wrapper, class, module, package export), a system (an existing subsystem or flow, e.g. the inquiry system, the audit log), a pattern (an established way the codebase does a thing), or a rule (a standing convention or process rule, e.g. "merge yourself once green", "no backwards compatibility"). Name that target.
|
|
358
|
+
|
|
359
|
+
For each item:
|
|
360
|
+
- target: null when the correction pushes toward nothing that already existed (a new requirement, a bug report, a fact about the world, a scope change). Otherwise an object:
|
|
361
|
+
- name: the thing itself, as the codebase would name it, at most 80 characters (e.g. "Inquiry handler autoApprove", "shared VCR utilities", "SELECT FOR UPDATE lock helper", "audit log records the actor"). Name the thing, not the advice.
|
|
362
|
+
- kind: "primitive" | "system" | "pattern" | "rule"
|
|
363
|
+
- concept: the concept cluster it belongs to; use one of the given clusters when one fits, else a new short kebab-case cluster.
|
|
364
|
+
- where: the file, package or doc it lives in, only when the item says so; else omit.
|
|
365
|
+
|
|
366
|
+
Respond with JSON only: { "results": { "<id>": { "target": null | { "name": string, "kind": string, "concept": string, "where"?: string } } } }`;
|
|
367
|
+
|
|
368
|
+
/** The labeler's input for a batch of positive probes. */
|
|
369
|
+
export function correctionTargetMessages(
|
|
370
|
+
probes: readonly Pick<WardenProbe, 'id' | 'ask' | 'moment' | 'hidden'>[],
|
|
371
|
+
): { role: 'system' | 'user'; content: string }[] {
|
|
372
|
+
const body = probes
|
|
373
|
+
.map(
|
|
374
|
+
(p) =>
|
|
375
|
+
`## ${p.id}\n### Request\n${p.ask?.slice(0, 1500) || '(not recorded)'}\n\n### Agent turn\n${p.moment.slice(-1500) || '(not recorded)'}\n\n### Correction\n${(p.hidden ?? '').slice(0, 1500)}`,
|
|
376
|
+
)
|
|
377
|
+
.join('\n\n');
|
|
378
|
+
return [
|
|
379
|
+
{ role: 'system', content: CORRECTION_TARGET_PROMPT },
|
|
380
|
+
{
|
|
381
|
+
role: 'user',
|
|
382
|
+
content: `Concept clusters: ${CONCEPT_CLUSTERS.join(', ')}\n\nLabel each item below.\n\n${body}`,
|
|
383
|
+
},
|
|
384
|
+
];
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
const TARGET_KINDS = new Set(['primitive', 'system', 'pattern', 'rule']);
|
|
388
|
+
|
|
389
|
+
/** One answer per id: a target, null for no existing target, undefined when missing or malformed. */
|
|
390
|
+
export function parseCorrectionTargets(
|
|
391
|
+
content: string,
|
|
392
|
+
ids: readonly string[],
|
|
393
|
+
): (CorrectionTarget | null | undefined)[] {
|
|
394
|
+
const start = content.indexOf('{');
|
|
395
|
+
const end = content.lastIndexOf('}');
|
|
396
|
+
let results: Record<string, { target?: unknown }> = {};
|
|
397
|
+
try {
|
|
398
|
+
results =
|
|
399
|
+
(JSON.parse(content.slice(start, end + 1)) as { results?: typeof results }).results ?? {};
|
|
400
|
+
} catch {
|
|
401
|
+
return ids.map(() => undefined);
|
|
402
|
+
}
|
|
403
|
+
return ids.map((id) => {
|
|
404
|
+
const item = results[id];
|
|
405
|
+
if (!item || !('target' in item)) return undefined;
|
|
406
|
+
if (item.target === null) return null;
|
|
407
|
+
const t = item.target as Record<string, unknown>;
|
|
408
|
+
if (typeof t.name !== 'string' || !t.name.trim() || typeof t.concept !== 'string')
|
|
409
|
+
return undefined;
|
|
410
|
+
return {
|
|
411
|
+
name: t.name.trim().slice(0, 120),
|
|
412
|
+
kind: (TARGET_KINDS.has(String(t.kind)) ? t.kind : 'pattern') as CorrectionTarget['kind'],
|
|
413
|
+
concept: t.concept
|
|
414
|
+
.trim()
|
|
415
|
+
.toLowerCase()
|
|
416
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
417
|
+
.replace(/^-|-$/g, ''),
|
|
418
|
+
...(typeof t.where === 'string' && t.where.trim()
|
|
419
|
+
? { where: t.where.trim().slice(0, 160) }
|
|
420
|
+
: {}),
|
|
421
|
+
};
|
|
422
|
+
});
|
|
423
|
+
}
|
|
424
|
+
|
|
425
|
+
export const TARGET_CANONICAL_PROMPT = `You merge names of existing codebase things (primitives, systems, patterns, rules) that developers' corrections pointed to. All names given share one concept cluster. Names that refer to the same thing get the same canonical name; different things stay apart (when unsure, keep them apart). A canonical name is the clearest of the merged names, at most 80 characters.
|
|
426
|
+
Respond with JSON only: { "canonical": { "<N1>": "<canonical name>", ... } } with every given key.`;
|
|
427
|
+
|
|
428
|
+
/** The canonicalization input: one concept cluster's distinct target names, keyed N1..Nn. */
|
|
429
|
+
export function targetCanonicalMessages(
|
|
430
|
+
concept: string,
|
|
431
|
+
names: readonly string[],
|
|
432
|
+
): { role: 'system' | 'user'; content: string }[] {
|
|
433
|
+
return [
|
|
434
|
+
{ role: 'system', content: TARGET_CANONICAL_PROMPT },
|
|
435
|
+
{
|
|
436
|
+
role: 'user',
|
|
437
|
+
content: `Cluster: ${concept}\n\n${names.map((n, i) => `N${i + 1}: ${n}`).join('\n')}`,
|
|
438
|
+
},
|
|
439
|
+
];
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
/** Canonical names in order; a missing key keeps the name as given. */
|
|
443
|
+
export function parseTargetCanonical(content: string, names: readonly string[]): string[] {
|
|
444
|
+
try {
|
|
445
|
+
const parsed = JSON.parse(
|
|
446
|
+
content.slice(content.indexOf('{'), content.lastIndexOf('}') + 1),
|
|
447
|
+
) as {
|
|
448
|
+
canonical?: Record<string, unknown>;
|
|
449
|
+
};
|
|
450
|
+
return names.map((name, i) => {
|
|
451
|
+
const value = parsed.canonical?.[`N${i + 1}`];
|
|
452
|
+
return typeof value === 'string' && value.trim() ? value.trim().slice(0, 120) : name;
|
|
453
|
+
});
|
|
454
|
+
} catch {
|
|
455
|
+
return [...names];
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
|
|
169
459
|
/** The judge's instructions: one remark against the user's real next message. */
|
|
170
|
-
export const WARDEN_JUDGE_PROMPT = `You grade one remark a narrow expert reviewer (a "Warden") made
|
|
171
|
-
|
|
460
|
+
export const WARDEN_JUDGE_PROMPT = `You grade one remark a narrow expert reviewer (a "Warden") made during an AI coding session.
|
|
461
|
+
The Warden spoke either on the user's message, before the agent worked, or on the agent's turn, after it.
|
|
462
|
+
You see the Warden's domain, what the Warden saw, the remark, and what the user actually said next.
|
|
463
|
+
When a TARGET is given (the existing rule, primitive, system or pattern the user pushed the agent toward), the test is the target, not wording: the remark matches when it names that same target (or points the agent to it unmistakably), however differently it is phrased.
|
|
172
464
|
Answer two questions:
|
|
173
|
-
- matches: does the remark say what the user's next message enforced: the same rule, or the same fix? A remark that is merely related, or generic, does not match.
|
|
465
|
+
- matches: with a target, does the remark name the same target? Without one, does the remark say what the user's next message enforced: the same rule, or the same fix? A remark that is merely related, or generic, does not match.
|
|
174
466
|
- inDomain: does the remark stay inside the Warden's stated domain?
|
|
175
467
|
Respond with JSON only: { "matches": boolean, "inDomain": boolean, "reason": string } where reason is one short sentence.`;
|
|
176
468
|
|
|
177
|
-
/** The judge's input for one remark. */
|
|
469
|
+
/** The judge's input for one remark: what the Warden saw at its trigger point, and what came next. */
|
|
178
470
|
export function wardenJudgeMessage(input: {
|
|
179
471
|
domain: string;
|
|
180
472
|
domainDescription: string;
|
|
473
|
+
/** The agent's turn; '' when the Warden spoke on the user's message only. */
|
|
181
474
|
moment: string;
|
|
475
|
+
/** The user's message the agent was answering, when recorded. */
|
|
476
|
+
ask?: string;
|
|
477
|
+
trigger?: WardenProbeTrigger;
|
|
478
|
+
/** What the user pushed toward, when labelled. */
|
|
479
|
+
target?: CorrectionTarget;
|
|
182
480
|
remark: string;
|
|
183
481
|
hidden: string;
|
|
184
482
|
}): string {
|
|
483
|
+
const before = input.trigger === 'user-message';
|
|
185
484
|
return [
|
|
186
485
|
`## Warden domain: ${input.domain}`,
|
|
187
486
|
input.domainDescription,
|
|
188
|
-
|
|
189
|
-
input.moment
|
|
190
|
-
|
|
487
|
+
...(input.ask ? ["## The user's message", input.ask] : []),
|
|
488
|
+
...(input.moment
|
|
489
|
+
? [
|
|
490
|
+
before ? '## Agent turn (after the remark; the Warden did not see it)' : '## Agent turn',
|
|
491
|
+
input.moment,
|
|
492
|
+
]
|
|
493
|
+
: []),
|
|
494
|
+
before
|
|
495
|
+
? "## Remark (made on the user's message, before the agent worked)"
|
|
496
|
+
: '## Remark (made on the agent turn)',
|
|
191
497
|
input.remark,
|
|
192
498
|
"## The user's actual next message",
|
|
193
499
|
input.hidden || '(the user let the turn pass without correcting it)',
|
|
500
|
+
...(input.target
|
|
501
|
+
? [
|
|
502
|
+
'## TARGET (what the user pushed the agent toward)',
|
|
503
|
+
`${input.target.canonical ?? input.target.name} (${input.target.kind}, ${input.target.concept}${input.target.where ? `, ${input.target.where}` : ''})`,
|
|
504
|
+
]
|
|
505
|
+
: []),
|
|
194
506
|
].join('\n\n');
|
|
195
507
|
}
|
|
196
508
|
|
|
@@ -210,3 +522,59 @@ export function parseWardenJudgement(text: string): WardenJudgement | undefined
|
|
|
210
522
|
return undefined;
|
|
211
523
|
}
|
|
212
524
|
}
|
|
525
|
+
|
|
526
|
+
/**
|
|
527
|
+
* The pointer judge: a Warden that classifies (relevant, and if so "look over here") is graded on
|
|
528
|
+
* where its pointer leads, not on advice quality. A pointer is as specific as the Warden can make
|
|
529
|
+
* it, and a general one is better than none (Aron, 2026-10-15): the exact thing, the target among a
|
|
530
|
+
* few candidates, or the right class.
|
|
531
|
+
*/
|
|
532
|
+
export const WARDEN_POINTER_JUDGE_PROMPT = `You grade one pointer a narrow classifier (a "Warden") gave a coding agent. The Warden only says whether something it knows applies to the agent's moment and, if so, points at it ("look over here: ..."); the agent then follows the pointer itself.
|
|
533
|
+
You see the agent's moment, the TARGET (the existing thing that actually applies, with where it is written) and the pointer.
|
|
534
|
+
Place the pointer on this ladder, leniently on wording (a doc section that holds the TARGET counts as naming it):
|
|
535
|
+
- "exact": it points at the TARGET itself, alone or clearly first.
|
|
536
|
+
- "candidates": it names at most three specific things to choose between, and the TARGET is one of them.
|
|
537
|
+
- "class": it names no specific TARGET but points at the right class or area (the family, doc or section the TARGET belongs to), near enough to find it.
|
|
538
|
+
- "wrong": it steers away from the TARGET (a different specific thing, an unrelated area, more than three candidates without the TARGET singled out).
|
|
539
|
+
Respond with JSON only: { "level": "exact" | "candidates" | "class" | "wrong", "reason": string } where reason is one short sentence.`;
|
|
540
|
+
|
|
541
|
+
export const POINTER_LEVELS = ['exact', 'candidates', 'class', 'wrong'] as const;
|
|
542
|
+
export type PointerLevel = (typeof POINTER_LEVELS)[number];
|
|
543
|
+
|
|
544
|
+
export interface WardenPointerJudgement {
|
|
545
|
+
readonly level: PointerLevel;
|
|
546
|
+
readonly reason?: string;
|
|
547
|
+
}
|
|
548
|
+
|
|
549
|
+
/** The pointer judge's input for one pointer. */
|
|
550
|
+
export function wardenPointerJudgeMessage(input: {
|
|
551
|
+
moment: string;
|
|
552
|
+
target: CorrectionTarget;
|
|
553
|
+
pointer: string;
|
|
554
|
+
}): string {
|
|
555
|
+
return [
|
|
556
|
+
'## Agent moment',
|
|
557
|
+
input.moment,
|
|
558
|
+
'## TARGET',
|
|
559
|
+
`${input.target.canonical ?? input.target.name} (${input.target.kind}${input.target.where ? `, ${input.target.where}` : ''})`,
|
|
560
|
+
'## Pointer',
|
|
561
|
+
input.pointer,
|
|
562
|
+
].join('\n\n');
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/** Parse a pointer judgement strictly; undefined when it is not the shape. */
|
|
566
|
+
export function parseWardenPointerJudgement(text: string): WardenPointerJudgement | undefined {
|
|
567
|
+
const trimmed = text.trim();
|
|
568
|
+
const fenced = /^```(?:json)?\s*\n([\s\S]*?)\n?\s*```$/.exec(trimmed);
|
|
569
|
+
try {
|
|
570
|
+
const value = JSON.parse(fenced ? fenced[1]! : trimmed) as Record<string, unknown>;
|
|
571
|
+
const level = POINTER_LEVELS.find((l) => l === value.level);
|
|
572
|
+
if (!level) return undefined;
|
|
573
|
+
return {
|
|
574
|
+
level,
|
|
575
|
+
...(typeof value.reason === 'string' ? { reason: value.reason.slice(0, 300) } : {}),
|
|
576
|
+
};
|
|
577
|
+
} catch {
|
|
578
|
+
return undefined;
|
|
579
|
+
}
|
|
580
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -36,6 +36,7 @@ export { HttpMemory } from './adapters/http-memory';
|
|
|
36
36
|
export {
|
|
37
37
|
claudemdSource,
|
|
38
38
|
formatInstructionFiles,
|
|
39
|
+
INSTRUCTION_FILE_LIMIT,
|
|
39
40
|
type InstructionFile,
|
|
40
41
|
type InstructionReadOptions,
|
|
41
42
|
instructionImports,
|
|
@@ -189,6 +190,30 @@ export {
|
|
|
189
190
|
scrubSecrets,
|
|
190
191
|
type TranscriptTurn,
|
|
191
192
|
} from './feedback/correction';
|
|
193
|
+
export {
|
|
194
|
+
DESIGN_SECTIONS,
|
|
195
|
+
type ExperimentRecord,
|
|
196
|
+
experimentProblems,
|
|
197
|
+
requireExperiment,
|
|
198
|
+
reviewPathOf,
|
|
199
|
+
TINY_MOMENTS,
|
|
200
|
+
} from './feedback/experiment-gate';
|
|
201
|
+
export {
|
|
202
|
+
AUTHORSHIP_MARKERS,
|
|
203
|
+
authorshipOf,
|
|
204
|
+
evidenceSnippet,
|
|
205
|
+
type FindingScope,
|
|
206
|
+
parseReviewFindings,
|
|
207
|
+
REUSE_HINT,
|
|
208
|
+
REVIEW_FINDINGS_PROMPT,
|
|
209
|
+
type ReviewFinding,
|
|
210
|
+
type ReviewMoment,
|
|
211
|
+
type ReviewReading,
|
|
212
|
+
reviewFindingMessages,
|
|
213
|
+
SCOPE_WEIGHT,
|
|
214
|
+
type SessionEvidence,
|
|
215
|
+
styleOf,
|
|
216
|
+
} from './feedback/review-findings';
|
|
192
217
|
export {
|
|
193
218
|
type DomainAgreement,
|
|
194
219
|
fixtureProbe,
|
|
@@ -222,15 +247,36 @@ export {
|
|
|
222
247
|
scorePrefilter,
|
|
223
248
|
} from './feedback/warden-prefilter';
|
|
224
249
|
export {
|
|
250
|
+
type Authorship,
|
|
225
251
|
type CharterScore,
|
|
252
|
+
CONCEPT_CLUSTERS,
|
|
253
|
+
CORRECTION_TARGET_PROMPT,
|
|
254
|
+
type CorrectionTarget,
|
|
255
|
+
correctionTargetMessages,
|
|
256
|
+
MISS_KINDS,
|
|
257
|
+
type MissKind,
|
|
258
|
+
POINTER_LEVELS,
|
|
259
|
+
type PointerLevel,
|
|
260
|
+
type ProbeSource,
|
|
261
|
+
parseCorrectionTargets,
|
|
262
|
+
parseTargetCanonical,
|
|
226
263
|
parseWardenJudgement,
|
|
264
|
+
parseWardenPointerJudgement,
|
|
265
|
+
probeTriggers,
|
|
227
266
|
scoreCharterRun,
|
|
267
|
+
TARGET_CANONICAL_PROMPT,
|
|
268
|
+
targetCanonicalMessages,
|
|
228
269
|
WARDEN_JUDGE_PROMPT,
|
|
270
|
+
WARDEN_POINTER_JUDGE_PROMPT,
|
|
229
271
|
type WardenJudgement,
|
|
272
|
+
type WardenPointerJudgement,
|
|
230
273
|
type WardenProbe,
|
|
231
274
|
type WardenProbeAnswer,
|
|
275
|
+
type WardenProbeTrigger,
|
|
232
276
|
type WardenProbeTurn,
|
|
233
277
|
wardenJudgeMessage,
|
|
278
|
+
wardenPointerJudgeMessage,
|
|
279
|
+
wilsonInterval,
|
|
234
280
|
} from './feedback/warden-probes';
|
|
235
281
|
export {
|
|
236
282
|
type FlowConfig,
|
|
@@ -345,6 +391,7 @@ export {
|
|
|
345
391
|
type ThreadMeta,
|
|
346
392
|
type ThreadName,
|
|
347
393
|
type ThreadReference,
|
|
394
|
+
type ThreadSetup,
|
|
348
395
|
type ThreadSpawn,
|
|
349
396
|
type ThreadStatus,
|
|
350
397
|
threadTitle,
|
package/src/thread.ts
CHANGED
|
@@ -96,6 +96,12 @@ export interface ThreadMeta {
|
|
|
96
96
|
/** 1 for a child of a main thread. */
|
|
97
97
|
depth?: number;
|
|
98
98
|
|
|
99
|
+
/**
|
|
100
|
+
* Start-up of a thread whose Wardens are held in its own process (docs/THREAD-TREE.md spec C):
|
|
101
|
+
* turns typed meanwhile wait as `queued` until it is `ready` (or `degraded`).
|
|
102
|
+
*/
|
|
103
|
+
setup?: ThreadSetup;
|
|
104
|
+
|
|
99
105
|
/** Project that owns this thread. Scopes memory sources and tool calls. */
|
|
100
106
|
projectId?: string;
|
|
101
107
|
|
|
@@ -109,6 +115,15 @@ export interface ThreadMeta {
|
|
|
109
115
|
archivedAt?: number;
|
|
110
116
|
}
|
|
111
117
|
|
|
118
|
+
export interface ThreadSetup {
|
|
119
|
+
status: 'starting' | 'ready' | 'degraded';
|
|
120
|
+
startedAt: number;
|
|
121
|
+
readyAt?: number;
|
|
122
|
+
/** Every configured Warden, those holding a spare, and those that did not by the deadline. */
|
|
123
|
+
wardens: { expected: number; held: number; failed: string[] };
|
|
124
|
+
runtime: 'claude-in-session' | 'codex-in-process' | 'side-session';
|
|
125
|
+
}
|
|
126
|
+
|
|
112
127
|
export interface Dispatch<T = unknown> {
|
|
113
128
|
readonly agentId: string;
|
|
114
129
|
readonly timestamp: number;
|
package/src/types.ts
CHANGED
|
@@ -114,6 +114,11 @@ export interface CompletionOpts {
|
|
|
114
114
|
* cached prefix; others ignore it. Messages are sent unchanged either way.
|
|
115
115
|
*/
|
|
116
116
|
stablePrefix?: string;
|
|
117
|
+
/**
|
|
118
|
+
* Text right after `stablePrefix` that changes only between turns (a Warden's thread knowledge and
|
|
119
|
+
* the thread state). Providers that pre-start a call shape hold it; others ignore it.
|
|
120
|
+
*/
|
|
121
|
+
threadPrefix?: string;
|
|
117
122
|
/**
|
|
118
123
|
* The executor's context split into standing parts and per-turn text. A provider that keeps a
|
|
119
124
|
* native session delivers standing parts once and then only what changed (session-delivery.ts);
|
|
@@ -175,6 +180,13 @@ export interface LLMProvider {
|
|
|
175
180
|
};
|
|
176
181
|
complete(messages: LLMMessage[], opts?: CompletionOpts): Promise<CompletionResult>;
|
|
177
182
|
stream?(messages: LLMMessage[], opts?: CompletionOpts): AsyncGenerator<LLMStreamEvent>;
|
|
183
|
+
/**
|
|
184
|
+
* Keep this call shape pre-started, so a later `complete` with the same `stablePrefix` and
|
|
185
|
+
* `threadPrefix` starts at once. `messages` end at those prefixes; no decision is made. Optional.
|
|
186
|
+
*/
|
|
187
|
+
prestart?(messages: LLMMessage[], opts: CompletionOpts): void;
|
|
188
|
+
/** A thread closed: release whatever the provider holds for it. Optional. */
|
|
189
|
+
releaseThread?(threadId: string): Promise<void>;
|
|
178
190
|
}
|
|
179
191
|
|
|
180
192
|
export interface EmbeddingProvider {
|