@hecer/yoke 1.2.1 → 1.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/CHANGELOG.md +66 -24
- package/README.md +186 -104
- package/TODOS.md +0 -3
- package/canon/loop/loop-spec.md +53 -24
- package/canon/loop/prd.schema.md +30 -6
- package/canon/manifest.yaml +1 -1
- package/canon/skills/authoring-prd/SKILL.md +30 -31
- package/dist/agents/contracts.js +50 -0
- package/dist/agents/process-incarnation.js +15 -0
- package/dist/agents/process-record.js +65 -0
- package/dist/agents/process-streams.js +40 -0
- package/dist/agents/process.js +177 -0
- package/dist/agents/providers.js +10 -7
- package/dist/agents/telemetry.js +62 -0
- package/dist/change/inbox.js +279 -0
- package/dist/cli.js +90 -4
- package/dist/loop/candidate-boundaries.js +43 -0
- package/dist/loop/candidate-cleanup.js +98 -0
- package/dist/loop/candidate-contracts.js +1 -0
- package/dist/loop/candidate-selection.js +84 -0
- package/dist/loop/candidates.js +228 -0
- package/dist/loop/claim-lease.js +131 -0
- package/dist/loop/claims.js +177 -40
- package/dist/loop/cleanup.js +117 -15
- package/dist/loop/decision.js +31 -0
- package/dist/loop/dispatcher.js +334 -0
- package/dist/loop/evidence.js +31 -0
- package/dist/loop/gates.js +10 -1
- package/dist/loop/loop.js +236 -33
- package/dist/loop/merge-queue.js +12 -6
- package/dist/loop/parallel-adapters.js +185 -0
- package/dist/loop/parallel-command.js +287 -0
- package/dist/loop/parallel.js +2 -4
- package/dist/loop/prd.js +63 -2
- package/dist/loop/reporter.js +86 -5
- package/dist/loop/run-command.js +227 -53
- package/dist/loop/runner.js +77 -34
- package/dist/loop/verify.js +11 -0
- package/dist/loop/watchdog.js +67 -8
- package/dist/loop/worker-cancellation.js +17 -0
- package/dist/loop/worker-cleanup.js +23 -0
- package/dist/loop/worker-contracts.js +1 -0
- package/dist/loop/worker.js +254 -0
- package/dist/prd/command.js +18 -5
- package/dist/quality/artifacts.js +59 -0
- package/dist/quality/candidate-comparison.js +130 -0
- package/dist/quality/command.js +316 -0
- package/dist/quality/loop.js +86 -0
- package/dist/quality/process-command.js +57 -0
- package/dist/quality/reference.js +187 -0
- package/dist/quality/repair.js +11 -0
- package/dist/quality/runner.js +66 -0
- package/dist/quality/types.js +60 -0
- package/dist/quality/verdict.js +142 -0
- package/dist/retrofit/config.js +13 -4
- package/dist/retrofit/gitignore.js +4 -0
- package/dist/review/command.js +27 -38
- package/dist/review/verdict.js +38 -7
- package/dist/routing/router.js +4 -1
- package/docs/MIGRATING-TO-1.4.md +70 -0
- package/docs/PUBLISHING.md +16 -2
- package/docs/superpowers/plans/2026-08-13-gauntlet-quality-loop.md +537 -0
- package/docs/superpowers/specs/2026-08-13-gauntlet-quality-loop-design.md +422 -0
- package/gemini-extension.json +1 -1
- package/package.json +6 -6
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
3
|
+
import { hasShellControlOperator as containsShellControlOperator, parseQualityCommand } from './process-command.js';
|
|
4
|
+
const DEFAULT_MAX_BYTES = 10 * 1024 * 1024;
|
|
5
|
+
const DEFAULT_MAX_REDIRECTS = 3;
|
|
6
|
+
export function hasShellControlOperator(command) {
|
|
7
|
+
return containsShellControlOperator(command);
|
|
8
|
+
}
|
|
9
|
+
export function acquireReference(input, adapters) {
|
|
10
|
+
const bytesResult = acquireBytes(input, adapters);
|
|
11
|
+
if (bytesResult.kind === 'rejected')
|
|
12
|
+
return bytesResult;
|
|
13
|
+
if (bytesResult.content.bytes.byteLength > (input.maxBytes ?? DEFAULT_MAX_BYTES)) {
|
|
14
|
+
return { kind: 'rejected', reason: 'oversize' };
|
|
15
|
+
}
|
|
16
|
+
const digest = createHash('sha256').update(bytesResult.content.bytes).digest('hex');
|
|
17
|
+
if (input.reference.digest && normalizeDigest(input.reference.digest) !== digest) {
|
|
18
|
+
return { kind: 'rejected', reason: 'digest-mismatch' };
|
|
19
|
+
}
|
|
20
|
+
const referencesDir = join(resolve(input.projectDir), '.yoke', 'references');
|
|
21
|
+
const storagePath = join(referencesDir, digest);
|
|
22
|
+
const provenance = {
|
|
23
|
+
kind: input.reference.kind,
|
|
24
|
+
source: input.reference.source,
|
|
25
|
+
acquiredAt: (input.now ?? (() => new Date().toISOString()))(),
|
|
26
|
+
...(bytesResult.content.contentType ? { contentType: bytesResult.content.contentType } : {}),
|
|
27
|
+
byteSize: bytesResult.content.bytes.byteLength,
|
|
28
|
+
sha256: digest,
|
|
29
|
+
adapterVersion: adapters.version ?? 'unknown',
|
|
30
|
+
...(bytesResult.content.parameters ? { parameters: bytesResult.content.parameters } : {}),
|
|
31
|
+
};
|
|
32
|
+
adapters.mkdir(storagePath);
|
|
33
|
+
adapters.writeFile(join(storagePath, 'content'), bytesResult.content.bytes);
|
|
34
|
+
adapters.writeFile(join(storagePath, 'provenance.json'), new TextEncoder().encode(JSON.stringify(provenance)));
|
|
35
|
+
return {
|
|
36
|
+
kind: 'acquired',
|
|
37
|
+
artifact: {
|
|
38
|
+
name: input.reference.name,
|
|
39
|
+
digest,
|
|
40
|
+
bytes: bytesResult.content.bytes.byteLength,
|
|
41
|
+
storagePath,
|
|
42
|
+
provenance,
|
|
43
|
+
},
|
|
44
|
+
};
|
|
45
|
+
}
|
|
46
|
+
function acquireBytes(input, adapters) {
|
|
47
|
+
if (input.reference.kind === 'file')
|
|
48
|
+
return acquireFile(input, adapters);
|
|
49
|
+
if (input.reference.kind === 'command')
|
|
50
|
+
return acquireCommand(input, adapters);
|
|
51
|
+
return acquireUrl(input, adapters);
|
|
52
|
+
}
|
|
53
|
+
function acquireFile(input, adapters) {
|
|
54
|
+
const source = input.reference.source;
|
|
55
|
+
if (containsTraversal(source))
|
|
56
|
+
return { kind: 'rejected', reason: 'path-traversal' };
|
|
57
|
+
if (isAbsolute(source))
|
|
58
|
+
return { kind: 'rejected', reason: 'outside-project' };
|
|
59
|
+
const projectRoot = resolve(input.projectDir);
|
|
60
|
+
const path = resolve(projectRoot, source);
|
|
61
|
+
if (!isInside(projectRoot, path))
|
|
62
|
+
return { kind: 'rejected', reason: 'outside-project' };
|
|
63
|
+
const canonicalRoot = adapters.realpath?.(projectRoot) ?? projectRoot;
|
|
64
|
+
const canonicalPath = adapters.realpath?.(path) ?? path;
|
|
65
|
+
if (!isInside(canonicalRoot, canonicalPath))
|
|
66
|
+
return { kind: 'rejected', reason: 'outside-project' };
|
|
67
|
+
const bytes = adapters.readFile(canonicalPath);
|
|
68
|
+
return bytes === null ? { kind: 'rejected', reason: 'missing' } : { kind: 'bytes', content: normalizeContent(bytes) };
|
|
69
|
+
}
|
|
70
|
+
function acquireCommand(input, adapters) {
|
|
71
|
+
if (hasShellControlOperator(input.reference.source))
|
|
72
|
+
return { kind: 'rejected', reason: 'shell-control' };
|
|
73
|
+
if (!parseQualityCommand(input.reference.source))
|
|
74
|
+
return { kind: 'rejected', reason: 'invalid-command' };
|
|
75
|
+
return { kind: 'bytes', content: normalizeContent(adapters.runCommand(input.reference.source)) };
|
|
76
|
+
}
|
|
77
|
+
function acquireUrl(input, adapters) {
|
|
78
|
+
let current = input.reference.source;
|
|
79
|
+
let redirects = 0;
|
|
80
|
+
const maxRedirects = input.maxRedirects ?? DEFAULT_MAX_REDIRECTS;
|
|
81
|
+
for (;;) {
|
|
82
|
+
if (!URL.canParse(current))
|
|
83
|
+
return { kind: 'rejected', reason: 'invalid-url' };
|
|
84
|
+
const url = new URL(current);
|
|
85
|
+
if (url.protocol !== 'http:' && url.protocol !== 'https:') {
|
|
86
|
+
return { kind: 'rejected', reason: 'unsupported-protocol' };
|
|
87
|
+
}
|
|
88
|
+
const addressResult = validateAddress(url.hostname, input.allowLocal === true, adapters);
|
|
89
|
+
if (addressResult.kind === 'rejected')
|
|
90
|
+
return addressResult;
|
|
91
|
+
const response = adapters.requestUrl(url.toString(), addressResult.addresses);
|
|
92
|
+
if (response.status >= 300 && response.status < 400) {
|
|
93
|
+
if (!response.location || !URL.canParse(response.location, url.toString())) {
|
|
94
|
+
return { kind: 'rejected', reason: 'invalid-redirect' };
|
|
95
|
+
}
|
|
96
|
+
if (redirects >= maxRedirects)
|
|
97
|
+
return { kind: 'rejected', reason: 'redirect-overflow' };
|
|
98
|
+
current = new URL(response.location, url).toString();
|
|
99
|
+
redirects += 1;
|
|
100
|
+
continue;
|
|
101
|
+
}
|
|
102
|
+
if (response.status < 200 || response.status >= 300)
|
|
103
|
+
return { kind: 'rejected', reason: 'http-status' };
|
|
104
|
+
let content = normalizeContent(response.body);
|
|
105
|
+
if (mediaType(content.contentType) === 'text/html')
|
|
106
|
+
content = htmlAsInertText(content);
|
|
107
|
+
if (content.contentType && !isAllowedUrlContentType(content.contentType)) {
|
|
108
|
+
return { kind: 'rejected', reason: 'unsupported-content-type' };
|
|
109
|
+
}
|
|
110
|
+
return { kind: 'bytes', content };
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
function validateAddress(hostname, allowLocal, adapters) {
|
|
114
|
+
const normalizedHostname = hostname.replace(/^\[|\]$/g, '');
|
|
115
|
+
if (!allowLocal && isPrivateAddress(normalizedHostname)) {
|
|
116
|
+
return { kind: 'rejected', reason: 'private-address' };
|
|
117
|
+
}
|
|
118
|
+
const addresses = adapters.resolveHostname(normalizedHostname);
|
|
119
|
+
if (addresses.length === 0)
|
|
120
|
+
return { kind: 'rejected', reason: 'unresolved-address' };
|
|
121
|
+
if (!allowLocal && addresses.some(isPrivateAddress)) {
|
|
122
|
+
return { kind: 'rejected', reason: 'private-address' };
|
|
123
|
+
}
|
|
124
|
+
return { kind: 'validated', addresses };
|
|
125
|
+
}
|
|
126
|
+
function isAllowedUrlContentType(value) {
|
|
127
|
+
const type = mediaType(value);
|
|
128
|
+
return type?.startsWith('image/') === true
|
|
129
|
+
|| type === 'text/plain'
|
|
130
|
+
|| type === 'application/json'
|
|
131
|
+
|| type === 'application/pdf'
|
|
132
|
+
|| type === 'application/octet-stream';
|
|
133
|
+
}
|
|
134
|
+
function mediaType(value) {
|
|
135
|
+
return value?.split(';', 1)[0]?.trim().toLowerCase();
|
|
136
|
+
}
|
|
137
|
+
function htmlAsInertText(content) {
|
|
138
|
+
const html = new TextDecoder().decode(content.bytes);
|
|
139
|
+
const text = html
|
|
140
|
+
.replace(/<(script|style)\b[^>]*>[\s\S]*?<\/\1>/giu, ' ')
|
|
141
|
+
.replace(/<!--[\s\S]*?-->/gu, ' ')
|
|
142
|
+
.replace(/<[^>]+>/gu, ' ')
|
|
143
|
+
.replace(/ /giu, ' ')
|
|
144
|
+
.replace(/&/giu, '&')
|
|
145
|
+
.replace(/</giu, '<')
|
|
146
|
+
.replace(/>/giu, '>')
|
|
147
|
+
.replace(/"/giu, '"')
|
|
148
|
+
.replace(/\s+/gu, ' ')
|
|
149
|
+
.trim();
|
|
150
|
+
return { bytes: new TextEncoder().encode(text), contentType: 'text/plain', ...(content.parameters ? { parameters: content.parameters } : {}) };
|
|
151
|
+
}
|
|
152
|
+
function containsTraversal(path) {
|
|
153
|
+
return path.split(/[\\/]+/).includes('..');
|
|
154
|
+
}
|
|
155
|
+
function normalizeContent(content) {
|
|
156
|
+
return content instanceof Uint8Array ? { bytes: content } : content;
|
|
157
|
+
}
|
|
158
|
+
function normalizeDigest(value) {
|
|
159
|
+
return value.replace(/^sha256:/i, '').toLowerCase();
|
|
160
|
+
}
|
|
161
|
+
function isInside(root, path) {
|
|
162
|
+
const relativePath = relative(root, path);
|
|
163
|
+
return relativePath === '' || (!relativePath.startsWith(`..${sep}`) && relativePath !== '..' && !isAbsolute(relativePath));
|
|
164
|
+
}
|
|
165
|
+
function isPrivateAddress(value) {
|
|
166
|
+
const address = value.toLowerCase();
|
|
167
|
+
if (address === 'localhost' || address.endsWith('.localhost') || address === '::1' || address === '::')
|
|
168
|
+
return true;
|
|
169
|
+
if (address.startsWith('fc') || address.startsWith('fd') || address.startsWith('fe80:'))
|
|
170
|
+
return true;
|
|
171
|
+
if (address.startsWith('::ffff:'))
|
|
172
|
+
return isPrivateAddress(address.slice('::ffff:'.length));
|
|
173
|
+
const octets = address.split('.');
|
|
174
|
+
if (octets.length !== 4 || octets.some(octet => !/^\d+$/.test(octet)))
|
|
175
|
+
return false;
|
|
176
|
+
const numbers = octets.map(Number);
|
|
177
|
+
if (numbers.some(number => number < 0 || number > 255))
|
|
178
|
+
return false;
|
|
179
|
+
const [first, second] = numbers;
|
|
180
|
+
return first === 0
|
|
181
|
+
|| first === 10
|
|
182
|
+
|| first === 127
|
|
183
|
+
|| (first === 100 && second >= 64 && second <= 127)
|
|
184
|
+
|| (first === 169 && second === 254)
|
|
185
|
+
|| (first === 172 && second >= 16 && second <= 31)
|
|
186
|
+
|| (first === 192 && second === 168);
|
|
187
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export function authorizeRepair(input) {
|
|
2
|
+
if (input.limits?.unbounded)
|
|
3
|
+
return { kind: 'authorized', round: input.usedRounds + 1 };
|
|
4
|
+
const maxRounds = input.limits?.maxRounds ?? 3;
|
|
5
|
+
const maxMinutes = input.limits?.maxMinutes ?? 60;
|
|
6
|
+
if (input.usedRounds >= maxRounds)
|
|
7
|
+
return { kind: 'blocked', reason: 'round-budget-exhausted' };
|
|
8
|
+
if (input.elapsedMs >= maxMinutes * 60 * 1000)
|
|
9
|
+
return { kind: 'blocked', reason: 'time-budget-exhausted' };
|
|
10
|
+
return { kind: 'authorized', round: input.usedRounds + 1 };
|
|
11
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { join } from 'node:path';
|
|
2
|
+
import { createHash } from 'node:crypto';
|
|
3
|
+
import { parseProviderResult } from '../agents/telemetry.js';
|
|
4
|
+
import { storyPathSegment } from '../loop/prd.js';
|
|
5
|
+
import { reduceComparisons } from './verdict.js';
|
|
6
|
+
export function runQualityCritic(input) {
|
|
7
|
+
const segment = input.evidenceScope ? `${input.evidenceScope}-round-${input.round}` : `round-${input.round}`;
|
|
8
|
+
const root = join(input.targetDir, '.yoke', 'proof', storyPathSegment(input.storyId), 'quality', segment);
|
|
9
|
+
input.mkdir(root);
|
|
10
|
+
const first = input.firstLabel();
|
|
11
|
+
const second = first === 'A' ? 'B' : 'A';
|
|
12
|
+
const attemptPrefix = input.attemptIdPrefix ?? String(input.round);
|
|
13
|
+
const normalRequest = request(input, `${attemptPrefix}-normal`, first);
|
|
14
|
+
const swappedRequest = request(input, `${attemptPrefix}-swapped`, second);
|
|
15
|
+
const normal = input.invoke(structuredClone(normalRequest));
|
|
16
|
+
if (!normal.ok)
|
|
17
|
+
return { kind: 'infrastructure', summary: normal.summary };
|
|
18
|
+
const swapped = input.invoke(structuredClone(swappedRequest));
|
|
19
|
+
if (!swapped.ok)
|
|
20
|
+
return { kind: 'infrastructure', summary: swapped.summary };
|
|
21
|
+
if (normal.actualModel && swapped.actualModel && normal.actualModel !== swapped.actualModel)
|
|
22
|
+
return { kind: 'infrastructure', summary: `critic model changed between comparisons: ${normal.actualModel} -> ${swapped.actualModel}` };
|
|
23
|
+
const actualModel = normal.actualModel ?? swapped.actualModel ?? input.model;
|
|
24
|
+
if (!actualModel)
|
|
25
|
+
return { kind: 'infrastructure', summary: 'critic did not report its model and no explicit model was configured' };
|
|
26
|
+
const expected = {
|
|
27
|
+
normal: expectation(normalRequest),
|
|
28
|
+
swapped: expectation(swappedRequest),
|
|
29
|
+
provenance: { provider: input.provider, model: actualModel },
|
|
30
|
+
};
|
|
31
|
+
input.writeFile(join(root, 'normal.verdict.json'), JSON.stringify({ provider: input.provider, model: actualModel, raw: normal.output }));
|
|
32
|
+
input.writeFile(join(root, 'swapped.verdict.json'), JSON.stringify({ provider: input.provider, model: actualModel, raw: swapped.output }));
|
|
33
|
+
const reduction = reduceComparisons({
|
|
34
|
+
expected,
|
|
35
|
+
normal: parseProviderResult(input.provider, normal.output),
|
|
36
|
+
swapped: parseProviderResult(input.provider, swapped.output),
|
|
37
|
+
});
|
|
38
|
+
return input.policy === 'advisory' ? { kind: 'skipped', summary: 'advisory comparison recorded' } : reduction;
|
|
39
|
+
}
|
|
40
|
+
function request(input, attemptId, candidateLabel) {
|
|
41
|
+
const issued = {
|
|
42
|
+
permissions: 'read-only',
|
|
43
|
+
attemptId,
|
|
44
|
+
candidateLabel,
|
|
45
|
+
referenceLabel: candidateLabel === 'A' ? 'B' : 'A',
|
|
46
|
+
trustedRubric: input.rubric,
|
|
47
|
+
reference: { ...input.reference, artifact: input.reference.artifact ?? 'reference.bin' },
|
|
48
|
+
candidate: {
|
|
49
|
+
digests: [...input.candidate.digests],
|
|
50
|
+
artifacts: input.candidate.artifacts ? [...input.candidate.artifacts] : input.candidate.digests.map((_, index) => `candidate-${index + 1}.bin`),
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
return { ...issued, promptDigest: digest(JSON.stringify(issued)), rubricDigest: digest(input.rubric) };
|
|
54
|
+
}
|
|
55
|
+
function expectation(request) {
|
|
56
|
+
return {
|
|
57
|
+
attemptId: request.attemptId,
|
|
58
|
+
candidateDigests: [...request.candidate.digests],
|
|
59
|
+
referenceDigest: request.reference.digest,
|
|
60
|
+
promptDigest: request.promptDigest,
|
|
61
|
+
rubricDigest: request.rubricDigest,
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
function digest(value) {
|
|
65
|
+
return createHash('sha256').update(value).digest('hex');
|
|
66
|
+
}
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
const QualityPolicySchema = z.enum(['blocking', 'advisory']);
|
|
3
|
+
const QualityAgentSchema = z.enum(['claude', 'codex', 'gemini']);
|
|
4
|
+
const RelativePathSchema = z.string().min(1).refine(value => !/^(?:[A-Za-z]:[\\/]|[\\/])/.test(value) && !value.split(/[\\/]+/).includes('..'), 'path must stay within the project');
|
|
5
|
+
const QualityReferenceSchema = z.object({
|
|
6
|
+
name: z.string().min(1),
|
|
7
|
+
source: z.string().min(1),
|
|
8
|
+
kind: z.enum(['url', 'file', 'command']),
|
|
9
|
+
digest: z.string().min(1).optional(),
|
|
10
|
+
}).superRefine((value, context) => {
|
|
11
|
+
if (value.kind === 'file' && !RelativePathSchema.safeParse(value.source).success)
|
|
12
|
+
context.addIssue({ code: 'custom', path: ['source'], message: 'file reference must stay within the project' });
|
|
13
|
+
});
|
|
14
|
+
const QualityCandidateSchema = z.union([
|
|
15
|
+
z.object({ kind: z.enum(['screenshots', 'files']), paths: z.array(RelativePathSchema).min(1) }),
|
|
16
|
+
z.object({ kind: z.enum(['command-output', 'benchmark']), command: z.string().min(1) }),
|
|
17
|
+
]);
|
|
18
|
+
export const ProjectQualityDefaultsSchema = z.object({
|
|
19
|
+
enabled: z.boolean().default(false),
|
|
20
|
+
policy: QualityPolicySchema.default('blocking'),
|
|
21
|
+
maxRounds: z.number().int().positive().default(3),
|
|
22
|
+
maxMinutes: z.number().int().positive().default(60),
|
|
23
|
+
consistencyChecks: z.literal(2).default(2),
|
|
24
|
+
maxParallelCandidates: z.number().int().positive().default(2),
|
|
25
|
+
criticAgent: QualityAgentSchema.optional(),
|
|
26
|
+
criticModel: z.string().min(1).optional(),
|
|
27
|
+
criticReasoningEffort: z.string().min(1).optional(),
|
|
28
|
+
repairAgent: QualityAgentSchema.optional(),
|
|
29
|
+
repairModel: z.string().min(1).optional(),
|
|
30
|
+
repairReasoningEffort: z.string().min(1).optional(),
|
|
31
|
+
critic: z.object({
|
|
32
|
+
agent: QualityAgentSchema.optional(),
|
|
33
|
+
model: z.string().min(1).optional(),
|
|
34
|
+
reasoningEffort: z.string().min(1).optional(),
|
|
35
|
+
}).optional(),
|
|
36
|
+
repair: z.object({
|
|
37
|
+
agent: QualityAgentSchema.optional(),
|
|
38
|
+
model: z.string().min(1).optional(),
|
|
39
|
+
reasoningEffort: z.string().min(1).optional(),
|
|
40
|
+
}).optional(),
|
|
41
|
+
});
|
|
42
|
+
export const StoryQualityDeclarationSchema = z.object({
|
|
43
|
+
reference: QualityReferenceSchema,
|
|
44
|
+
candidate: QualityCandidateSchema,
|
|
45
|
+
rubric: z.string().min(1),
|
|
46
|
+
policy: QualityPolicySchema.optional(),
|
|
47
|
+
});
|
|
48
|
+
export function resolveQualityPolicy(input) {
|
|
49
|
+
const overrides = input.overrides;
|
|
50
|
+
const unbounded = overrides?.qualityUnbounded === true;
|
|
51
|
+
const enabled = input.declaration !== undefined && (unbounded || (overrides?.quality ?? input.defaults?.enabled ?? false));
|
|
52
|
+
const policy = overrides?.qualityPolicy ?? input.declaration?.policy ?? input.defaults?.policy ?? 'blocking';
|
|
53
|
+
const maxRounds = overrides?.qualityRounds ?? input.defaults?.maxRounds ?? 3;
|
|
54
|
+
const maxMinutes = overrides?.qualityMinutes ?? input.defaults?.maxMinutes ?? 60;
|
|
55
|
+
return {
|
|
56
|
+
enabled,
|
|
57
|
+
policy,
|
|
58
|
+
limits: unbounded ? { unbounded: true } : { maxRounds, maxMinutes },
|
|
59
|
+
};
|
|
60
|
+
}
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
export const QualityLabelSchema = z.enum(['A', 'B']);
|
|
3
|
+
const DigestSchema = z.string().min(1);
|
|
4
|
+
const ComparedArtifactSchema = z.object({ label: QualityLabelSchema, digest: DigestSchema });
|
|
5
|
+
const ProvenanceSchema = z.object({
|
|
6
|
+
provider: z.string().min(1),
|
|
7
|
+
model: z.string().min(1),
|
|
8
|
+
promptDigest: DigestSchema,
|
|
9
|
+
rubricDigest: DigestSchema,
|
|
10
|
+
referenceDigest: DigestSchema,
|
|
11
|
+
candidateDigest: DigestSchema,
|
|
12
|
+
});
|
|
13
|
+
export const QualityVerdictSchema = z.object({
|
|
14
|
+
schemaVersion: z.literal(1),
|
|
15
|
+
attemptId: z.string().min(1),
|
|
16
|
+
winner: z.enum(['candidate', 'reference']),
|
|
17
|
+
biggestGap: z.string().min(1),
|
|
18
|
+
evidence: z.array(z.string().min(1)).min(1),
|
|
19
|
+
confidence: z.enum(['high', 'medium', 'low']),
|
|
20
|
+
candidate: ComparedArtifactSchema,
|
|
21
|
+
reference: ComparedArtifactSchema,
|
|
22
|
+
provenance: ProvenanceSchema,
|
|
23
|
+
}).superRefine((value, context) => {
|
|
24
|
+
if (value.candidate.label === value.reference.label) {
|
|
25
|
+
context.addIssue({ code: z.ZodIssueCode.custom, message: 'candidate and reference labels must differ' });
|
|
26
|
+
}
|
|
27
|
+
if (value.candidate.digest !== value.provenance.candidateDigest || value.reference.digest !== value.provenance.referenceDigest) {
|
|
28
|
+
context.addIssue({ code: z.ZodIssueCode.custom, message: 'artifact digests must match provenance' });
|
|
29
|
+
}
|
|
30
|
+
});
|
|
31
|
+
const CandidatePairVerdictSchema = z.object({
|
|
32
|
+
schemaVersion: z.literal(1),
|
|
33
|
+
attemptId: z.string().min(1),
|
|
34
|
+
winner: QualityLabelSchema,
|
|
35
|
+
evidence: z.array(z.string().min(1)).min(1),
|
|
36
|
+
confidence: z.enum(['high', 'medium', 'low']),
|
|
37
|
+
left: ComparedArtifactSchema,
|
|
38
|
+
right: ComparedArtifactSchema,
|
|
39
|
+
provenance: z.object({
|
|
40
|
+
leftDigest: DigestSchema,
|
|
41
|
+
rightDigest: DigestSchema,
|
|
42
|
+
provider: z.string().min(1),
|
|
43
|
+
model: z.string().min(1),
|
|
44
|
+
promptDigest: DigestSchema,
|
|
45
|
+
rubricDigest: DigestSchema,
|
|
46
|
+
}),
|
|
47
|
+
}).superRefine((value, context) => {
|
|
48
|
+
if (value.left.label === value.right.label) {
|
|
49
|
+
context.addIssue({ code: z.ZodIssueCode.custom, message: 'candidate labels must differ' });
|
|
50
|
+
}
|
|
51
|
+
if (value.left.digest !== value.provenance.leftDigest || value.right.digest !== value.provenance.rightDigest) {
|
|
52
|
+
context.addIssue({ code: z.ZodIssueCode.custom, message: 'candidate digests must match provenance' });
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
export function assignBlindLabels(selectCandidateLabel) {
|
|
56
|
+
const candidate = selectCandidateLabel();
|
|
57
|
+
return { candidate, reference: candidate === 'A' ? 'B' : 'A' };
|
|
58
|
+
}
|
|
59
|
+
export function reduceComparisons(input) {
|
|
60
|
+
const normal = QualityVerdictSchema.safeParse(input.normal);
|
|
61
|
+
const swapped = QualityVerdictSchema.safeParse(input.swapped);
|
|
62
|
+
if (!normal.success || !swapped.success)
|
|
63
|
+
return { kind: 'inconsistent', reason: 'invalid-verdict' };
|
|
64
|
+
if (normal.data.attemptId === swapped.data.attemptId)
|
|
65
|
+
return { kind: 'inconsistent', reason: 'not-fresh' };
|
|
66
|
+
if (normal.data.attemptId !== input.expected.normal.attemptId || swapped.data.attemptId !== input.expected.swapped.attemptId) {
|
|
67
|
+
return { kind: 'inconsistent', reason: 'attempt-mismatch' };
|
|
68
|
+
}
|
|
69
|
+
if (normal.data.candidate.label === swapped.data.candidate.label || normal.data.reference.label === swapped.data.reference.label) {
|
|
70
|
+
return { kind: 'inconsistent', reason: 'labels-not-swapped' };
|
|
71
|
+
}
|
|
72
|
+
if (normal.data.candidate.digest !== swapped.data.candidate.digest || normal.data.reference.digest !== swapped.data.reference.digest) {
|
|
73
|
+
return { kind: 'inconsistent', reason: 'digest-mismatch' };
|
|
74
|
+
}
|
|
75
|
+
if (!input.expected.normal.candidateDigests.includes(normal.data.candidate.digest)
|
|
76
|
+
|| !input.expected.swapped.candidateDigests.includes(swapped.data.candidate.digest)
|
|
77
|
+
|| normal.data.reference.digest !== input.expected.normal.referenceDigest
|
|
78
|
+
|| swapped.data.reference.digest !== input.expected.swapped.referenceDigest) {
|
|
79
|
+
return { kind: 'inconsistent', reason: 'digest-mismatch' };
|
|
80
|
+
}
|
|
81
|
+
if (normal.data.provenance.provider !== input.expected.provenance.provider
|
|
82
|
+
|| swapped.data.provenance.provider !== input.expected.provenance.provider
|
|
83
|
+
|| normal.data.provenance.model !== input.expected.provenance.model
|
|
84
|
+
|| swapped.data.provenance.model !== input.expected.provenance.model
|
|
85
|
+
|| normal.data.provenance.promptDigest !== input.expected.normal.promptDigest
|
|
86
|
+
|| swapped.data.provenance.promptDigest !== input.expected.swapped.promptDigest
|
|
87
|
+
|| normal.data.provenance.rubricDigest !== input.expected.normal.rubricDigest
|
|
88
|
+
|| swapped.data.provenance.rubricDigest !== input.expected.swapped.rubricDigest) {
|
|
89
|
+
return { kind: 'inconsistent', reason: 'provenance-mismatch' };
|
|
90
|
+
}
|
|
91
|
+
if (normal.data.winner !== swapped.data.winner)
|
|
92
|
+
return { kind: 'inconsistent', reason: 'winner-disagrees' };
|
|
93
|
+
if (normal.data.winner === 'reference') {
|
|
94
|
+
return { kind: 'lose', reason: 'reference-selected', biggestGap: normal.data.biggestGap, evidence: normal.data.evidence };
|
|
95
|
+
}
|
|
96
|
+
if (normal.data.confidence === 'low' || swapped.data.confidence === 'low') {
|
|
97
|
+
return { kind: 'lose', reason: 'low-confidence', biggestGap: normal.data.biggestGap, evidence: normal.data.evidence };
|
|
98
|
+
}
|
|
99
|
+
return {
|
|
100
|
+
kind: 'pass',
|
|
101
|
+
candidateDigest: normal.data.candidate.digest,
|
|
102
|
+
referenceDigest: normal.data.reference.digest,
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
export function reduceCandidatePairComparisons(input) {
|
|
106
|
+
const normal = CandidatePairVerdictSchema.safeParse(input.normal);
|
|
107
|
+
const swapped = CandidatePairVerdictSchema.safeParse(input.swapped);
|
|
108
|
+
if (!normal.success || !swapped.success)
|
|
109
|
+
return { kind: 'inconsistent', reason: 'invalid-verdict' };
|
|
110
|
+
const normalMismatch = candidatePairMismatch(normal.data, input.expected.normal);
|
|
111
|
+
if (normalMismatch)
|
|
112
|
+
return { kind: 'inconsistent', reason: normalMismatch };
|
|
113
|
+
const swappedMismatch = candidatePairMismatch(swapped.data, input.expected.swapped);
|
|
114
|
+
if (swappedMismatch)
|
|
115
|
+
return { kind: 'inconsistent', reason: swappedMismatch };
|
|
116
|
+
if (normal.data.confidence === 'low' || swapped.data.confidence === 'low') {
|
|
117
|
+
return { kind: 'inconsistent', reason: 'low-confidence' };
|
|
118
|
+
}
|
|
119
|
+
const normalWinner = normal.data.winner === normal.data.left.label
|
|
120
|
+
? { candidateId: input.expected.normal.candidateIds.left, digest: input.expected.normal.request.left.digest }
|
|
121
|
+
: { candidateId: input.expected.normal.candidateIds.right, digest: input.expected.normal.request.right.digest };
|
|
122
|
+
const swappedWinner = swapped.data.winner === swapped.data.left.label
|
|
123
|
+
? { candidateId: input.expected.swapped.candidateIds.left, digest: input.expected.swapped.request.left.digest }
|
|
124
|
+
: { candidateId: input.expected.swapped.candidateIds.right, digest: input.expected.swapped.request.right.digest };
|
|
125
|
+
return normalWinner.candidateId === swappedWinner.candidateId && normalWinner.digest === swappedWinner.digest
|
|
126
|
+
? { kind: 'selected', winnerCandidateId: normalWinner.candidateId, winnerDigest: normalWinner.digest, normal: normal.data, swapped: swapped.data }
|
|
127
|
+
: { kind: 'inconsistent', reason: 'winner-disagrees' };
|
|
128
|
+
}
|
|
129
|
+
function candidatePairMismatch(verdict, expected) {
|
|
130
|
+
if (verdict.attemptId !== expected.request.attemptId)
|
|
131
|
+
return 'attempt-mismatch';
|
|
132
|
+
if (verdict.left.label !== expected.request.left.label || verdict.right.label !== expected.request.right.label)
|
|
133
|
+
return 'label-mismatch';
|
|
134
|
+
if (verdict.left.digest !== expected.request.left.digest || verdict.right.digest !== expected.request.right.digest)
|
|
135
|
+
return 'digest-mismatch';
|
|
136
|
+
if (verdict.provenance.provider !== expected.provenance.provider
|
|
137
|
+
|| verdict.provenance.model !== expected.provenance.model
|
|
138
|
+
|| verdict.provenance.promptDigest !== expected.provenance.promptDigest
|
|
139
|
+
|| verdict.provenance.rubricDigest !== expected.provenance.rubricDigest)
|
|
140
|
+
return 'provenance-mismatch';
|
|
141
|
+
return null;
|
|
142
|
+
}
|
package/dist/retrofit/config.js
CHANGED
|
@@ -2,7 +2,8 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
|
2
2
|
import { join, dirname } from 'node:path';
|
|
3
3
|
import { parse, stringify } from 'yaml';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
-
|
|
5
|
+
import { AgentSchema, PermissionProfileSchema } from '../agents/contracts.js';
|
|
6
|
+
import { ProjectQualityDefaultsSchema } from '../quality/types.js';
|
|
6
7
|
const CodeGraphSchema = z.enum(['graphify', 'serena']);
|
|
7
8
|
const SmokeFlowSchema = z.object({ name: z.string().min(1), path: z.string().min(1), landmark: z.string().optional() });
|
|
8
9
|
const SmokeSchema = z.object({ baseUrl: z.string().min(1), flows: z.array(SmokeFlowSchema).min(1) });
|
|
@@ -30,7 +31,7 @@ export const YokeConfigSchema = z.object({
|
|
|
30
31
|
model: z.string().min(1).optional(),
|
|
31
32
|
reasoningEffort: z.string().min(1).optional(),
|
|
32
33
|
bare: z.boolean().optional(),
|
|
33
|
-
permissions:
|
|
34
|
+
permissions: PermissionProfileSchema.optional(),
|
|
34
35
|
}).optional(),
|
|
35
36
|
routing: z.object({
|
|
36
37
|
enabled: z.boolean(),
|
|
@@ -53,17 +54,25 @@ export const YokeConfigSchema = z.object({
|
|
|
53
54
|
suppressionsVersion: z.literal(1).optional(),
|
|
54
55
|
suppressions: z.array(z.object({ ruleId: z.string().min(1), file: z.string().min(1).optional(), reason: z.string(), expires: z.string().optional() })).optional(),
|
|
55
56
|
}).optional(),
|
|
56
|
-
verify: z.object({
|
|
57
|
+
verify: z.object({
|
|
58
|
+
command: z.string().min(1).optional(),
|
|
59
|
+
retries: z.number().int().nonnegative().optional(),
|
|
60
|
+
requireCriteria: z.boolean().optional(),
|
|
61
|
+
}).optional(),
|
|
62
|
+
// Runs only when no open stories remain. This proves the current integrated
|
|
63
|
+
// system, without introducing release objects or a persistent stale graph.
|
|
64
|
+
completion: z.object({ command: z.string().min(1), retries: z.number().int().nonnegative().optional() }).optional(),
|
|
57
65
|
// Optional performance budget gate: a benchmark command that must exit 0 for a
|
|
58
66
|
// story to land (runs after verify). Benchmarks are noisy → retried like verify.
|
|
59
67
|
perf: z.object({ command: z.string().min(1), retries: z.number().int().nonnegative().optional() }).optional(),
|
|
60
68
|
codeGraph: CodeGraphSchema.optional(),
|
|
61
69
|
smoke: SmokeSchema.optional(),
|
|
70
|
+
quality: ProjectQualityDefaultsSchema.optional(),
|
|
62
71
|
// Opt-in: upgrade yoke at loop START when a newer version is cached (never mid-run).
|
|
63
72
|
update: z.object({ auto: z.boolean() }).optional(),
|
|
64
73
|
});
|
|
65
74
|
export function defaultConfig(canonVersion) {
|
|
66
|
-
return { canonVersion, agents: [], loop: { enabled: false } };
|
|
75
|
+
return { canonVersion, agents: [], loop: { enabled: false }, verify: { requireCriteria: true } };
|
|
67
76
|
}
|
|
68
77
|
export function configPath(targetDir) {
|
|
69
78
|
return join(targetDir, '.yoke', 'config.yaml');
|
|
@@ -11,6 +11,8 @@ export const YOKE_IGNORE_LINES = [
|
|
|
11
11
|
'.yoke/loop.lock.*.tmp',
|
|
12
12
|
'.yoke/loop.pause',
|
|
13
13
|
'.yoke/runner.pid',
|
|
14
|
+
'.yoke/provider-processes/',
|
|
15
|
+
'.yoke/claims/',
|
|
14
16
|
'.yoke/ambiguity.md',
|
|
15
17
|
'.yoke/decision-request.yaml',
|
|
16
18
|
'.yoke/pending-decision.yaml',
|
|
@@ -19,6 +21,8 @@ export const YOKE_IGNORE_LINES = [
|
|
|
19
21
|
'.yoke/decision-*.yaml.*.tmp',
|
|
20
22
|
'.yoke/story-durations.json',
|
|
21
23
|
'.yoke/proof/',
|
|
24
|
+
'.yoke/references/',
|
|
25
|
+
'.yoke/changes/',
|
|
22
26
|
];
|
|
23
27
|
const HEADER = '# Yoke runtime artifacts (managed by yoke retrofit)';
|
|
24
28
|
// Idempotently ensure each Yoke runtime path is gitignored. Appends only the
|
package/dist/review/command.js
CHANGED
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import { agentInvocation, buildStandaloneReviewPrompt, buildWatchdogInvocation,
|
|
1
|
+
import { agentInvocation, buildStandaloneReviewPrompt, buildWatchdogInvocation, runCapturedAgent, repositoryFingerprint, isAgentAvailable, } from '../loop/runner.js';
|
|
2
|
+
import { parseProviderResult } from '../agents/telemetry.js';
|
|
2
3
|
import { resolveIdleMs } from '../loop/run-command.js';
|
|
3
|
-
import { existsSync, mkdirSync, rmSync, rmdirSync } from 'node:fs';
|
|
4
|
-
import { join } from 'node:path';
|
|
5
4
|
import { loadConfig } from '../retrofit/config.js';
|
|
6
|
-
import {
|
|
5
|
+
import { parseReviewVerdict } from './verdict.js';
|
|
7
6
|
// Resolve to the first available agent, preferring a *second* model so the review
|
|
8
7
|
// is genuinely cross-model. claude last => a Claude-only box degrades to self-review.
|
|
9
8
|
const RESOLUTION_ORDER = ['codex', 'gemini', 'claude'];
|
|
@@ -33,51 +32,41 @@ export function runReview(targetDir, opts = {}) {
|
|
|
33
32
|
const scope = opts.base
|
|
34
33
|
? `the diff ${opts.base}..HEAD`
|
|
35
34
|
: 'the uncommitted working-tree changes (working tree + staged)';
|
|
36
|
-
const
|
|
37
|
-
const yokeDir = join(targetDir, '.yoke');
|
|
38
|
-
const createdYokeDir = !existsSync(yokeDir);
|
|
39
|
-
mkdirSync(yokeDir, { recursive: true });
|
|
40
|
-
rmSync(verdictPath, { force: true });
|
|
41
|
-
const prompt = buildStandaloneReviewPrompt(scope, opts.focus, verdictPath);
|
|
35
|
+
const prompt = buildStandaloneReviewPrompt(scope, opts.focus, undefined, reviewer);
|
|
42
36
|
const idleMs = resolveIdleMs(opts.timeoutMinutes, undefined);
|
|
43
37
|
// Pass the *agent* invocation to the runner so callers (and tests) see the
|
|
44
38
|
// reviewer command. The default runner adds the watchdog wrapper before exec;
|
|
45
39
|
// an injected run() gets the raw invocation.
|
|
46
|
-
const inv = agentInvocation(reviewer, prompt, targetDir, '
|
|
40
|
+
const inv = agentInvocation(reviewer, prompt, targetDir, 'read-only');
|
|
47
41
|
const say = opts.json ? console.error : console.log;
|
|
48
42
|
say(`Reviewing ${scope} with ${reviewer}...`);
|
|
49
|
-
const
|
|
43
|
+
const before = repositoryFingerprint(targetDir);
|
|
44
|
+
const run = opts.run ?? ((i) => runCapturedAgent(reviewer, buildWatchdogInvocation(i, idleMs)));
|
|
50
45
|
const processResult = run(inv);
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
verdict = readReviewVerdict(verdictPath);
|
|
54
|
-
}
|
|
55
|
-
catch (error) {
|
|
56
|
-
say(`✗ ${reviewer} produced no valid verdict (${error.message})${processResult.success ? '' : `; process: ${processResult.summary}`}`);
|
|
57
|
-
if (createdYokeDir) {
|
|
58
|
-
try {
|
|
59
|
-
rmdirSync(yokeDir);
|
|
60
|
-
}
|
|
61
|
-
catch { }
|
|
62
|
-
}
|
|
46
|
+
if (repositoryFingerprint(targetDir) !== before) {
|
|
47
|
+
say(`✗ ${reviewer} modified the repository during a read-only review`);
|
|
63
48
|
return 1;
|
|
64
49
|
}
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
50
|
+
try {
|
|
51
|
+
const actualModel = opts.run ? undefined : processResult.tokens?.model;
|
|
52
|
+
if (!opts.run && processResult.success && !actualModel)
|
|
53
|
+
throw new Error('review provider did not report its model');
|
|
54
|
+
const verdict = parseReviewVerdict(parseProviderResult(reviewer, processResult.output), { provider: reviewer, ...(actualModel ? { model: actualModel } : {}) });
|
|
55
|
+
if (opts.json)
|
|
56
|
+
console.log(JSON.stringify({ reviewer, process: processResult, verdict }));
|
|
57
|
+
if (!processResult.success) {
|
|
58
|
+
say(`✗ ${reviewer} process failed (${processResult.summary}); verdict: ${verdict.summary}`);
|
|
59
|
+
return 1;
|
|
68
60
|
}
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
say(`✗ ${reviewer} process failed (${processResult.summary}); verdict: ${verdict.summary}`);
|
|
61
|
+
if (verdict.approved) {
|
|
62
|
+
say(`✓ ${reviewer} approved: ${verdict.summary}`);
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
65
|
+
say(`✗ ${reviewer} rejected: ${verdict.summary}`);
|
|
75
66
|
return 1;
|
|
76
67
|
}
|
|
77
|
-
|
|
78
|
-
say(
|
|
79
|
-
return
|
|
68
|
+
catch (error) {
|
|
69
|
+
say(`✗ ${reviewer} produced no valid verdict (${error.message})${processResult.success ? '' : `; process: ${processResult.summary}`}`);
|
|
70
|
+
return 1;
|
|
80
71
|
}
|
|
81
|
-
say(`✗ ${reviewer} rejected: ${verdict.summary}`);
|
|
82
|
-
return 1;
|
|
83
72
|
}
|