@tobi-or-not/quorum 0.1.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +5 -0
- package/dist/bench/fingerprint.d.ts +36 -0
- package/dist/bench/fingerprint.js +79 -0
- package/dist/bench/records.d.ts +62 -0
- package/dist/bench/records.js +161 -0
- package/dist/bench/report.d.ts +110 -0
- package/dist/bench/report.js +332 -0
- package/dist/bench/run-dir.d.ts +101 -0
- package/dist/bench/run-dir.js +379 -0
- package/dist/bench/runner.d.ts +80 -0
- package/dist/bench/runner.js +160 -0
- package/dist/bench/score.d.ts +30 -0
- package/dist/bench/score.js +38 -0
- package/dist/bench/stats.d.ts +51 -0
- package/dist/bench/stats.js +83 -0
- package/dist/bench/suite.d.ts +87 -0
- package/dist/bench/suite.js +286 -0
- package/dist/cli/bench.d.ts +32 -0
- package/dist/cli/bench.js +463 -0
- package/dist/cli/bin.js +9 -1
- package/dist/cli/io.d.ts +45 -0
- package/dist/cli/io.js +45 -0
- package/dist/cli/main.d.ts +7 -37
- package/dist/cli/main.js +47 -60
- package/dist/cli/print-config.js +38 -9
- package/dist/cli/render.d.ts +10 -0
- package/dist/cli/render.js +37 -16
- package/dist/config/claude-code.js +42 -11
- package/dist/config/env.js +135 -32
- package/dist/config/from-config.js +43 -9
- package/dist/config/load.d.ts +43 -0
- package/dist/config/load.js +205 -46
- package/dist/config/profiles.js +87 -20
- package/dist/config/resolve.js +194 -47
- package/dist/config/schema.js +98 -23
- package/dist/core/diff.d.ts +10 -0
- package/dist/core/diff.js +90 -17
- package/dist/core/errors.js +14 -5
- package/dist/core/flags.js +30 -6
- package/dist/core/output.js +49 -13
- package/dist/core/policy.js +6 -2
- package/dist/core/profile.js +42 -15
- package/dist/core/prompts.d.ts +3 -1
- package/dist/core/prompts.js +28 -8
- package/dist/core/quorum.js +128 -33
- package/dist/core/sanitize.d.ts +16 -1
- package/dist/core/sanitize.js +31 -5
- package/dist/core/stages/context.js +30 -5
- package/dist/core/stages/panel.js +55 -9
- package/dist/core/stages/research.js +13 -3
- package/dist/core/verify.js +6 -2
- package/dist/index.js +5 -1
- package/dist/providers/anthropic.js +28 -5
- package/dist/providers/build.js +57 -16
- package/dist/providers/claude-code.js +206 -51
- package/dist/providers/gemini.js +61 -14
- package/dist/providers/http.js +32 -9
- package/dist/providers/messages.js +14 -2
- package/dist/providers/ollama.js +31 -8
- package/dist/providers/openai-compatible.js +51 -15
- package/dist/providers/options.js +120 -40
- package/dist/providers/presets.js +143 -16
- package/dist/providers/reply.js +15 -5
- package/dist/providers/size.js +11 -3
- package/package.json +6 -5
package/dist/core/quorum.js
CHANGED
|
@@ -50,7 +50,10 @@ export function createQuorum(options, ports) {
|
|
|
50
50
|
sink: ports.events,
|
|
51
51
|
};
|
|
52
52
|
const warnings = Object.freeze(options.profiles.flatMap(profile => profileWarnings(profile)));
|
|
53
|
-
return Object.freeze({
|
|
53
|
+
return Object.freeze({
|
|
54
|
+
review: (input, options) => review(compiled, input, options),
|
|
55
|
+
warnings,
|
|
56
|
+
});
|
|
54
57
|
}
|
|
55
58
|
/**
|
|
56
59
|
* One review of a diff by every profile of the quorum, concurrently, each through research, dedup
|
|
@@ -82,7 +85,9 @@ async function review(compiled, input, options = {}) {
|
|
|
82
85
|
throw new ConfigError(issues);
|
|
83
86
|
options.signal?.throwIfAborted();
|
|
84
87
|
const internal = new AbortController();
|
|
85
|
-
const signal = options.signal === undefined
|
|
88
|
+
const signal = options.signal === undefined
|
|
89
|
+
? internal.signal
|
|
90
|
+
: AbortSignal.any([options.signal, internal.signal]);
|
|
86
91
|
const fail = (error) => {
|
|
87
92
|
if (!signal.aborted)
|
|
88
93
|
internal.abort(error);
|
|
@@ -90,9 +95,15 @@ async function review(compiled, input, options = {}) {
|
|
|
90
95
|
const startedAt = new Date().toISOString();
|
|
91
96
|
const started = performance.now();
|
|
92
97
|
const { runs, sink } = compiled;
|
|
93
|
-
const sections = input.diff.trim() === ''
|
|
94
|
-
|
|
95
|
-
|
|
98
|
+
const sections = input.diff.trim() === ''
|
|
99
|
+
? undefined
|
|
100
|
+
: splitDiff(input.diff);
|
|
101
|
+
const flags = sections === undefined
|
|
102
|
+
? []
|
|
103
|
+
: diffFlags(sections);
|
|
104
|
+
const tally = {
|
|
105
|
+
invalidFindings: new Map(),
|
|
106
|
+
};
|
|
96
107
|
const emit = (event) => {
|
|
97
108
|
if (signal.aborted)
|
|
98
109
|
return;
|
|
@@ -109,8 +120,21 @@ async function review(compiled, input, options = {}) {
|
|
|
109
120
|
throw error;
|
|
110
121
|
}
|
|
111
122
|
};
|
|
112
|
-
const ctx = {
|
|
113
|
-
|
|
123
|
+
const ctx = {
|
|
124
|
+
limit: createLimiter(compiled.concurrency),
|
|
125
|
+
emit,
|
|
126
|
+
signal,
|
|
127
|
+
fail,
|
|
128
|
+
maxFindings: compiled.maxFindings,
|
|
129
|
+
};
|
|
130
|
+
emit({
|
|
131
|
+
type: 'run:start',
|
|
132
|
+
profiles: runs.map(run => run.id),
|
|
133
|
+
providers: Object.fromEntries(runs.map(run => [
|
|
134
|
+
run.id,
|
|
135
|
+
providersOf(run),
|
|
136
|
+
])),
|
|
137
|
+
});
|
|
114
138
|
for (const flag of flags)
|
|
115
139
|
emit({ type: 'flag', flag });
|
|
116
140
|
const outcomes = await reviewAll(ctx, runs, input.diff, sections).catch((error) => {
|
|
@@ -129,12 +153,24 @@ async function review(compiled, input, options = {}) {
|
|
|
129
153
|
version: packageJson.version,
|
|
130
154
|
startedAt,
|
|
131
155
|
durationMs,
|
|
132
|
-
profiles: Object.fromEntries(runs.map(run => [
|
|
133
|
-
|
|
156
|
+
profiles: Object.fromEntries(runs.map(run => [
|
|
157
|
+
run.id,
|
|
158
|
+
profileMeta(run, tally.invalidFindings.get(run.id) ?? 0),
|
|
159
|
+
])),
|
|
160
|
+
...(tally.usage === undefined
|
|
161
|
+
? {}
|
|
162
|
+
: { usage: tally.usage }),
|
|
134
163
|
incomplete,
|
|
135
164
|
flags,
|
|
136
165
|
};
|
|
137
|
-
emit({
|
|
166
|
+
emit({
|
|
167
|
+
type: 'run:end',
|
|
168
|
+
kept: kept.length,
|
|
169
|
+
discarded: discarded.length,
|
|
170
|
+
unreviewed: unreviewed.length,
|
|
171
|
+
incomplete: incomplete.length,
|
|
172
|
+
durationMs,
|
|
173
|
+
});
|
|
138
174
|
return { kept, discarded, unreviewed, meta };
|
|
139
175
|
}
|
|
140
176
|
/** Run every profile through its stages at once on the diff's sections; a blank diff, which has none, runs no stage. */
|
|
@@ -148,7 +184,13 @@ async function reviewProfile(ctx, run, diff, sections) {
|
|
|
148
184
|
const researched = await stage(ctx, run.id, 'research', () => research(ctx, run, diff, sections), outcome => outcome.findings.length);
|
|
149
185
|
const candidates = await stage(ctx, run.id, 'dedup', () => dedup(researched.findings), found => found.length);
|
|
150
186
|
const judged = await stage(ctx, run.id, 'panel', () => panel(ctx, run, candidates, sections, diff), outcome => outcome.kept.length + outcome.discarded.length + outcome.unreviewed.length);
|
|
151
|
-
return {
|
|
187
|
+
return {
|
|
188
|
+
...judged,
|
|
189
|
+
incomplete: [
|
|
190
|
+
...researched.incomplete,
|
|
191
|
+
...judged.incomplete,
|
|
192
|
+
],
|
|
193
|
+
};
|
|
152
194
|
}
|
|
153
195
|
/**
|
|
154
196
|
* Every problem with one review's input and options, so an untyped caller gets issues, never a
|
|
@@ -171,17 +213,30 @@ function reviewIssues(input, options) {
|
|
|
171
213
|
/** The usage of one more call added to a running total; `costUsd` is summed over the calls that reported one. */
|
|
172
214
|
function addUsage(total, call) {
|
|
173
215
|
const sum = { inputTokens: (total?.inputTokens ?? 0) + call.inputTokens, outputTokens: (total?.outputTokens ?? 0) + call.outputTokens };
|
|
174
|
-
const costUsd = call.costUsd === undefined
|
|
216
|
+
const costUsd = call.costUsd === undefined
|
|
217
|
+
? total?.costUsd
|
|
218
|
+
: (total?.costUsd ?? 0) + call.costUsd;
|
|
175
219
|
if (costUsd !== undefined)
|
|
176
220
|
sum.costUsd = costUsd;
|
|
177
221
|
return sum;
|
|
178
222
|
}
|
|
179
223
|
/** Who reviews for a profile, as new objects the caller may keep or change: never the frozen refs of the run. */
|
|
180
224
|
function providersOf(run) {
|
|
181
|
-
return {
|
|
225
|
+
return {
|
|
226
|
+
research: { ...run.research.ref },
|
|
227
|
+
lenses: Object.fromEntries(run.lenses.map(lens => [
|
|
228
|
+
lens.id,
|
|
229
|
+
{ ...lens.ref },
|
|
230
|
+
])),
|
|
231
|
+
};
|
|
182
232
|
}
|
|
183
233
|
function profileMeta(run, invalidFindings) {
|
|
184
|
-
return {
|
|
234
|
+
return {
|
|
235
|
+
providers: providersOf(run),
|
|
236
|
+
policy: { ...run.thresholds },
|
|
237
|
+
gating: run.gating,
|
|
238
|
+
invalidFindings,
|
|
239
|
+
};
|
|
185
240
|
}
|
|
186
241
|
/**
|
|
187
242
|
* Compile a checked profile and its checked ports into the frozen run every review reads: the vote
|
|
@@ -197,7 +252,10 @@ function compile(profile, ports) {
|
|
|
197
252
|
}));
|
|
198
253
|
return Object.freeze({
|
|
199
254
|
id: profile.id,
|
|
200
|
-
research: Object.freeze({
|
|
255
|
+
research: Object.freeze({
|
|
256
|
+
instructions: profile.prompts.research,
|
|
257
|
+
...target(ports.research),
|
|
258
|
+
}),
|
|
201
259
|
lenses: Object.freeze(lenses),
|
|
202
260
|
thresholds: Object.freeze(thresholdsFor(profile)),
|
|
203
261
|
gating: profile.gating ?? true,
|
|
@@ -205,7 +263,10 @@ function compile(profile, ports) {
|
|
|
205
263
|
});
|
|
206
264
|
}
|
|
207
265
|
function target(provider) {
|
|
208
|
-
return {
|
|
266
|
+
return {
|
|
267
|
+
provider,
|
|
268
|
+
ref: Object.freeze({ id: provider.id, model: provider.model }),
|
|
269
|
+
};
|
|
209
270
|
}
|
|
210
271
|
/**
|
|
211
272
|
* Every problem with the options and the ports, in this order: options and ports that are not
|
|
@@ -219,8 +280,13 @@ function inputIssues(options, ports) {
|
|
|
219
280
|
issues.push({ path: 'ports', message: 'must be an object' });
|
|
220
281
|
if (!isRecord(options))
|
|
221
282
|
return issues;
|
|
222
|
-
if (!Array.isArray(options.profiles))
|
|
223
|
-
return [
|
|
283
|
+
if (!Array.isArray(options.profiles)) {
|
|
284
|
+
return [
|
|
285
|
+
...issues,
|
|
286
|
+
{ path: 'profiles', message: 'must be an array of profiles' },
|
|
287
|
+
...limitIssues(options),
|
|
288
|
+
];
|
|
289
|
+
}
|
|
224
290
|
// `Array.from` reads a hole as `undefined`, so it is reported like any other profile that is not an object.
|
|
225
291
|
const profiles = profileListIssues(Array.from(options.profiles));
|
|
226
292
|
issues.push(...profiles.issues, ...limitIssues(options));
|
|
@@ -243,7 +309,9 @@ function profileListIssues(profiles) {
|
|
|
243
309
|
profiles.forEach((profile, index) => {
|
|
244
310
|
const problems = profileIssues(profile, `profiles.${index}`);
|
|
245
311
|
issues.push(...problems);
|
|
246
|
-
const id = isRecord(profile) && typeof profile.id === 'string'
|
|
312
|
+
const id = isRecord(profile) && typeof profile.id === 'string'
|
|
313
|
+
? profile.id
|
|
314
|
+
: undefined;
|
|
247
315
|
if (id === undefined)
|
|
248
316
|
return;
|
|
249
317
|
const repeated = named.has(id);
|
|
@@ -261,8 +329,12 @@ function limitIssues(options) {
|
|
|
261
329
|
const value = options[path];
|
|
262
330
|
if (value === undefined || (typeof value === 'number' && Number.isInteger(value) && value >= 1))
|
|
263
331
|
return [];
|
|
264
|
-
const got = typeof value === 'number'
|
|
265
|
-
|
|
332
|
+
const got = typeof value === 'number'
|
|
333
|
+
? `, got ${value}`
|
|
334
|
+
: '';
|
|
335
|
+
return [
|
|
336
|
+
{ path, message: `must be a positive integer${got}` },
|
|
337
|
+
];
|
|
266
338
|
});
|
|
267
339
|
}
|
|
268
340
|
/** The ports of each checked profile in order, then the ports no profile of the run names, then the sink. */
|
|
@@ -288,16 +360,30 @@ function portIssues(ports, profiles, named) {
|
|
|
288
360
|
function profilePortIssues(entries, profile) {
|
|
289
361
|
const path = `ports.profiles.${profile.id}`;
|
|
290
362
|
const entry = own(entries, profile.id);
|
|
291
|
-
if (entry === undefined)
|
|
292
|
-
return [
|
|
293
|
-
|
|
294
|
-
|
|
363
|
+
if (entry === undefined) {
|
|
364
|
+
return [
|
|
365
|
+
{ path, message: 'is missing' },
|
|
366
|
+
];
|
|
367
|
+
}
|
|
368
|
+
if (!isRecord(entry)) {
|
|
369
|
+
return [
|
|
370
|
+
{ path, message: 'must be an object with a research provider and voters' },
|
|
371
|
+
];
|
|
372
|
+
}
|
|
295
373
|
const issues = providerIssues(`${path}.research`, entry.research);
|
|
296
374
|
const { voters } = entry;
|
|
297
|
-
if (voters === undefined)
|
|
298
|
-
return [
|
|
299
|
-
|
|
300
|
-
|
|
375
|
+
if (voters === undefined) {
|
|
376
|
+
return [
|
|
377
|
+
...issues,
|
|
378
|
+
{ path: `${path}.voters`, message: 'is missing' },
|
|
379
|
+
];
|
|
380
|
+
}
|
|
381
|
+
if (!isRecord(voters)) {
|
|
382
|
+
return [
|
|
383
|
+
...issues,
|
|
384
|
+
{ path: `${path}.voters`, message: 'must be an object keyed by lens id' },
|
|
385
|
+
];
|
|
386
|
+
}
|
|
301
387
|
const lensIds = new Set(profile.lenses.map(lens => lens.id));
|
|
302
388
|
for (const lens of profile.lenses)
|
|
303
389
|
issues.push(...providerIssues(`${path}.voters.${lens.id}`, own(voters, lens.id)));
|
|
@@ -308,9 +394,16 @@ function profilePortIssues(entries, profile) {
|
|
|
308
394
|
return issues;
|
|
309
395
|
}
|
|
310
396
|
function providerIssues(path, value) {
|
|
311
|
-
if (value === undefined)
|
|
312
|
-
return [
|
|
313
|
-
|
|
397
|
+
if (value === undefined) {
|
|
398
|
+
return [
|
|
399
|
+
{ path, message: 'is missing' },
|
|
400
|
+
];
|
|
401
|
+
}
|
|
402
|
+
return isProvider(value)
|
|
403
|
+
? []
|
|
404
|
+
: [
|
|
405
|
+
{ path, message: PROVIDER_SHAPE },
|
|
406
|
+
];
|
|
314
407
|
}
|
|
315
408
|
/** A provider is a non-null object with a string id, a string model and a complete function. */
|
|
316
409
|
function isProvider(value) {
|
|
@@ -318,5 +411,7 @@ function isProvider(value) {
|
|
|
318
411
|
}
|
|
319
412
|
/** What `record` holds under `key` as its own property: an id such as `constructor` never finds an inherited member. */
|
|
320
413
|
function own(record, key) {
|
|
321
|
-
return Object.hasOwn(record, key)
|
|
414
|
+
return Object.hasOwn(record, key)
|
|
415
|
+
? record[key]
|
|
416
|
+
: undefined;
|
|
322
417
|
}
|
package/dist/core/sanitize.d.ts
CHANGED
|
@@ -9,7 +9,9 @@
|
|
|
9
9
|
/**
|
|
10
10
|
* The body of the hidden-character class, for a character class under the `u` flag: controls (C0,
|
|
11
11
|
* DEL, C1), format characters, unpaired surrogate halves, other default-ignorable code points, and
|
|
12
|
-
* line and paragraph separators. Every check for hidden characters
|
|
12
|
+
* line and paragraph separators. Every check for hidden characters in the product code and in the
|
|
13
|
+
* docs test is built from it; the two spec oracles are not, on purpose, so that they stay
|
|
14
|
+
* independent of the class they check.
|
|
13
15
|
*/
|
|
14
16
|
export declare const HIDDEN_CLASS: string;
|
|
15
17
|
/** `U+`, then the code point of the first character in upper-case hexadecimal, four digits at least. */
|
|
@@ -24,6 +26,12 @@ export declare function hiddenLabels(text: string): string[];
|
|
|
24
26
|
export declare function stripHidden(text: string): string;
|
|
25
27
|
/** The text with each hidden character replaced by a space: what a reader sees when an invisible character separates two words. */
|
|
26
28
|
export declare function spaceHidden(text: string): string;
|
|
29
|
+
/**
|
|
30
|
+
* The text with each hidden character shown as `<U+XXXX>` and nothing else changed: for text that
|
|
31
|
+
* reaches a model or a reader but is no line of a log, such as a file name cited in a prompt.
|
|
32
|
+
* `inertLine` adds the workflow-command rules.
|
|
33
|
+
*/
|
|
34
|
+
export declare function showHidden(text: string): string;
|
|
27
35
|
/**
|
|
28
36
|
* One line of a terminal or a CI log. Every hidden character is shown as `<U+XXXX>`, so the line
|
|
29
37
|
* holds no line break and starts no escape sequence, and no character of the class reorders or
|
|
@@ -43,3 +51,10 @@ export declare function inertLine(text: string): string;
|
|
|
43
51
|
* value with no JSON form, such as `undefined`, throws instead of returning `undefined`.
|
|
44
52
|
*/
|
|
45
53
|
export declare function inertJson(value: unknown): string;
|
|
54
|
+
/**
|
|
55
|
+
* The JSON of `value` on one line: `JSON.stringify(value)`, escaped as `inertJson` escapes. The line
|
|
56
|
+
* holds no line break, since `JSON.stringify` writes a line feed inside a string as `\n` and the
|
|
57
|
+
* escaping covers every other hidden character, so it is one line of a file that is read line by line.
|
|
58
|
+
* It is lossless, and a value with no JSON form throws.
|
|
59
|
+
*/
|
|
60
|
+
export declare function inertJsonLine(value: unknown): string;
|
package/dist/core/sanitize.js
CHANGED
|
@@ -9,7 +9,9 @@
|
|
|
9
9
|
/**
|
|
10
10
|
* The body of the hidden-character class, for a character class under the `u` flag: controls (C0,
|
|
11
11
|
* DEL, C1), format characters, unpaired surrogate halves, other default-ignorable code points, and
|
|
12
|
-
* line and paragraph separators. Every check for hidden characters
|
|
12
|
+
* line and paragraph separators. Every check for hidden characters in the product code and in the
|
|
13
|
+
* docs test is built from it; the two spec oracles are not, on purpose, so that they stay
|
|
14
|
+
* independent of the class they check.
|
|
13
15
|
*/
|
|
14
16
|
export const HIDDEN_CLASS = String.raw `\p{Cc}\p{Cf}\p{Cs}\p{Default_Ignorable_Code_Point}\p{Zl}\p{Zp}`;
|
|
15
17
|
/** The hidden class, matched one character at a time over the whole text. */
|
|
@@ -41,6 +43,14 @@ export function stripHidden(text) {
|
|
|
41
43
|
export function spaceHidden(text) {
|
|
42
44
|
return text.replace(HIDDEN, ' ');
|
|
43
45
|
}
|
|
46
|
+
/**
|
|
47
|
+
* The text with each hidden character shown as `<U+XXXX>` and nothing else changed: for text that
|
|
48
|
+
* reaches a model or a reader but is no line of a log, such as a file name cited in a prompt.
|
|
49
|
+
* `inertLine` adds the workflow-command rules.
|
|
50
|
+
*/
|
|
51
|
+
export function showHidden(text) {
|
|
52
|
+
return text.replace(HIDDEN, char => `<${codePointLabel(char)}>`);
|
|
53
|
+
}
|
|
44
54
|
/**
|
|
45
55
|
* One line of a terminal or a CI log. Every hidden character is shown as `<U+XXXX>`, so the line
|
|
46
56
|
* holds no line break and starts no escape sequence, and no character of the class reorders or
|
|
@@ -51,8 +61,7 @@ export function spaceHidden(text) {
|
|
|
51
61
|
* inert pieces printed side by side can still meet as `::` or `##[`.
|
|
52
62
|
*/
|
|
53
63
|
export function inertLine(text) {
|
|
54
|
-
return text
|
|
55
|
-
.replace(HIDDEN, char => `<${codePointLabel(char)}>`)
|
|
64
|
+
return showHidden(text)
|
|
56
65
|
.replace(COMMAND_START, '$1<U+003A>')
|
|
57
66
|
.replace(LEGACY_COMMAND, '##<U+005B>');
|
|
58
67
|
}
|
|
@@ -65,11 +74,28 @@ export function inertLine(text) {
|
|
|
65
74
|
* value with no JSON form, such as `undefined`, throws instead of returning `undefined`.
|
|
66
75
|
*/
|
|
67
76
|
export function inertJson(value) {
|
|
68
|
-
|
|
77
|
+
return escapeJson(JSON.stringify(value, null, 2));
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* The JSON of `value` on one line: `JSON.stringify(value)`, escaped as `inertJson` escapes. The line
|
|
81
|
+
* holds no line break, since `JSON.stringify` writes a line feed inside a string as `\n` and the
|
|
82
|
+
* escaping covers every other hidden character, so it is one line of a file that is read line by line.
|
|
83
|
+
* It is lossless, and a value with no JSON form throws.
|
|
84
|
+
*/
|
|
85
|
+
export function inertJsonLine(value) {
|
|
86
|
+
return escapeJson(JSON.stringify(value));
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* A JSON text with every hidden character it leaves raw, but a line feed, written as escapes, and
|
|
90
|
+
* each `##[` broken. `undefined` is what `JSON.stringify` returns for a value with no JSON form.
|
|
91
|
+
*/
|
|
92
|
+
function escapeJson(json) {
|
|
69
93
|
if (json === undefined)
|
|
70
94
|
throw new TypeError('the value has no JSON form');
|
|
71
95
|
return json
|
|
72
|
-
.replace(HIDDEN, char => (char === '\n'
|
|
96
|
+
.replace(HIDDEN, char => (char === '\n'
|
|
97
|
+
? char
|
|
98
|
+
: escapeUnits(char)))
|
|
73
99
|
.replace(LEGACY_COMMAND, '##\\u005b');
|
|
74
100
|
}
|
|
75
101
|
/** The \uXXXX escape of each UTF-16 code unit of `char`. */
|
|
@@ -8,7 +8,9 @@ function ownData(value, key) {
|
|
|
8
8
|
return undefined;
|
|
9
9
|
try {
|
|
10
10
|
const descriptor = Object.getOwnPropertyDescriptor(value, key);
|
|
11
|
-
return descriptor !== undefined && Object.hasOwn(descriptor, 'value')
|
|
11
|
+
return descriptor !== undefined && Object.hasOwn(descriptor, 'value')
|
|
12
|
+
? descriptor.value
|
|
13
|
+
: undefined;
|
|
12
14
|
}
|
|
13
15
|
catch {
|
|
14
16
|
return undefined;
|
|
@@ -27,7 +29,9 @@ export function isTooLarge(error) {
|
|
|
27
29
|
/** What a refusal as too large says: its own data string `message`, else a fixed text, never anything else from the object. */
|
|
28
30
|
export function refusalMessage(refusal) {
|
|
29
31
|
const message = ownData(refusal, 'message');
|
|
30
|
-
return typeof message === 'string'
|
|
32
|
+
return typeof message === 'string'
|
|
33
|
+
? message
|
|
34
|
+
: REFUSAL_FALLBACK;
|
|
31
35
|
}
|
|
32
36
|
/** Whole milliseconds since `started`, a `performance.now()` reading. */
|
|
33
37
|
export function elapsed(started) {
|
|
@@ -63,14 +67,29 @@ export async function callProvider(ctx, requester, target, buildRequest) {
|
|
|
63
67
|
throw new ProviderError({ providerId: target.ref.id, kind: 'invalid-response', message: 'returned no reply text' });
|
|
64
68
|
}
|
|
65
69
|
catch (error) {
|
|
66
|
-
ctx.emit({
|
|
70
|
+
ctx.emit({
|
|
71
|
+
type: 'call',
|
|
72
|
+
...requester,
|
|
73
|
+
provider: target.ref,
|
|
74
|
+
durationMs: elapsed(started),
|
|
75
|
+
ok: false,
|
|
76
|
+
});
|
|
67
77
|
// Still inside the slot: the queued call it is handed to next sees the abort and never starts.
|
|
68
78
|
if (requester.role === 'research' && !isTooLarge(error))
|
|
69
79
|
ctx.fail(error);
|
|
70
80
|
throw error;
|
|
71
81
|
}
|
|
72
82
|
const usage = checkedUsage(response.usage);
|
|
73
|
-
ctx.emit({
|
|
83
|
+
ctx.emit({
|
|
84
|
+
type: 'call',
|
|
85
|
+
...requester,
|
|
86
|
+
provider: target.ref,
|
|
87
|
+
durationMs: elapsed(started),
|
|
88
|
+
ok: true,
|
|
89
|
+
...(usage === undefined
|
|
90
|
+
? {}
|
|
91
|
+
: { usage }),
|
|
92
|
+
});
|
|
74
93
|
return response.text;
|
|
75
94
|
});
|
|
76
95
|
}
|
|
@@ -86,7 +105,13 @@ export async function stage(ctx, profile, name, step, count) {
|
|
|
86
105
|
ctx.emit({ type: 'stage:start', profile, stage: name });
|
|
87
106
|
const started = performance.now();
|
|
88
107
|
const result = await step();
|
|
89
|
-
ctx.emit({
|
|
108
|
+
ctx.emit({
|
|
109
|
+
type: 'stage:end',
|
|
110
|
+
profile,
|
|
111
|
+
stage: name,
|
|
112
|
+
durationMs: elapsed(started),
|
|
113
|
+
count: count(result),
|
|
114
|
+
});
|
|
90
115
|
return result;
|
|
91
116
|
}
|
|
92
117
|
/**
|
|
@@ -25,7 +25,12 @@ import { callProvider, isTooLarge, refusalMessage } from "./context.js";
|
|
|
25
25
|
*/
|
|
26
26
|
export async function panel(ctx, run, findings, sections, diff) {
|
|
27
27
|
const judged = await Promise.all(findings.map(finding => judge(ctx, run, finding, diffSectionFor(sections, finding.file, diff))));
|
|
28
|
-
const outcome = {
|
|
28
|
+
const outcome = {
|
|
29
|
+
kept: [],
|
|
30
|
+
discarded: [],
|
|
31
|
+
unreviewed: [],
|
|
32
|
+
incomplete: [],
|
|
33
|
+
};
|
|
29
34
|
for (const { verdict, status, gaps } of judged) {
|
|
30
35
|
outcome[status].push(verdict);
|
|
31
36
|
outcome.incomplete.push(...gaps);
|
|
@@ -35,16 +40,36 @@ export async function panel(ctx, run, findings, sections, diff) {
|
|
|
35
40
|
async function judge(ctx, run, finding, diffContext) {
|
|
36
41
|
const casts = await Promise.all(run.lenses.map(lens => castVote(ctx, run, finding, lens, diffContext)));
|
|
37
42
|
const votes = casts.map(({ lensVote }) => lensVote);
|
|
38
|
-
const refusals = casts.flatMap(({ refusal }) => refusal === undefined
|
|
43
|
+
const refusals = casts.flatMap(({ refusal }) => refusal === undefined
|
|
44
|
+
? []
|
|
45
|
+
: [refusal]);
|
|
39
46
|
const keptVotes = votes.filter(lensVote => lensVote.vote?.keep).length;
|
|
40
|
-
const verdict = {
|
|
47
|
+
const verdict = {
|
|
48
|
+
...finding,
|
|
49
|
+
keptVotes,
|
|
50
|
+
totalVotes: votes.length,
|
|
51
|
+
confidence: confidenceFor(keptVotes, run.thresholds),
|
|
52
|
+
votes,
|
|
53
|
+
};
|
|
41
54
|
const status = verdictStatus(verdict, run.thresholds);
|
|
42
55
|
const { confidence } = verdict;
|
|
43
|
-
ctx.emit({
|
|
56
|
+
ctx.emit({
|
|
57
|
+
type: 'verdict',
|
|
58
|
+
profile: run.id,
|
|
59
|
+
findingId: finding.id,
|
|
60
|
+
keptVotes,
|
|
61
|
+
totalVotes: votes.length,
|
|
62
|
+
...(confidence === undefined
|
|
63
|
+
? {}
|
|
64
|
+
: { confidence }),
|
|
65
|
+
status,
|
|
66
|
+
});
|
|
44
67
|
const gaps = [];
|
|
45
68
|
if (refusals.length > 0)
|
|
46
69
|
gaps.push(requestTooLarge(run.id, finding, refusals, votes.length));
|
|
47
|
-
const undecided = status === 'kept'
|
|
70
|
+
const undecided = status === 'kept'
|
|
71
|
+
? undecidedLevel(verdict, run.thresholds)
|
|
72
|
+
: undefined;
|
|
48
73
|
if (undecided !== undefined)
|
|
49
74
|
gaps.push(voteFailed(run.id, verdict, undecided));
|
|
50
75
|
for (const entry of gaps)
|
|
@@ -65,18 +90,39 @@ async function castVote(ctx, run, finding, lens, diffContext) {
|
|
|
65
90
|
let refusal;
|
|
66
91
|
try {
|
|
67
92
|
const text = await callProvider(ctx, { profile: run.id, role: 'vote', lens: lens.id }, lens, () => voteRequest(lens.instructions, finding, diffContext, run.guidelines));
|
|
68
|
-
lensVote = {
|
|
93
|
+
lensVote = {
|
|
94
|
+
angle: lens.id,
|
|
95
|
+
vote: parseVoteReply(text),
|
|
96
|
+
error: null,
|
|
97
|
+
};
|
|
69
98
|
}
|
|
70
99
|
catch (error) {
|
|
71
100
|
// Once the review is aborted, a failed call is not the voter's failure: the review rejects with the abort's reason.
|
|
72
101
|
ctx.signal.throwIfAborted();
|
|
73
|
-
lensVote = {
|
|
102
|
+
lensVote = {
|
|
103
|
+
angle: lens.id,
|
|
104
|
+
vote: null,
|
|
105
|
+
error: error instanceof Error
|
|
106
|
+
? error.message
|
|
107
|
+
: String(error),
|
|
108
|
+
};
|
|
74
109
|
if (isTooLarge(error))
|
|
75
110
|
refusal = refusalMessage(error);
|
|
76
111
|
}
|
|
77
112
|
const { error } = lensVote;
|
|
78
|
-
ctx.emit({
|
|
79
|
-
|
|
113
|
+
ctx.emit({
|
|
114
|
+
type: 'vote',
|
|
115
|
+
profile: run.id,
|
|
116
|
+
findingId: finding.id,
|
|
117
|
+
lens: lens.id,
|
|
118
|
+
keep: lensVote.vote?.keep ?? null,
|
|
119
|
+
...(error === null
|
|
120
|
+
? {}
|
|
121
|
+
: { error }),
|
|
122
|
+
});
|
|
123
|
+
return refusal === undefined
|
|
124
|
+
? { lensVote }
|
|
125
|
+
: { lensVote, refusal };
|
|
80
126
|
}
|
|
81
127
|
/**
|
|
82
128
|
* The gap a finding leaves when its provider refused some of its vote requests as too large. The file
|
|
@@ -37,7 +37,11 @@ export function checkReply(text, sections, profile, maxFindings = DEFAULT_MAX_FI
|
|
|
37
37
|
}
|
|
38
38
|
const parsed = parseResearchReply(text, maxFindings);
|
|
39
39
|
const checked = verifyFindings(parsed.findings, sections);
|
|
40
|
-
const findings = checked.findings.map((finding) => ({
|
|
40
|
+
const findings = checked.findings.map((finding) => ({
|
|
41
|
+
id: findingId(profile, finding.file, finding.line, finding.category),
|
|
42
|
+
profile,
|
|
43
|
+
...finding,
|
|
44
|
+
}));
|
|
41
45
|
const incomplete = parsed.invalid.map((detail) => ({ profile, reason: 'finding-unreadable', detail }));
|
|
42
46
|
if (parsed.listed > maxFindings) {
|
|
43
47
|
incomplete.push({
|
|
@@ -48,7 +52,9 @@ export function checkReply(text, sections, profile, maxFindings = DEFAULT_MAX_FI
|
|
|
48
52
|
}
|
|
49
53
|
if (parsed.listed > 0 && findings.length === 0) {
|
|
50
54
|
const reported = `${parsed.listed} finding(s) reported`;
|
|
51
|
-
const detail = parsed.listed > maxFindings
|
|
55
|
+
const detail = parsed.listed > maxFindings
|
|
56
|
+
? `${reported}, the first ${maxFindings} read, none usable`
|
|
57
|
+
: `${reported}, none usable`;
|
|
52
58
|
incomplete.push({ profile, reason: 'research-unusable', detail });
|
|
53
59
|
}
|
|
54
60
|
return { findings, invalid: checked.invalid, incomplete };
|
|
@@ -78,7 +84,11 @@ export async function research(ctx, run, diff, sections) {
|
|
|
78
84
|
ctx.signal.throwIfAborted();
|
|
79
85
|
const entry = { profile: run.id, reason: 'request-too-large', detail: `the research request was not sent: ${refusalMessage(error)}` };
|
|
80
86
|
ctx.emit({ type: 'incomplete', entry });
|
|
81
|
-
return {
|
|
87
|
+
return {
|
|
88
|
+
findings: [],
|
|
89
|
+
invalid: [],
|
|
90
|
+
incomplete: [entry],
|
|
91
|
+
};
|
|
82
92
|
}
|
|
83
93
|
let outcome;
|
|
84
94
|
try {
|
package/dist/core/verify.js
CHANGED
|
@@ -32,7 +32,9 @@ export function verifyFindings(details, sections) {
|
|
|
32
32
|
invalid.push(`${plainText(detail.file, MAX_REPORTED_FILE)}:${detail.line} is not in the diff`);
|
|
33
33
|
continue;
|
|
34
34
|
}
|
|
35
|
-
const file = located.path === ''
|
|
35
|
+
const file = located.path === ''
|
|
36
|
+
? plainText(detail.file, MAX_FILE)
|
|
37
|
+
: located.path;
|
|
36
38
|
if (file === '') {
|
|
37
39
|
invalid.push(`line ${detail.line} names no file`);
|
|
38
40
|
continue;
|
|
@@ -72,6 +74,8 @@ const isHighSurrogate = (unit) => unit >= 0xD800 && unit <= 0xDBFF;
|
|
|
72
74
|
function cap(value, max) {
|
|
73
75
|
if (value.length <= max)
|
|
74
76
|
return value;
|
|
75
|
-
const keep = isHighSurrogate(value.charCodeAt(max - 2))
|
|
77
|
+
const keep = isHighSurrogate(value.charCodeAt(max - 2))
|
|
78
|
+
? max - 2
|
|
79
|
+
: max - 1;
|
|
76
80
|
return `${value.slice(0, keep)}…`;
|
|
77
81
|
}
|
package/dist/index.js
CHANGED
|
@@ -29,6 +29,10 @@ export { presets } from "./providers/presets.js";
|
|
|
29
29
|
*/
|
|
30
30
|
export async function runQuorum(diff, onProgress) {
|
|
31
31
|
const { config } = await loadConfig();
|
|
32
|
-
const quorum = await fromConfig(config, {
|
|
32
|
+
const quorum = await fromConfig(config, {
|
|
33
|
+
events: onProgress === undefined
|
|
34
|
+
? undefined
|
|
35
|
+
: progressSink(onProgress),
|
|
36
|
+
});
|
|
33
37
|
return quorum.review({ diff });
|
|
34
38
|
}
|
|
@@ -15,9 +15,26 @@ const ANTHROPIC_VERSION = '2026-10-21';
|
|
|
15
15
|
* header offers a tool by itself.
|
|
16
16
|
*/
|
|
17
17
|
const RULES = {
|
|
18
|
-
reservedBodyKeys: [
|
|
19
|
-
|
|
20
|
-
|
|
18
|
+
reservedBodyKeys: [
|
|
19
|
+
'model',
|
|
20
|
+
'messages',
|
|
21
|
+
'system',
|
|
22
|
+
'stream',
|
|
23
|
+
],
|
|
24
|
+
toolBodyKeys: [
|
|
25
|
+
'tools',
|
|
26
|
+
'tool_choice',
|
|
27
|
+
'mcp_servers',
|
|
28
|
+
'container',
|
|
29
|
+
'thread',
|
|
30
|
+
'fallbacks',
|
|
31
|
+
'fallback_credit_token',
|
|
32
|
+
'context_management',
|
|
33
|
+
'compaction',
|
|
34
|
+
],
|
|
35
|
+
replyLimitPaths: [
|
|
36
|
+
['max_tokens'],
|
|
37
|
+
],
|
|
21
38
|
fixedHeaders: ['anthropic-version'],
|
|
22
39
|
structuredOutputs: ['prompt', 'schema'],
|
|
23
40
|
};
|
|
@@ -45,7 +62,9 @@ function readReply(reply, { id, model }, redact) {
|
|
|
45
62
|
throw fail('invalid-response', 'reply has no text');
|
|
46
63
|
return {
|
|
47
64
|
text,
|
|
48
|
-
model: typeof reply.model === 'string' && reply.model !== ''
|
|
65
|
+
model: typeof reply.model === 'string' && reply.model !== ''
|
|
66
|
+
? reply.model
|
|
67
|
+
: model,
|
|
49
68
|
usage: tokenUsage(reply.usage, ['input_tokens', 'cache_creation_input_tokens', 'cache_read_input_tokens'], ['output_tokens']),
|
|
50
69
|
};
|
|
51
70
|
}
|
|
@@ -81,7 +100,11 @@ export function anthropic(options) {
|
|
|
81
100
|
system,
|
|
82
101
|
messages,
|
|
83
102
|
temperature: temperature ?? undefined,
|
|
84
|
-
output_config: structuredOutput === 'schema'
|
|
103
|
+
output_config: structuredOutput === 'schema'
|
|
104
|
+
? {
|
|
105
|
+
format: { type: 'json_schema', schema: request.output.schema },
|
|
106
|
+
}
|
|
107
|
+
: undefined,
|
|
85
108
|
}, extraBody),
|
|
86
109
|
signal: request.signal,
|
|
87
110
|
redact,
|