@tobi-or-not/quorum 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/bench/fingerprint.d.ts +8 -5
- package/dist/bench/fingerprint.js +28 -9
- package/dist/bench/records.d.ts +168 -0
- package/dist/bench/records.js +106 -1
- package/dist/bench/runner.d.ts +18 -0
- package/dist/bench/runner.js +30 -4
- package/dist/cli/bench.d.ts +11 -4
- package/dist/cli/bench.js +34 -17
- package/dist/cli/bin.js +2 -2
- package/dist/cli/io.d.ts +5 -0
- package/dist/cli/main.d.ts +67 -24
- package/dist/cli/main.js +288 -59
- package/dist/cli/print-config.d.ts +8 -6
- package/dist/cli/print-config.js +26 -8
- package/dist/cli/render.d.ts +9 -3
- package/dist/cli/render.js +43 -2
- package/dist/config/claude-code.d.ts +2 -6
- package/dist/config/claude-code.js +6 -44
- package/dist/config/command.d.ts +10 -0
- package/dist/config/command.js +47 -0
- package/dist/config/env.d.ts +32 -19
- package/dist/config/env.js +95 -32
- package/dist/config/from-config.js +21 -3
- package/dist/config/resolve.d.ts +13 -1
- package/dist/config/resolve.js +27 -1
- package/dist/config/schema.d.ts +152 -5
- package/dist/config/schema.js +156 -12
- package/dist/core/clock.d.ts +17 -0
- package/dist/core/clock.js +14 -0
- package/dist/core/coverage.d.ts +74 -0
- package/dist/core/coverage.js +221 -0
- package/dist/core/diff.d.ts +101 -0
- package/dist/core/diff.js +178 -35
- package/dist/core/errors.d.ts +12 -3
- package/dist/core/errors.js +21 -2
- package/dist/core/estimate.d.ts +45 -0
- package/dist/core/estimate.js +188 -0
- package/dist/core/exclude.d.ts +42 -0
- package/dist/core/exclude.js +189 -0
- package/dist/core/flags.d.ts +14 -1
- package/dist/core/flags.js +56 -0
- package/dist/core/inputs.d.ts +38 -0
- package/dist/core/inputs.js +228 -0
- package/dist/core/limits.d.ts +83 -0
- package/dist/core/limits.js +169 -0
- package/dist/core/output.d.ts +3 -2
- package/dist/core/output.js +3 -2
- package/dist/core/parts.d.ts +146 -0
- package/dist/core/parts.js +388 -0
- package/dist/core/ports.d.ts +67 -2
- package/dist/core/ports.js +36 -1
- package/dist/core/prompts.d.ts +22 -3
- package/dist/core/prompts.js +43 -7
- package/dist/core/quorum.d.ts +26 -8
- package/dist/core/quorum.js +310 -73
- package/dist/core/retry.d.ts +36 -0
- package/dist/core/retry.js +46 -0
- package/dist/core/secrets.d.ts +98 -0
- package/dist/core/secrets.js +507 -0
- package/dist/core/stages/context.d.ts +131 -23
- package/dist/core/stages/context.js +331 -46
- package/dist/core/stages/panel.d.ts +13 -6
- package/dist/core/stages/panel.js +79 -19
- package/dist/core/stages/research.d.ts +110 -25
- package/dist/core/stages/research.js +338 -44
- package/dist/core/tokens.d.ts +31 -0
- package/dist/core/tokens.js +81 -0
- package/dist/core/types.d.ts +280 -25
- package/dist/core/verify.d.ts +6 -0
- package/dist/core/verify.js +6 -3
- package/dist/git/run.d.ts +223 -0
- package/dist/git/run.js +608 -0
- package/dist/git/tree.d.ts +43 -0
- package/dist/git/tree.js +171 -0
- package/dist/index.d.ts +26 -2
- package/dist/index.js +34 -2
- package/dist/providers/anthropic.d.ts +3 -1
- package/dist/providers/anthropic.js +10 -3
- package/dist/providers/claude-code.d.ts +56 -11
- package/dist/providers/claude-code.js +70 -25
- package/dist/providers/gemini.d.ts +5 -1
- package/dist/providers/gemini.js +65 -11
- package/dist/providers/http.d.ts +49 -4
- package/dist/providers/http.js +155 -14
- package/dist/providers/ollama.d.ts +4 -1
- package/dist/providers/ollama.js +13 -5
- package/dist/providers/openai-compatible.d.ts +4 -1
- package/dist/providers/openai-compatible.js +13 -5
- package/dist/providers/options.d.ts +12 -3
- package/dist/providers/options.js +30 -10
- package/dist/providers/size.d.ts +19 -15
- package/dist/providers/size.js +33 -73
- package/dist/sources/added.d.ts +49 -0
- package/dist/sources/added.js +134 -0
- package/dist/sources/diff.d.ts +13 -0
- package/dist/sources/diff.js +63 -0
- package/dist/sources/local.d.ts +41 -0
- package/dist/sources/local.js +251 -0
- package/dist/sources/paths.d.ts +45 -0
- package/dist/sources/paths.js +230 -0
- package/dist/sources/range.d.ts +21 -0
- package/dist/sources/range.js +75 -0
- package/dist/sources/read.d.ts +69 -0
- package/dist/sources/read.js +37 -0
- package/dist/sources/repository.d.ts +9 -0
- package/dist/sources/repository.js +28 -0
- package/dist/sources/skips.d.ts +49 -0
- package/dist/sources/skips.js +126 -0
- package/package.json +1 -1
- package/schema/quorum.schema.json +49 -1
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
|
+
import { OutputError } from "../errors.js";
|
|
3
|
+
import { isLimit } from "../limits.js";
|
|
2
4
|
import { parseResearchReply } from "../output.js";
|
|
5
|
+
import { listedFiles, packParts, partText } from "../parts.js";
|
|
3
6
|
import { researchRequest } from "../prompts.js";
|
|
7
|
+
import { tokenBound } from "../tokens.js";
|
|
4
8
|
import { verifyFindings } from "../verify.js";
|
|
5
|
-
import { callProvider, isTooLarge, refusalMessage } from "./context.js";
|
|
9
|
+
import { callProvider, isTooLarge, providerFailure, refusalMessage, roomOf } from "./context.js";
|
|
6
10
|
/** How many reported findings a research reply may send to the panel; the rest is reported as incomplete. */
|
|
7
11
|
export const DEFAULT_MAX_FINDINGS = 50;
|
|
8
12
|
/**
|
|
@@ -14,45 +18,134 @@ export const DEFAULT_MAX_FINDINGS = 50;
|
|
|
14
18
|
export function findingId(profile, file, line, category) {
|
|
15
19
|
return createHash('sha256').update(JSON.stringify([profile, file, line, category])).digest('hex').slice(0, 12);
|
|
16
20
|
}
|
|
21
|
+
/** The lines of `text`, the last one included. */
|
|
22
|
+
function lineCount(text) {
|
|
23
|
+
let lines = 1;
|
|
24
|
+
for (let at = text.indexOf('\n'); at !== -1; at = text.indexOf('\n', at + 1))
|
|
25
|
+
lines++;
|
|
26
|
+
return lines;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Plan a profile's research of the filtered `diff`, whose sections are `sections`, before any stage runs.
|
|
30
|
+
* The research is one request holding the whole diff, which is what 0.5.0 sent, when:
|
|
31
|
+
* 1. the research provider states no room: no request is built to ask, so no nonce is drawn;
|
|
32
|
+
* 2. the room of the one-request research with an empty diff is no safe integer or is negative: its guard
|
|
33
|
+
* then refuses the request as it always did;
|
|
34
|
+
* 3. the diff's `tokenBound` is within the lesser of `partTokens` and that room;
|
|
35
|
+
* 4. the room of a part's request with an empty diff, its parts block at its widest (a position and a
|
|
36
|
+
* count of as many digits as the diff has lines, which no packing has more parts than), is no safe
|
|
37
|
+
* integer or is negative;
|
|
38
|
+
* 5. the packing sets a hunk aside while the one request still holds the whole diff: that hunk fits the
|
|
39
|
+
* one request but not a part's, which carries the parts block, and 0.5.0 read it, so nothing it read is
|
|
40
|
+
* left unread now.
|
|
41
|
+
* Otherwise it is the packing of the sections into parts of at most `partTokens` that fit that room (see
|
|
42
|
+
* `packParts`), with the change's files its requests name: at most 100, and at most a quarter of the
|
|
43
|
+
* lesser of `partTokens` and the one request's room, so the list stays small beside the diff of a part. When that
|
|
44
|
+
* packing sets a unit aside and the list names a file, the sections are packed again at the room of a parts
|
|
45
|
+
* block that lists no name (`and <n> more` alone), asked at the same widest position, because the list
|
|
46
|
+
* may be what left the unit out: the second packing and its block are kept when it sets fewer units aside,
|
|
47
|
+
* else the first, and the rule above (5) applies to the packing kept.
|
|
48
|
+
*/
|
|
49
|
+
export function planResearch(run, diff, sections, partTokens) {
|
|
50
|
+
const whole = { sections };
|
|
51
|
+
if (run.research.room === undefined)
|
|
52
|
+
return whole;
|
|
53
|
+
const { instructions } = run.research;
|
|
54
|
+
const one = roomOf(run.research, researchRequest(instructions, '', run.guidelines));
|
|
55
|
+
if (one === undefined || one < 0)
|
|
56
|
+
return whole;
|
|
57
|
+
const tokens = tokenBound(diff);
|
|
58
|
+
if (tokens <= Math.min(partTokens, one))
|
|
59
|
+
return whole;
|
|
60
|
+
const widest = lineCount(diff);
|
|
61
|
+
const roomWith = (names) => roomOf(run.research, researchRequest(instructions, '', run.guidelines, { index: widest, count: widest, ...names }));
|
|
62
|
+
let listed = listedFiles(sections, Math.floor(Math.min(partTokens, one) / 4));
|
|
63
|
+
const room = roomWith(listed);
|
|
64
|
+
if (room === undefined || room < 0)
|
|
65
|
+
return whole;
|
|
66
|
+
let packing = packParts(sections, { partTokens, room });
|
|
67
|
+
if (packing.tooLarge.length > 0 && listed.files.length > 0) {
|
|
68
|
+
// The names a request lists take room a hunk may need: pack again with none listed, and keep that when it leaves fewer units out.
|
|
69
|
+
const bare = { files: [], unlisted: listed.files.length + listed.unlisted };
|
|
70
|
+
const bareRoom = roomWith(bare);
|
|
71
|
+
const again = bareRoom !== undefined && bareRoom >= 0
|
|
72
|
+
? packParts(sections, { partTokens, room: bareRoom })
|
|
73
|
+
: undefined;
|
|
74
|
+
if (again !== undefined && again.tooLarge.length < packing.tooLarge.length) {
|
|
75
|
+
packing = again;
|
|
76
|
+
listed = bare;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
if (packing.tooLarge.length > 0 && tokens <= one)
|
|
80
|
+
return whole;
|
|
81
|
+
return { sections, packing, listed };
|
|
82
|
+
}
|
|
17
83
|
/**
|
|
18
|
-
* Turn
|
|
19
|
-
*
|
|
20
|
-
* only the first `maxFindings` findings it lists are read and checked against the diff (the work
|
|
21
|
-
* spent on a reply built to flood the run stays bounded, whatever the entries hold), and nothing
|
|
22
|
-
* the harness read is dropped silently: a finding that does not match the reply schema (what
|
|
23
|
-
* planted text that closes a string early leaves behind), findings beyond the cap, and a reply
|
|
24
|
-
* whose findings were all unusable, are recorded as incomplete, which fails the run (ADR-0012).
|
|
25
|
-
* A finding that cites code outside the diff is only dropped with its reason: it was read, and
|
|
26
|
-
* it is wrong. `invalid` holds only the reasons of the findings the check against the diff
|
|
27
|
-
* dropped, so an unreadable finding is reported once, in `incomplete`. Every finding it keeps is
|
|
28
|
-
* named by `findingId` and tagged with `profile`, and every incomplete entry it records is tagged
|
|
29
|
-
* with `profile` too.
|
|
84
|
+
* Turn one research reply into what the panel receives: `checkReplies` of that reply alone, unlabelled, so
|
|
85
|
+
* its gaps and its errors are the ones `checkReplies` states.
|
|
30
86
|
*
|
|
31
87
|
* @throws OutputError when the reply is not one JSON object holding a `findings` array and nothing else
|
|
32
88
|
* @throws RangeError when `maxFindings` is not a positive integer
|
|
33
89
|
*/
|
|
34
90
|
export function checkReply(text, sections, profile, maxFindings = DEFAULT_MAX_FINDINGS) {
|
|
91
|
+
return checkReplies([
|
|
92
|
+
{ text },
|
|
93
|
+
], sections, profile, maxFindings);
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Turn the research replies of a profile, in the parts' order, into what the panel receives. Each reply is
|
|
97
|
+
* parsed strictly (a malformed envelope, or one with a key other than `findings`, throws, so it never reads
|
|
98
|
+
* as "no findings"), only the first `maxFindings` findings the replies list in all are read and checked
|
|
99
|
+
* against the diff, the earlier replies' first (the work spent on a reply built to flood the run stays
|
|
100
|
+
* bounded, whatever the entries hold), and nothing the harness read is dropped silently: a finding that
|
|
101
|
+
* does not match the reply schema (what planted text that closes a string early leaves behind), findings
|
|
102
|
+
* beyond the cap, counted over all the replies in one gap, and replies whose findings were all unusable,
|
|
103
|
+
* are recorded as incomplete, which fails the run (ADR-0012). A finding that cites code outside the diff,
|
|
104
|
+
* the whole diff of every part, is only dropped with its reason: it was read, and it is wrong. `invalid`
|
|
105
|
+
* holds only the reasons of the findings the check against the diff dropped, so an unreadable finding is
|
|
106
|
+
* reported once, in `incomplete`, with its reply's `label` before the reason when it has one. Every
|
|
107
|
+
* finding it keeps is named by `findingId` and tagged with `profile`, and every incomplete entry it
|
|
108
|
+
* records is tagged with `profile` too.
|
|
109
|
+
*
|
|
110
|
+
* @throws OutputError when a reply is not one JSON object holding a `findings` array and nothing else
|
|
111
|
+
* @throws RangeError when `maxFindings` is not a positive integer
|
|
112
|
+
*/
|
|
113
|
+
export function checkReplies(replies, sections, profile, maxFindings = DEFAULT_MAX_FINDINGS) {
|
|
35
114
|
if (!Number.isInteger(maxFindings) || maxFindings < 1) {
|
|
36
115
|
throw new RangeError(`maxFindings must be a positive integer, got ${maxFindings}`);
|
|
37
116
|
}
|
|
38
|
-
const
|
|
39
|
-
const
|
|
117
|
+
const details = [];
|
|
118
|
+
const unreadable = [];
|
|
119
|
+
let listed = 0;
|
|
120
|
+
for (const { text, label } of replies) {
|
|
121
|
+
// A reply past the cap is parsed for its envelope and its count alone: its entries are not read.
|
|
122
|
+
const left = maxFindings - details.length - unreadable.length;
|
|
123
|
+
const parsed = parseResearchReply(text, Math.max(left, 1));
|
|
124
|
+
listed += parsed.listed;
|
|
125
|
+
if (left < 1)
|
|
126
|
+
continue;
|
|
127
|
+
details.push(...parsed.findings);
|
|
128
|
+
unreadable.push(...parsed.invalid.map(reason => label === undefined
|
|
129
|
+
? reason
|
|
130
|
+
: `${label}: ${reason}`));
|
|
131
|
+
}
|
|
132
|
+
const checked = verifyFindings(details, sections);
|
|
40
133
|
const findings = checked.findings.map((finding) => ({
|
|
41
134
|
id: findingId(profile, finding.file, finding.line, finding.category),
|
|
42
135
|
profile,
|
|
43
136
|
...finding,
|
|
44
137
|
}));
|
|
45
|
-
const incomplete =
|
|
46
|
-
if (
|
|
138
|
+
const incomplete = unreadable.map((detail) => ({ profile, reason: 'finding-unreadable', detail }));
|
|
139
|
+
if (listed > maxFindings) {
|
|
47
140
|
incomplete.push({
|
|
48
141
|
profile,
|
|
49
142
|
reason: 'findings-capped',
|
|
50
|
-
detail: `${
|
|
143
|
+
detail: `${listed - maxFindings} of ${listed} reported findings were not reviewed (the cap is ${maxFindings})`,
|
|
51
144
|
});
|
|
52
145
|
}
|
|
53
|
-
if (
|
|
54
|
-
const reported = `${
|
|
55
|
-
const detail =
|
|
146
|
+
if (listed > 0 && findings.length === 0) {
|
|
147
|
+
const reported = `${listed} finding(s) reported`;
|
|
148
|
+
const detail = listed > maxFindings
|
|
56
149
|
? `${reported}, the first ${maxFindings} read, none usable`
|
|
57
150
|
: `${reported}, none usable`;
|
|
58
151
|
incomplete.push({ profile, reason: 'research-unusable', detail });
|
|
@@ -60,47 +153,248 @@ export function checkReply(text, sections, profile, maxFindings = DEFAULT_MAX_FI
|
|
|
60
153
|
return { findings, invalid: checked.invalid, incomplete };
|
|
61
154
|
}
|
|
62
155
|
/**
|
|
63
|
-
* The research stage of one profile:
|
|
64
|
-
* research instructions, the diff and the guidelines,
|
|
65
|
-
*
|
|
66
|
-
*
|
|
67
|
-
*
|
|
68
|
-
* records one `request-too-large` gap, and the other profiles run. Any other failed research
|
|
69
|
-
* call, or a reply that is not a valid envelope, fails the profile and aborts the review where it
|
|
70
|
-
* fails (`callProvider` aborts it for the call), so no other profile starts a call after it.
|
|
156
|
+
* The research stage of one profile, as `plan` lays it out: a change within one request is researched by one
|
|
157
|
+
* call to its research provider, with the profile's research instructions, the diff and the guidelines, and a
|
|
158
|
+
* large change is researched in parts, one call each, whose replies `checkReplies` turns into what the panel
|
|
159
|
+
* receives. Each finding dropped for citing code outside the diff is reported by a `finding:invalid` event,
|
|
160
|
+
* and each gap by an `incomplete` event.
|
|
71
161
|
*
|
|
72
|
-
*
|
|
73
|
-
*
|
|
162
|
+
* A change in one request (no packing, or a packing of one part, whose text is that part's) goes through
|
|
163
|
+
* one call. A research request its provider refused as too large (a `too-large` `ProviderError`) was not
|
|
164
|
+
* sent: the profile reviews nothing, records one `request-too-large` gap, and the other profiles run. For
|
|
165
|
+
* any other failed research, the call after its retries or a reply that is not a valid envelope, a gating
|
|
166
|
+
* profile fails and aborts the review where it fails (`onFailure` aborts it for the call), so no other
|
|
167
|
+
* profile starts a call after it; an advisory profile's failure is its own `research-failed` gap, in words
|
|
168
|
+
* the code writes, and the other profiles run. A research request a limit stopped (the token cap refused it
|
|
169
|
+
* or left no room for it, or the review's deadline ended it) is no failure of either kind: the outcome holds
|
|
170
|
+
* no finding and names the limit in `parts`, which the review turns into the profile's one gap for that
|
|
171
|
+
* limit, a gating profile's included. A packing of none, where no part holds any of the diff, makes no call, and
|
|
172
|
+
* neither does a part whose text is blank (white space only, as the text before a diff's first header can be,
|
|
173
|
+
* when every other unit was set aside): there is nothing to ask a model, so its outcome is `reviewed`.
|
|
174
|
+
*
|
|
175
|
+
* A packing of several parts runs `ctx.concurrency` workers of this profile, each taking the next part, in
|
|
176
|
+
* order, while one is left, and a call of each through the review's one limit, so the review's concurrency
|
|
177
|
+
* holds across profiles. Each part fails as a one-request research does, and a reply's envelope is checked
|
|
178
|
+
* as it arrives: a gating profile's failed call or unreadable envelope aborts the review where it fails, so
|
|
179
|
+
* no queued part starts and the review rejects with the first reason, and an advisory profile's is that
|
|
180
|
+
* part's `research-failed`, while its other parts go on. Once a part ends with a limit, the profile's parts
|
|
181
|
+
* not yet taken are stopped by the same limit without a call: each would be built and counted only to be
|
|
182
|
+
* refused, and the gap counts them, failing closed. Once every part has answered, the replies are read in
|
|
183
|
+
* the parts' order, up to `maxFindings` findings in all, each checked against the whole diff, and a finding
|
|
184
|
+
* two parts report is dropped by the dedup stage. The gaps come in this order, each counting parts: the
|
|
185
|
+
* requests refused as too large, then the parts that failed, then the reading's. The events follow: the
|
|
186
|
+
* `finding:invalid` ones, then the `incomplete` ones.
|
|
187
|
+
*
|
|
188
|
+
* A review that is aborted rejects with its reason, never with a gap.
|
|
189
|
+
*
|
|
190
|
+
* @throws ProviderError, or whatever a custom provider throws, when a gating profile's research call fails for any other reason than a refusal as too large
|
|
191
|
+
* @throws OutputError when a gating profile's reply is not one JSON object holding a `findings` array and nothing else
|
|
74
192
|
*/
|
|
75
|
-
export async function research(ctx, run, diff, sections) {
|
|
193
|
+
export async function research(ctx, run, plan, diff, sections) {
|
|
194
|
+
const { packing } = plan;
|
|
195
|
+
if (packing === undefined)
|
|
196
|
+
return researchInOne(ctx, run, diff, sections);
|
|
197
|
+
if (packing.parts.length === 0)
|
|
198
|
+
return nothingRead([]);
|
|
199
|
+
if (packing.parts.length === 1) {
|
|
200
|
+
const text = partText(plan.sections, packing, packing.parts[0]);
|
|
201
|
+
return isBlank(text)
|
|
202
|
+
? nothingRead(['reviewed'])
|
|
203
|
+
: researchInOne(ctx, run, text, sections);
|
|
204
|
+
}
|
|
205
|
+
return researchInParts(ctx, run, plan, packing, sections);
|
|
206
|
+
}
|
|
207
|
+
/** Whether `text` holds nothing but white space: a part of it asks the model nothing, as the text before a diff's first header can be. */
|
|
208
|
+
function isBlank(text) {
|
|
209
|
+
return text.trim() === '';
|
|
210
|
+
}
|
|
211
|
+
/** The outcome of a research that read no reply and found no gap: no finding, `parts` as given. */
|
|
212
|
+
function nothingRead(parts) {
|
|
213
|
+
return {
|
|
214
|
+
findings: [],
|
|
215
|
+
invalid: [],
|
|
216
|
+
incomplete: [],
|
|
217
|
+
parts,
|
|
218
|
+
};
|
|
219
|
+
}
|
|
220
|
+
const isSpoken = (result) => 'words' in result;
|
|
221
|
+
const isStopped = (result) => result.outcome === 'token-cap' || result.outcome === 'review-deadline';
|
|
222
|
+
/**
|
|
223
|
+
* One research request, sent and its reply's envelope checked, and what became of it: the one place where a
|
|
224
|
+
* failure is told from a gap, for a research of one request and for each part of a research in parts alike.
|
|
225
|
+
* `onFailure` aborts the review for a gating profile's failure (the call's, not a refusal as too large nor a
|
|
226
|
+
* limit's) inside the call's slot, so no queued request starts after it. In this order:
|
|
227
|
+
* - a limit that stopped the call is the limit's name;
|
|
228
|
+
* - a refusal as too large is `request-too-large` with the refusal's message;
|
|
229
|
+
* - any other failure of the call rethrows for a gating profile, and is `research-failed` with the words
|
|
230
|
+
* `researchFailed` writes for an advisory one;
|
|
231
|
+
* - a reply that is no findings envelope aborts the review and rethrows for a gating profile, and for any
|
|
232
|
+
* error that is no `OutputError`; for an advisory profile it is `research-failed` with the reason;
|
|
233
|
+
* - else `reviewed`, with the reply's text.
|
|
234
|
+
* A limit, a refusal and an advisory failure are each told only after the review's abort check, so a review
|
|
235
|
+
* that is aborted rejects with its reason, never with a gap.
|
|
236
|
+
*/
|
|
237
|
+
async function askResearch(ctx, run, request) {
|
|
238
|
+
let attempts = 1;
|
|
76
239
|
let text;
|
|
77
240
|
try {
|
|
78
|
-
text = await callProvider(ctx, { profile: run.id, role: 'research' }, run.research, () =>
|
|
241
|
+
text = await callProvider(ctx, { profile: run.id, role: 'research' }, run.research, request, (error, made) => {
|
|
242
|
+
attempts = made;
|
|
243
|
+
if (run.gating && !isTooLarge(error) && !isLimit(error))
|
|
244
|
+
ctx.fail(error);
|
|
245
|
+
});
|
|
79
246
|
}
|
|
80
247
|
catch (error) {
|
|
81
|
-
if (
|
|
248
|
+
if (isLimit(error)) {
|
|
249
|
+
ctx.signal.throwIfAborted();
|
|
250
|
+
return { outcome: error.limit };
|
|
251
|
+
}
|
|
252
|
+
if (isTooLarge(error)) {
|
|
253
|
+
ctx.signal.throwIfAborted();
|
|
254
|
+
return {
|
|
255
|
+
outcome: 'request-too-large',
|
|
256
|
+
words: refusalMessage(error),
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
// A gating profile's failure aborted the review where it failed (`onFailure`).
|
|
260
|
+
if (run.gating)
|
|
82
261
|
throw error;
|
|
83
|
-
// An aborted review rejects with its reason, never with a gap.
|
|
84
262
|
ctx.signal.throwIfAborted();
|
|
85
|
-
const entry = { profile: run.id, reason: 'request-too-large', detail: `the research request was not sent: ${refusalMessage(error)}` };
|
|
86
|
-
ctx.emit({ type: 'incomplete', entry });
|
|
87
263
|
return {
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
incomplete: [entry],
|
|
264
|
+
outcome: 'research-failed',
|
|
265
|
+
words: researchFailed(error, attempts),
|
|
91
266
|
};
|
|
92
267
|
}
|
|
93
|
-
let outcome;
|
|
94
268
|
try {
|
|
95
|
-
|
|
269
|
+
parseResearchReply(text, 1);
|
|
96
270
|
}
|
|
97
271
|
catch (error) {
|
|
98
|
-
|
|
99
|
-
|
|
272
|
+
if (run.gating || !(error instanceof OutputError)) {
|
|
273
|
+
ctx.fail(error);
|
|
274
|
+
throw error;
|
|
275
|
+
}
|
|
276
|
+
ctx.signal.throwIfAborted();
|
|
277
|
+
return { outcome: 'research-failed', words: `the research reply could not be read: ${error.reason}` };
|
|
100
278
|
}
|
|
279
|
+
return { outcome: 'reviewed', text };
|
|
280
|
+
}
|
|
281
|
+
/** One research request holding `diff`, its outcome's `parts` the one part's, and its failure told as a gap in today's words. */
|
|
282
|
+
async function researchInOne(ctx, run, diff, sections) {
|
|
283
|
+
const result = await askResearch(ctx, run, () => researchRequest(run.research.instructions, diff, run.guidelines));
|
|
284
|
+
if (isSpoken(result)) {
|
|
285
|
+
return gap(ctx, {
|
|
286
|
+
profile: run.id,
|
|
287
|
+
reason: result.outcome,
|
|
288
|
+
detail: result.outcome === 'request-too-large'
|
|
289
|
+
? `the research request was not sent: ${result.words}`
|
|
290
|
+
: result.words,
|
|
291
|
+
}, result.outcome);
|
|
292
|
+
}
|
|
293
|
+
if (result.outcome !== 'reviewed')
|
|
294
|
+
return nothingRead([result.outcome]);
|
|
295
|
+
return reported(ctx, run, {
|
|
296
|
+
...checkReply(result.text, sections, run.id, ctx.maxFindings),
|
|
297
|
+
parts: ['reviewed'],
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
/** A research in parts: its workers, then its replies read in order, as `research` says. */
|
|
301
|
+
async function researchInParts(ctx, run, plan, packing, sections) {
|
|
302
|
+
const count = packing.parts.length;
|
|
303
|
+
const results = [];
|
|
304
|
+
let next = 0;
|
|
305
|
+
let stoppedBy;
|
|
306
|
+
const work = async () => {
|
|
307
|
+
while (next < count) {
|
|
308
|
+
const index = next++;
|
|
309
|
+
const result = stoppedBy === undefined
|
|
310
|
+
? await askPart(ctx, run, plan, packing, index)
|
|
311
|
+
: { outcome: stoppedBy };
|
|
312
|
+
results[index] = result;
|
|
313
|
+
if (isStopped(result))
|
|
314
|
+
stoppedBy ??= result.outcome;
|
|
315
|
+
}
|
|
316
|
+
};
|
|
317
|
+
await Promise.all(Array.from({
|
|
318
|
+
length: Math.min(ctx.concurrency, count),
|
|
319
|
+
}, work));
|
|
320
|
+
const replies = results.flatMap((result, index) => result.outcome === 'reviewed'
|
|
321
|
+
? [
|
|
322
|
+
{ text: result.text, label: `part ${index + 1}` },
|
|
323
|
+
]
|
|
324
|
+
: []);
|
|
325
|
+
const read = checkReplies(replies, sections, run.id, ctx.maxFindings);
|
|
326
|
+
return reported(ctx, run, {
|
|
327
|
+
...read,
|
|
328
|
+
incomplete: [
|
|
329
|
+
...partGaps(run.id, results),
|
|
330
|
+
...read.incomplete,
|
|
331
|
+
],
|
|
332
|
+
parts: results.map(result => result.outcome),
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
/** What a part that holds nothing but white space is read as, since no model is asked about it: a reply with no finding. */
|
|
336
|
+
const NO_FINDINGS = { outcome: 'reviewed', text: '{"findings":[]}' };
|
|
337
|
+
/** One part of a research in parts, asked as a request of its own, unless its text is blank. */
|
|
338
|
+
function askPart(ctx, run, plan, packing, index) {
|
|
339
|
+
const text = partText(plan.sections, packing, packing.parts[index]);
|
|
340
|
+
if (isBlank(text))
|
|
341
|
+
return Promise.resolve(NO_FINDINGS);
|
|
342
|
+
const { files, unlisted } = plan.listed ?? { files: [], unlisted: 0 };
|
|
343
|
+
return askResearch(ctx, run, () => researchRequest(run.research.instructions, text, run.guidelines, {
|
|
344
|
+
index: index + 1,
|
|
345
|
+
count: packing.parts.length,
|
|
346
|
+
files,
|
|
347
|
+
unlisted,
|
|
348
|
+
}));
|
|
349
|
+
}
|
|
350
|
+
/** The gaps of the parts that were not read, one per reason and profile counting its parts: the refused requests, then the failed parts, each naming the first one's words. */
|
|
351
|
+
function partGaps(profile, results) {
|
|
352
|
+
const spoken = results.filter(result => isSpoken(result));
|
|
353
|
+
const gaps = [];
|
|
354
|
+
const refused = spoken.filter(result => result.outcome === 'request-too-large');
|
|
355
|
+
if (refused.length > 0) {
|
|
356
|
+
gaps.push({ profile, reason: 'request-too-large', detail: `${refused.length} of ${results.length} research requests were not sent: ${refused[0].words}` });
|
|
357
|
+
}
|
|
358
|
+
const failed = spoken.filter(result => result.outcome === 'research-failed');
|
|
359
|
+
if (failed.length > 0) {
|
|
360
|
+
gaps.push({ profile, reason: 'research-failed', detail: `${failed.length} of ${results.length} research parts failed: ${failed[0].words}` });
|
|
361
|
+
}
|
|
362
|
+
return gaps;
|
|
363
|
+
}
|
|
364
|
+
/** Sends the events of a finished research, the `finding:invalid` ones then the `incomplete` ones, and returns its outcome. */
|
|
365
|
+
function reported(ctx, run, outcome) {
|
|
101
366
|
for (const reason of outcome.invalid)
|
|
102
367
|
ctx.emit({ type: 'finding:invalid', profile: run.id, reason });
|
|
103
368
|
for (const entry of outcome.incomplete)
|
|
104
369
|
ctx.emit({ type: 'incomplete', entry });
|
|
105
370
|
return outcome;
|
|
106
371
|
}
|
|
372
|
+
/** A profile that reviews nothing: the one gap it records, reported by an `incomplete` event. */
|
|
373
|
+
function gap(ctx, entry, outcome) {
|
|
374
|
+
ctx.emit({ type: 'incomplete', entry });
|
|
375
|
+
return {
|
|
376
|
+
findings: [],
|
|
377
|
+
invalid: [],
|
|
378
|
+
incomplete: [entry],
|
|
379
|
+
parts: [outcome],
|
|
380
|
+
};
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* What a failed research call says in a gap, in words the code writes: `the research call failed`,
|
|
384
|
+
* then ` (<kind>)` when `providerFailure` reads a kind of `ProviderError` in the failure, with ` <status>`
|
|
385
|
+
* after an `http` kind when it reads an HTTP status too, then ` after <n> attempts` when the call made
|
|
386
|
+
* more than one. The provider's message, which can quote a server, is never read: a gating review
|
|
387
|
+
* rejects with it, and a gap carries nothing a server or a model wrote.
|
|
388
|
+
*/
|
|
389
|
+
function researchFailed(error, attempts) {
|
|
390
|
+
const failure = providerFailure(error);
|
|
391
|
+
const words = ['the research call failed'];
|
|
392
|
+
if (failure !== undefined) {
|
|
393
|
+
words.push(failure.status === undefined
|
|
394
|
+
? `(${failure.kind})`
|
|
395
|
+
: `(${failure.kind} ${failure.status})`);
|
|
396
|
+
}
|
|
397
|
+
if (attempts > 1)
|
|
398
|
+
words.push(`after ${attempts} attempts`);
|
|
399
|
+
return words.join(' ');
|
|
400
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { CompletionRequest } from './ports.ts';
|
|
2
|
+
/**
|
|
3
|
+
* Tokens kept for what a server adds around the messages: the chat template's role markers, a thinking
|
|
4
|
+
* or Harmony preamble, a tokenizer's dummy prefix. Measured at 10 to 29 tokens for Llama 3 and Qwen 2.5;
|
|
5
|
+
* the margin is many times that, since a preamble can be longer than any measured.
|
|
6
|
+
*/
|
|
7
|
+
export declare const TEMPLATE_TOKENS = 512;
|
|
8
|
+
/**
|
|
9
|
+
* Tokens kept for the reply when `maxTokens` is not set, and the limit every adapter sends as the server's
|
|
10
|
+
* then, so the budget the guard keeps is the one the server enforces.
|
|
11
|
+
*/
|
|
12
|
+
export declare const DEFAULT_REPLY_TOKENS = 16000;
|
|
13
|
+
/** The length of `text` in UTF-8 bytes. A lone surrogate counts the 3 bytes of U+FFFD, as the encoder writes it. */
|
|
14
|
+
export declare function utf8Length(text: string): number;
|
|
15
|
+
/**
|
|
16
|
+
* A bound of the tokens in `text`, one per UTF-8 byte in any normalization: the sum over its code points of
|
|
17
|
+
* each one's longest form. The text itself is never normalized, since that takes quadratic time on a long run of
|
|
18
|
+
* combining marks (canonical reordering) and a gigabyte on a run of U+FDFA. It is a bound because
|
|
19
|
+
* decomposition works on one code point at a time, reordering only permutes code points, and composing
|
|
20
|
+
* never lengthens UTF-8 (true of every code point, which `size.spec.ts` checks), so no NFC, NFKC, NFD or
|
|
21
|
+
* NFKD form of the whole text is longer. Normalizing each character as NFC, or the text in chunks, is not
|
|
22
|
+
* a bound: a mark that composes first, or one that crosses a chunk, can lengthen the whole.
|
|
23
|
+
*/
|
|
24
|
+
export declare function tokenBound(text: string): number;
|
|
25
|
+
/**
|
|
26
|
+
* A bound of the tokens in the text of a request: `tokenBound` of the system text, of every message and of
|
|
27
|
+
* the reply schema's JSON, the schema counted once more than the body carries it, since a server may add it
|
|
28
|
+
* to the prompt. The size guard counts what an adapter sends with the same bound, so the core's count and an
|
|
29
|
+
* adapter's never drift.
|
|
30
|
+
*/
|
|
31
|
+
export declare function requestTokens(request: Pick<CompletionRequest, 'system' | 'messages' | 'output'>): number;
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tokens kept for what a server adds around the messages: the chat template's role markers, a thinking
|
|
3
|
+
* or Harmony preamble, a tokenizer's dummy prefix. Measured at 10 to 29 tokens for Llama 3 and Qwen 2.5;
|
|
4
|
+
* the margin is many times that, since a preamble can be longer than any measured.
|
|
5
|
+
*/
|
|
6
|
+
export const TEMPLATE_TOKENS = 512;
|
|
7
|
+
/**
|
|
8
|
+
* Tokens kept for the reply when `maxTokens` is not set, and the limit every adapter sends as the server's
|
|
9
|
+
* then, so the budget the guard keeps is the one the server enforces.
|
|
10
|
+
*/
|
|
11
|
+
export const DEFAULT_REPLY_TOKENS = 16_000;
|
|
12
|
+
const encoder = new TextEncoder();
|
|
13
|
+
/** The length of `text` in UTF-8 bytes. A lone surrogate counts the 3 bytes of U+FFFD, as the encoder writes it. */
|
|
14
|
+
export function utf8Length(text) {
|
|
15
|
+
return encoder.encode(text).length;
|
|
16
|
+
}
|
|
17
|
+
/**
|
|
18
|
+
* The bytes of the longest form of each code point, 0 until it is first seen, so a repeated character is
|
|
19
|
+
* normalized once. A fixed table of 1.1 MB, where a map grew past 200 MB on a text holding every code
|
|
20
|
+
* point; the longest form, U+FDFA's, is 33 bytes, so a byte holds any of them.
|
|
21
|
+
*/
|
|
22
|
+
const longestFormBytes = new Uint8Array(0x110000);
|
|
23
|
+
/**
|
|
24
|
+
* The most UTF-8 bytes a code point takes as sent, decomposed (NFD) or compatibility-decomposed (NFKD): U+0F81
|
|
25
|
+
* is 3 bytes and 6 decomposed, U+00E9 is 2 and 3, a Hangul syllable up to 3 times its size, U+FDFA is 3 and 33.
|
|
26
|
+
* A lone surrogate is 3, the bytes of the U+FFFD the encoder writes for it.
|
|
27
|
+
*/
|
|
28
|
+
function longestFormLength(codePoint) {
|
|
29
|
+
if (codePoint >= 0xD800 && codePoint <= 0xDFFF)
|
|
30
|
+
return 3;
|
|
31
|
+
const cached = longestFormBytes[codePoint];
|
|
32
|
+
if (cached !== undefined && cached !== 0)
|
|
33
|
+
return cached;
|
|
34
|
+
const char = String.fromCodePoint(codePoint);
|
|
35
|
+
const bytes = Math.max(utf8Length(char), utf8Length(char.normalize('NFD')), utf8Length(char.normalize('NFKD')));
|
|
36
|
+
longestFormBytes[codePoint] = bytes;
|
|
37
|
+
return bytes;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* A bound of the tokens in `text`, one per UTF-8 byte in any normalization: the sum over its code points of
|
|
41
|
+
* each one's longest form. The text itself is never normalized, since that takes quadratic time on a long run of
|
|
42
|
+
* combining marks (canonical reordering) and a gigabyte on a run of U+FDFA. It is a bound because
|
|
43
|
+
* decomposition works on one code point at a time, reordering only permutes code points, and composing
|
|
44
|
+
* never lengthens UTF-8 (true of every code point, which `size.spec.ts` checks), so no NFC, NFKC, NFD or
|
|
45
|
+
* NFKD form of the whole text is longer. Normalizing each character as NFC, or the text in chunks, is not
|
|
46
|
+
* a bound: a mark that composes first, or one that crosses a chunk, can lengthen the whole.
|
|
47
|
+
*/
|
|
48
|
+
export function tokenBound(text) {
|
|
49
|
+
let bytes = 0;
|
|
50
|
+
for (let index = 0; index < text.length; index++) {
|
|
51
|
+
const unit = text.charCodeAt(index);
|
|
52
|
+
if (unit < 0x80) {
|
|
53
|
+
bytes += 1;
|
|
54
|
+
continue;
|
|
55
|
+
}
|
|
56
|
+
const low = index + 1 < text.length
|
|
57
|
+
? text.charCodeAt(index + 1)
|
|
58
|
+
: 0;
|
|
59
|
+
const pair = unit >= 0xD800 && unit <= 0xDBFF && low >= 0xDC00 && low <= 0xDFFF;
|
|
60
|
+
bytes += longestFormLength(pair
|
|
61
|
+
? 0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)
|
|
62
|
+
: unit);
|
|
63
|
+
if (pair)
|
|
64
|
+
index++;
|
|
65
|
+
}
|
|
66
|
+
return bytes;
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* A bound of the tokens in the text of a request: `tokenBound` of the system text, of every message and of
|
|
70
|
+
* the reply schema's JSON, the schema counted once more than the body carries it, since a server may add it
|
|
71
|
+
* to the prompt. The size guard counts what an adapter sends with the same bound, so the core's count and an
|
|
72
|
+
* adapter's never drift.
|
|
73
|
+
*/
|
|
74
|
+
export function requestTokens(request) {
|
|
75
|
+
const texts = [
|
|
76
|
+
request.system,
|
|
77
|
+
...request.messages.map(message => message.content),
|
|
78
|
+
JSON.stringify(request.output.schema),
|
|
79
|
+
];
|
|
80
|
+
return texts.reduce((sum, text) => sum + tokenBound(text), 0);
|
|
81
|
+
}
|