@tobi-or-not/quorum 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +2 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +7 -4
  5. package/dist/bench/fingerprint.js +22 -8
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +241 -13
  9. package/dist/bench/records.js +64 -11
  10. package/dist/bench/report.d.ts +44 -11
  11. package/dist/bench/report.js +238 -32
  12. package/dist/bench/run-dir.d.ts +52 -26
  13. package/dist/bench/run-dir.js +124 -45
  14. package/dist/bench/runner.d.ts +47 -17
  15. package/dist/bench/runner.js +82 -21
  16. package/dist/bench/score.d.ts +38 -7
  17. package/dist/bench/score.js +64 -17
  18. package/dist/bench/stats.d.ts +61 -7
  19. package/dist/bench/stats.js +82 -7
  20. package/dist/bench/suite.d.ts +54 -9
  21. package/dist/bench/suite.js +264 -96
  22. package/dist/cli/bench.d.ts +27 -9
  23. package/dist/cli/bench.js +174 -83
  24. package/dist/cli/bin.js +2 -2
  25. package/dist/cli/io.d.ts +9 -2
  26. package/dist/cli/io.js +20 -2
  27. package/dist/cli/main.d.ts +34 -14
  28. package/dist/cli/main.js +100 -38
  29. package/dist/cli/print-config.d.ts +8 -6
  30. package/dist/cli/print-config.js +21 -8
  31. package/dist/cli/render.d.ts +2 -2
  32. package/dist/cli/render.js +21 -2
  33. package/dist/config/env.d.ts +32 -19
  34. package/dist/config/env.js +95 -32
  35. package/dist/config/from-config.js +15 -3
  36. package/dist/config/load.d.ts +3 -0
  37. package/dist/config/load.js +17 -8
  38. package/dist/config/resolve.d.ts +9 -1
  39. package/dist/config/resolve.js +17 -1
  40. package/dist/config/schema.d.ts +141 -5
  41. package/dist/config/schema.js +125 -12
  42. package/dist/core/clock.d.ts +17 -0
  43. package/dist/core/clock.js +14 -0
  44. package/dist/core/coverage.d.ts +49 -0
  45. package/dist/core/coverage.js +144 -0
  46. package/dist/core/diff.d.ts +34 -0
  47. package/dist/core/diff.js +140 -18
  48. package/dist/core/errors.d.ts +18 -3
  49. package/dist/core/errors.js +26 -2
  50. package/dist/core/exclude.d.ts +42 -0
  51. package/dist/core/exclude.js +189 -0
  52. package/dist/core/limits.d.ts +82 -0
  53. package/dist/core/limits.js +163 -0
  54. package/dist/core/output.d.ts +3 -2
  55. package/dist/core/output.js +3 -2
  56. package/dist/core/ports.d.ts +57 -2
  57. package/dist/core/ports.js +36 -1
  58. package/dist/core/quorum.d.ts +24 -8
  59. package/dist/core/quorum.js +187 -56
  60. package/dist/core/retry.d.ts +36 -0
  61. package/dist/core/retry.js +46 -0
  62. package/dist/core/stages/context.d.ts +126 -22
  63. package/dist/core/stages/context.js +319 -41
  64. package/dist/core/stages/panel.d.ts +5 -4
  65. package/dist/core/stages/panel.js +30 -11
  66. package/dist/core/stages/research.d.ts +15 -6
  67. package/dist/core/stages/research.js +75 -18
  68. package/dist/core/tokens.d.ts +31 -0
  69. package/dist/core/tokens.js +81 -0
  70. package/dist/core/types.d.ts +77 -10
  71. package/dist/index.d.ts +1 -1
  72. package/dist/providers/anthropic.d.ts +2 -1
  73. package/dist/providers/anthropic.js +14 -6
  74. package/dist/providers/claude-code.d.ts +51 -11
  75. package/dist/providers/claude-code.js +69 -33
  76. package/dist/providers/gemini.d.ts +4 -1
  77. package/dist/providers/gemini.js +72 -15
  78. package/dist/providers/http.d.ts +49 -4
  79. package/dist/providers/http.js +155 -14
  80. package/dist/providers/ollama.d.ts +3 -1
  81. package/dist/providers/ollama.js +17 -9
  82. package/dist/providers/openai-compatible.d.ts +3 -1
  83. package/dist/providers/openai-compatible.js +17 -9
  84. package/dist/providers/options.d.ts +12 -3
  85. package/dist/providers/options.js +30 -10
  86. package/dist/providers/reply.d.ts +6 -2
  87. package/dist/providers/reply.js +13 -3
  88. package/dist/providers/size.d.ts +10 -14
  89. package/dist/providers/size.js +15 -69
  90. package/package.json +1 -1
  91. package/schema/quorum.schema.json +34 -1
@@ -1,8 +1,9 @@
1
1
  import { diffSectionFor } from "../diff.js";
2
+ import { isLimit } from "../limits.js";
2
3
  import { parseVoteReply } from "../output.js";
3
4
  import { confidenceFor, undecidedLevel, verdictStatus } from "../policy.js";
4
5
  import { voteRequest } from "../prompts.js";
5
- import { callProvider, isTooLarge, refusalMessage } from "./context.js";
6
+ import { callProvider, isTooLarge, providerFailure, refusalMessage } from "./context.js";
6
7
  /**
7
8
  * The panel stage of one profile: every finding goes to every lens, each vote to the lens's own
8
9
  * provider, and the quorum verdict is computed in code. Each voter receives the diff section of the
@@ -15,10 +16,11 @@ import { callProvider, isTooLarge, refusalMessage } from "./context.js";
15
16
  * self-reported confidence gets to override: with the security profile, unanimous 3/3 is a ceiling
16
17
  * of 'high', a 2/3 split is capped at 'medium' no matter how certain any one voter sounded.
17
18
  *
18
- * A failed vote counts as a reject and is recorded with its error, never as a crash, but a vote cut
19
- * by the review's abort is not a failed vote: it rejects the panel with the abort's reason at once,
20
- * without waiting for the votes still in flight. A vote request its provider refused as too large
21
- * is a failed vote that also records a `request-too-large` gap. Each vote emits a `vote` event.
19
+ * A failed vote counts as a reject and is recorded with its error and its failure, never as a crash,
20
+ * but a vote cut by the review's abort is not a failed vote: it rejects the panel with the abort's
21
+ * reason at once, without waiting for the votes still in flight. A vote request its provider refused
22
+ * as too large is a failed vote that also records a `request-too-large` gap. Each vote emits a `vote`
23
+ * event.
22
24
  * Once a finding's votes are in, its verdict emits a `verdict` event, then its gaps are reported by
23
25
  * `incomplete` events: one `request-too-large` gap when any vote request was refused, and a
24
26
  * `vote-failed` gap when a kept verdict's failed votes leave a higher level undecided (ADR-0012).
@@ -79,21 +81,26 @@ async function judge(ctx, run, finding, diffContext) {
79
81
  /**
80
82
  * One lens's vote on one finding, from the lens's own provider. A failed call or an unusable reply
81
83
  * is a vote of null with its error: the message of an `Error`, the text of anything else thrown.
82
- * A request its provider refused as too large is that failed vote too, and carries the refusal's
83
- * message besides, for the finding's `request-too-large` gap. Once the review is aborted, the vote
84
- * rethrows the abort's reason instead, whatever the call raised, so an aborted review never
85
- * resolves with a finding the abort left unreviewed. The `vote` event is emitted after the vote's
86
- * own `try`, so a sink that throws on it is never recorded as a failed vote.
84
+ * It also records its failure: `voteFailureOf` the error for a failed call (a call the token cap or the
85
+ * review's deadline stopped is a failed vote of kind `token-cap` or `review-deadline`), `unusable-reply`
86
+ * for a reply that came back and could not be read as a vote; a cast vote's failure is null. A request its provider refused as too large is
87
+ * that failed vote too, and carries the refusal's message besides, for the finding's `request-too-large`
88
+ * gap. Once the review is aborted, the vote rethrows the abort's reason instead, whatever the call
89
+ * raised, so an aborted review never resolves with a finding the abort left unreviewed. The `vote` event
90
+ * is emitted after the vote's own `try`, so a sink that throws on it is never recorded as a failed vote.
87
91
  */
88
92
  async function castVote(ctx, run, finding, lens, diffContext) {
89
93
  let lensVote;
90
94
  let refusal;
95
+ let replied = false;
91
96
  try {
92
97
  const text = await callProvider(ctx, { profile: run.id, role: 'vote', lens: lens.id }, lens, () => voteRequest(lens.instructions, finding, diffContext, run.guidelines));
98
+ replied = true;
93
99
  lensVote = {
94
100
  angle: lens.id,
95
101
  vote: parseVoteReply(text),
96
102
  error: null,
103
+ failure: null,
97
104
  };
98
105
  }
99
106
  catch (error) {
@@ -105,11 +112,14 @@ async function castVote(ctx, run, finding, lens, diffContext) {
105
112
  error: error instanceof Error
106
113
  ? error.message
107
114
  : String(error),
115
+ failure: replied
116
+ ? { kind: 'unusable-reply' }
117
+ : voteFailureOf(error),
108
118
  };
109
119
  if (isTooLarge(error))
110
120
  refusal = refusalMessage(error);
111
121
  }
112
- const { error } = lensVote;
122
+ const { error, failure } = lensVote;
113
123
  ctx.emit({
114
124
  type: 'vote',
115
125
  profile: run.id,
@@ -119,11 +129,20 @@ async function castVote(ctx, run, finding, lens, diffContext) {
119
129
  ...(error === null
120
130
  ? {}
121
131
  : { error }),
132
+ ...(failure === null
133
+ ? {}
134
+ : { failure }),
122
135
  });
123
136
  return refusal === undefined
124
137
  ? { lensVote }
125
138
  : { lensVote, refusal };
126
139
  }
140
+ /** What a call that failed says of itself as a vote's failure: the limit that stopped it, else what `providerFailure` reads, else `unknown`. */
141
+ function voteFailureOf(error) {
142
+ if (isLimit(error))
143
+ return { kind: error.limit };
144
+ return providerFailure(error) ?? { kind: 'unknown' };
145
+ }
127
146
  /**
128
147
  * The gap a finding leaves when its provider refused some of its vote requests as too large. The file
129
148
  * name is JSON-quoted, as in `voteFailed`; the detail names the first refusal in lens order.
@@ -1,3 +1,4 @@
1
+ import type { LimitName } from '../limits.ts';
1
2
  import type { Finding, IncompleteEntry } from '../types.ts';
2
3
  import type { ProfileRun, RunContext } from './context.ts';
3
4
  /** How many reported findings a research reply may send to the panel; the rest is reported as incomplete. */
@@ -9,8 +10,10 @@ export interface ResearchOutcome {
9
10
  * code outside the diff, or names no file). An unreadable finding is reported in `incomplete` only.
10
11
  */
11
12
  invalid: string[];
12
- /** What the run could not review: findings the harness could not read, findings beyond the cap, a reply whose findings were all unusable, or a research request refused as too large. */
13
+ /** What the run could not review: findings the harness could not read, findings beyond the cap, a reply whose findings were all unusable, a research request refused as too large, or an advisory profile's research that failed. */
13
14
  incomplete: IncompleteEntry[];
15
+ /** The limit that stopped the research request, so the profile reviews nothing. Its gap is written once the profile's panel has run, with the votes the limit stopped. */
16
+ stopped?: LimitName;
14
17
  }
15
18
  /**
16
19
  * The id of a finding, stable across runs: the first 12 hex characters of SHA-256 over the JSON
@@ -43,11 +46,17 @@ export declare function checkReply(text: string, sections: ReadonlyMap<string, s
43
46
  * panel receives. Each finding dropped for citing code outside the diff is reported by a
44
47
  * `finding:invalid` event, and each gap by an `incomplete` event. A research request its provider
45
48
  * refused as too large (a `too-large` `ProviderError`) was not sent: the profile reviews nothing,
46
- * records one `request-too-large` gap, and the other profiles run. Any other failed research
47
- * call, or a reply that is not a valid envelope, fails the profile and aborts the review where it
48
- * fails (`callProvider` aborts it for the call), so no other profile starts a call after it.
49
+ * records one `request-too-large` gap, and the other profiles run. For any other failed research,
50
+ * the call after its retries or a reply that is not a valid envelope, a gating profile fails and
51
+ * aborts the review where it fails (`onFailure` aborts it for the call), so no other profile starts
52
+ * a call after it; an advisory profile's failure is its own `research-failed` gap, in words the code
53
+ * writes, and the other profiles run. A research request a limit stopped (the token cap refused it or left
54
+ * no room for it, or the review's deadline ended it) is no failure of either kind: the outcome holds no
55
+ * finding and names the limit in `stopped`, which the review turns into the profile's one gap for that
56
+ * limit, a gating profile's included.
57
+ * A review that is aborted rejects with its reason, never with a gap.
49
58
  *
50
- * @throws ProviderError, or whatever a custom provider throws, when the research call fails for any other reason than a refusal as too large
51
- * @throws OutputError when the reply is not one JSON object holding a `findings` array and nothing else
59
+ * @throws ProviderError, or whatever a custom provider throws, when a gating profile's research call fails for any other reason than a refusal as too large
60
+ * @throws OutputError when a gating profile's reply is not one JSON object holding a `findings` array and nothing else
52
61
  */
53
62
  export declare function research(ctx: RunContext, run: ProfileRun, diff: string, sections: ReadonlyMap<string, string>): Promise<ResearchOutcome>;
@@ -1,8 +1,10 @@
1
1
  import { createHash } from 'node:crypto';
2
+ import { OutputError } from "../errors.js";
3
+ import { isLimit } from "../limits.js";
2
4
  import { parseResearchReply } from "../output.js";
3
5
  import { researchRequest } from "../prompts.js";
4
6
  import { verifyFindings } from "../verify.js";
5
- import { callProvider, isTooLarge, refusalMessage } from "./context.js";
7
+ import { callProvider, isTooLarge, providerFailure, refusalMessage } from "./context.js";
6
8
  /** How many reported findings a research reply may send to the panel; the rest is reported as incomplete. */
7
9
  export const DEFAULT_MAX_FINDINGS = 50;
8
10
  /**
@@ -65,38 +67,65 @@ export function checkReply(text, sections, profile, maxFindings = DEFAULT_MAX_FI
65
67
  * panel receives. Each finding dropped for citing code outside the diff is reported by a
66
68
  * `finding:invalid` event, and each gap by an `incomplete` event. A research request its provider
67
69
  * refused as too large (a `too-large` `ProviderError`) was not sent: the profile reviews nothing,
68
- * records one `request-too-large` gap, and the other profiles run. Any other failed research
69
- * call, or a reply that is not a valid envelope, fails the profile and aborts the review where it
70
- * fails (`callProvider` aborts it for the call), so no other profile starts a call after it.
70
+ * records one `request-too-large` gap, and the other profiles run. For any other failed research,
71
+ * the call after its retries or a reply that is not a valid envelope, a gating profile fails and
72
+ * aborts the review where it fails (`onFailure` aborts it for the call), so no other profile starts
73
+ * a call after it; an advisory profile's failure is its own `research-failed` gap, in words the code
74
+ * writes, and the other profiles run. A research request a limit stopped (the token cap refused it or left
75
+ * no room for it, or the review's deadline ended it) is no failure of either kind: the outcome holds no
76
+ * finding and names the limit in `stopped`, which the review turns into the profile's one gap for that
77
+ * limit, a gating profile's included.
78
+ * A review that is aborted rejects with its reason, never with a gap.
71
79
  *
72
- * @throws ProviderError, or whatever a custom provider throws, when the research call fails for any other reason than a refusal as too large
73
- * @throws OutputError when the reply is not one JSON object holding a `findings` array and nothing else
80
+ * @throws ProviderError, or whatever a custom provider throws, when a gating profile's research call fails for any other reason than a refusal as too large
81
+ * @throws OutputError when a gating profile's reply is not one JSON object holding a `findings` array and nothing else
74
82
  */
75
83
  export async function research(ctx, run, diff, sections) {
84
+ let attempts = 1;
76
85
  let text;
77
86
  try {
78
- text = await callProvider(ctx, { profile: run.id, role: 'research' }, run.research, () => researchRequest(run.research.instructions, diff, run.guidelines));
87
+ text = await callProvider(ctx, { profile: run.id, role: 'research' }, run.research, () => researchRequest(run.research.instructions, diff, run.guidelines), (error, made) => {
88
+ attempts = made;
89
+ if (run.gating && !isTooLarge(error) && !isLimit(error))
90
+ ctx.fail(error);
91
+ });
79
92
  }
80
93
  catch (error) {
81
- if (!isTooLarge(error))
94
+ // A review that is aborted rejects with its reason, never with a gap.
95
+ if (isLimit(error)) {
96
+ ctx.signal.throwIfAborted();
97
+ return {
98
+ findings: [],
99
+ invalid: [],
100
+ incomplete: [],
101
+ stopped: error.limit,
102
+ };
103
+ }
104
+ if (isTooLarge(error)) {
105
+ ctx.signal.throwIfAborted();
106
+ return gap(ctx, { profile: run.id, reason: 'request-too-large', detail: `the research request was not sent: ${refusalMessage(error)}` });
107
+ }
108
+ // A gating profile's failure aborted the review where it failed (`onFailure`).
109
+ if (run.gating)
82
110
  throw error;
83
- // An aborted review rejects with its reason, never with a gap.
84
111
  ctx.signal.throwIfAborted();
85
- const entry = { profile: run.id, reason: 'request-too-large', detail: `the research request was not sent: ${refusalMessage(error)}` };
86
- ctx.emit({ type: 'incomplete', entry });
87
- return {
88
- findings: [],
89
- invalid: [],
90
- incomplete: [entry],
91
- };
112
+ return gap(ctx, {
113
+ profile: run.id,
114
+ reason: 'research-failed',
115
+ detail: researchFailed(error, attempts),
116
+ });
92
117
  }
93
118
  let outcome;
94
119
  try {
95
120
  outcome = checkReply(text, sections, run.id, ctx.maxFindings);
96
121
  }
97
122
  catch (error) {
98
- ctx.fail(error);
99
- throw error;
123
+ if (run.gating || !(error instanceof OutputError)) {
124
+ ctx.fail(error);
125
+ throw error;
126
+ }
127
+ ctx.signal.throwIfAborted();
128
+ return gap(ctx, { profile: run.id, reason: 'research-failed', detail: `the research reply could not be read: ${error.reason}` });
100
129
  }
101
130
  for (const reason of outcome.invalid)
102
131
  ctx.emit({ type: 'finding:invalid', profile: run.id, reason });
@@ -104,3 +133,31 @@ export async function research(ctx, run, diff, sections) {
104
133
  ctx.emit({ type: 'incomplete', entry });
105
134
  return outcome;
106
135
  }
136
+ /** A profile that reviews nothing: the one gap it records, reported by an `incomplete` event. */
137
+ function gap(ctx, entry) {
138
+ ctx.emit({ type: 'incomplete', entry });
139
+ return {
140
+ findings: [],
141
+ invalid: [],
142
+ incomplete: [entry],
143
+ };
144
+ }
145
+ /**
146
+ * What a failed research call says in a gap, in words the code writes: `the research call failed`,
147
+ * then ` (<kind>)` when `providerFailure` reads a kind of `ProviderError` in the failure, with ` <status>`
148
+ * after an `http` kind when it reads an HTTP status too, then ` after <n> attempts` when the call made
149
+ * more than one. The provider's message, which can quote a server, is never read: a gating review
150
+ * rejects with it, and a gap carries nothing a server or a model wrote.
151
+ */
152
+ function researchFailed(error, attempts) {
153
+ const failure = providerFailure(error);
154
+ const words = ['the research call failed'];
155
+ if (failure !== undefined) {
156
+ words.push(failure.status === undefined
157
+ ? `(${failure.kind})`
158
+ : `(${failure.kind} ${failure.status})`);
159
+ }
160
+ if (attempts > 1)
161
+ words.push(`after ${attempts} attempts`);
162
+ return words.join(' ');
163
+ }
@@ -0,0 +1,31 @@
1
+ import type { CompletionRequest } from './ports.ts';
2
+ /**
3
+ * Tokens kept for what a server adds around the messages: the chat template's role markers, a thinking
4
+ * or Harmony preamble, a tokenizer's dummy prefix. Measured at 10 to 29 tokens for Llama 3 and Qwen 2.5;
5
+ * the margin is many times that, since a preamble can be longer than any measured.
6
+ */
7
+ export declare const TEMPLATE_TOKENS = 512;
8
+ /**
9
+ * Tokens kept for the reply when `maxTokens` is not set, and the limit every adapter sends as the server's
10
+ * then, so the budget the guard keeps is the one the server enforces.
11
+ */
12
+ export declare const DEFAULT_REPLY_TOKENS = 16000;
13
+ /** The length of `text` in UTF-8 bytes. A lone surrogate counts the 3 bytes of U+FFFD, as the encoder writes it. */
14
+ export declare function utf8Length(text: string): number;
15
+ /**
16
+ * A bound of the tokens in `text`, one per UTF-8 byte in any normalization: the sum over its code points of
17
+ * each one's longest form. The text itself is never normalized, since that takes quadratic time on a long run of
18
+ * combining marks (canonical reordering) and a gigabyte on a run of U+FDFA. It is a bound because
19
+ * decomposition works on one code point at a time, reordering only permutes code points, and composing
20
+ * never lengthens UTF-8 (true of every code point, which `size.spec.ts` checks), so no NFC, NFKC, NFD or
21
+ * NFKD form of the whole text is longer. Normalizing each character as NFC, or the text in chunks, is not
22
+ * a bound: a mark that composes first, or one that crosses a chunk, can lengthen the whole.
23
+ */
24
+ export declare function tokenBound(text: string): number;
25
+ /**
26
+ * A bound of the tokens in the text of a request: `tokenBound` of the system text, of every message and of
27
+ * the reply schema's JSON, the schema counted once more than the body carries it, since a server may add it
28
+ * to the prompt. The size guard counts what an adapter sends with the same bound, so the core's count and an
29
+ * adapter's never drift.
30
+ */
31
+ export declare function requestTokens(request: Pick<CompletionRequest, 'system' | 'messages' | 'output'>): number;
@@ -0,0 +1,81 @@
1
+ /**
2
+ * Tokens kept for what a server adds around the messages: the chat template's role markers, a thinking
3
+ * or Harmony preamble, a tokenizer's dummy prefix. Measured at 10 to 29 tokens for Llama 3 and Qwen 2.5;
4
+ * the margin is many times that, since a preamble can be longer than any measured.
5
+ */
6
+ export const TEMPLATE_TOKENS = 512;
7
+ /**
8
+ * Tokens kept for the reply when `maxTokens` is not set, and the limit every adapter sends as the server's
9
+ * then, so the budget the guard keeps is the one the server enforces.
10
+ */
11
+ export const DEFAULT_REPLY_TOKENS = 16_000;
12
+ const encoder = new TextEncoder();
13
+ /** The length of `text` in UTF-8 bytes. A lone surrogate counts the 3 bytes of U+FFFD, as the encoder writes it. */
14
+ export function utf8Length(text) {
15
+ return encoder.encode(text).length;
16
+ }
17
+ /**
18
+ * The bytes of the longest form of each code point, 0 until it is first seen, so a repeated character is
19
+ * normalized once. A fixed table of 1.1 MB, where a map grew past 200 MB on a text holding every code
20
+ * point; the longest form, U+FDFA's, is 33 bytes, so a byte holds any of them.
21
+ */
22
+ const longestFormBytes = new Uint8Array(0x110000);
23
+ /**
24
+ * The most UTF-8 bytes a code point takes as sent, decomposed (NFD) or compatibility-decomposed (NFKD): U+0F81
25
+ * is 3 bytes and 6 decomposed, U+00E9 is 2 and 3, a Hangul syllable up to 3 times its size, U+FDFA is 3 and 33.
26
+ * A lone surrogate is 3, the bytes of the U+FFFD the encoder writes for it.
27
+ */
28
+ function longestFormLength(codePoint) {
29
+ if (codePoint >= 0xD800 && codePoint <= 0xDFFF)
30
+ return 3;
31
+ const cached = longestFormBytes[codePoint];
32
+ if (cached !== undefined && cached !== 0)
33
+ return cached;
34
+ const char = String.fromCodePoint(codePoint);
35
+ const bytes = Math.max(utf8Length(char), utf8Length(char.normalize('NFD')), utf8Length(char.normalize('NFKD')));
36
+ longestFormBytes[codePoint] = bytes;
37
+ return bytes;
38
+ }
39
+ /**
40
+ * A bound of the tokens in `text`, one per UTF-8 byte in any normalization: the sum over its code points of
41
+ * each one's longest form. The text itself is never normalized, since that takes quadratic time on a long run of
42
+ * combining marks (canonical reordering) and a gigabyte on a run of U+FDFA. It is a bound because
43
+ * decomposition works on one code point at a time, reordering only permutes code points, and composing
44
+ * never lengthens UTF-8 (true of every code point, which `size.spec.ts` checks), so no NFC, NFKC, NFD or
45
+ * NFKD form of the whole text is longer. Normalizing each character as NFC, or the text in chunks, is not
46
+ * a bound: a mark that composes first, or one that crosses a chunk, can lengthen the whole.
47
+ */
48
+ export function tokenBound(text) {
49
+ let bytes = 0;
50
+ for (let index = 0; index < text.length; index++) {
51
+ const unit = text.charCodeAt(index);
52
+ if (unit < 0x80) {
53
+ bytes += 1;
54
+ continue;
55
+ }
56
+ const low = index + 1 < text.length
57
+ ? text.charCodeAt(index + 1)
58
+ : 0;
59
+ const pair = unit >= 0xD800 && unit <= 0xDBFF && low >= 0xDC00 && low <= 0xDFFF;
60
+ bytes += longestFormLength(pair
61
+ ? 0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)
62
+ : unit);
63
+ if (pair)
64
+ index++;
65
+ }
66
+ return bytes;
67
+ }
68
+ /**
69
+ * A bound of the tokens in the text of a request: `tokenBound` of the system text, of every message and of
70
+ * the reply schema's JSON, the schema counted once more than the body carries it, since a server may add it
71
+ * to the prompt. The size guard counts what an adapter sends with the same bound, so the core's count and an
72
+ * adapter's never drift.
73
+ */
74
+ export function requestTokens(request) {
75
+ const texts = [
76
+ request.system,
77
+ ...request.messages.map(message => message.content),
78
+ JSON.stringify(request.output.schema),
79
+ ];
80
+ return texts.reduce((sum, text) => sum + tokenBound(text), 0);
81
+ }
@@ -40,12 +40,31 @@ export interface Lens {
40
40
  /** What the voter judging from this lens is asked, filled into the vote template's `{{lens.question}}`. */
41
41
  readonly question: string;
42
42
  }
43
- /** One lens's vote, or the error if that call failed. */
43
+ /**
44
+ * What a vote failed with: a provider's kind of failure, the limit that stopped it (`token-cap` or
45
+ * `review-deadline`), `unusable-reply` for a reply the panel could not read as a vote, or `unknown` for
46
+ * anything else thrown.
47
+ * The kinds a gating profile's `--allow-incomplete` lifts are `network`, `timeout` and an `http` failure of
48
+ * status 408, 409, 429 or 5xx.
49
+ */
50
+ export type VoteFailureKind = 'http' | 'network' | 'process' | 'timeout' | 'aborted' | 'refusal' | 'truncated' | 'invalid-response' | 'unsupported' | 'too-large' | 'token-cap' | 'review-deadline' | 'unusable-reply' | 'unknown';
51
+ /** The facts of a failed vote the code read, never a word a server or a model wrote. */
52
+ export interface VoteFailure {
53
+ kind: VoteFailureKind;
54
+ /** An `http` failure's HTTP status, an integer from 100 to 599; absent for every other kind, and for an `http` failure that states none. */
55
+ status?: number;
56
+ }
57
+ /**
58
+ * One lens's vote, or the error if that call failed. A failed vote also records its `failure`: what it
59
+ * failed with, which the CLI's `--allow-incomplete` reads to lift only a vote that failed in passing.
60
+ */
44
61
  export interface LensVote {
45
62
  /** The id of the lens the vote was cast from. */
46
63
  angle: string;
47
64
  vote: Vote | null;
48
65
  error: string | null;
66
+ /** What a failed vote failed with; `null` for a vote that was cast. */
67
+ failure: VoteFailure | null;
49
68
  }
50
69
  /** Instruction templates, one per stage. `{{name}}` placeholders are filled by renderTemplate. */
51
70
  export interface Prompts {
@@ -96,15 +115,24 @@ export interface Verdict extends Finding {
96
115
  */
97
116
  export type VerdictStatus = 'kept' | 'discarded' | 'unreviewed';
98
117
  /**
99
- * Why part of a review is missing, as an `IncompleteEntry` records it. `request-too-large`: a
100
- * provider refused a request that may not fit its model's context, so it was not sent.
118
+ * Why part of a review is missing, as an `IncompleteEntry` records it. `research-failed`: an
119
+ * advisory profile's research call failed after its retries, or its reply was no findings
120
+ * envelope, so the profile reviewed nothing; a gating profile's ends the review instead.
121
+ * `request-too-large`: a provider refused a request that may not fit its model's context, so it was
122
+ * not sent. `binary` and `submodule`: the diff carries no content for a file, shown only as binary
123
+ * or as a submodule's commit; `meta.leftOut` lists the files. `token-cap`: the review's token cap
124
+ * stopped part of the profile's work, its research request or some of its votes, which were not sent;
125
+ * `review-deadline`: the review reached its deadline, which stopped part of the profile's work, its
126
+ * research request or some of its votes, in flight or not yet sent; one entry per limit and profile counts them.
101
127
  */
102
- export type IncompleteReason = 'research-unusable' | 'findings-capped' | 'finding-unreadable' | 'vote-failed' | 'request-too-large';
128
+ export type IncompleteReason = 'research-unusable' | 'research-failed' | 'findings-capped' | 'finding-unreadable' | 'vote-failed' | 'request-too-large' | 'token-cap' | 'review-deadline' | 'binary' | 'submodule';
103
129
  /**
104
- * Work the run could not review: a research reply with findings but none usable, findings beyond
105
- * the cap, a finding the harness could not read, a kept finding whose failed votes could have
106
- * raised it to a higher level, or a request its provider refused as too large to send. It is
107
- * reported and counted, never dropped silently (ADR-0012).
130
+ * Work the run could not review: a research reply with findings but none usable, an advisory
131
+ * profile's research that failed (`research-failed`), findings beyond the cap, a finding the
132
+ * harness could not read, a kept finding whose failed votes could have raised it to a higher
133
+ * level, a request its provider refused as too large to send, work the review's token cap or deadline
134
+ * stopped, or a file whose change the diff does not carry, binary or a submodule's commit. It is reported and
135
+ * counted, never dropped silently (ADR-0012).
108
136
  */
109
137
  export interface IncompleteEntry {
110
138
  /** The id of the profile whose run has the gap. */
@@ -114,6 +142,14 @@ export interface IncompleteEntry {
114
142
  file?: string;
115
143
  detail: string;
116
144
  }
145
+ /** Why a file is in `meta.leftOut`: an `exclude` pattern matches it, or its change reaches the models only as binary, or as a submodule's commit. */
146
+ export type LeftOutReason = 'excluded' | 'binary' | 'submodule';
147
+ /** A file the review left out: one an `exclude` pattern matches, which is no gap, or one whose change the diff does not carry, which is a gap of every profile. */
148
+ export interface LeftOutFile {
149
+ /** The diff's own path, decoded; '' for a part of the diff that no `diff --git` header names. */
150
+ file: string;
151
+ reason: LeftOutReason;
152
+ }
117
153
  /** What an advisory flag points at. */
118
154
  export type FlagCode = 'hidden-characters' | 'ai-addressed-text';
119
155
  /**
@@ -163,6 +199,8 @@ export interface QuorumMeta {
163
199
  usage?: Usage;
164
200
  /** Gaps that are not findings. An entry here, or an unreviewed finding, makes the review incomplete. */
165
201
  incomplete: IncompleteEntry[];
202
+ /** The files the review left out, in the diff's order, once per file and reason. Each, but an excluded one, is also a gap of every profile. */
203
+ leftOut: LeftOutFile[];
166
204
  /** Advisory signals the harness read from the diff: where a human should look. They change no verdict and no exit code. */
167
205
  flags: Flag[];
168
206
  }
@@ -182,6 +220,20 @@ export interface QuorumOptions {
182
220
  readonly concurrency?: number;
183
221
  /** How many findings of a research reply each profile sends to the panel; `DEFAULT_MAX_FINDINGS` when unset. */
184
222
  readonly maxFindings?: number;
223
+ /** Path patterns of the files left out of every request; none when unset. */
224
+ readonly exclude?: readonly string[];
225
+ /**
226
+ * The most tokens, input and output, one review may spend: a whole number from 1 to `Number.MAX_SAFE_INTEGER`,
227
+ * 2000000 when unset. Each attempt reserves its worst case before it is sent and settles at what it spent; what
228
+ * the cap stops is a `token-cap` gap of its profile.
229
+ */
230
+ readonly maxReviewTokens?: number;
231
+ /**
232
+ * How long one review runs, in milliseconds: a whole number from 1000 to 86400000, 1800000 (30 minutes) when
233
+ * unset. At it every call in flight and every wait is stopped at once and no new call starts, and what was not
234
+ * done is a `review-deadline` gap of its profile: the review resolves with its findings so far. It is no abort.
235
+ */
236
+ readonly reviewTimeoutMs?: number;
185
237
  }
186
238
  /** A quorum built by createQuorum: its profiles compiled once, ready to review any number of diffs. */
187
239
  export interface Quorum {
@@ -211,6 +263,11 @@ export type QuorumEvent = {
211
263
  profiles: string[];
212
264
  providers: Record<string, ProfileProviders>;
213
265
  }
266
+ /** The files the review leaves out, as `meta.leftOut` holds them: sent once right after `run:start` when there is one. */
267
+ | {
268
+ type: 'left-out';
269
+ files: LeftOutFile[];
270
+ }
214
271
  /** An advisory flag read from the diff, sent before any stage starts. */
215
272
  | {
216
273
  type: 'flag';
@@ -230,7 +287,14 @@ export type QuorumEvent = {
230
287
  durationMs: number;
231
288
  count: number;
232
289
  }
233
- /** One per provider call, failed ones included: its latency inside its slot and its usage, when valid. */
290
+ /**
291
+ * One per provider call, failed ones included, sent when the call ends. The call runs its
292
+ * attempts inside its slot, retried as the retry policy decides, so `durationMs` runs from its
293
+ * first attempt to its end, the waits between attempts included; `attempts` is their number,
294
+ * only when the call made more than one; `usage` is summed over the attempts that reported
295
+ * valid counts; `spent` is the tokens the review's cap counted for the call: each attempt's
296
+ * reported usage, nothing for one the server refused before any work, else its reservation.
297
+ */
234
298
  | {
235
299
  type: 'call';
236
300
  profile: string;
@@ -239,7 +303,9 @@ export type QuorumEvent = {
239
303
  provider: ProviderRef;
240
304
  durationMs: number;
241
305
  ok: boolean;
306
+ attempts?: number;
242
307
  usage?: Usage;
308
+ spent: number;
243
309
  }
244
310
  /** A finding was dropped for citing code outside the diff, with the reason. */
245
311
  | {
@@ -252,7 +318,7 @@ export type QuorumEvent = {
252
318
  type: 'incomplete';
253
319
  entry: IncompleteEntry;
254
320
  }
255
- /** One lens voted on one finding; `keep` is null, with the error, when the vote failed. */
321
+ /** One lens voted on one finding; `keep` is null, with the error and the failure's kind and status, when the vote failed. */
256
322
  | {
257
323
  type: 'vote';
258
324
  profile: string;
@@ -260,6 +326,7 @@ export type QuorumEvent = {
260
326
  lens: string;
261
327
  keep: boolean | null;
262
328
  error?: string;
329
+ failure?: VoteFailure;
263
330
  }
264
331
  /** The panel decided one finding: its vote split, the confidence it earned and where it lands. */
265
332
  | {
package/dist/index.d.ts CHANGED
@@ -13,7 +13,7 @@ export type { PolicyRules, Rule } from './core/policy.ts';
13
13
  export type { ChatMessage, CompletionRequest, CompletionResponse, EventSink, JsonSchema, ProfilePorts, Provider, QuorumPorts } from './core/ports.ts';
14
14
  export { createQuorum } from './core/quorum.ts';
15
15
  export { inertJson, inertLine } from './core/sanitize.ts';
16
- export type { Confidence, Finding, Flag, FlagCode, IncompleteEntry, IncompleteReason, Lens, LensVote, Profile, ProfileMeta, ProfileProviders, Prompts, ProviderRef, Quorum, QuorumEvent, QuorumMeta, QuorumOptions, QuorumResult, StageName, Thresholds, Usage, Verdict, VerdictStatus, Vote, } from './core/types.ts';
16
+ export type { Confidence, Finding, Flag, FlagCode, IncompleteEntry, IncompleteReason, LeftOutFile, LeftOutReason, Lens, LensVote, Profile, ProfileMeta, ProfileProviders, Prompts, ProviderRef, Quorum, QuorumEvent, QuorumMeta, QuorumOptions, QuorumResult, StageName, Thresholds, Usage, Verdict, VerdictStatus, Vote, VoteFailure, VoteFailureKind, } from './core/types.ts';
17
17
  export { securityProfile } from './profiles/security.ts';
18
18
  export { anthropic } from './providers/anthropic.ts';
19
19
  export type { AnthropicOptions } from './providers/anthropic.ts';
@@ -7,7 +7,8 @@ export type AnthropicOptions = HttpProviderOptions;
7
7
  * there is one, travels only in the header `auth` names, `Authorization: Bearer` by default (invariant
8
8
  * I1). It never sends an assistant prefill, which current models reject. It refuses, before any request,
9
9
  * a request that may not fit `contextTokens`. It always sends a token limit, since the Messages API
10
- * requires one: `maxTokens`, else the reply budget of the size guard.
10
+ * requires one: `maxTokens`, else the reply budget of the size guard. It states its worst case from the
11
+ * count its size guard keeps: the text it sends, the template reserve and that budget.
11
12
  *
12
13
  * @throws ConfigError with every problem of the options, before any request
13
14
  */
@@ -1,8 +1,9 @@
1
+ import { DEFAULT_RETRIES, DEFAULT_TIMEOUT_MS } from "../core/ports.js";
1
2
  import { postJson } from "./http.js";
2
3
  import { shapeMessages } from "./messages.js";
3
4
  import { isRecord, mergeBody, prepareHttp } from "./options.js";
4
5
  import { endedEarly, joinedText, replyFailure, tokenUsage } from "./reply.js";
5
- import { checkFits, contextBudget, DEFAULT_REPLY_TOKENS } from "./size.js";
6
+ import { checkFits, contextBudget, DEFAULT_REPLY_TOKENS, worstCaseTokens } from "./size.js";
6
7
  /**
7
8
  * The API version every request names. For this call it differs from `2023-06-01` only in refusing any
8
9
  * `temperature`, `top_p` or `top_k`, so no temperature is sent unless one is set.
@@ -43,11 +44,14 @@ const FINISHED = new Set(['end_turn', 'stop_sequence']);
43
44
  /**
44
45
  * Turn a Messages reply into text, or a `ProviderError`. Every property is read defensively: the reply
45
46
  * is untrusted. The stop reason comes first, since a refused or cut-off reply can still hold some text.
47
+ * The counts are read once, after the body check, and every failure from then on carries them, since the
48
+ * server generated those tokens.
46
49
  */
47
50
  function readReply(reply, { id, model }, redact) {
48
- const fail = replyFailure(id);
49
51
  if (!isRecord(reply))
50
- throw fail('invalid-response', 'reply is not a JSON object');
52
+ throw replyFailure(id)('invalid-response', 'reply is not a JSON object');
53
+ const usage = tokenUsage(reply.usage, ['input_tokens', 'cache_creation_input_tokens', 'cache_read_input_tokens'], ['output_tokens']);
54
+ const fail = replyFailure(id, usage);
51
55
  const reason = reply.stop_reason;
52
56
  if (reason === 'refusal')
53
57
  throw fail('refusal', 'model refused');
@@ -65,7 +69,7 @@ function readReply(reply, { id, model }, redact) {
65
69
  model: typeof reply.model === 'string' && reply.model !== ''
66
70
  ? reply.model
67
71
  : model,
68
- usage: tokenUsage(reply.usage, ['input_tokens', 'cache_creation_input_tokens', 'cache_read_input_tokens'], ['output_tokens']),
72
+ usage,
69
73
  };
70
74
  }
71
75
  /**
@@ -73,12 +77,13 @@ function readReply(reply, { id, model }, redact) {
73
77
  * there is one, travels only in the header `auth` names, `Authorization: Bearer` by default (invariant
74
78
  * I1). It never sends an assistant prefill, which current models reject. It refuses, before any request,
75
79
  * a request that may not fit `contextTokens`. It always sends a token limit, since the Messages API
76
- * requires one: `maxTokens`, else the reply budget of the size guard.
80
+ * requires one: `maxTokens`, else the reply budget of the size guard. It states its worst case from the
81
+ * count its size guard keeps: the text it sends, the template reserve and that budget.
77
82
  *
78
83
  * @throws ConfigError with every problem of the options, before any request
79
84
  */
80
85
  export function anthropic(options) {
81
- const { id, model, auth = 'bearer', temperature, maxTokens = DEFAULT_REPLY_TOKENS, contextTokens, systemPrompt = 'native', structuredOutput = 'prompt' } = options;
86
+ const { id, model, auth = 'bearer', temperature, maxTokens = DEFAULT_REPLY_TOKENS, contextTokens, systemPrompt = 'native', structuredOutput = 'prompt', timeoutMs = DEFAULT_TIMEOUT_MS, retries = DEFAULT_RETRIES } = options;
82
87
  const { url, headers, redact, extraBody } = prepareHttp(options, { ...RULES, auth }, '/v1/messages');
83
88
  // The version goes last, so no header can replace it.
84
89
  const sent = { ...headers, 'anthropic-version': ANTHROPIC_VERSION };
@@ -86,6 +91,9 @@ export function anthropic(options) {
86
91
  return {
87
92
  id,
88
93
  model,
94
+ timeoutMs,
95
+ retries,
96
+ worstCase: request => worstCaseTokens(shapeMessages(request, systemPrompt), request.output, budget),
89
97
  async complete(request) {
90
98
  const { system, messages } = shapeMessages(request, systemPrompt);
91
99
  checkFits(id, { system, messages }, request.output, budget);