@tobi-or-not/quorum 0.3.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. package/README.md +2 -1
  2. package/dist/bench/attacks.d.ts +103 -0
  3. package/dist/bench/attacks.js +272 -0
  4. package/dist/bench/fingerprint.d.ts +7 -4
  5. package/dist/bench/fingerprint.js +22 -8
  6. package/dist/bench/independence.d.ts +28 -0
  7. package/dist/bench/independence.js +67 -0
  8. package/dist/bench/records.d.ts +241 -13
  9. package/dist/bench/records.js +64 -11
  10. package/dist/bench/report.d.ts +44 -11
  11. package/dist/bench/report.js +238 -32
  12. package/dist/bench/run-dir.d.ts +52 -26
  13. package/dist/bench/run-dir.js +124 -45
  14. package/dist/bench/runner.d.ts +47 -17
  15. package/dist/bench/runner.js +82 -21
  16. package/dist/bench/score.d.ts +38 -7
  17. package/dist/bench/score.js +64 -17
  18. package/dist/bench/stats.d.ts +61 -7
  19. package/dist/bench/stats.js +82 -7
  20. package/dist/bench/suite.d.ts +54 -9
  21. package/dist/bench/suite.js +264 -96
  22. package/dist/cli/bench.d.ts +27 -9
  23. package/dist/cli/bench.js +174 -83
  24. package/dist/cli/bin.js +2 -2
  25. package/dist/cli/io.d.ts +9 -2
  26. package/dist/cli/io.js +20 -2
  27. package/dist/cli/main.d.ts +34 -14
  28. package/dist/cli/main.js +100 -38
  29. package/dist/cli/print-config.d.ts +8 -6
  30. package/dist/cli/print-config.js +21 -8
  31. package/dist/cli/render.d.ts +2 -2
  32. package/dist/cli/render.js +21 -2
  33. package/dist/config/env.d.ts +32 -19
  34. package/dist/config/env.js +95 -32
  35. package/dist/config/from-config.js +15 -3
  36. package/dist/config/load.d.ts +3 -0
  37. package/dist/config/load.js +17 -8
  38. package/dist/config/resolve.d.ts +9 -1
  39. package/dist/config/resolve.js +17 -1
  40. package/dist/config/schema.d.ts +141 -5
  41. package/dist/config/schema.js +125 -12
  42. package/dist/core/clock.d.ts +17 -0
  43. package/dist/core/clock.js +14 -0
  44. package/dist/core/coverage.d.ts +49 -0
  45. package/dist/core/coverage.js +144 -0
  46. package/dist/core/diff.d.ts +34 -0
  47. package/dist/core/diff.js +140 -18
  48. package/dist/core/errors.d.ts +18 -3
  49. package/dist/core/errors.js +26 -2
  50. package/dist/core/exclude.d.ts +42 -0
  51. package/dist/core/exclude.js +189 -0
  52. package/dist/core/limits.d.ts +82 -0
  53. package/dist/core/limits.js +163 -0
  54. package/dist/core/output.d.ts +3 -2
  55. package/dist/core/output.js +3 -2
  56. package/dist/core/ports.d.ts +57 -2
  57. package/dist/core/ports.js +36 -1
  58. package/dist/core/quorum.d.ts +24 -8
  59. package/dist/core/quorum.js +187 -56
  60. package/dist/core/retry.d.ts +36 -0
  61. package/dist/core/retry.js +46 -0
  62. package/dist/core/stages/context.d.ts +126 -22
  63. package/dist/core/stages/context.js +319 -41
  64. package/dist/core/stages/panel.d.ts +5 -4
  65. package/dist/core/stages/panel.js +30 -11
  66. package/dist/core/stages/research.d.ts +15 -6
  67. package/dist/core/stages/research.js +75 -18
  68. package/dist/core/tokens.d.ts +31 -0
  69. package/dist/core/tokens.js +81 -0
  70. package/dist/core/types.d.ts +77 -10
  71. package/dist/index.d.ts +1 -1
  72. package/dist/providers/anthropic.d.ts +2 -1
  73. package/dist/providers/anthropic.js +14 -6
  74. package/dist/providers/claude-code.d.ts +51 -11
  75. package/dist/providers/claude-code.js +69 -33
  76. package/dist/providers/gemini.d.ts +4 -1
  77. package/dist/providers/gemini.js +72 -15
  78. package/dist/providers/http.d.ts +49 -4
  79. package/dist/providers/http.js +155 -14
  80. package/dist/providers/ollama.d.ts +3 -1
  81. package/dist/providers/ollama.js +17 -9
  82. package/dist/providers/openai-compatible.d.ts +3 -1
  83. package/dist/providers/openai-compatible.js +17 -9
  84. package/dist/providers/options.d.ts +12 -3
  85. package/dist/providers/options.js +30 -10
  86. package/dist/providers/reply.d.ts +6 -2
  87. package/dist/providers/reply.js +13 -3
  88. package/dist/providers/size.d.ts +10 -14
  89. package/dist/providers/size.js +15 -69
  90. package/package.json +1 -1
  91. package/schema/quorum.schema.json +34 -1
@@ -1,16 +1,6 @@
1
1
  import { ProviderError } from "../core/errors.js";
2
- /**
3
- * Tokens kept for what a server adds around the messages: the chat template's role markers, a thinking
4
- * or Harmony preamble, a tokenizer's dummy prefix. Measured at 10 to 29 tokens for Llama 3 and Qwen 2.5;
5
- * the margin is many times that, since a preamble can be longer than any measured.
6
- */
7
- export const TEMPLATE_TOKENS = 512;
8
- /**
9
- * Tokens kept for the reply when `maxTokens` is not set: the limit `anthropic` sends then, and room for a
10
- * thinking model's longest reply. Every other adapter sends no limit, so there it is room the guard keeps.
11
- */
12
- export const DEFAULT_REPLY_TOKENS = 16_000;
13
- const encoder = new TextEncoder();
2
+ import { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS, tokenBound } from "../core/tokens.js";
3
+ export { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS, utf8Length } from "../core/tokens.js";
14
4
  /**
15
5
  * The budget a request is checked against: the context size, the reply budget, `maxTokens` else
16
6
  * `DEFAULT_REPLY_TOKENS`, and the template reserve, which an adapter whose server adds more context than a
@@ -19,61 +9,6 @@ const encoder = new TextEncoder();
19
9
  export function contextBudget(contextTokens, maxTokens, templateTokens = TEMPLATE_TOKENS) {
20
10
  return { contextTokens, replyTokens: maxTokens ?? DEFAULT_REPLY_TOKENS, templateTokens };
21
11
  }
22
- /** The length of `text` in UTF-8 bytes. A lone surrogate counts the 3 bytes of U+FFFD, as the encoder writes it. */
23
- export function utf8Length(text) {
24
- return encoder.encode(text).length;
25
- }
26
- /**
27
- * The bytes of the longest form of each code point, 0 until it is first seen, so a repeated character is
28
- * normalized once. A fixed table of 1.1 MB, where a map grew past 200 MB on a text holding every code
29
- * point; the longest form, U+FDFA's, is 33 bytes, so a byte holds any of them.
30
- */
31
- const longestFormBytes = new Uint8Array(0x110000);
32
- /**
33
- * The most UTF-8 bytes a code point takes as sent, decomposed (NFD) or compatibility-decomposed (NFKD): U+0F81
34
- * is 3 bytes and 6 decomposed, U+00E9 is 2 and 3, a Hangul syllable up to 3 times its size, U+FDFA is 3 and 33.
35
- * A lone surrogate is 3, the bytes of the U+FFFD the encoder writes for it.
36
- */
37
- function longestFormLength(codePoint) {
38
- if (codePoint >= 0xD800 && codePoint <= 0xDFFF)
39
- return 3;
40
- const cached = longestFormBytes[codePoint];
41
- if (cached !== undefined && cached !== 0)
42
- return cached;
43
- const char = String.fromCodePoint(codePoint);
44
- const bytes = Math.max(utf8Length(char), utf8Length(char.normalize('NFD')), utf8Length(char.normalize('NFKD')));
45
- longestFormBytes[codePoint] = bytes;
46
- return bytes;
47
- }
48
- /**
49
- * A bound of the UTF-8 bytes of `text` in any normalization: the sum over its code points of each one's
50
- * longest form. The text itself is never normalized, since that takes quadratic time on a long run of
51
- * combining marks (canonical reordering) and a gigabyte on a run of U+FDFA. It is a bound because
52
- * decomposition works on one code point at a time, reordering only permutes code points, and composing
53
- * never lengthens UTF-8 (true of every code point, which `size.spec.ts` checks), so no NFC, NFKC, NFD or
54
- * NFKD form of the whole text is longer. Normalizing each character as NFC, or the text in chunks, is not
55
- * a bound: a mark that composes first, or one that crosses a chunk, can lengthen the whole.
56
- */
57
- function normalizedLength(text) {
58
- let bytes = 0;
59
- for (let index = 0; index < text.length; index++) {
60
- const unit = text.charCodeAt(index);
61
- if (unit < 0x80) {
62
- bytes += 1;
63
- continue;
64
- }
65
- const low = index + 1 < text.length
66
- ? text.charCodeAt(index + 1)
67
- : 0;
68
- const pair = unit >= 0xD800 && unit <= 0xDBFF && low >= 0xDC00 && low <= 0xDFFF;
69
- bytes += longestFormLength(pair
70
- ? 0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)
71
- : unit);
72
- if (pair)
73
- index++;
74
- }
75
- return bytes;
76
- }
77
12
  /**
78
13
  * A bound of the tokens in the text of a request: one per UTF-8 byte of the system text, of every message,
79
14
  * and of the reply schema's JSON, each counted at the sum of its code points' longest forms. One token per
@@ -83,7 +18,8 @@ function normalizedLength(text) {
83
18
  * the count is not the raw bytes. One that folds case is not covered; no chat model's tokenizer is known
84
19
  * to. The price is that accented letters count one more byte per accent and Hangul syllables up to three
85
20
  * times their size. The schema counts once more than the body carries it, since a server may add it to the
86
- * prompt.
21
+ * prompt. The core counts a request's text with the same bound (`requestTokens`), so the worst case an adapter
22
+ * states and the core's own count never drift.
87
23
  */
88
24
  export function textTokens(shaped, output) {
89
25
  const texts = [
@@ -91,7 +27,17 @@ export function textTokens(shaped, output) {
91
27
  ...shaped.messages.map(message => message.content),
92
28
  JSON.stringify(output.schema),
93
29
  ];
94
- return texts.reduce((sum, text) => sum + normalizedLength(text), 0);
30
+ return texts.reduce((sum, text) => sum + tokenBound(text), 0);
31
+ }
32
+ /**
33
+ * The most tokens, input and output, that one attempt of a request may spend as the provider is billed for
34
+ * it, from what the guard counts: each of `turns` model turns sends the text and the template reserve, and
35
+ * the reply of every turn but the last is the input of the next, so the replies count `2 * turns - 1` times.
36
+ * One turn is the text, the template reserve and the reply budget, which `checkFits` keeps within the context.
37
+ */
38
+ export function worstCaseTokens(shaped, output, budget, turns = 1) {
39
+ const template = budget.templateTokens ?? TEMPLATE_TOKENS;
40
+ return turns * (textTokens(shaped, output) + template) + (2 * turns - 1) * budget.replyTokens;
95
41
  }
96
42
  /**
97
43
  * Refuse, before it is sent, a request that may not fit the model's context: some endpoints cut an oversized
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "@tobi-or-not/quorum",
3
3
  "type": "module",
4
- "version": "0.3.0",
4
+ "version": "0.5.0",
5
5
  "packageManager": "bun@1.4.4",
6
6
  "description": "Multi-agent code review for diffs: a model proposes candidate findings, then a panel of blind voters judges each before it's reported.",
7
7
  "license": "Apache-2.0",
@@ -153,7 +153,7 @@
153
153
  ]
154
154
  },
155
155
  "maxTokens": {
156
- "description": "The most tokens the reply may hold",
156
+ "description": "The most tokens the reply may hold, thinking included, sent as the server's limit; 16000 when unset. On claude-code, Claude Code's own limit on each reply",
157
157
  "type": "integer",
158
158
  "minimum": 1,
159
159
  "maximum": 9007199254740991
@@ -174,6 +174,18 @@
174
174
  "inline"
175
175
  ]
176
176
  },
177
+ "timeoutMs": {
178
+ "description": "The deadline of one attempt of a call, in milliseconds; 600000 when unset. On claude-code, Claude Code's wait for each attempt's response headers, and the whole run's deadline is timeoutMs times retries plus one",
179
+ "type": "integer",
180
+ "minimum": 1000,
181
+ "maximum": 86400000
182
+ },
183
+ "retries": {
184
+ "description": "How many times a call that failed in passing is tried again; 2 when unset",
185
+ "type": "integer",
186
+ "minimum": 0,
187
+ "maximum": 5
188
+ },
177
189
  "args": {
178
190
  "description": "On claude-code, extra claude flags, each with its value: --effort, --fallback-model, --max-budget-usd, or --betas with auth api-key",
179
191
  "type": "array",
@@ -214,6 +226,27 @@
214
226
  "minimum": 1,
215
227
  "maximum": 9007199254740991
216
228
  },
229
+ "maxReviewTokens": {
230
+ "description": "The most tokens, input and output, one review may spend; 2000000 when unset. Set here or in QUORUM_MAX_REVIEW_TOKENS",
231
+ "type": "integer",
232
+ "minimum": 1,
233
+ "maximum": 9007199254740991
234
+ },
235
+ "reviewTimeoutMs": {
236
+ "description": "The deadline of one review, in milliseconds; 1800000 when unset. Set here or in QUORUM_REVIEW_TIMEOUT_MS",
237
+ "type": "integer",
238
+ "minimum": 1000,
239
+ "maximum": 86400000
240
+ },
241
+ "exclude": {
242
+ "description": "The files left out of every request, by path pattern, each listed as excluded in meta.leftOut; nothing is excluded by default",
243
+ "maxItems": 100,
244
+ "type": "array",
245
+ "items": {
246
+ "type": "string",
247
+ "description": "A path pattern: * and ? within a name, ** for whole directories, no / to match a file name in any directory, a leading / for the root"
248
+ }
249
+ },
217
250
  "profiles": {
218
251
  "description": "The review profiles, by id: a built-in profile, one that extends it, or one of your own. The ones --profile names run, else the ones QUORUM_PROFILES names, else every one listed here, else the security profile alone",
219
252
  "type": "object",