@tobi-or-not/quorum 0.3.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -1
- package/dist/bench/attacks.d.ts +103 -0
- package/dist/bench/attacks.js +272 -0
- package/dist/bench/fingerprint.d.ts +3 -2
- package/dist/bench/fingerprint.js +3 -2
- package/dist/bench/independence.d.ts +28 -0
- package/dist/bench/independence.js +67 -0
- package/dist/bench/records.d.ts +171 -13
- package/dist/bench/records.js +34 -11
- package/dist/bench/report.d.ts +44 -11
- package/dist/bench/report.js +238 -32
- package/dist/bench/run-dir.d.ts +52 -26
- package/dist/bench/run-dir.js +124 -45
- package/dist/bench/runner.d.ts +29 -17
- package/dist/bench/runner.js +53 -18
- package/dist/bench/score.d.ts +38 -7
- package/dist/bench/score.js +64 -17
- package/dist/bench/stats.d.ts +61 -7
- package/dist/bench/stats.js +82 -7
- package/dist/bench/suite.d.ts +54 -9
- package/dist/bench/suite.js +264 -96
- package/dist/cli/bench.d.ts +20 -9
- package/dist/cli/bench.js +148 -74
- package/dist/cli/io.d.ts +9 -2
- package/dist/cli/io.js +20 -2
- package/dist/cli/main.js +2 -19
- package/dist/config/load.d.ts +3 -0
- package/dist/config/load.js +17 -8
- package/dist/core/diff.d.ts +10 -0
- package/dist/core/diff.js +77 -0
- package/dist/core/errors.d.ts +6 -0
- package/dist/core/errors.js +5 -0
- package/dist/core/stages/context.d.ts +12 -3
- package/dist/core/stages/context.js +23 -3
- package/dist/providers/anthropic.js +6 -3
- package/dist/providers/claude-code.js +11 -10
- package/dist/providers/gemini.js +9 -4
- package/dist/providers/ollama.js +6 -4
- package/dist/providers/openai-compatible.js +6 -4
- package/dist/providers/reply.d.ts +6 -2
- package/dist/providers/reply.js +13 -3
- package/package.json +1 -1
package/dist/core/diff.js
CHANGED
|
@@ -271,6 +271,41 @@ export function locate(sections, file, line) {
|
|
|
271
271
|
}
|
|
272
272
|
return undefined;
|
|
273
273
|
}
|
|
274
|
+
/**
|
|
275
|
+
* The first line from `start` to `end` that no hunk of `file` holds, or undefined when each sits in
|
|
276
|
+
* one: the answer `locate` gives for each line in turn, found by a binary search over the spans the
|
|
277
|
+
* hunks hold, merged and sorted once per section, so a range costs a search and not a walk over every
|
|
278
|
+
* hunk for each of its lines. The file is resolved as `locate` resolves it, and a file the diff does
|
|
279
|
+
* not hold has no line in a hunk, so the answer is `start`.
|
|
280
|
+
*
|
|
281
|
+
* @throws {RangeError} unless `start` and `end` are safe integers with `start` at most `end`
|
|
282
|
+
*/
|
|
283
|
+
export function firstLineOutside(sections, file, start, end) {
|
|
284
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(end) || start > end) {
|
|
285
|
+
throw new RangeError('start and end must be safe integers, start at most end');
|
|
286
|
+
}
|
|
287
|
+
const section = sectionFor(sections, file);
|
|
288
|
+
if (!section)
|
|
289
|
+
return start;
|
|
290
|
+
const spans = spansOf(sections, section.path, section.text);
|
|
291
|
+
let before = 0;
|
|
292
|
+
let after = spans.length;
|
|
293
|
+
while (before < after) {
|
|
294
|
+
const middle = (before + after) >>> 1;
|
|
295
|
+
if (spans[middle].start <= start)
|
|
296
|
+
before = middle + 1;
|
|
297
|
+
else
|
|
298
|
+
after = middle;
|
|
299
|
+
}
|
|
300
|
+
const span = before === 0
|
|
301
|
+
? undefined
|
|
302
|
+
: spans[before - 1];
|
|
303
|
+
if (span === undefined || span.end < start)
|
|
304
|
+
return start;
|
|
305
|
+
return span.end >= end
|
|
306
|
+
? undefined
|
|
307
|
+
: span.end + 1;
|
|
308
|
+
}
|
|
274
309
|
/**
|
|
275
310
|
* The lines the diff adds to `path`, in the order of the diff: the rows of its hunks marked `+`,
|
|
276
311
|
* read from the hunks `locate` reads, so they are cached with them. `path` is the diff's own path,
|
|
@@ -315,6 +350,48 @@ function hunksOf(sections, path, text) {
|
|
|
315
350
|
}
|
|
316
351
|
return hunks;
|
|
317
352
|
}
|
|
353
|
+
/** The merged spans of each section, built once, in a cache that is dropped with the sections map like `parsedSections`. */
|
|
354
|
+
const coveredSections = new WeakMap();
|
|
355
|
+
/** The new-file lines a hunk holds for `locate`: those it shows from its start, up to its count, or its start alone for a hunk that counts no new line; none for a hunk that shows no row. */
|
|
356
|
+
function heldSpan(hunk) {
|
|
357
|
+
if (hunk.rows.length === 0)
|
|
358
|
+
return undefined;
|
|
359
|
+
const shown = hunk.count === 0
|
|
360
|
+
? 1
|
|
361
|
+
: Math.min(hunk.count, hunk.rowOfLine.length);
|
|
362
|
+
return shown === 0
|
|
363
|
+
? undefined
|
|
364
|
+
: { start: hunk.start, end: hunk.start + shown - 1 };
|
|
365
|
+
}
|
|
366
|
+
/** The spans the hunks of a section hold, sorted by start, each merged with the one before it when it starts at most a line after that one's end, so no line between two spans is in either. */
|
|
367
|
+
function spansOf(sections, path, text) {
|
|
368
|
+
let covered = coveredSections.get(sections);
|
|
369
|
+
if (!covered) {
|
|
370
|
+
covered = new Map();
|
|
371
|
+
coveredSections.set(sections, covered);
|
|
372
|
+
}
|
|
373
|
+
let spans = covered.get(path);
|
|
374
|
+
if (!spans) {
|
|
375
|
+
const held = hunksOf(sections, path, text).flatMap(hunk => heldSpan(hunk) ?? []);
|
|
376
|
+
held.sort((first, second) => first.start - second.start);
|
|
377
|
+
const merged = [];
|
|
378
|
+
for (const span of held) {
|
|
379
|
+
const last = merged.at(-1);
|
|
380
|
+
if (last !== undefined && span.start <= last.end + 1) {
|
|
381
|
+
merged[merged.length - 1] = {
|
|
382
|
+
start: last.start,
|
|
383
|
+
end: Math.max(last.end, span.end),
|
|
384
|
+
};
|
|
385
|
+
}
|
|
386
|
+
else {
|
|
387
|
+
merged.push(span);
|
|
388
|
+
}
|
|
389
|
+
}
|
|
390
|
+
spans = merged;
|
|
391
|
+
covered.set(path, spans);
|
|
392
|
+
}
|
|
393
|
+
return spans;
|
|
394
|
+
}
|
|
318
395
|
/**
|
|
319
396
|
* Read the hunks of one file section. The header's line counts say where a hunk ends, so file
|
|
320
397
|
* headers and `` markers are never mistaken for hunk lines, and a line
|
package/dist/core/errors.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { Usage } from './types.ts';
|
|
1
2
|
/** Base class for every error quorum raises on purpose. */
|
|
2
3
|
export declare class QuorumError extends Error {
|
|
3
4
|
constructor(message: string, options?: ErrorOptions);
|
|
@@ -31,6 +32,8 @@ export type ProviderErrorKind = 'http' | 'network' | 'process' | 'timeout' | 'ab
|
|
|
31
32
|
/**
|
|
32
33
|
* A provider call that failed. The message starts with the provider id. Adapters never put request
|
|
33
34
|
* headers, keys or the runtime's own error text in it; they quote server text through `snippet`.
|
|
35
|
+
* `usage` holds the counts of a reply the provider read and then refused, which the review counts as
|
|
36
|
+
* the failed call's usage.
|
|
34
37
|
*/
|
|
35
38
|
export declare class ProviderError extends QuorumError {
|
|
36
39
|
readonly providerId: string;
|
|
@@ -38,11 +41,14 @@ export declare class ProviderError extends QuorumError {
|
|
|
38
41
|
readonly status: number | undefined;
|
|
39
42
|
/** True for network errors, timeouts, 408, 409, 429 and 5xx; a caller may retry the call. */
|
|
40
43
|
readonly retryable: boolean;
|
|
44
|
+
/** What the reply the call failed on reported, when the provider read one: the review counts it as the failed call's usage. */
|
|
45
|
+
readonly usage: Usage | undefined;
|
|
41
46
|
constructor(details: {
|
|
42
47
|
providerId: string;
|
|
43
48
|
kind: ProviderErrorKind;
|
|
44
49
|
message: string;
|
|
45
50
|
status?: number;
|
|
51
|
+
usage?: Usage;
|
|
46
52
|
cause?: unknown;
|
|
47
53
|
});
|
|
48
54
|
}
|
package/dist/core/errors.js
CHANGED
|
@@ -61,6 +61,8 @@ function isRetryable(kind, status) {
|
|
|
61
61
|
/**
|
|
62
62
|
* A provider call that failed. The message starts with the provider id. Adapters never put request
|
|
63
63
|
* headers, keys or the runtime's own error text in it; they quote server text through `snippet`.
|
|
64
|
+
* `usage` holds the counts of a reply the provider read and then refused, which the review counts as
|
|
65
|
+
* the failed call's usage.
|
|
64
66
|
*/
|
|
65
67
|
export class ProviderError extends QuorumError {
|
|
66
68
|
providerId;
|
|
@@ -68,6 +70,8 @@ export class ProviderError extends QuorumError {
|
|
|
68
70
|
status;
|
|
69
71
|
/** True for network errors, timeouts, 408, 409, 429 and 5xx; a caller may retry the call. */
|
|
70
72
|
retryable;
|
|
73
|
+
/** What the reply the call failed on reported, when the provider read one: the review counts it as the failed call's usage. */
|
|
74
|
+
usage;
|
|
71
75
|
constructor(details) {
|
|
72
76
|
super(`${details.providerId}: ${details.message}`, details.cause === undefined
|
|
73
77
|
? undefined
|
|
@@ -76,5 +80,6 @@ export class ProviderError extends QuorumError {
|
|
|
76
80
|
this.kind = details.kind;
|
|
77
81
|
this.status = details.status;
|
|
78
82
|
this.retryable = isRetryable(details.kind, details.status);
|
|
83
|
+
this.usage = details.usage;
|
|
79
84
|
}
|
|
80
85
|
}
|
|
@@ -61,6 +61,14 @@ export interface RunContext {
|
|
|
61
61
|
export declare function isTooLarge(error: unknown): boolean;
|
|
62
62
|
/** What a refusal as too large says: its own data string `message`, else a fixed text, never anything else from the object. */
|
|
63
63
|
export declare function refusalMessage(refusal: unknown): string;
|
|
64
|
+
/**
|
|
65
|
+
* The usage a failed call's thrown value carries, when its counts can be trusted: its own data property
|
|
66
|
+
* `usage`, and that object's own data `inputTokens`, `outputTokens` and `costUsd`, checked as a reply's
|
|
67
|
+
* usage is. It is matched by those properties, not by class, as `isTooLarge` is, so a `ProviderError`
|
|
68
|
+
* from a second copy of quorum or a library provider's own error object counts too. It never runs a
|
|
69
|
+
* getter and never throws: an inherited property, a getter or a throwing proxy reads as no usage.
|
|
70
|
+
*/
|
|
71
|
+
export declare function failureUsage(error: unknown): Usage | undefined;
|
|
64
72
|
/** Who requests a provider call, as its `call` event names it: `lens` only for a vote. */
|
|
65
73
|
export interface Requester {
|
|
66
74
|
readonly profile: string;
|
|
@@ -83,9 +91,10 @@ export declare function elapsed(started: number): number;
|
|
|
83
91
|
* once the request is built: the provider's latency, not the time the call waited for a slot or
|
|
84
92
|
* took to build its request. A reply whose text is not a string is a `ProviderError`. Each call
|
|
85
93
|
* emits one `call` event, which names the provider by the ref captured at creation, never by the
|
|
86
|
-
* live object: failed, before the error is rethrown,
|
|
87
|
-
* valid
|
|
88
|
-
* that throws on it is never reported as a
|
|
94
|
+
* live object: failed, before the error is rethrown, with the usage its error carries when its counts
|
|
95
|
+
* are valid, or ok, with the usage when its counts are valid. The ok event is emitted outside the
|
|
96
|
+
* `try` that catches the provider's failure, so a sink that throws on it is never reported as a
|
|
97
|
+
* failed call.
|
|
89
98
|
*/
|
|
90
99
|
export declare function callProvider(ctx: RunContext, requester: Requester, target: Target, buildRequest: () => CompletionRequest): Promise<string>;
|
|
91
100
|
/**
|
|
@@ -33,6 +33,21 @@ export function refusalMessage(refusal) {
|
|
|
33
33
|
? message
|
|
34
34
|
: REFUSAL_FALLBACK;
|
|
35
35
|
}
|
|
36
|
+
/**
|
|
37
|
+
* The usage a failed call's thrown value carries, when its counts can be trusted: its own data property
|
|
38
|
+
* `usage`, and that object's own data `inputTokens`, `outputTokens` and `costUsd`, checked as a reply's
|
|
39
|
+
* usage is. It is matched by those properties, not by class, as `isTooLarge` is, so a `ProviderError`
|
|
40
|
+
* from a second copy of quorum or a library provider's own error object counts too. It never runs a
|
|
41
|
+
* getter and never throws: an inherited property, a getter or a throwing proxy reads as no usage.
|
|
42
|
+
*/
|
|
43
|
+
export function failureUsage(error) {
|
|
44
|
+
const usage = ownData(error, 'usage');
|
|
45
|
+
return checkedUsage({
|
|
46
|
+
inputTokens: ownData(usage, 'inputTokens'),
|
|
47
|
+
outputTokens: ownData(usage, 'outputTokens'),
|
|
48
|
+
costUsd: ownData(usage, 'costUsd'),
|
|
49
|
+
});
|
|
50
|
+
}
|
|
36
51
|
/** Whole milliseconds since `started`, a `performance.now()` reading. */
|
|
37
52
|
export function elapsed(started) {
|
|
38
53
|
return Math.round(performance.now() - started);
|
|
@@ -51,9 +66,10 @@ export function elapsed(started) {
|
|
|
51
66
|
* once the request is built: the provider's latency, not the time the call waited for a slot or
|
|
52
67
|
* took to build its request. A reply whose text is not a string is a `ProviderError`. Each call
|
|
53
68
|
* emits one `call` event, which names the provider by the ref captured at creation, never by the
|
|
54
|
-
* live object: failed, before the error is rethrown,
|
|
55
|
-
* valid
|
|
56
|
-
* that throws on it is never reported as a
|
|
69
|
+
* live object: failed, before the error is rethrown, with the usage its error carries when its counts
|
|
70
|
+
* are valid, or ok, with the usage when its counts are valid. The ok event is emitted outside the
|
|
71
|
+
* `try` that catches the provider's failure, so a sink that throws on it is never reported as a
|
|
72
|
+
* failed call.
|
|
57
73
|
*/
|
|
58
74
|
export async function callProvider(ctx, requester, target, buildRequest) {
|
|
59
75
|
return ctx.limit(async () => {
|
|
@@ -67,12 +83,16 @@ export async function callProvider(ctx, requester, target, buildRequest) {
|
|
|
67
83
|
throw new ProviderError({ providerId: target.ref.id, kind: 'invalid-response', message: 'returned no reply text' });
|
|
68
84
|
}
|
|
69
85
|
catch (error) {
|
|
86
|
+
const failedUsage = failureUsage(error);
|
|
70
87
|
ctx.emit({
|
|
71
88
|
type: 'call',
|
|
72
89
|
...requester,
|
|
73
90
|
provider: target.ref,
|
|
74
91
|
durationMs: elapsed(started),
|
|
75
92
|
ok: false,
|
|
93
|
+
...(failedUsage === undefined
|
|
94
|
+
? {}
|
|
95
|
+
: { usage: failedUsage }),
|
|
76
96
|
});
|
|
77
97
|
// Still inside the slot: the queued call it is handed to next sees the abort and never starts.
|
|
78
98
|
if (requester.role === 'research' && !isTooLarge(error))
|
|
@@ -43,11 +43,14 @@ const FINISHED = new Set(['end_turn', 'stop_sequence']);
|
|
|
43
43
|
/**
|
|
44
44
|
* Turn a Messages reply into text, or a `ProviderError`. Every property is read defensively: the reply
|
|
45
45
|
* is untrusted. The stop reason comes first, since a refused or cut-off reply can still hold some text.
|
|
46
|
+
* The counts are read once, after the body check, and every failure from then on carries them, since the
|
|
47
|
+
* server generated those tokens.
|
|
46
48
|
*/
|
|
47
49
|
function readReply(reply, { id, model }, redact) {
|
|
48
|
-
const fail = replyFailure(id);
|
|
49
50
|
if (!isRecord(reply))
|
|
50
|
-
throw
|
|
51
|
+
throw replyFailure(id)('invalid-response', 'reply is not a JSON object');
|
|
52
|
+
const usage = tokenUsage(reply.usage, ['input_tokens', 'cache_creation_input_tokens', 'cache_read_input_tokens'], ['output_tokens']);
|
|
53
|
+
const fail = replyFailure(id, usage);
|
|
51
54
|
const reason = reply.stop_reason;
|
|
52
55
|
if (reason === 'refusal')
|
|
53
56
|
throw fail('refusal', 'model refused');
|
|
@@ -65,7 +68,7 @@ function readReply(reply, { id, model }, redact) {
|
|
|
65
68
|
model: typeof reply.model === 'string' && reply.model !== ''
|
|
66
69
|
? reply.model
|
|
67
70
|
: model,
|
|
68
|
-
usage
|
|
71
|
+
usage,
|
|
69
72
|
};
|
|
70
73
|
}
|
|
71
74
|
/**
|
|
@@ -492,10 +492,10 @@ function replyText(result, structuredOutput, fail) {
|
|
|
492
492
|
/**
|
|
493
493
|
* The failure of a result that reports an error: `http` with the status when `api_error_status` is one from
|
|
494
494
|
* 400 to 599, so 429 and 5xx stay retryable, else `process` with the exit code, naming the reason when it
|
|
495
|
-
* is a short word.
|
|
495
|
+
* is a short word. It carries `usage`, the counts the result reports.
|
|
496
496
|
*/
|
|
497
|
-
function errorFailure(result, run, { id, redact }) {
|
|
498
|
-
const fail = replyFailure(id);
|
|
497
|
+
function errorFailure(result, run, { id, redact }, usage) {
|
|
498
|
+
const fail = replyFailure(id, usage);
|
|
499
499
|
const status = result.api_error_status;
|
|
500
500
|
const detail = failureDetail(result, redact);
|
|
501
501
|
if (typeof status === 'number' && Number.isInteger(status) && status >= 400 && status <= 599)
|
|
@@ -518,29 +518,30 @@ function signalName(signal) {
|
|
|
518
518
|
* refusal, an error (anything but an `is_error` of `false` is one), a child a signal ended or a non-zero
|
|
519
519
|
* exit after a success, a tool the server ran, a reply the token limit cut off, then the text. Claude Code
|
|
520
520
|
* reports an API error as exit 1 with `subtype: "success"` and `is_error: true`, so the JSON on stdout is
|
|
521
|
-
* read before the exit code.
|
|
521
|
+
* read before the exit code. Every failure carries the usage the result reports, summed as `sumUsage`
|
|
522
|
+
* does, since the model generated and billed those tokens whatever came after.
|
|
522
523
|
*/
|
|
523
524
|
function readFinished(result, run, context) {
|
|
524
|
-
const
|
|
525
|
+
const modelUsage = isRecord(result.modelUsage)
|
|
526
|
+
? Object.values(result.modelUsage).filter(isRecord)
|
|
527
|
+
: [];
|
|
528
|
+
const usage = sumUsage(modelUsage, result.total_cost_usd);
|
|
529
|
+
const fail = replyFailure(context.id, usage);
|
|
525
530
|
if (isPromptTooLong(result))
|
|
526
531
|
throw fail('too-large', 'Claude Code refused the prompt as too long for the model\'s context');
|
|
527
532
|
if (result.stop_reason === 'refusal')
|
|
528
533
|
throw fail('refusal', 'model refused');
|
|
529
534
|
if (result.is_error !== false || result.subtype !== 'success')
|
|
530
|
-
throw errorFailure(result, run, context);
|
|
535
|
+
throw errorFailure(result, run, context, usage);
|
|
531
536
|
if (run.signal !== null)
|
|
532
537
|
throw fail('process', `claude was stopped by ${signalName(run.signal)} after its result`);
|
|
533
538
|
if (run.code !== null && run.code !== 0)
|
|
534
539
|
throw fail('process', `claude exited ${run.code} after its result`, run.code);
|
|
535
|
-
const modelUsage = isRecord(result.modelUsage)
|
|
536
|
-
? Object.values(result.modelUsage).filter(isRecord)
|
|
537
|
-
: [];
|
|
538
540
|
if (ranServerTool(result, modelUsage))
|
|
539
541
|
throw fail('invalid-response', 'the server ran a tool for this reply, which quorum never allows');
|
|
540
542
|
if (result.stop_reason === 'max_tokens')
|
|
541
543
|
throw fail('truncated', 'reply was cut off by the token limit');
|
|
542
544
|
const text = replyText(result, context.structuredOutput, fail);
|
|
543
|
-
const usage = sumUsage(modelUsage, result.total_cost_usd);
|
|
544
545
|
const models = isRecord(result.modelUsage)
|
|
545
546
|
? Object.keys(result.modelUsage)
|
|
546
547
|
: [];
|
package/dist/providers/gemini.js
CHANGED
|
@@ -73,12 +73,17 @@ function generationConfig({ temperature, maxTokens, structuredOutput }, output)
|
|
|
73
73
|
}
|
|
74
74
|
/**
|
|
75
75
|
* Turn a `generateContent` reply into text, or a `ProviderError`. Every property is read defensively: the
|
|
76
|
-
* reply is untrusted. A blocked prompt comes first, since its reply has no candidates.
|
|
76
|
+
* reply is untrusted. A blocked prompt comes first, since its reply has no candidates. The counts are read
|
|
77
|
+
* once, after the body check, and every failure from then on carries them, since the server generated those
|
|
78
|
+
* tokens. Gemini leaves a zero count out of its JSON, so a candidates count the reply leaves out reads as 0.
|
|
77
79
|
*/
|
|
78
80
|
function readReply(reply, { id, model }, redact) {
|
|
79
|
-
const fail = replyFailure(id);
|
|
80
81
|
if (!isRecord(reply))
|
|
81
|
-
throw
|
|
82
|
+
throw replyFailure(id)('invalid-response', 'reply is not a JSON object');
|
|
83
|
+
const usage = isRecord(reply.usageMetadata)
|
|
84
|
+
? tokenUsage({ candidatesTokenCount: 0, ...reply.usageMetadata }, ['promptTokenCount'], ['candidatesTokenCount', 'thoughtsTokenCount'])
|
|
85
|
+
: undefined;
|
|
86
|
+
const fail = replyFailure(id, usage);
|
|
82
87
|
const blocked = isRecord(reply.promptFeedback)
|
|
83
88
|
? reply.promptFeedback.blockReason
|
|
84
89
|
: undefined;
|
|
@@ -113,7 +118,7 @@ function readReply(reply, { id, model }, redact) {
|
|
|
113
118
|
model: typeof reply.modelVersion === 'string' && reply.modelVersion !== ''
|
|
114
119
|
? reply.modelVersion
|
|
115
120
|
: model,
|
|
116
|
-
usage
|
|
121
|
+
usage,
|
|
117
122
|
};
|
|
118
123
|
}
|
|
119
124
|
/**
|
package/dist/providers/ollama.js
CHANGED
|
@@ -63,12 +63,14 @@ function format(mode, output) {
|
|
|
63
63
|
/**
|
|
64
64
|
* Turn an `/api/chat` reply into text, or a `ProviderError`. Every property is read defensively: the
|
|
65
65
|
* reply is untrusted. A server error comes first: a local server answers one with an error status, but
|
|
66
|
-
* the reply of a remote model could carry one in a 200.
|
|
66
|
+
* the reply of a remote model could carry one in a 200. The counts are read once, after the body check,
|
|
67
|
+
* and every failure from then on carries them, since the server generated those tokens.
|
|
67
68
|
*/
|
|
68
69
|
function readReply(reply, { id, model }, redact) {
|
|
69
|
-
const fail = replyFailure(id);
|
|
70
70
|
if (!isRecord(reply))
|
|
71
|
-
throw
|
|
71
|
+
throw replyFailure(id)('invalid-response', 'reply is not a JSON object');
|
|
72
|
+
const usage = tokenUsage(reply, ['prompt_eval_count'], ['eval_count']);
|
|
73
|
+
const fail = replyFailure(id, usage);
|
|
72
74
|
if (typeof reply.error === 'string')
|
|
73
75
|
throw fail('invalid-response', `server error: ${snippet(reply.error, MAX_MODEL_TEXT, redact)}`);
|
|
74
76
|
if (reply.done !== true || !DONE.has(reply.done_reason))
|
|
@@ -85,7 +87,7 @@ function readReply(reply, { id, model }, redact) {
|
|
|
85
87
|
model: typeof reply.model === 'string' && reply.model !== ''
|
|
86
88
|
? reply.model
|
|
87
89
|
: model,
|
|
88
|
-
usage
|
|
90
|
+
usage,
|
|
89
91
|
};
|
|
90
92
|
}
|
|
91
93
|
/**
|
|
@@ -85,12 +85,14 @@ function textOf(content) {
|
|
|
85
85
|
* reply is untrusted, and a malformed shape must fail as a provider error, not as a `TypeError`. A
|
|
86
86
|
* server error comes first, since some servers answer 200 with one beside an empty choice (OpenRouter,
|
|
87
87
|
* for a failure during generation), as `{ error: { message, code } }` or `{ error: "text" }`. A tool the
|
|
88
|
-
* server ran comes next, before a refusal or a finish reason.
|
|
88
|
+
* server ran comes next, before a refusal or a finish reason. The counts are read once, after the body
|
|
89
|
+
* check, and every failure from then on carries them, since the server generated those tokens.
|
|
89
90
|
*/
|
|
90
91
|
function readReply(reply, { id, model }, redact) {
|
|
91
|
-
const fail = replyFailure(id);
|
|
92
92
|
if (!isRecord(reply))
|
|
93
|
-
throw
|
|
93
|
+
throw replyFailure(id)('invalid-response', 'reply is not a JSON object');
|
|
94
|
+
const usage = tokenUsage(reply.usage, ['prompt_tokens'], ['completion_tokens']);
|
|
95
|
+
const fail = replyFailure(id, usage);
|
|
94
96
|
const { error } = reply;
|
|
95
97
|
if (typeof error === 'string' || isRecord(error)) {
|
|
96
98
|
const text = typeof error === 'string'
|
|
@@ -137,7 +139,7 @@ function readReply(reply, { id, model }, redact) {
|
|
|
137
139
|
model: typeof reply.model === 'string' && reply.model !== ''
|
|
138
140
|
? reply.model
|
|
139
141
|
: model,
|
|
140
|
-
usage
|
|
142
|
+
usage,
|
|
141
143
|
};
|
|
142
144
|
}
|
|
143
145
|
/**
|
|
@@ -3,8 +3,12 @@ import type { ProviderErrorKind } from '../core/errors.ts';
|
|
|
3
3
|
import type { Usage } from '../core/types.ts';
|
|
4
4
|
/** The most characters of model text, or of server text in a 2xx reply, that an error message quotes. */
|
|
5
5
|
export declare const MAX_MODEL_TEXT = 200;
|
|
6
|
-
/**
|
|
7
|
-
|
|
6
|
+
/**
|
|
7
|
+
* The failures of one provider: a `ProviderError` of `kind`, with its message and, for `http`, its status.
|
|
8
|
+
* Each carries `usage`, the counts of a reply the adapter read before it failed, so the review counts the
|
|
9
|
+
* tokens of a reply it refuses.
|
|
10
|
+
*/
|
|
11
|
+
export declare function replyFailure(providerId: string, usage?: Usage): (kind: ProviderErrorKind, message: string, status?: number) => ProviderError;
|
|
8
12
|
/**
|
|
9
13
|
* The token counts of `source`, or undefined unless both are known. Each list names the key of its
|
|
10
14
|
* count, which must hold a number, then the keys of the parts the protocol counts apart, added when
|
package/dist/providers/reply.js
CHANGED
|
@@ -5,9 +5,19 @@ import { isRecord } from "./options.js";
|
|
|
5
5
|
export const MAX_MODEL_TEXT = 200;
|
|
6
6
|
/** The most characters of a stop reason that an error message quotes. */
|
|
7
7
|
const MAX_REASON = 40;
|
|
8
|
-
/**
|
|
9
|
-
|
|
10
|
-
|
|
8
|
+
/**
|
|
9
|
+
* The failures of one provider: a `ProviderError` of `kind`, with its message and, for `http`, its status.
|
|
10
|
+
* Each carries `usage`, the counts of a reply the adapter read before it failed, so the review counts the
|
|
11
|
+
* tokens of a reply it refuses.
|
|
12
|
+
*/
|
|
13
|
+
export function replyFailure(providerId, usage) {
|
|
14
|
+
return (kind, message, status) => new ProviderError({
|
|
15
|
+
providerId,
|
|
16
|
+
kind,
|
|
17
|
+
message,
|
|
18
|
+
status,
|
|
19
|
+
usage,
|
|
20
|
+
});
|
|
11
21
|
}
|
|
12
22
|
/** The count `source` holds under the first key, plus each other key's when it is a number; undefined unless the first is one. */
|
|
13
23
|
function countOf(source, [first = '', ...others]) {
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@tobi-or-not/quorum",
|
|
3
3
|
"type": "module",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.4.0",
|
|
5
5
|
"packageManager": "bun@1.4.4",
|
|
6
6
|
"description": "Multi-agent code review for diffs: a model proposes candidate findings, then a panel of blind voters judges each before it's reported.",
|
|
7
7
|
"license": "Apache-2.0",
|