@tobi-or-not/quorum 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -1
- package/dist/bench/fingerprint.d.ts +8 -5
- package/dist/bench/fingerprint.js +28 -9
- package/dist/bench/records.d.ts +168 -0
- package/dist/bench/records.js +106 -1
- package/dist/bench/runner.d.ts +18 -0
- package/dist/bench/runner.js +30 -4
- package/dist/cli/bench.d.ts +11 -4
- package/dist/cli/bench.js +34 -17
- package/dist/cli/bin.js +2 -2
- package/dist/cli/io.d.ts +5 -0
- package/dist/cli/main.d.ts +67 -24
- package/dist/cli/main.js +288 -59
- package/dist/cli/print-config.d.ts +8 -6
- package/dist/cli/print-config.js +26 -8
- package/dist/cli/render.d.ts +9 -3
- package/dist/cli/render.js +43 -2
- package/dist/config/claude-code.d.ts +2 -6
- package/dist/config/claude-code.js +6 -44
- package/dist/config/command.d.ts +10 -0
- package/dist/config/command.js +47 -0
- package/dist/config/env.d.ts +32 -19
- package/dist/config/env.js +95 -32
- package/dist/config/from-config.js +21 -3
- package/dist/config/resolve.d.ts +13 -1
- package/dist/config/resolve.js +27 -1
- package/dist/config/schema.d.ts +152 -5
- package/dist/config/schema.js +156 -12
- package/dist/core/clock.d.ts +17 -0
- package/dist/core/clock.js +14 -0
- package/dist/core/coverage.d.ts +74 -0
- package/dist/core/coverage.js +221 -0
- package/dist/core/diff.d.ts +101 -0
- package/dist/core/diff.js +178 -35
- package/dist/core/errors.d.ts +12 -3
- package/dist/core/errors.js +21 -2
- package/dist/core/estimate.d.ts +45 -0
- package/dist/core/estimate.js +188 -0
- package/dist/core/exclude.d.ts +42 -0
- package/dist/core/exclude.js +189 -0
- package/dist/core/flags.d.ts +14 -1
- package/dist/core/flags.js +56 -0
- package/dist/core/inputs.d.ts +38 -0
- package/dist/core/inputs.js +228 -0
- package/dist/core/limits.d.ts +83 -0
- package/dist/core/limits.js +169 -0
- package/dist/core/output.d.ts +3 -2
- package/dist/core/output.js +3 -2
- package/dist/core/parts.d.ts +146 -0
- package/dist/core/parts.js +388 -0
- package/dist/core/ports.d.ts +67 -2
- package/dist/core/ports.js +36 -1
- package/dist/core/prompts.d.ts +22 -3
- package/dist/core/prompts.js +43 -7
- package/dist/core/quorum.d.ts +26 -8
- package/dist/core/quorum.js +310 -73
- package/dist/core/retry.d.ts +36 -0
- package/dist/core/retry.js +46 -0
- package/dist/core/secrets.d.ts +98 -0
- package/dist/core/secrets.js +507 -0
- package/dist/core/stages/context.d.ts +131 -23
- package/dist/core/stages/context.js +331 -46
- package/dist/core/stages/panel.d.ts +13 -6
- package/dist/core/stages/panel.js +79 -19
- package/dist/core/stages/research.d.ts +110 -25
- package/dist/core/stages/research.js +338 -44
- package/dist/core/tokens.d.ts +31 -0
- package/dist/core/tokens.js +81 -0
- package/dist/core/types.d.ts +280 -25
- package/dist/core/verify.d.ts +6 -0
- package/dist/core/verify.js +6 -3
- package/dist/git/run.d.ts +223 -0
- package/dist/git/run.js +608 -0
- package/dist/git/tree.d.ts +43 -0
- package/dist/git/tree.js +171 -0
- package/dist/index.d.ts +26 -2
- package/dist/index.js +34 -2
- package/dist/providers/anthropic.d.ts +3 -1
- package/dist/providers/anthropic.js +10 -3
- package/dist/providers/claude-code.d.ts +56 -11
- package/dist/providers/claude-code.js +70 -25
- package/dist/providers/gemini.d.ts +5 -1
- package/dist/providers/gemini.js +65 -11
- package/dist/providers/http.d.ts +49 -4
- package/dist/providers/http.js +155 -14
- package/dist/providers/ollama.d.ts +4 -1
- package/dist/providers/ollama.js +13 -5
- package/dist/providers/openai-compatible.d.ts +4 -1
- package/dist/providers/openai-compatible.js +13 -5
- package/dist/providers/options.d.ts +12 -3
- package/dist/providers/options.js +30 -10
- package/dist/providers/size.d.ts +19 -15
- package/dist/providers/size.js +33 -73
- package/dist/sources/added.d.ts +49 -0
- package/dist/sources/added.js +134 -0
- package/dist/sources/diff.d.ts +13 -0
- package/dist/sources/diff.js +63 -0
- package/dist/sources/local.d.ts +41 -0
- package/dist/sources/local.js +251 -0
- package/dist/sources/paths.d.ts +45 -0
- package/dist/sources/paths.js +230 -0
- package/dist/sources/range.d.ts +21 -0
- package/dist/sources/range.js +75 -0
- package/dist/sources/read.d.ts +69 -0
- package/dist/sources/read.js +37 -0
- package/dist/sources/repository.d.ts +9 -0
- package/dist/sources/repository.js +28 -0
- package/dist/sources/skips.d.ts +49 -0
- package/dist/sources/skips.js +126 -0
- package/package.json +1 -1
- package/schema/quorum.schema.json +49 -1
|
@@ -10,7 +10,10 @@ export type OllamaOptions = HttpProviderOptions;
|
|
|
10
10
|
* stop as a truncation, instead of part of the prompt dropped so the reply can go on (Ollama 0.12.6 or
|
|
11
11
|
* later; an older server ignores both fields). It sends `contextTokens` as `options.num_ctx`, so the server
|
|
12
12
|
* loads the context the size guard counts with, and refuses, before any request, a request that may not
|
|
13
|
-
* fit it.
|
|
13
|
+
* fit it. It sends the reply's limit, `maxTokens` else the budget of the size guard, as `options.num_predict`,
|
|
14
|
+
* so the server bounds a reply's length. It states its worst case from the count its size guard keeps: the
|
|
15
|
+
* text it sends, the template reserve and that budget, and the room its guard leaves from the same count:
|
|
16
|
+
* `contextTokens` less those three.
|
|
14
17
|
*
|
|
15
18
|
* @throws ConfigError with every problem of the options, before any request
|
|
16
19
|
*/
|
package/dist/providers/ollama.js
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { issuePath } from "../core/errors.js";
|
|
2
|
+
import { DEFAULT_RETRIES, DEFAULT_TIMEOUT_MS } from "../core/ports.js";
|
|
2
3
|
import { postJson, snippet } from "./http.js";
|
|
3
4
|
import { shapeMessages } from "./messages.js";
|
|
4
5
|
import { bodyKeyForm, isRecord, mergeBody, prepareHttp, spellingIssues } from "./options.js";
|
|
5
6
|
import { MAX_MODEL_TEXT, replyFailure, tokenUsage } from "./reply.js";
|
|
6
|
-
import { checkFits, contextBudget } from "./size.js";
|
|
7
|
+
import { checkFits, contextBudget, roomTokens, worstCaseTokens } from "./size.js";
|
|
7
8
|
/**
|
|
8
9
|
* The protocol's rules. The tool fields: `tools`, its only one, and `_debug_render_only`, which answers
|
|
9
10
|
* 200 with the rendered template instead of a reply.
|
|
@@ -98,21 +99,28 @@ function readReply(reply, { id, model }, redact) {
|
|
|
98
99
|
* stop as a truncation, instead of part of the prompt dropped so the reply can go on (Ollama 0.12.6 or
|
|
99
100
|
* later; an older server ignores both fields). It sends `contextTokens` as `options.num_ctx`, so the server
|
|
100
101
|
* loads the context the size guard counts with, and refuses, before any request, a request that may not
|
|
101
|
-
* fit it.
|
|
102
|
+
* fit it. It sends the reply's limit, `maxTokens` else the budget of the size guard, as `options.num_predict`,
|
|
103
|
+
* so the server bounds a reply's length. It states its worst case from the count its size guard keeps: the
|
|
104
|
+
* text it sends, the template reserve and that budget, and the room its guard leaves from the same count:
|
|
105
|
+
* `contextTokens` less those three.
|
|
102
106
|
*
|
|
103
107
|
* @throws ConfigError with every problem of the options, before any request
|
|
104
108
|
*/
|
|
105
109
|
export function ollama(options) {
|
|
106
|
-
const { id, model, auth = 'bearer', temperature = 0, maxTokens, contextTokens, systemPrompt = 'native', structuredOutput = 'json' } = options;
|
|
110
|
+
const { id, model, auth = 'bearer', temperature = 0, maxTokens, contextTokens, systemPrompt = 'native', structuredOutput = 'json', timeoutMs = DEFAULT_TIMEOUT_MS, retries = DEFAULT_RETRIES } = options;
|
|
107
111
|
const { url, headers, redact, extraBody } = prepareHttp(options, { ...RULES, auth }, '/api/chat', numCtxIssues(id, options.extraBody));
|
|
108
112
|
const budget = contextBudget(contextTokens, maxTokens);
|
|
109
113
|
return {
|
|
110
114
|
id,
|
|
111
115
|
model,
|
|
116
|
+
timeoutMs,
|
|
117
|
+
retries,
|
|
118
|
+
worstCase: request => worstCaseTokens(shapeMessages(request, systemPrompt), request.output, budget),
|
|
119
|
+
room: request => roomTokens(shapeMessages(request, systemPrompt), request.output, budget),
|
|
112
120
|
async complete(request) {
|
|
113
121
|
const { system, messages } = shapeMessages(request, systemPrompt);
|
|
114
122
|
checkFits(id, { system, messages }, request.output, budget);
|
|
115
|
-
// JSON.stringify drops the undefined fields: no format for `prompt`, no temperature for null
|
|
123
|
+
// JSON.stringify drops the undefined fields: no format for `prompt`, no temperature for null.
|
|
116
124
|
const reply = await postJson({
|
|
117
125
|
providerId: id,
|
|
118
126
|
url,
|
|
@@ -128,7 +136,7 @@ export function ollama(options) {
|
|
|
128
136
|
stream: false,
|
|
129
137
|
truncate: false,
|
|
130
138
|
shift: false,
|
|
131
|
-
options: { temperature: temperature ?? undefined, num_predict:
|
|
139
|
+
options: { temperature: temperature ?? undefined, num_predict: budget.replyTokens, num_ctx: contextTokens },
|
|
132
140
|
format: format(structuredOutput, request.output),
|
|
133
141
|
}, extraBody),
|
|
134
142
|
signal: request.signal,
|
|
@@ -8,7 +8,10 @@ export interface OpenAICompatibleOptions extends HttpProviderOptions {
|
|
|
8
8
|
/**
|
|
9
9
|
* A provider for any OpenAI-compatible `/chat/completions` endpoint. The key, when there is one, travels
|
|
10
10
|
* only in the header `auth` names, `Authorization: Bearer` by default (invariant I1). It refuses, before
|
|
11
|
-
* any request, a request that may not fit `contextTokens`.
|
|
11
|
+
* any request, a request that may not fit `contextTokens`. It sends the reply's limit, `maxTokens` else the
|
|
12
|
+
* budget of the size guard, under `maxTokensField`, so the server bounds a reply's length. It states its
|
|
13
|
+
* worst case from the count its size guard keeps: the text it sends, the template reserve and that budget,
|
|
14
|
+
* and the room its guard leaves from the same count: `contextTokens` less those three.
|
|
12
15
|
*
|
|
13
16
|
* @throws ConfigError with every problem of the options, before any request
|
|
14
17
|
*/
|
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { DEFAULT_RETRIES, DEFAULT_TIMEOUT_MS } from "../core/ports.js";
|
|
1
2
|
import { postJson, snippet } from "./http.js";
|
|
2
3
|
import { shapeMessages } from "./messages.js";
|
|
3
4
|
import { isRecord, mergeBody, prepareHttp } from "./options.js";
|
|
4
5
|
import { endedEarly, joinedText, MAX_MODEL_TEXT, replyFailure, tokenUsage } from "./reply.js";
|
|
5
|
-
import { checkFits, contextBudget } from "./size.js";
|
|
6
|
+
import { checkFits, contextBudget, roomTokens, worstCaseTokens } from "./size.js";
|
|
6
7
|
/**
|
|
7
8
|
* The protocol's rules. The tool fields are the union over the OpenAI-compatible vendors: OpenAI's and
|
|
8
9
|
* the others' function tools and web search, OpenRouter's plugins, server-tool loop, presets and model
|
|
@@ -145,12 +146,15 @@ function readReply(reply, { id, model }, redact) {
|
|
|
145
146
|
/**
|
|
146
147
|
* A provider for any OpenAI-compatible `/chat/completions` endpoint. The key, when there is one, travels
|
|
147
148
|
* only in the header `auth` names, `Authorization: Bearer` by default (invariant I1). It refuses, before
|
|
148
|
-
* any request, a request that may not fit `contextTokens`.
|
|
149
|
+
* any request, a request that may not fit `contextTokens`. It sends the reply's limit, `maxTokens` else the
|
|
150
|
+
* budget of the size guard, under `maxTokensField`, so the server bounds a reply's length. It states its
|
|
151
|
+
* worst case from the count its size guard keeps: the text it sends, the template reserve and that budget,
|
|
152
|
+
* and the room its guard leaves from the same count: `contextTokens` less those three.
|
|
149
153
|
*
|
|
150
154
|
* @throws ConfigError with every problem of the options, before any request
|
|
151
155
|
*/
|
|
152
156
|
export function openaiCompatible(options) {
|
|
153
|
-
const { id, model, auth = 'bearer', temperature = 0, maxTokens, contextTokens, maxTokensField = 'max_tokens', systemPrompt = 'native', structuredOutput = 'json' } = options;
|
|
157
|
+
const { id, model, auth = 'bearer', temperature = 0, maxTokens, contextTokens, maxTokensField = 'max_tokens', systemPrompt = 'native', structuredOutput = 'json', timeoutMs = DEFAULT_TIMEOUT_MS, retries = DEFAULT_RETRIES } = options;
|
|
154
158
|
const fieldIssues = MAX_TOKENS_FIELDS.has(maxTokensField)
|
|
155
159
|
? []
|
|
156
160
|
: [
|
|
@@ -161,10 +165,14 @@ export function openaiCompatible(options) {
|
|
|
161
165
|
return {
|
|
162
166
|
id,
|
|
163
167
|
model,
|
|
168
|
+
timeoutMs,
|
|
169
|
+
retries,
|
|
170
|
+
worstCase: request => worstCaseTokens(shapeMessages(request, systemPrompt), request.output, budget),
|
|
171
|
+
room: request => roomTokens(shapeMessages(request, systemPrompt), request.output, budget),
|
|
164
172
|
async complete(request) {
|
|
165
173
|
const { system, messages } = shapeMessages(request, systemPrompt);
|
|
166
174
|
checkFits(id, { system, messages }, request.output, budget);
|
|
167
|
-
// JSON.stringify drops the undefined fields: no response_format for `prompt`, no temperature for null
|
|
175
|
+
// JSON.stringify drops the undefined fields: no response_format for `prompt`, no temperature for null.
|
|
168
176
|
const reply = await postJson({
|
|
169
177
|
providerId: id,
|
|
170
178
|
url,
|
|
@@ -179,7 +187,7 @@ export function openaiCompatible(options) {
|
|
|
179
187
|
],
|
|
180
188
|
response_format: responseFormat(structuredOutput, request.output),
|
|
181
189
|
temperature: temperature ?? undefined,
|
|
182
|
-
[maxTokensField]:
|
|
190
|
+
[maxTokensField]: budget.replyTokens,
|
|
183
191
|
}, extraBody),
|
|
184
192
|
signal: request.signal,
|
|
185
193
|
redact,
|
|
@@ -44,10 +44,14 @@ export interface HttpProviderOptions {
|
|
|
44
44
|
extraBody?: Readonly<Record<string, unknown>>;
|
|
45
45
|
/** `null` leaves the field out, for models that reject it. */
|
|
46
46
|
temperature?: number | null;
|
|
47
|
-
/** The most tokens the reply may hold. */
|
|
47
|
+
/** The most tokens the reply may hold, thinking included, sent as the server's limit. Default 16000. */
|
|
48
48
|
maxTokens?: number;
|
|
49
49
|
systemPrompt?: SystemPrompt;
|
|
50
50
|
structuredOutput?: StructuredOutput;
|
|
51
|
+
/** The deadline of one attempt, in milliseconds, from 1000 to 86400000. Default 600000. */
|
|
52
|
+
timeoutMs?: number;
|
|
53
|
+
/** The retries of a call that failed in passing, from 0 to 5. Default 2. */
|
|
54
|
+
retries?: number;
|
|
51
55
|
}
|
|
52
56
|
/** What an adapter declares of its protocol, which the shared checks hold the options to. */
|
|
53
57
|
export interface AdapterRules {
|
|
@@ -97,6 +101,8 @@ export declare function bodyKeyForm(key: string): string;
|
|
|
97
101
|
export declare function spellingIssues(id: string, key: string, canonical: string): ConfigIssue[];
|
|
98
102
|
/** The words of a list as a message names them: commas between them, and the conjunction before the last. */
|
|
99
103
|
export declare function wordList(words: readonly string[], conjunction?: 'or' | 'and'): string;
|
|
104
|
+
/** The problem of `maxTokens`, the reply's limit, when it is set and is not a whole number of at least 1. */
|
|
105
|
+
export declare function maxTokensIssues(id: string, maxTokens: unknown): ConfigIssue[];
|
|
100
106
|
/**
|
|
101
107
|
* The problems of `contextTokens`: not set, not a whole number of at least 1, or no more than the reply budget
|
|
102
108
|
* and the template reserve (`TEMPLATE_TOKENS` unless the adapter names one), which leaves no room for a
|
|
@@ -104,6 +110,11 @@ export declare function wordList(words: readonly string[], conjunction?: 'or' |
|
|
|
104
110
|
* one has its own issue.
|
|
105
111
|
*/
|
|
106
112
|
export declare function contextTokensIssues(id: string, contextTokens: number | undefined, maxTokens: number | undefined, templateTokens?: number): ConfigIssue[];
|
|
113
|
+
/**
|
|
114
|
+
* The problems of the deadline and the retries an adapter takes, each at `<id>.<field>` when it is set
|
|
115
|
+
* and out of range: the deadline first. No message repeats a value.
|
|
116
|
+
*/
|
|
117
|
+
export declare function attemptIssues(id: string, timeoutMs: unknown, retries: unknown): ConfigIssue[];
|
|
107
118
|
/**
|
|
108
119
|
* The problems of the options every HTTP adapter shares, under the rules the adapter declares. No
|
|
109
120
|
* message repeats a value: a key, a header value or a URL can be a secret.
|
|
@@ -121,7 +132,5 @@ export declare function prepareHttp(options: HttpProviderOptions, rules: Adapter
|
|
|
121
132
|
export declare function requestHeaders(auth: Auth, apiKey: string | undefined, headers?: Readonly<Record<string, string>>): Record<string, string>;
|
|
122
133
|
/** `body` with `extra` deep-merged into it: objects merge key by key, anything else, arrays included, replaces. */
|
|
123
134
|
export declare function mergeBody(body: Record<string, unknown>, extra: Readonly<Record<string, unknown>> | undefined): Record<string, unknown>;
|
|
124
|
-
/** `fields`, or undefined when every one of them is undefined, so the body leaves out an object that would be empty. */
|
|
125
|
-
export declare function unlessEmpty(fields: Record<string, unknown>): Record<string, unknown> | undefined;
|
|
126
135
|
/** Whether `value` is a non-null object that is not an array. */
|
|
127
136
|
export declare function isRecord(value: unknown): value is Record<string, unknown>;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ConfigError, issuePath } from "../core/errors.js";
|
|
2
|
+
import { retriesProblem, timeoutProblem } from "../core/ports.js";
|
|
2
3
|
import { endpointUrl, redactionsOf } from "./http.js";
|
|
3
4
|
import { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS } from "./size.js";
|
|
4
5
|
const HTTP_URL_WITH_HOST = /^https?:\/\/[^/\\?#\s]/i;
|
|
@@ -209,6 +210,14 @@ export function wordList(words, conjunction = 'or') {
|
|
|
209
210
|
function isTokenCount(value) {
|
|
210
211
|
return Number.isSafeInteger(value) && value >= 1;
|
|
211
212
|
}
|
|
213
|
+
/** The problem of `maxTokens`, the reply's limit, when it is set and is not a whole number of at least 1. */
|
|
214
|
+
export function maxTokensIssues(id, maxTokens) {
|
|
215
|
+
if (maxTokens === undefined || (typeof maxTokens === 'number' && isTokenCount(maxTokens)))
|
|
216
|
+
return [];
|
|
217
|
+
return [
|
|
218
|
+
{ path: `${id}.maxTokens`, message: 'must be a whole number of at least 1' },
|
|
219
|
+
];
|
|
220
|
+
}
|
|
212
221
|
/**
|
|
213
222
|
* The problems of `contextTokens`: not set, not a whole number of at least 1, or no more than the reply budget
|
|
214
223
|
* and the template reserve (`TEMPLATE_TOKENS` unless the adapter names one), which leaves no room for a
|
|
@@ -236,12 +245,30 @@ export function contextTokensIssues(id, contextTokens, maxTokens, templateTokens
|
|
|
236
245
|
{ path, message: `must be more than ${reply + templateTokens}, the ${reply} tokens of the reply budget and ${templateTokens} for the chat template` },
|
|
237
246
|
];
|
|
238
247
|
}
|
|
248
|
+
/**
|
|
249
|
+
* The problems of the deadline and the retries an adapter takes, each at `<id>.<field>` when it is set
|
|
250
|
+
* and out of range: the deadline first. No message repeats a value.
|
|
251
|
+
*/
|
|
252
|
+
export function attemptIssues(id, timeoutMs, retries) {
|
|
253
|
+
const issues = [];
|
|
254
|
+
const deadline = timeoutMs === undefined
|
|
255
|
+
? undefined
|
|
256
|
+
: timeoutProblem(timeoutMs);
|
|
257
|
+
if (deadline !== undefined)
|
|
258
|
+
issues.push({ path: `${id}.timeoutMs`, message: deadline });
|
|
259
|
+
const count = retries === undefined
|
|
260
|
+
? undefined
|
|
261
|
+
: retriesProblem(retries);
|
|
262
|
+
if (count !== undefined)
|
|
263
|
+
issues.push({ path: `${id}.retries`, message: count });
|
|
264
|
+
return issues;
|
|
265
|
+
}
|
|
239
266
|
/**
|
|
240
267
|
* The problems of the options every HTTP adapter shares, under the rules the adapter declares. No
|
|
241
268
|
* message repeats a value: a key, a header value or a URL can be a secret.
|
|
242
269
|
*/
|
|
243
270
|
export function httpOptionIssues(options, rules) {
|
|
244
|
-
const { id, baseUrl, apiKey, headers, query, extraBody, temperature, maxTokens, contextTokens, systemPrompt, structuredOutput } = options;
|
|
271
|
+
const { id, baseUrl, apiKey, headers, query, extraBody, temperature, maxTokens, contextTokens, systemPrompt, structuredOutput, timeoutMs, retries } = options;
|
|
245
272
|
const { auth, structuredOutputs } = rules;
|
|
246
273
|
const fixedHeaders = rules.fixedHeaders.map(name => name.toLowerCase());
|
|
247
274
|
const issues = [
|
|
@@ -275,15 +302,14 @@ export function httpOptionIssues(options, rules) {
|
|
|
275
302
|
if (temperature !== undefined && temperature !== null && !Number.isFinite(temperature)) {
|
|
276
303
|
issues.push({ path: `${id}.temperature`, message: 'must be a finite number, or null to leave it out' });
|
|
277
304
|
}
|
|
278
|
-
|
|
279
|
-
issues.push({ path: `${id}.maxTokens`, message: 'must be a whole number of at least 1' });
|
|
280
|
-
}
|
|
305
|
+
issues.push(...maxTokensIssues(id, maxTokens));
|
|
281
306
|
issues.push(...contextTokensIssues(id, contextTokens, maxTokens));
|
|
282
307
|
if (systemPrompt !== undefined && !SYSTEM_PROMPTS.has(systemPrompt))
|
|
283
308
|
issues.push({ path: `${id}.systemPrompt`, message: 'must be native or inline' });
|
|
284
309
|
if (structuredOutput !== undefined && !structuredOutputs.includes(structuredOutput)) {
|
|
285
310
|
issues.push({ path: `${id}.structuredOutput`, message: `must be ${wordList(structuredOutputs)}` });
|
|
286
311
|
}
|
|
312
|
+
issues.push(...attemptIssues(id, timeoutMs, retries));
|
|
287
313
|
return issues;
|
|
288
314
|
}
|
|
289
315
|
/**
|
|
@@ -339,12 +365,6 @@ export function mergeBody(body, extra) {
|
|
|
339
365
|
}
|
|
340
366
|
return merged;
|
|
341
367
|
}
|
|
342
|
-
/** `fields`, or undefined when every one of them is undefined, so the body leaves out an object that would be empty. */
|
|
343
|
-
export function unlessEmpty(fields) {
|
|
344
|
-
return Object.values(fields).every(value => value === undefined)
|
|
345
|
-
? undefined
|
|
346
|
-
: fields;
|
|
347
|
-
}
|
|
348
368
|
/** Whether `value` is a non-null object that is not an array. */
|
|
349
369
|
export function isRecord(value) {
|
|
350
370
|
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
package/dist/providers/size.d.ts
CHANGED
|
@@ -1,16 +1,6 @@
|
|
|
1
1
|
import type { CompletionRequest } from '../core/ports.ts';
|
|
2
2
|
import type { ShapedMessages } from './messages.ts';
|
|
3
|
-
|
|
4
|
-
* Tokens kept for what a server adds around the messages: the chat template's role markers, a thinking
|
|
5
|
-
* or Harmony preamble, a tokenizer's dummy prefix. Measured at 10 to 29 tokens for Llama 3 and Qwen 2.5;
|
|
6
|
-
* the margin is many times that, since a preamble can be longer than any measured.
|
|
7
|
-
*/
|
|
8
|
-
export declare const TEMPLATE_TOKENS = 512;
|
|
9
|
-
/**
|
|
10
|
-
* Tokens kept for the reply when `maxTokens` is not set: the limit `anthropic` sends then, and room for a
|
|
11
|
-
* thinking model's longest reply. Every other adapter sends no limit, so there it is room the guard keeps.
|
|
12
|
-
*/
|
|
13
|
-
export declare const DEFAULT_REPLY_TOKENS = 16000;
|
|
3
|
+
export { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS, utf8Length } from '../core/tokens.ts';
|
|
14
4
|
/** The model's context window, the tokens a reply may take of it, and the tokens kept for what is added around the messages. */
|
|
15
5
|
export interface ContextBudget {
|
|
16
6
|
readonly contextTokens: number;
|
|
@@ -24,8 +14,6 @@ export interface ContextBudget {
|
|
|
24
14
|
* chat template raises.
|
|
25
15
|
*/
|
|
26
16
|
export declare function contextBudget(contextTokens: number, maxTokens: number | undefined, templateTokens?: number): ContextBudget;
|
|
27
|
-
/** The length of `text` in UTF-8 bytes. A lone surrogate counts the 3 bytes of U+FFFD, as the encoder writes it. */
|
|
28
|
-
export declare function utf8Length(text: string): number;
|
|
29
17
|
/**
|
|
30
18
|
* A bound of the tokens in the text of a request: one per UTF-8 byte of the system text, of every message,
|
|
31
19
|
* and of the reply schema's JSON, each counted at the sum of its code points' longest forms. One token per
|
|
@@ -35,14 +23,30 @@ export declare function utf8Length(text: string): number;
|
|
|
35
23
|
* the count is not the raw bytes. One that folds case is not covered; no chat model's tokenizer is known
|
|
36
24
|
* to. The price is that accented letters count one more byte per accent and Hangul syllables up to three
|
|
37
25
|
* times their size. The schema counts once more than the body carries it, since a server may add it to the
|
|
38
|
-
* prompt.
|
|
26
|
+
* prompt. The core counts a request's text with the same bound (`requestTokens`), so the worst case an adapter
|
|
27
|
+
* states and the core's own count never drift.
|
|
39
28
|
*/
|
|
40
29
|
export declare function textTokens(shaped: ShapedMessages, output: CompletionRequest['output']): number;
|
|
30
|
+
/**
|
|
31
|
+
* The most tokens, input and output, that one attempt of a request may spend as the provider is billed for
|
|
32
|
+
* it, from what the guard counts: each of `turns` model turns sends the text and the template reserve, and
|
|
33
|
+
* the reply of every turn but the last is the input of the next, so the replies count `2 * turns - 1` times.
|
|
34
|
+
* One turn is the text, the template reserve and the reply budget, which `checkFits` keeps within the context.
|
|
35
|
+
*/
|
|
36
|
+
export declare function worstCaseTokens(shaped: ShapedMessages, output: CompletionRequest['output'], budget: ContextBudget, turns?: number): number;
|
|
37
|
+
/**
|
|
38
|
+
* How many more tokens of text a request could hold and still fit the model's context, from what the guard
|
|
39
|
+
* counts: the context less the text, the budget's template reserve (`TEMPLATE_TOKENS` unless it names one)
|
|
40
|
+
* and the reply budget. It is negative exactly when the request does not fit, and `checkFits` reads the same
|
|
41
|
+
* count, so the guard and the room an adapter states never drift: a request at a room of 0 is sent, one at -1
|
|
42
|
+
* is refused.
|
|
43
|
+
*/
|
|
44
|
+
export declare function roomTokens(shaped: ShapedMessages, output: CompletionRequest['output'], budget: ContextBudget): number;
|
|
41
45
|
/**
|
|
42
46
|
* Refuse, before it is sent, a request that may not fit the model's context: some endpoints cut an oversized
|
|
43
47
|
* prompt in silence and answer 200, which would leave a review of text the model never read. The request
|
|
44
48
|
* fits only when its text, the budget's template reserve (`TEMPLATE_TOKENS` unless it names one) and the
|
|
45
|
-
* reply budget together stay within `contextTokens
|
|
49
|
+
* reply budget together stay within `contextTokens`, which is when its room is not negative.
|
|
46
50
|
* The message holds the provider id and numbers, never the text.
|
|
47
51
|
*
|
|
48
52
|
* @throws ProviderError of kind `too-large`, which is not retryable
|
package/dist/providers/size.js
CHANGED
|
@@ -1,16 +1,6 @@
|
|
|
1
1
|
import { ProviderError } from "../core/errors.js";
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
* or Harmony preamble, a tokenizer's dummy prefix. Measured at 10 to 29 tokens for Llama 3 and Qwen 2.5;
|
|
5
|
-
* the margin is many times that, since a preamble can be longer than any measured.
|
|
6
|
-
*/
|
|
7
|
-
export const TEMPLATE_TOKENS = 512;
|
|
8
|
-
/**
|
|
9
|
-
* Tokens kept for the reply when `maxTokens` is not set: the limit `anthropic` sends then, and room for a
|
|
10
|
-
* thinking model's longest reply. Every other adapter sends no limit, so there it is room the guard keeps.
|
|
11
|
-
*/
|
|
12
|
-
export const DEFAULT_REPLY_TOKENS = 16_000;
|
|
13
|
-
const encoder = new TextEncoder();
|
|
2
|
+
import { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS, tokenBound } from "../core/tokens.js";
|
|
3
|
+
export { DEFAULT_REPLY_TOKENS, TEMPLATE_TOKENS, utf8Length } from "../core/tokens.js";
|
|
14
4
|
/**
|
|
15
5
|
* The budget a request is checked against: the context size, the reply budget, `maxTokens` else
|
|
16
6
|
* `DEFAULT_REPLY_TOKENS`, and the template reserve, which an adapter whose server adds more context than a
|
|
@@ -19,61 +9,6 @@ const encoder = new TextEncoder();
|
|
|
19
9
|
export function contextBudget(contextTokens, maxTokens, templateTokens = TEMPLATE_TOKENS) {
|
|
20
10
|
return { contextTokens, replyTokens: maxTokens ?? DEFAULT_REPLY_TOKENS, templateTokens };
|
|
21
11
|
}
|
|
22
|
-
/** The length of `text` in UTF-8 bytes. A lone surrogate counts the 3 bytes of U+FFFD, as the encoder writes it. */
|
|
23
|
-
export function utf8Length(text) {
|
|
24
|
-
return encoder.encode(text).length;
|
|
25
|
-
}
|
|
26
|
-
/**
|
|
27
|
-
* The bytes of the longest form of each code point, 0 until it is first seen, so a repeated character is
|
|
28
|
-
* normalized once. A fixed table of 1.1 MB, where a map grew past 200 MB on a text holding every code
|
|
29
|
-
* point; the longest form, U+FDFA's, is 33 bytes, so a byte holds any of them.
|
|
30
|
-
*/
|
|
31
|
-
const longestFormBytes = new Uint8Array(0x110000);
|
|
32
|
-
/**
|
|
33
|
-
* The most UTF-8 bytes a code point takes as sent, decomposed (NFD) or compatibility-decomposed (NFKD): U+0F81
|
|
34
|
-
* is 3 bytes and 6 decomposed, U+00E9 is 2 and 3, a Hangul syllable up to 3 times its size, U+FDFA is 3 and 33.
|
|
35
|
-
* A lone surrogate is 3, the bytes of the U+FFFD the encoder writes for it.
|
|
36
|
-
*/
|
|
37
|
-
function longestFormLength(codePoint) {
|
|
38
|
-
if (codePoint >= 0xD800 && codePoint <= 0xDFFF)
|
|
39
|
-
return 3;
|
|
40
|
-
const cached = longestFormBytes[codePoint];
|
|
41
|
-
if (cached !== undefined && cached !== 0)
|
|
42
|
-
return cached;
|
|
43
|
-
const char = String.fromCodePoint(codePoint);
|
|
44
|
-
const bytes = Math.max(utf8Length(char), utf8Length(char.normalize('NFD')), utf8Length(char.normalize('NFKD')));
|
|
45
|
-
longestFormBytes[codePoint] = bytes;
|
|
46
|
-
return bytes;
|
|
47
|
-
}
|
|
48
|
-
/**
|
|
49
|
-
* A bound of the UTF-8 bytes of `text` in any normalization: the sum over its code points of each one's
|
|
50
|
-
* longest form. The text itself is never normalized, since that takes quadratic time on a long run of
|
|
51
|
-
* combining marks (canonical reordering) and a gigabyte on a run of U+FDFA. It is a bound because
|
|
52
|
-
* decomposition works on one code point at a time, reordering only permutes code points, and composing
|
|
53
|
-
* never lengthens UTF-8 (true of every code point, which `size.spec.ts` checks), so no NFC, NFKC, NFD or
|
|
54
|
-
* NFKD form of the whole text is longer. Normalizing each character as NFC, or the text in chunks, is not
|
|
55
|
-
* a bound: a mark that composes first, or one that crosses a chunk, can lengthen the whole.
|
|
56
|
-
*/
|
|
57
|
-
function normalizedLength(text) {
|
|
58
|
-
let bytes = 0;
|
|
59
|
-
for (let index = 0; index < text.length; index++) {
|
|
60
|
-
const unit = text.charCodeAt(index);
|
|
61
|
-
if (unit < 0x80) {
|
|
62
|
-
bytes += 1;
|
|
63
|
-
continue;
|
|
64
|
-
}
|
|
65
|
-
const low = index + 1 < text.length
|
|
66
|
-
? text.charCodeAt(index + 1)
|
|
67
|
-
: 0;
|
|
68
|
-
const pair = unit >= 0xD800 && unit <= 0xDBFF && low >= 0xDC00 && low <= 0xDFFF;
|
|
69
|
-
bytes += longestFormLength(pair
|
|
70
|
-
? 0x10000 + ((unit - 0xD800) << 10) + (low - 0xDC00)
|
|
71
|
-
: unit);
|
|
72
|
-
if (pair)
|
|
73
|
-
index++;
|
|
74
|
-
}
|
|
75
|
-
return bytes;
|
|
76
|
-
}
|
|
77
12
|
/**
|
|
78
13
|
* A bound of the tokens in the text of a request: one per UTF-8 byte of the system text, of every message,
|
|
79
14
|
* and of the reply schema's JSON, each counted at the sum of its code points' longest forms. One token per
|
|
@@ -83,7 +18,8 @@ function normalizedLength(text) {
|
|
|
83
18
|
* the count is not the raw bytes. One that folds case is not covered; no chat model's tokenizer is known
|
|
84
19
|
* to. The price is that accented letters count one more byte per accent and Hangul syllables up to three
|
|
85
20
|
* times their size. The schema counts once more than the body carries it, since a server may add it to the
|
|
86
|
-
* prompt.
|
|
21
|
+
* prompt. The core counts a request's text with the same bound (`requestTokens`), so the worst case an adapter
|
|
22
|
+
* states and the core's own count never drift.
|
|
87
23
|
*/
|
|
88
24
|
export function textTokens(shaped, output) {
|
|
89
25
|
const texts = [
|
|
@@ -91,21 +27,45 @@ export function textTokens(shaped, output) {
|
|
|
91
27
|
...shaped.messages.map(message => message.content),
|
|
92
28
|
JSON.stringify(output.schema),
|
|
93
29
|
];
|
|
94
|
-
return texts.reduce((sum, text) => sum +
|
|
30
|
+
return texts.reduce((sum, text) => sum + tokenBound(text), 0);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* The most tokens, input and output, that one attempt of a request may spend as the provider is billed for
|
|
34
|
+
* it, from what the guard counts: each of `turns` model turns sends the text and the template reserve, and
|
|
35
|
+
* the reply of every turn but the last is the input of the next, so the replies count `2 * turns - 1` times.
|
|
36
|
+
* One turn is the text, the template reserve and the reply budget, which `checkFits` keeps within the context.
|
|
37
|
+
*/
|
|
38
|
+
export function worstCaseTokens(shaped, output, budget, turns = 1) {
|
|
39
|
+
const template = budget.templateTokens ?? TEMPLATE_TOKENS;
|
|
40
|
+
return turns * (textTokens(shaped, output) + template) + (2 * turns - 1) * budget.replyTokens;
|
|
41
|
+
}
|
|
42
|
+
/** What a request needs of its context: the tokens of its text, the template reserve, the reply budget, and their sum. */
|
|
43
|
+
function neededOf(shaped, output, budget) {
|
|
44
|
+
const text = textTokens(shaped, output);
|
|
45
|
+
const template = budget.templateTokens ?? TEMPLATE_TOKENS;
|
|
46
|
+
return { text, template, need: text + template + budget.replyTokens };
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* How many more tokens of text a request could hold and still fit the model's context, from what the guard
|
|
50
|
+
* counts: the context less the text, the budget's template reserve (`TEMPLATE_TOKENS` unless it names one)
|
|
51
|
+
* and the reply budget. It is negative exactly when the request does not fit, and `checkFits` reads the same
|
|
52
|
+
* count, so the guard and the room an adapter states never drift: a request at a room of 0 is sent, one at -1
|
|
53
|
+
* is refused.
|
|
54
|
+
*/
|
|
55
|
+
export function roomTokens(shaped, output, budget) {
|
|
56
|
+
return budget.contextTokens - neededOf(shaped, output, budget).need;
|
|
95
57
|
}
|
|
96
58
|
/**
|
|
97
59
|
* Refuse, before it is sent, a request that may not fit the model's context: some endpoints cut an oversized
|
|
98
60
|
* prompt in silence and answer 200, which would leave a review of text the model never read. The request
|
|
99
61
|
* fits only when its text, the budget's template reserve (`TEMPLATE_TOKENS` unless it names one) and the
|
|
100
|
-
* reply budget together stay within `contextTokens
|
|
62
|
+
* reply budget together stay within `contextTokens`, which is when its room is not negative.
|
|
101
63
|
* The message holds the provider id and numbers, never the text.
|
|
102
64
|
*
|
|
103
65
|
* @throws ProviderError of kind `too-large`, which is not retryable
|
|
104
66
|
*/
|
|
105
67
|
export function checkFits(providerId, shaped, output, budget) {
|
|
106
|
-
const text =
|
|
107
|
-
const template = budget.templateTokens ?? TEMPLATE_TOKENS;
|
|
108
|
-
const need = text + template + budget.replyTokens;
|
|
68
|
+
const { text, template, need } = neededOf(shaped, output, budget);
|
|
109
69
|
if (need <= budget.contextTokens)
|
|
110
70
|
return;
|
|
111
71
|
throw new ProviderError({
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The all-added writer: files that have no diff of their own (a path review's, the untracked files of a
|
|
3
|
+
* local review) written as git writes a new file, so the core reads them as it reads any diff. The form
|
|
4
|
+
* is git's new-file form less its `index` line, with each name quoted as git quotes it, and a file cut
|
|
5
|
+
* into hunks of at most `HUNK_BYTES` at line ends, so research in parts spreads a large file across
|
|
6
|
+
* parts. It is a reading form and not a patch: `git apply` refuses a new file cut in several hunks.
|
|
7
|
+
*
|
|
8
|
+
* @see ../../docs/95-reference/40-architecture.md for where it sits in the layers.
|
|
9
|
+
*/
|
|
10
|
+
/** The most bytes of rows a hunk holds, line feeds included, a longer line aside, which goes alone in its hunk. */
|
|
11
|
+
export declare const HUNK_BYTES = 4096;
|
|
12
|
+
/** How many leading bytes git looks at for a NUL to call a file binary. */
|
|
13
|
+
export declare const BINARY_PROBE_BYTES = 8000;
|
|
14
|
+
/** The modes git gives a new file: a file, an executable, a link and a gitlink (a submodule's commit). */
|
|
15
|
+
export type AddedMode = '100644' | '100755' | '120000' | '160000';
|
|
16
|
+
/** One file to write. */
|
|
17
|
+
export interface AddedFile {
|
|
18
|
+
/** The path as git names it: bytes, not text, since a name can hold any byte but NUL. */
|
|
19
|
+
readonly path: Uint8Array;
|
|
20
|
+
readonly mode: AddedMode;
|
|
21
|
+
/** The file's bytes, or a link's target; empty for a gitlink, which holds another repository's commit. */
|
|
22
|
+
readonly content: Uint8Array;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* The inside of git's quoting of a name, as `core.quotePath=true` writes it: `\a \b \t \n \v \f \r \" \\`
|
|
26
|
+
* for those nine bytes and three octal digits for every other byte below 0x20 and from 0x7F up.
|
|
27
|
+
* `undefined` when git writes the name as it is.
|
|
28
|
+
*/
|
|
29
|
+
export declare function quotedPath(raw: Uint8Array): string | undefined;
|
|
30
|
+
/**
|
|
31
|
+
* The name `diffSections` reads back from the header this writer writes for `raw`: the name itself when
|
|
32
|
+
* git writes it as it is, else `decodeQuoted`, the core's own reading of git's quoted names, of its
|
|
33
|
+
* quoting, so writer and reader cannot drift.
|
|
34
|
+
*/
|
|
35
|
+
export declare function pathName(raw: Uint8Array): string;
|
|
36
|
+
/** Whether git would call `content` binary: a NUL among its first `BINARY_PROBE_BYTES` bytes. */
|
|
37
|
+
export declare function isBinary(content: Uint8Array): boolean;
|
|
38
|
+
/** The text of a blob: a byte that is not UTF-8 becomes U+FFFD, as `decodeDiff` makes it of git's own output, and a leading byte-order mark stays. */
|
|
39
|
+
export declare function blobText(content: Uint8Array): string;
|
|
40
|
+
/**
|
|
41
|
+
* One file as git's new-file diff less its `index` line, with no final line feed: the header, the mode,
|
|
42
|
+
* then `--- /dev/null`, `+++` and the hunks, or git's binary line for a file with a NUL among its first
|
|
43
|
+
* 8000 bytes, or nothing more for an empty file and a gitlink. The `+++` line ends with a tab when its
|
|
44
|
+
* label holds a space, as git writes it, and a last line with no line feed is followed by git's
|
|
45
|
+
* ``.
|
|
46
|
+
*/
|
|
47
|
+
export declare function addedSection(file: AddedFile): string;
|
|
48
|
+
/** The sections of `files`, joined by line feeds and ending with one: `''` for no file. */
|
|
49
|
+
export declare function addedDiff(files: readonly AddedFile[]): string;
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
import { decodeQuoted } from "../core/diff.js";
|
|
2
|
+
import { utf8Length } from "../core/tokens.js";
|
|
3
|
+
/**
|
|
4
|
+
* The all-added writer: files that have no diff of their own (a path review's, the untracked files of a
|
|
5
|
+
* local review) written as git writes a new file, so the core reads them as it reads any diff. The form
|
|
6
|
+
* is git's new-file form less its `index` line, with each name quoted as git quotes it, and a file cut
|
|
7
|
+
* into hunks of at most `HUNK_BYTES` at line ends, so research in parts spreads a large file across
|
|
8
|
+
* parts. It is a reading form and not a patch: `git apply` refuses a new file cut in several hunks.
|
|
9
|
+
*
|
|
10
|
+
* @see ../../docs/95-reference/40-architecture.md for where it sits in the layers.
|
|
11
|
+
*/
|
|
12
|
+
/** The most bytes of rows a hunk holds, line feeds included, a longer line aside, which goes alone in its hunk. */
|
|
13
|
+
export const HUNK_BYTES = 4096;
|
|
14
|
+
/** How many leading bytes git looks at for a NUL to call a file binary. */
|
|
15
|
+
export const BINARY_PROBE_BYTES = 8000;
|
|
16
|
+
/** Reads a name git writes as it is, which holds printable ASCII only. */
|
|
17
|
+
const ASCII_DECODER = new TextDecoder();
|
|
18
|
+
const NAMED = {
|
|
19
|
+
0x07: 'a',
|
|
20
|
+
0x08: 'b',
|
|
21
|
+
0x09: 't',
|
|
22
|
+
0x0A: 'n',
|
|
23
|
+
0x0B: 'v',
|
|
24
|
+
0x0C: 'f',
|
|
25
|
+
0x0D: 'r',
|
|
26
|
+
0x22: '"',
|
|
27
|
+
0x5C: '\\',
|
|
28
|
+
};
|
|
29
|
+
// A byte's escape under core.quotePath=true, as git's cq_lookup writes it, or undefined when git writes it as it is.
|
|
30
|
+
function escapeOf(byte) {
|
|
31
|
+
return byte in NAMED
|
|
32
|
+
? `\\${NAMED[byte]}`
|
|
33
|
+
: byte < 0x20 || byte >= 0x7F
|
|
34
|
+
? `\\${byte.toString(8).padStart(3, '0')}`
|
|
35
|
+
: undefined;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* The inside of git's quoting of a name, as `core.quotePath=true` writes it: `\a \b \t \n \v \f \r \" \\`
|
|
39
|
+
* for those nine bytes and three octal digits for every other byte below 0x20 and from 0x7F up.
|
|
40
|
+
* `undefined` when git writes the name as it is.
|
|
41
|
+
*/
|
|
42
|
+
export function quotedPath(raw) {
|
|
43
|
+
const escapes = Array.from(raw, escapeOf);
|
|
44
|
+
return escapes.some(escape => escape !== undefined)
|
|
45
|
+
? escapes.map((escape, at) => escape ?? String.fromCharCode(raw[at])).join('')
|
|
46
|
+
: undefined;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* The name `diffSections` reads back from the header this writer writes for `raw`: the name itself when
|
|
50
|
+
* git writes it as it is, else `decodeQuoted`, the core's own reading of git's quoted names, of its
|
|
51
|
+
* quoting, so writer and reader cannot drift.
|
|
52
|
+
*/
|
|
53
|
+
export function pathName(raw) {
|
|
54
|
+
const quoted = quotedPath(raw);
|
|
55
|
+
return quoted === undefined
|
|
56
|
+
? ASCII_DECODER.decode(raw)
|
|
57
|
+
: decodeQuoted(quoted);
|
|
58
|
+
}
|
|
59
|
+
/** Whether git would call `content` binary: a NUL among its first `BINARY_PROBE_BYTES` bytes. */
|
|
60
|
+
export function isBinary(content) {
|
|
61
|
+
return content.subarray(0, BINARY_PROBE_BYTES).includes(0);
|
|
62
|
+
}
|
|
63
|
+
/** A label of a header: the name behind `a/` or `b/`, the whole label quoted when any byte of the name needs it. */
|
|
64
|
+
function labelOf(prefix, raw) {
|
|
65
|
+
const quoted = quotedPath(raw);
|
|
66
|
+
return quoted === undefined
|
|
67
|
+
? `${prefix}${ASCII_DECODER.decode(raw)}`
|
|
68
|
+
: `"${prefix}${quoted}"`;
|
|
69
|
+
}
|
|
70
|
+
const BLOB_DECODER = new TextDecoder('utf-8', { ignoreBOM: true });
|
|
71
|
+
/** The text of a blob: a byte that is not UTF-8 becomes U+FFFD, as `decodeDiff` makes it of git's own output, and a leading byte-order mark stays. */
|
|
72
|
+
export function blobText(content) {
|
|
73
|
+
return BLOB_DECODER.decode(content);
|
|
74
|
+
}
|
|
75
|
+
/** The lines of a text, split at line feeds, with no empty piece after a last one. */
|
|
76
|
+
function linesOf(text) {
|
|
77
|
+
const lines = text.split('\n');
|
|
78
|
+
if (lines.at(-1) === '')
|
|
79
|
+
lines.pop();
|
|
80
|
+
return lines;
|
|
81
|
+
}
|
|
82
|
+
/** A hunk's header: no old lines, and `count` new ones from line `first`, with no count for a single line, as git writes it. */
|
|
83
|
+
function hunkHeader(first, count) {
|
|
84
|
+
return count === 1
|
|
85
|
+
? `@@ -0,0 +${first} @@`
|
|
86
|
+
: `@@ -0,0 +${first},${count} @@`;
|
|
87
|
+
}
|
|
88
|
+
/** The hunks of `lines`: whole lines while their UTF-8 bytes, line feeds included, stay within `HUNK_BYTES`, a longer line alone. */
|
|
89
|
+
function hunksOf(lines) {
|
|
90
|
+
const hunks = [];
|
|
91
|
+
for (let start = 0; start < lines.length;) {
|
|
92
|
+
let end = start;
|
|
93
|
+
for (let bytes = 0; end < lines.length; end++) {
|
|
94
|
+
const size = utf8Length(lines[end]) + 1;
|
|
95
|
+
if (end > start && bytes + size > HUNK_BYTES)
|
|
96
|
+
break;
|
|
97
|
+
bytes += size;
|
|
98
|
+
}
|
|
99
|
+
hunks.push([
|
|
100
|
+
hunkHeader(start + 1, end - start),
|
|
101
|
+
...lines.slice(start, end).map(line => `+${line}`),
|
|
102
|
+
].join('\n'));
|
|
103
|
+
start = end;
|
|
104
|
+
}
|
|
105
|
+
return hunks;
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* One file as git's new-file diff less its `index` line, with no final line feed: the header, the mode,
|
|
109
|
+
* then `--- /dev/null`, `+++` and the hunks, or git's binary line for a file with a NUL among its first
|
|
110
|
+
* 8000 bytes, or nothing more for an empty file and a gitlink. The `+++` line ends with a tab when its
|
|
111
|
+
* label holds a space, as git writes it, and a last line with no line feed is followed by git's
|
|
112
|
+
* ``.
|
|
113
|
+
*/
|
|
114
|
+
export function addedSection(file) {
|
|
115
|
+
const oldLabel = labelOf('a/', file.path);
|
|
116
|
+
const newLabel = labelOf('b/', file.path);
|
|
117
|
+
const lines = [`diff --git ${oldLabel} ${newLabel}`, `new file mode ${file.mode}`];
|
|
118
|
+
if (file.content.length === 0)
|
|
119
|
+
return lines.join('\n');
|
|
120
|
+
if (isBinary(file.content))
|
|
121
|
+
return [...lines, `Binary files /dev/null and ${newLabel} differ`].join('\n');
|
|
122
|
+
const text = blobText(file.content);
|
|
123
|
+
const tab = newLabel.includes(' ')
|
|
124
|
+
? '\t'
|
|
125
|
+
: '';
|
|
126
|
+
lines.push('--- /dev/null', `+++ ${newLabel}${tab}`, ...hunksOf(linesOf(text)));
|
|
127
|
+
if (!text.endsWith('\n'))
|
|
128
|
+
lines.push('\');
|
|
129
|
+
return lines.join('\n');
|
|
130
|
+
}
|
|
131
|
+
/** The sections of `files`, joined by line feeds and ending with one: `''` for no file. */
|
|
132
|
+
export function addedDiff(files) {
|
|
133
|
+
return files.map(file => `${addedSection(file)}\n`).join('');
|
|
134
|
+
}
|