@draht/coding-agent 2026.10.23 → 2026.10.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +21 -0
- package/README.md +10 -0
- package/dist/bundle/chunks/{anthropic-messages-VEMVBETR.js → anthropic-messages-OR5LCIS6.js} +1 -1
- package/dist/bundle/chunks/{azure-openai-responses-EXUP6SLM.js → azure-openai-responses-D6DVM232.js} +1 -1
- package/dist/bundle/chunks/bedrock-converse-stream.js +1 -1
- package/dist/bundle/chunks/{chunk-CHRFWNPF.js → chunk-7JJWQ53O.js} +81 -77
- package/dist/bundle/chunks/{chunk-DHZBWXZK.js → chunk-AEG546ID.js} +1 -1
- package/dist/bundle/chunks/{chunk-FXLM6UQV.js → chunk-CKPJSVRD.js} +1 -1
- package/dist/bundle/chunks/{chunk-CI37W6HM.js → chunk-DLEJ4D4T.js} +1 -1
- package/dist/bundle/chunks/{chunk-QG2WW2SA.js → chunk-JQF3BGRS.js} +1 -1
- package/dist/bundle/chunks/{chunk-DLVZ3EK7.js → chunk-NUUJKLOX.js} +1 -1
- package/dist/bundle/chunks/{chunk-YSIQCH4Z.js → chunk-QJADVGUY.js} +1 -1
- package/dist/bundle/chunks/{chunk-RGI7SF5G.js → chunk-RMLEJL4F.js} +1 -1
- package/dist/bundle/chunks/{chunk-JRHPLIE5.js → chunk-X4UBYFTC.js} +1 -1
- package/dist/bundle/chunks/{cloudflare-workers-ai-system-one-47PK45EQ.js → cloudflare-workers-ai-system-one-YVG4FYXV.js} +1 -1
- package/dist/bundle/chunks/{google-generative-ai-PBPH45HB.js → google-generative-ai-IHELIMAC.js} +1 -1
- package/dist/bundle/chunks/{google-vertex-ZPCFQNU6.js → google-vertex-3QCFWSNG.js} +1 -1
- package/dist/bundle/chunks/{llama-cpp-classify-MWIQKGBR.js → llama-cpp-classify-6J5XKC7Q.js} +1 -1
- package/dist/bundle/chunks/{mistral-conversations-7ODWQ37T.js → mistral-conversations-D3JRG6BL.js} +1 -1
- package/dist/bundle/chunks/{openai-codex-responses-7Y2SP2GS.js → openai-codex-responses-CBOPSGUQ.js} +1 -1
- package/dist/bundle/chunks/{openai-completions-QNWYJ4H4.js → openai-completions-LDMDMEF2.js} +1 -1
- package/dist/bundle/chunks/{openai-decisions-4PKBAPR3.js → openai-decisions-NDYRKL2E.js} +1 -1
- package/dist/bundle/chunks/{openai-responses-73WU3TQM.js → openai-responses-EZWJOXWT.js} +1 -1
- package/dist/bundle/chunks/{openrouter-images-IMWDSQ5F.js → openrouter-images-ZIDF5HXA.js} +1 -1
- package/dist/bundle/chunks/{pi-messages-AMNZ7HKZ.js → pi-messages-R75QWB4G.js} +1 -1
- package/dist/bundle/chunks/{typesafe-system-one-AXQRIVVX.js → typesafe-system-one-YUNLPYVG.js} +1 -1
- package/dist/bundle/chunks/{virtual-modules-Z4F3ZUX3.js → virtual-modules-4DJB4LTP.js} +1 -1
- package/dist/bundle/cli-runtime.js +1 -1
- package/dist/bundle/index.js +1 -1
- package/dist/bundle/rpc-entry.js +1 -1
- package/dist/core/agent-session.d.ts.map +1 -1
- package/dist/core/agent-session.js +1 -0
- package/dist/core/agent-session.js.map +1 -1
- package/dist/core/builtins/budget-guidance.d.ts +2 -0
- package/dist/core/builtins/budget-guidance.d.ts.map +1 -0
- package/dist/core/builtins/budget-guidance.js +2 -0
- package/dist/core/builtins/budget-guidance.js.map +1 -0
- package/dist/core/builtins/budgets.d.ts +3 -0
- package/dist/core/builtins/budgets.d.ts.map +1 -0
- package/dist/core/builtins/budgets.js +272 -0
- package/dist/core/builtins/budgets.js.map +1 -0
- package/dist/core/builtins/index.d.ts.map +1 -1
- package/dist/core/builtins/index.js +2 -0
- package/dist/core/builtins/index.js.map +1 -1
- package/dist/core/builtins/subagent.d.ts +3 -1
- package/dist/core/builtins/subagent.d.ts.map +1 -1
- package/dist/core/builtins/subagent.js +6 -3
- package/dist/core/builtins/subagent.js.map +1 -1
- package/dist/core/extensions/loader.d.ts.map +1 -1
- package/dist/core/extensions/loader.js +32 -3
- package/dist/core/extensions/loader.js.map +1 -1
- package/dist/core/extensions/runner.d.ts.map +1 -1
- package/dist/core/extensions/runner.js +2 -2
- package/dist/core/extensions/runner.js.map +1 -1
- package/dist/core/extensions/types.d.ts +9 -1
- package/dist/core/extensions/types.d.ts.map +1 -1
- package/dist/core/extensions/types.js.map +1 -1
- package/dist/core/mcp-servers.d.ts +3 -0
- package/dist/core/mcp-servers.d.ts.map +1 -1
- package/dist/core/mcp-servers.js.map +1 -1
- package/dist/core/multi-agent/permission-gate.d.ts +37 -8
- package/dist/core/multi-agent/permission-gate.d.ts.map +1 -1
- package/dist/core/multi-agent/permission-gate.js +158 -17
- package/dist/core/multi-agent/permission-gate.js.map +1 -1
- package/dist/extensions/mcp/config.d.ts +3 -0
- package/dist/extensions/mcp/config.d.ts.map +1 -1
- package/dist/extensions/mcp/config.js.map +1 -1
- package/dist/extensions/mcp/index.d.ts.map +1 -1
- package/dist/extensions/mcp/index.js +29 -7
- package/dist/extensions/mcp/index.js.map +1 -1
- package/dist/extensions/mcp/tools.d.ts +3 -0
- package/dist/extensions/mcp/tools.d.ts.map +1 -1
- package/dist/extensions/mcp/tools.js +1 -1
- package/dist/extensions/mcp/tools.js.map +1 -1
- package/dist/hooks/checks/budget-command.md +22 -0
- package/dist/hooks/checks/budgets-cl100k.json.gz +0 -0
- package/dist/hooks/checks/budgets-runtime.LICENSE +95 -0
- package/dist/hooks/checks/budgets-runtime.cjs +3382 -0
- package/docs/budgets.md +122 -0
- package/docs/docs.json +4 -0
- package/docs/environment-variables.md +1 -1
- package/docs/extensions.md +3 -1
- package/docs/keybindings.md +1 -1
- package/docs/mcp.md +13 -1
- package/docs/slash-commands.md +1 -0
- package/eval/drahtbank/runtime-checks.d.mts +1 -1
- package/eval/drahtbank/runtime-checks.mjs +38 -4
- package/hooks/checks/budget-command.md +22 -0
- package/hooks/checks/budgets-cl100k.json.gz +0 -0
- package/hooks/checks/budgets-runtime.LICENSE +95 -0
- package/hooks/checks/budgets-runtime.cjs +3382 -0
- package/package.json +9 -9
|
@@ -0,0 +1,3382 @@
|
|
|
1
|
+
// Generated by scripts/sync-budgets-runtime.mjs. Do not edit.
|
|
2
|
+
"use strict";
|
|
3
|
+
var __defProp = Object.defineProperty;
|
|
4
|
+
var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
|
|
5
|
+
var __getOwnPropNames = Object.getOwnPropertyNames;
|
|
6
|
+
var __hasOwnProp = Object.prototype.hasOwnProperty;
|
|
7
|
+
var __export = (target, all) => {
|
|
8
|
+
for (var name in all)
|
|
9
|
+
__defProp(target, name, { get: all[name], enumerable: true });
|
|
10
|
+
};
|
|
11
|
+
var __copyProps = (to, from, except, desc) => {
|
|
12
|
+
if (from && typeof from === "object" || typeof from === "function") {
|
|
13
|
+
for (let key of __getOwnPropNames(from))
|
|
14
|
+
if (!__hasOwnProp.call(to, key) && key !== except)
|
|
15
|
+
__defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
|
|
16
|
+
}
|
|
17
|
+
return to;
|
|
18
|
+
};
|
|
19
|
+
var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
|
|
20
|
+
|
|
21
|
+
// packages/coding-agent/src/core/budgets/cli.ts
|
|
22
|
+
var cli_exports = {};
|
|
23
|
+
__export(cli_exports, {
|
|
24
|
+
auditBudget: () => auditBudget,
|
|
25
|
+
beginTask: () => beginTask,
|
|
26
|
+
budgetConfigured: () => budgetConfigured,
|
|
27
|
+
checkBudget: () => checkBudget,
|
|
28
|
+
runHook: () => runHook,
|
|
29
|
+
stopBudget: () => stopBudget
|
|
30
|
+
});
|
|
31
|
+
module.exports = __toCommonJS(cli_exports);
|
|
32
|
+
var import_node_fs8 = require("node:fs");
|
|
33
|
+
var import_node_path9 = require("node:path");
|
|
34
|
+
|
|
35
|
+
// packages/coding-agent/src/core/budgets/runtime.ts
|
|
36
|
+
var import_node_crypto4 = require("node:crypto");
|
|
37
|
+
var import_node_fs6 = require("node:fs");
|
|
38
|
+
var import_node_path6 = require("node:path");
|
|
39
|
+
|
|
40
|
+
// node_modules/diff/libesm/diff/base.js
|
|
41
|
+
var Diff = class {
|
|
42
|
+
diff(oldStr, newStr, options = {}) {
|
|
43
|
+
let callback;
|
|
44
|
+
if (typeof options === "function") {
|
|
45
|
+
callback = options;
|
|
46
|
+
options = {};
|
|
47
|
+
} else if ("callback" in options) {
|
|
48
|
+
callback = options.callback;
|
|
49
|
+
}
|
|
50
|
+
const oldString = this.castInput(oldStr, options);
|
|
51
|
+
const newString = this.castInput(newStr, options);
|
|
52
|
+
const oldTokens = this.removeEmpty(this.tokenize(oldString, options));
|
|
53
|
+
const newTokens = this.removeEmpty(this.tokenize(newString, options));
|
|
54
|
+
return this.diffWithOptionsObj(oldTokens, newTokens, options, callback);
|
|
55
|
+
}
|
|
56
|
+
diffWithOptionsObj(oldTokens, newTokens, options, callback) {
|
|
57
|
+
var _a;
|
|
58
|
+
const done = (value) => {
|
|
59
|
+
value = this.postProcess(value, options);
|
|
60
|
+
if (callback) {
|
|
61
|
+
setTimeout(function() {
|
|
62
|
+
callback(value);
|
|
63
|
+
}, 0);
|
|
64
|
+
return void 0;
|
|
65
|
+
} else {
|
|
66
|
+
return value;
|
|
67
|
+
}
|
|
68
|
+
};
|
|
69
|
+
const newLen = newTokens.length, oldLen = oldTokens.length;
|
|
70
|
+
let editLength = 1;
|
|
71
|
+
let maxEditLength = newLen + oldLen;
|
|
72
|
+
if (options.maxEditLength != null) {
|
|
73
|
+
maxEditLength = Math.min(maxEditLength, options.maxEditLength);
|
|
74
|
+
}
|
|
75
|
+
const maxExecutionTime = (_a = options.timeout) !== null && _a !== void 0 ? _a : Infinity;
|
|
76
|
+
const abortAfterTimestamp = Date.now() + maxExecutionTime;
|
|
77
|
+
const bestPath = [{ oldPos: -1, lastComponent: void 0 }];
|
|
78
|
+
let newPos = this.extractCommon(bestPath[0], newTokens, oldTokens, 0, options);
|
|
79
|
+
if (bestPath[0].oldPos + 1 >= oldLen && newPos + 1 >= newLen) {
|
|
80
|
+
return done(this.buildValues(bestPath[0].lastComponent, newTokens, oldTokens));
|
|
81
|
+
}
|
|
82
|
+
let minDiagonalToConsider = -Infinity, maxDiagonalToConsider = Infinity;
|
|
83
|
+
const execEditLength = () => {
|
|
84
|
+
for (let diagonalPath = Math.max(minDiagonalToConsider, -editLength); diagonalPath <= Math.min(maxDiagonalToConsider, editLength); diagonalPath += 2) {
|
|
85
|
+
let basePath;
|
|
86
|
+
const removePath = bestPath[diagonalPath - 1], addPath = bestPath[diagonalPath + 1];
|
|
87
|
+
if (removePath) {
|
|
88
|
+
bestPath[diagonalPath - 1] = void 0;
|
|
89
|
+
}
|
|
90
|
+
let canAdd = false;
|
|
91
|
+
if (addPath) {
|
|
92
|
+
const addPathNewPos = addPath.oldPos - diagonalPath;
|
|
93
|
+
canAdd = addPath && 0 <= addPathNewPos && addPathNewPos < newLen;
|
|
94
|
+
}
|
|
95
|
+
const canRemove = removePath && removePath.oldPos + 1 < oldLen;
|
|
96
|
+
if (!canAdd && !canRemove) {
|
|
97
|
+
bestPath[diagonalPath] = void 0;
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (!canRemove || canAdd && removePath.oldPos < addPath.oldPos) {
|
|
101
|
+
basePath = this.addToPath(addPath, true, false, 0, options);
|
|
102
|
+
} else {
|
|
103
|
+
basePath = this.addToPath(removePath, false, true, 1, options);
|
|
104
|
+
}
|
|
105
|
+
newPos = this.extractCommon(basePath, newTokens, oldTokens, diagonalPath, options);
|
|
106
|
+
if (basePath.oldPos + 1 >= oldLen && newPos + 1 >= newLen) {
|
|
107
|
+
return done(this.buildValues(basePath.lastComponent, newTokens, oldTokens)) || true;
|
|
108
|
+
} else {
|
|
109
|
+
bestPath[diagonalPath] = basePath;
|
|
110
|
+
if (basePath.oldPos + 1 >= oldLen) {
|
|
111
|
+
maxDiagonalToConsider = Math.min(maxDiagonalToConsider, diagonalPath - 1);
|
|
112
|
+
}
|
|
113
|
+
if (newPos + 1 >= newLen) {
|
|
114
|
+
minDiagonalToConsider = Math.max(minDiagonalToConsider, diagonalPath + 1);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
editLength++;
|
|
119
|
+
};
|
|
120
|
+
if (callback) {
|
|
121
|
+
(function exec() {
|
|
122
|
+
setTimeout(function() {
|
|
123
|
+
if (editLength > maxEditLength || Date.now() > abortAfterTimestamp) {
|
|
124
|
+
return callback(void 0);
|
|
125
|
+
}
|
|
126
|
+
if (!execEditLength()) {
|
|
127
|
+
exec();
|
|
128
|
+
}
|
|
129
|
+
}, 0);
|
|
130
|
+
})();
|
|
131
|
+
} else {
|
|
132
|
+
while (editLength <= maxEditLength && Date.now() <= abortAfterTimestamp) {
|
|
133
|
+
const ret = execEditLength();
|
|
134
|
+
if (ret) {
|
|
135
|
+
return ret;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
addToPath(path, added, removed, oldPosInc, options) {
|
|
141
|
+
const last = path.lastComponent;
|
|
142
|
+
if (last && !options.oneChangePerToken && last.added === added && last.removed === removed) {
|
|
143
|
+
return {
|
|
144
|
+
oldPos: path.oldPos + oldPosInc,
|
|
145
|
+
lastComponent: { count: last.count + 1, added, removed, previousComponent: last.previousComponent }
|
|
146
|
+
};
|
|
147
|
+
} else {
|
|
148
|
+
return {
|
|
149
|
+
oldPos: path.oldPos + oldPosInc,
|
|
150
|
+
lastComponent: { count: 1, added, removed, previousComponent: last }
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
extractCommon(basePath, newTokens, oldTokens, diagonalPath, options) {
|
|
155
|
+
const newLen = newTokens.length, oldLen = oldTokens.length;
|
|
156
|
+
let oldPos = basePath.oldPos, newPos = oldPos - diagonalPath, commonCount = 0;
|
|
157
|
+
while (newPos + 1 < newLen && oldPos + 1 < oldLen && this.equals(oldTokens[oldPos + 1], newTokens[newPos + 1], options)) {
|
|
158
|
+
newPos++;
|
|
159
|
+
oldPos++;
|
|
160
|
+
commonCount++;
|
|
161
|
+
if (options.oneChangePerToken) {
|
|
162
|
+
basePath.lastComponent = { count: 1, previousComponent: basePath.lastComponent, added: false, removed: false };
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
if (commonCount && !options.oneChangePerToken) {
|
|
166
|
+
basePath.lastComponent = { count: commonCount, previousComponent: basePath.lastComponent, added: false, removed: false };
|
|
167
|
+
}
|
|
168
|
+
basePath.oldPos = oldPos;
|
|
169
|
+
return newPos;
|
|
170
|
+
}
|
|
171
|
+
equals(left, right, options) {
|
|
172
|
+
if (options.comparator) {
|
|
173
|
+
return options.comparator(left, right);
|
|
174
|
+
} else {
|
|
175
|
+
return left === right || !!options.ignoreCase && left.toLowerCase() === right.toLowerCase();
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
removeEmpty(array) {
|
|
179
|
+
const ret = [];
|
|
180
|
+
for (let i = 0; i < array.length; i++) {
|
|
181
|
+
if (array[i]) {
|
|
182
|
+
ret.push(array[i]);
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
return ret;
|
|
186
|
+
}
|
|
187
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
188
|
+
castInput(value, options) {
|
|
189
|
+
return value;
|
|
190
|
+
}
|
|
191
|
+
// eslint-disable-next-line @typescript-eslint/no-unused-vars
|
|
192
|
+
tokenize(value, options) {
|
|
193
|
+
return Array.from(value);
|
|
194
|
+
}
|
|
195
|
+
join(chars) {
|
|
196
|
+
return chars.join("");
|
|
197
|
+
}
|
|
198
|
+
postProcess(changeObjects, options) {
|
|
199
|
+
return changeObjects;
|
|
200
|
+
}
|
|
201
|
+
get useLongestToken() {
|
|
202
|
+
return false;
|
|
203
|
+
}
|
|
204
|
+
buildValues(lastComponent, newTokens, oldTokens) {
|
|
205
|
+
const components = [];
|
|
206
|
+
let nextComponent;
|
|
207
|
+
while (lastComponent) {
|
|
208
|
+
components.push(lastComponent);
|
|
209
|
+
nextComponent = lastComponent.previousComponent;
|
|
210
|
+
delete lastComponent.previousComponent;
|
|
211
|
+
lastComponent = nextComponent;
|
|
212
|
+
}
|
|
213
|
+
components.reverse();
|
|
214
|
+
const componentLen = components.length;
|
|
215
|
+
let componentPos = 0, newPos = 0, oldPos = 0;
|
|
216
|
+
for (; componentPos < componentLen; componentPos++) {
|
|
217
|
+
const component = components[componentPos];
|
|
218
|
+
if (!component.removed) {
|
|
219
|
+
if (!component.added && this.useLongestToken) {
|
|
220
|
+
let value = newTokens.slice(newPos, newPos + component.count);
|
|
221
|
+
value = value.map(function(value2, i) {
|
|
222
|
+
const oldValue = oldTokens[oldPos + i];
|
|
223
|
+
return oldValue.length > value2.length ? oldValue : value2;
|
|
224
|
+
});
|
|
225
|
+
component.value = this.join(value);
|
|
226
|
+
} else {
|
|
227
|
+
component.value = this.join(newTokens.slice(newPos, newPos + component.count));
|
|
228
|
+
}
|
|
229
|
+
newPos += component.count;
|
|
230
|
+
if (!component.added) {
|
|
231
|
+
oldPos += component.count;
|
|
232
|
+
}
|
|
233
|
+
} else {
|
|
234
|
+
component.value = this.join(oldTokens.slice(oldPos, oldPos + component.count));
|
|
235
|
+
oldPos += component.count;
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
return components;
|
|
239
|
+
}
|
|
240
|
+
};
|
|
241
|
+
|
|
242
|
+
// node_modules/diff/libesm/diff/line.js
|
|
243
|
+
var LineDiff = class extends Diff {
|
|
244
|
+
constructor() {
|
|
245
|
+
super(...arguments);
|
|
246
|
+
this.tokenize = tokenize;
|
|
247
|
+
}
|
|
248
|
+
equals(left, right, options) {
|
|
249
|
+
if (options.ignoreWhitespace) {
|
|
250
|
+
if (!options.newlineIsToken || !left.includes("\n")) {
|
|
251
|
+
left = left.trim();
|
|
252
|
+
}
|
|
253
|
+
if (!options.newlineIsToken || !right.includes("\n")) {
|
|
254
|
+
right = right.trim();
|
|
255
|
+
}
|
|
256
|
+
} else if (options.ignoreNewlineAtEof && !options.newlineIsToken) {
|
|
257
|
+
if (left.endsWith("\n")) {
|
|
258
|
+
left = left.slice(0, -1);
|
|
259
|
+
}
|
|
260
|
+
if (right.endsWith("\n")) {
|
|
261
|
+
right = right.slice(0, -1);
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
return super.equals(left, right, options);
|
|
265
|
+
}
|
|
266
|
+
};
|
|
267
|
+
var lineDiff = new LineDiff();
|
|
268
|
+
function diffLines(oldStr, newStr, options) {
|
|
269
|
+
return lineDiff.diff(oldStr, newStr, options);
|
|
270
|
+
}
|
|
271
|
+
function tokenize(value, options) {
|
|
272
|
+
if (options.stripTrailingCr) {
|
|
273
|
+
value = value.replace(/\r\n/g, "\n");
|
|
274
|
+
}
|
|
275
|
+
const retLines = [], linesAndNewlines = value.split(/(\n|\r\n)/);
|
|
276
|
+
if (!linesAndNewlines[linesAndNewlines.length - 1]) {
|
|
277
|
+
linesAndNewlines.pop();
|
|
278
|
+
}
|
|
279
|
+
for (let i = 0; i < linesAndNewlines.length; i++) {
|
|
280
|
+
const line = linesAndNewlines[i];
|
|
281
|
+
if (i % 2 && !options.newlineIsToken) {
|
|
282
|
+
retLines[retLines.length - 1] += line;
|
|
283
|
+
} else {
|
|
284
|
+
retLines.push(line);
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
return retLines;
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
// node_modules/gpt-tokenizer/esm/constants.js
|
|
291
|
+
var DEFAULT_MERGE_CACHE_SIZE = 1e5;
|
|
292
|
+
|
|
293
|
+
// node_modules/gpt-tokenizer/esm/utfUtil.js
|
|
294
|
+
var isAscii = (codePoint) => codePoint <= 127;
|
|
295
|
+
var HIGH_SURROGATE_START = 55296;
|
|
296
|
+
var HIGH_SURROGATE_END = 56319;
|
|
297
|
+
function endsWithIncompleteUtfPairSurrogate(string) {
|
|
298
|
+
if (string.length === 0) return false;
|
|
299
|
+
const lastCharCode = string.charCodeAt(string.length - 1);
|
|
300
|
+
return lastCharCode >= HIGH_SURROGATE_START && lastCharCode <= HIGH_SURROGATE_END;
|
|
301
|
+
}
|
|
302
|
+
function isValidUTF8(bytes) {
|
|
303
|
+
let i = 0;
|
|
304
|
+
while (i < bytes.length) {
|
|
305
|
+
const byte1 = bytes[i];
|
|
306
|
+
let numBytes = 0;
|
|
307
|
+
let codePoint = 0;
|
|
308
|
+
if (byte1 <= 127) {
|
|
309
|
+
numBytes = 1;
|
|
310
|
+
codePoint = byte1;
|
|
311
|
+
} else if ((byte1 & 224) === 192) {
|
|
312
|
+
numBytes = 2;
|
|
313
|
+
codePoint = byte1 & 31;
|
|
314
|
+
if (byte1 <= 193) return false;
|
|
315
|
+
} else if ((byte1 & 240) === 224) {
|
|
316
|
+
numBytes = 3;
|
|
317
|
+
codePoint = byte1 & 15;
|
|
318
|
+
} else if ((byte1 & 248) === 240) {
|
|
319
|
+
numBytes = 4;
|
|
320
|
+
codePoint = byte1 & 7;
|
|
321
|
+
if (byte1 > 244) return false;
|
|
322
|
+
} else return false;
|
|
323
|
+
if (i + numBytes > bytes.length) return false;
|
|
324
|
+
for (let j = 1; j < numBytes; j++) {
|
|
325
|
+
const byte = bytes[i + j];
|
|
326
|
+
if (byte === void 0 || (byte & 192) !== 128) return false;
|
|
327
|
+
codePoint = codePoint << 6 | byte & 63;
|
|
328
|
+
}
|
|
329
|
+
if (numBytes === 2 && codePoint < 128) return false;
|
|
330
|
+
if (numBytes === 3 && codePoint < 2048) return false;
|
|
331
|
+
if (numBytes === 4 && codePoint < 65536) return false;
|
|
332
|
+
if (codePoint >= 55296 && codePoint <= 57343) return false;
|
|
333
|
+
if (codePoint > 1114111) return false;
|
|
334
|
+
i += numBytes;
|
|
335
|
+
}
|
|
336
|
+
return true;
|
|
337
|
+
}
|
|
338
|
+
function startsWithByteOrderMark(bytes) {
|
|
339
|
+
return bytes.length >= 3 && bytes[0] === 239 && bytes[1] === 187 && bytes[2] === 191;
|
|
340
|
+
}
|
|
341
|
+
var textDecoder = new TextDecoder("utf8", {
|
|
342
|
+
fatal: false,
|
|
343
|
+
ignoreBOM: true
|
|
344
|
+
});
|
|
345
|
+
function tryConvertToString(arr) {
|
|
346
|
+
if (!isValidUTF8(arr)) return;
|
|
347
|
+
return textDecoder.decode(arr);
|
|
348
|
+
}
|
|
349
|
+
function compareUint8Arrays(a, b) {
|
|
350
|
+
const len = Math.min(a.length, b.length);
|
|
351
|
+
for (let i = 0; i < len; i++) if (a[i] !== b[i]) return a[i] - b[i];
|
|
352
|
+
return a.length - b.length;
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// node_modules/gpt-tokenizer/esm/util.js
|
|
356
|
+
function getMaxValueFromMap(map) {
|
|
357
|
+
let max = 0;
|
|
358
|
+
map.forEach((val) => {
|
|
359
|
+
max = Math.max(max, val);
|
|
360
|
+
});
|
|
361
|
+
return max;
|
|
362
|
+
}
|
|
363
|
+
function escapeRegExp(string) {
|
|
364
|
+
return string.replace(/[$()*+.?[\\\]^{|}]/g, "\\$&");
|
|
365
|
+
}
|
|
366
|
+
function getSpecialTokenRegex(tokens) {
|
|
367
|
+
const inner = [...tokens].map(escapeRegExp).join("|");
|
|
368
|
+
return new RegExp(`(${inner})`);
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/unicode16Data.js
|
|
372
|
+
var UNICODE_16_LETTER_RANGES = String.raw`\u0041-\u005A\u0061-\u007A\u00AA\u00B5\u00BA\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0370-\u0374\u0376-\u0377\u037A-\u037D\u037F\u0386\u0388-\u038A\u038C\u038E-\u03A1\u03A3-\u03F5\u03F7-\u0481\u048A-\u052F\u0531-\u0556\u0559\u0560-\u0588\u05D0-\u05EA\u05EF-\u05F2\u0620-\u064A\u066E-\u066F\u0671-\u06D3\u06D5\u06E5-\u06E6\u06EE-\u06EF\u06FA-\u06FC\u06FF\u0710\u0712-\u072F\u074D-\u07A5\u07B1\u07CA-\u07EA\u07F4-\u07F5\u07FA\u0800-\u0815\u081A\u0824\u0828\u0840-\u0858\u0860-\u086A\u0870-\u0887\u0889-\u088E\u08A0-\u08C9\u0904-\u0939\u093D\u0950\u0958-\u0961\u0971-\u0980\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BD\u09CE\u09DC-\u09DD\u09DF-\u09E1\u09F0-\u09F1\u09FC\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33\u0A35-\u0A36\u0A38-\u0A39\u0A59-\u0A5C\u0A5E\u0A72-\u0A74\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABD\u0AD0\u0AE0-\u0AE1\u0AF9\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32-\u0B33\u0B35-\u0B39\u0B3D\u0B5C-\u0B5D\u0B5F-\u0B61\u0B71\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BD0\u0C05-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3D\u0C58-\u0C5A\u0C5D\u0C60-\u0C61\u0C80\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBD\u0CDD-\u0CDE\u0CE0-\u0CE1\u0CF1-\u0CF2\u0D04-\u0D0C\u0D0E-\u0D10\u0D12-\u0D3A\u0D3D\u0D4E\u0D54-\u0D56\u0D5F-\u0D61\u0D7A-\u0D7F\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0E01-\u0E30\u0E32-\u0E33\u0E40-\u0E46\u0E81-\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EB0\u0EB2-\u0EB3\u0EBD\u0EC0-\u0EC4\u0EC6\u0EDC-\u0EDF\u0F00\u0F40-\u0F47\u0F49-\u0F6C\u0F88-\u0F8C\u1000-\u102A\u103F\u1050-\u1055\u105A-\u105D\u1061\u1065-\u1066\u106E-\u1070\u1075-\u1081\u108E\u10A0-\u10C5\u10C7\u10CD\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u1380-\u138F\u13A0-\u13F5\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16F1-\u16F8\u1700-\u1711\u171F-\u1731\u1740-\u1751\u1760-\u176C\u176E-\u1770\u1780-\u17B3\u17D7\u17DC\u1820-\u1878\u1880-\u1884\u1887-\u18A8\u18AA\u18B0-\u18F5\u1900-\u191E\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u1A00-\u1A16\u1A20-\u1A54\u1AA7\u1B05-\u1B33\u1B45-\u1B4C\u1B83-\u1BA0\u1BAE-\u1BAF\u1BBA-\u1BE5\u1C00-\u1C23\u1C4D-\u1C4F\u1C5A-\u1C7D\u1C80-\u1C8A\u1C90-\u1CBA\u1CBD-\u1CBF\u1CE9-\u1CEC\u1CEE-\u1CF3\u1CF5-\u1CF6\u1CFA\u1D00-\u1DBF\u1E00-\u1F15\u1F18-\u1F1D\u1F20-\u1F45\u1F48-\u1F4D\u1F50-\u1F57\u1F59\u1F5B\u1F5D\u1F5F-\u1F7D\u1F80-\u1FB4\u1FB6-\u1FBC\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FCC\u1FD0-\u1FD3\u1FD6-\u1FDB\u1FE0-\u1FEC\u1FF2-\u1FF4\u1FF6-\u1FFC\u2071\u207F\u2090-\u209C\u2102\u2107\u210A-\u2113\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u212F-\u2139\u213C-\u213F\u2145-\u2149\u214E\u2183-\u2184\u2C00-\u2CE4\u2CEB-\u2CEE\u2CF2-\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D80-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2E2F\u3005-\u3006\u3031-\u3035\u303B-\u303C\u3041-\u3096\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u31A0-\u31BF\u31F0-\u31FF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A-\uA62B\uA640-\uA66E\uA67F-\uA69D\uA6A0-\uA6E5\uA717-\uA71F\uA722-\uA788\uA78B-\uA7CD\uA7D0-\uA7D1\uA7D3\uA7D5-\uA7DC\uA7F2-\uA801\uA803-\uA805\uA807-\uA80A\uA80C-\uA822\uA840-\uA873\uA882-\uA8B3\uA8F2-\uA8F7\uA8FB\uA8FD-\uA8FE\uA90A-\uA925\uA930-\uA946\uA960-\uA97C\uA984-\uA9B2\uA9CF\uA9E0-\uA9E4\uA9E6-\uA9EF\uA9FA-\uA9FE\uAA00-\uAA28\uAA40-\uAA42\uAA44-\uAA4B\uAA60-\uAA76\uAA7A\uAA7E-\uAAAF\uAAB1\uAAB5-\uAAB6\uAAB9-\uAABD\uAAC0\uAAC2\uAADB-\uAADD\uAAE0-\uAAEA\uAAF2-\uAAF4\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABE2\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D\uFB1F-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC\u{10000}-\u{1000B}\u{1000D}-\u{10026}\u{10028}-\u{1003A}\u{1003C}-\u{1003D}\u{1003F}-\u{1004D}\u{10050}-\u{1005D}\u{10080}-\u{100FA}\u{10280}-\u{1029C}\u{102A0}-\u{102D0}\u{10300}-\u{1031F}\u{1032D}-\u{10340}\u{10342}-\u{10349}\u{10350}-\u{10375}\u{10380}-\u{1039D}\u{103A0}-\u{103C3}\u{103C8}-\u{103CF}\u{10400}-\u{1049D}\u{104B0}-\u{104D3}\u{104D8}-\u{104FB}\u{10500}-\u{10527}\u{10530}-\u{10563}\u{10570}-\u{1057A}\u{1057C}-\u{1058A}\u{1058C}-\u{10592}\u{10594}-\u{10595}\u{10597}-\u{105A1}\u{105A3}-\u{105B1}\u{105B3}-\u{105B9}\u{105BB}-\u{105BC}\u{105C0}-\u{105F3}\u{10600}-\u{10736}\u{10740}-\u{10755}\u{10760}-\u{10767}\u{10780}-\u{10785}\u{10787}-\u{107B0}\u{107B2}-\u{107BA}\u{10800}-\u{10805}\u{10808}\u{1080A}-\u{10835}\u{10837}-\u{10838}\u{1083C}\u{1083F}-\u{10855}\u{10860}-\u{10876}\u{10880}-\u{1089E}\u{108E0}-\u{108F2}\u{108F4}-\u{108F5}\u{10900}-\u{10915}\u{10920}-\u{10939}\u{10980}-\u{109B7}\u{109BE}-\u{109BF}\u{10A00}\u{10A10}-\u{10A13}\u{10A15}-\u{10A17}\u{10A19}-\u{10A35}\u{10A60}-\u{10A7C}\u{10A80}-\u{10A9C}\u{10AC0}-\u{10AC7}\u{10AC9}-\u{10AE4}\u{10B00}-\u{10B35}\u{10B40}-\u{10B55}\u{10B60}-\u{10B72}\u{10B80}-\u{10B91}\u{10C00}-\u{10C48}\u{10C80}-\u{10CB2}\u{10CC0}-\u{10CF2}\u{10D00}-\u{10D23}\u{10D4A}-\u{10D65}\u{10D6F}-\u{10D85}\u{10E80}-\u{10EA9}\u{10EB0}-\u{10EB1}\u{10EC2}-\u{10EC4}\u{10F00}-\u{10F1C}\u{10F27}\u{10F30}-\u{10F45}\u{10F70}-\u{10F81}\u{10FB0}-\u{10FC4}\u{10FE0}-\u{10FF6}\u{11003}-\u{11037}\u{11071}-\u{11072}\u{11075}\u{11083}-\u{110AF}\u{110D0}-\u{110E8}\u{11103}-\u{11126}\u{11144}\u{11147}\u{11150}-\u{11172}\u{11176}\u{11183}-\u{111B2}\u{111C1}-\u{111C4}\u{111DA}\u{111DC}\u{11200}-\u{11211}\u{11213}-\u{1122B}\u{1123F}-\u{11240}\u{11280}-\u{11286}\u{11288}\u{1128A}-\u{1128D}\u{1128F}-\u{1129D}\u{1129F}-\u{112A8}\u{112B0}-\u{112DE}\u{11305}-\u{1130C}\u{1130F}-\u{11310}\u{11313}-\u{11328}\u{1132A}-\u{11330}\u{11332}-\u{11333}\u{11335}-\u{11339}\u{1133D}\u{11350}\u{1135D}-\u{11361}\u{11380}-\u{11389}\u{1138B}\u{1138E}\u{11390}-\u{113B5}\u{113B7}\u{113D1}\u{113D3}\u{11400}-\u{11434}\u{11447}-\u{1144A}\u{1145F}-\u{11461}\u{11480}-\u{114AF}\u{114C4}-\u{114C5}\u{114C7}\u{11580}-\u{115AE}\u{115D8}-\u{115DB}\u{11600}-\u{1162F}\u{11644}\u{11680}-\u{116AA}\u{116B8}\u{11700}-\u{1171A}\u{11740}-\u{11746}\u{11800}-\u{1182B}\u{118A0}-\u{118DF}\u{118FF}-\u{11906}\u{11909}\u{1190C}-\u{11913}\u{11915}-\u{11916}\u{11918}-\u{1192F}\u{1193F}\u{11941}\u{119A0}-\u{119A7}\u{119AA}-\u{119D0}\u{119E1}\u{119E3}\u{11A00}\u{11A0B}-\u{11A32}\u{11A3A}\u{11A50}\u{11A5C}-\u{11A89}\u{11A9D}\u{11AB0}-\u{11AF8}\u{11BC0}-\u{11BE0}\u{11C00}-\u{11C08}\u{11C0A}-\u{11C2E}\u{11C40}\u{11C72}-\u{11C8F}\u{11D00}-\u{11D06}\u{11D08}-\u{11D09}\u{11D0B}-\u{11D30}\u{11D46}\u{11D60}-\u{11D65}\u{11D67}-\u{11D68}\u{11D6A}-\u{11D89}\u{11D98}\u{11EE0}-\u{11EF2}\u{11F02}\u{11F04}-\u{11F10}\u{11F12}-\u{11F33}\u{11FB0}\u{12000}-\u{12399}\u{12480}-\u{12543}\u{12F90}-\u{12FF0}\u{13000}-\u{1342F}\u{13441}-\u{13446}\u{13460}-\u{143FA}\u{14400}-\u{14646}\u{16100}-\u{1611D}\u{16800}-\u{16A38}\u{16A40}-\u{16A5E}\u{16A70}-\u{16ABE}\u{16AD0}-\u{16AED}\u{16B00}-\u{16B2F}\u{16B40}-\u{16B43}\u{16B63}-\u{16B77}\u{16B7D}-\u{16B8F}\u{16D40}-\u{16D6C}\u{16E40}-\u{16E7F}\u{16F00}-\u{16F4A}\u{16F50}\u{16F93}-\u{16F9F}\u{16FE0}-\u{16FE1}\u{16FE3}\u{17000}-\u{187F7}\u{18800}-\u{18CD5}\u{18CFF}-\u{18D08}\u{1AFF0}-\u{1AFF3}\u{1AFF5}-\u{1AFFB}\u{1AFFD}-\u{1AFFE}\u{1B000}-\u{1B122}\u{1B132}\u{1B150}-\u{1B152}\u{1B155}\u{1B164}-\u{1B167}\u{1B170}-\u{1B2FB}\u{1BC00}-\u{1BC6A}\u{1BC70}-\u{1BC7C}\u{1BC80}-\u{1BC88}\u{1BC90}-\u{1BC99}\u{1D400}-\u{1D454}\u{1D456}-\u{1D49C}\u{1D49E}-\u{1D49F}\u{1D4A2}\u{1D4A5}-\u{1D4A6}\u{1D4A9}-\u{1D4AC}\u{1D4AE}-\u{1D4B9}\u{1D4BB}\u{1D4BD}-\u{1D4C3}\u{1D4C5}-\u{1D505}\u{1D507}-\u{1D50A}\u{1D50D}-\u{1D514}\u{1D516}-\u{1D51C}\u{1D51E}-\u{1D539}\u{1D53B}-\u{1D53E}\u{1D540}-\u{1D544}\u{1D546}\u{1D54A}-\u{1D550}\u{1D552}-\u{1D6A5}\u{1D6A8}-\u{1D6C0}\u{1D6C2}-\u{1D6DA}\u{1D6DC}-\u{1D6FA}\u{1D6FC}-\u{1D714}\u{1D716}-\u{1D734}\u{1D736}-\u{1D74E}\u{1D750}-\u{1D76E}\u{1D770}-\u{1D788}\u{1D78A}-\u{1D7A8}\u{1D7AA}-\u{1D7C2}\u{1D7C4}-\u{1D7CB}\u{1DF00}-\u{1DF1E}\u{1DF25}-\u{1DF2A}\u{1E030}-\u{1E06D}\u{1E100}-\u{1E12C}\u{1E137}-\u{1E13D}\u{1E14E}\u{1E290}-\u{1E2AD}\u{1E2C0}-\u{1E2EB}\u{1E4D0}-\u{1E4EB}\u{1E5D0}-\u{1E5ED}\u{1E5F0}\u{1E7E0}-\u{1E7E6}\u{1E7E8}-\u{1E7EB}\u{1E7ED}-\u{1E7EE}\u{1E7F0}-\u{1E7FE}\u{1E800}-\u{1E8C4}\u{1E900}-\u{1E943}\u{1E94B}\u{1EE00}-\u{1EE03}\u{1EE05}-\u{1EE1F}\u{1EE21}-\u{1EE22}\u{1EE24}\u{1EE27}\u{1EE29}-\u{1EE32}\u{1EE34}-\u{1EE37}\u{1EE39}\u{1EE3B}\u{1EE42}\u{1EE47}\u{1EE49}\u{1EE4B}\u{1EE4D}-\u{1EE4F}\u{1EE51}-\u{1EE52}\u{1EE54}\u{1EE57}\u{1EE59}\u{1EE5B}\u{1EE5D}\u{1EE5F}\u{1EE61}-\u{1EE62}\u{1EE64}\u{1EE67}-\u{1EE6A}\u{1EE6C}-\u{1EE72}\u{1EE74}-\u{1EE77}\u{1EE79}-\u{1EE7C}\u{1EE7E}\u{1EE80}-\u{1EE89}\u{1EE8B}-\u{1EE9B}\u{1EEA1}-\u{1EEA3}\u{1EEA5}-\u{1EEA9}\u{1EEAB}-\u{1EEBB}\u{20000}-\u{2A6DF}\u{2A700}-\u{2B739}\u{2B740}-\u{2B81D}\u{2B820}-\u{2CEA1}\u{2CEB0}-\u{2EBE0}\u{2EBF0}-\u{2EE5D}\u{2F800}-\u{2FA1D}\u{30000}-\u{3134A}\u{31350}-\u{323AF}`;
|
|
373
|
+
var UNICODE_16_NUMBER_RANGES = String.raw`\u0030-\u0039\u00B2-\u00B3\u00B9\u00BC-\u00BE\u0660-\u0669\u06F0-\u06F9\u07C0-\u07C9\u0966-\u096F\u09E6-\u09EF\u09F4-\u09F9\u0A66-\u0A6F\u0AE6-\u0AEF\u0B66-\u0B6F\u0B72-\u0B77\u0BE6-\u0BF2\u0C66-\u0C6F\u0C78-\u0C7E\u0CE6-\u0CEF\u0D58-\u0D5E\u0D66-\u0D78\u0DE6-\u0DEF\u0E50-\u0E59\u0ED0-\u0ED9\u0F20-\u0F33\u1040-\u1049\u1090-\u1099\u1369-\u137C\u16EE-\u16F0\u17E0-\u17E9\u17F0-\u17F9\u1810-\u1819\u1946-\u194F\u19D0-\u19DA\u1A80-\u1A89\u1A90-\u1A99\u1B50-\u1B59\u1BB0-\u1BB9\u1C40-\u1C49\u1C50-\u1C59\u2070\u2074-\u2079\u2080-\u2089\u2150-\u2182\u2185-\u2189\u2460-\u249B\u24EA-\u24FF\u2776-\u2793\u2CFD\u3007\u3021-\u3029\u3038-\u303A\u3192-\u3195\u3220-\u3229\u3248-\u324F\u3251-\u325F\u3280-\u3289\u32B1-\u32BF\uA620-\uA629\uA6E6-\uA6EF\uA830-\uA835\uA8D0-\uA8D9\uA900-\uA909\uA9D0-\uA9D9\uA9F0-\uA9F9\uAA50-\uAA59\uABF0-\uABF9\uFF10-\uFF19\u{10107}-\u{10133}\u{10140}-\u{10178}\u{1018A}-\u{1018B}\u{102E1}-\u{102FB}\u{10320}-\u{10323}\u{10341}\u{1034A}\u{103D1}-\u{103D5}\u{104A0}-\u{104A9}\u{10858}-\u{1085F}\u{10879}-\u{1087F}\u{108A7}-\u{108AF}\u{108FB}-\u{108FF}\u{10916}-\u{1091B}\u{109BC}-\u{109BD}\u{109C0}-\u{109CF}\u{109D2}-\u{109FF}\u{10A40}-\u{10A48}\u{10A7D}-\u{10A7E}\u{10A9D}-\u{10A9F}\u{10AEB}-\u{10AEF}\u{10B58}-\u{10B5F}\u{10B78}-\u{10B7F}\u{10BA9}-\u{10BAF}\u{10CFA}-\u{10CFF}\u{10D30}-\u{10D39}\u{10D40}-\u{10D49}\u{10E60}-\u{10E7E}\u{10F1D}-\u{10F26}\u{10F51}-\u{10F54}\u{10FC5}-\u{10FCB}\u{11052}-\u{1106F}\u{110F0}-\u{110F9}\u{11136}-\u{1113F}\u{111D0}-\u{111D9}\u{111E1}-\u{111F4}\u{112F0}-\u{112F9}\u{11450}-\u{11459}\u{114D0}-\u{114D9}\u{11650}-\u{11659}\u{116C0}-\u{116C9}\u{116D0}-\u{116E3}\u{11730}-\u{1173B}\u{118E0}-\u{118F2}\u{11950}-\u{11959}\u{11BF0}-\u{11BF9}\u{11C50}-\u{11C6C}\u{11D50}-\u{11D59}\u{11DA0}-\u{11DA9}\u{11F50}-\u{11F59}\u{11FC0}-\u{11FD4}\u{12400}-\u{1246E}\u{16130}-\u{16139}\u{16A60}-\u{16A69}\u{16AC0}-\u{16AC9}\u{16B50}-\u{16B59}\u{16B5B}-\u{16B61}\u{16D70}-\u{16D79}\u{16E80}-\u{16E96}\u{1CCF0}-\u{1CCF9}\u{1D2C0}-\u{1D2D3}\u{1D2E0}-\u{1D2F3}\u{1D360}-\u{1D378}\u{1D7CE}-\u{1D7FF}\u{1E140}-\u{1E149}\u{1E2F0}-\u{1E2F9}\u{1E4F0}-\u{1E4F9}\u{1E5F1}-\u{1E5FA}\u{1E8C7}-\u{1E8CF}\u{1E950}-\u{1E959}\u{1EC71}-\u{1ECAB}\u{1ECAD}-\u{1ECAF}\u{1ECB1}-\u{1ECB4}\u{1ED01}-\u{1ED2D}\u{1ED2F}-\u{1ED3D}\u{1F100}-\u{1F10C}\u{1FBF0}-\u{1FBF9}`;
|
|
374
|
+
var UNICODE_16_O200K_UPPER_RANGES = String.raw`\u0041-\u005A\u00AA\u00BA\u00C0-\u00D6\u00D8-\u00DE\u0100\u0102\u0104\u0106\u0108\u010A\u010C\u010E\u0110\u0112\u0114\u0116\u0118\u011A\u011C\u011E\u0120\u0122\u0124\u0126\u0128\u012A\u012C\u012E\u0130\u0132\u0134\u0136\u0139\u013B\u013D\u013F\u0141\u0143\u0145\u0147\u014A\u014C\u014E\u0150\u0152\u0154\u0156\u0158\u015A\u015C\u015E\u0160\u0162\u0164\u0166\u0168\u016A\u016C\u016E\u0170\u0172\u0174\u0176\u0178-\u0179\u017B\u017D\u0181-\u0182\u0184\u0186-\u0187\u0189-\u018B\u018E-\u0191\u0193-\u0194\u0196-\u0198\u019C-\u019D\u019F-\u01A0\u01A2\u01A4\u01A6-\u01A7\u01A9\u01AC\u01AE-\u01AF\u01B1-\u01B3\u01B5\u01B7-\u01B8\u01BB-\u01BC\u01C0-\u01C5\u01C7-\u01C8\u01CA-\u01CB\u01CD\u01CF\u01D1\u01D3\u01D5\u01D7\u01D9\u01DB\u01DE\u01E0\u01E2\u01E4\u01E6\u01E8\u01EA\u01EC\u01EE\u01F1-\u01F2\u01F4\u01F6-\u01F8\u01FA\u01FC\u01FE\u0200\u0202\u0204\u0206\u0208\u020A\u020C\u020E\u0210\u0212\u0214\u0216\u0218\u021A\u021C\u021E\u0220\u0222\u0224\u0226\u0228\u022A\u022C\u022E\u0230\u0232\u023A-\u023B\u023D-\u023E\u0241\u0243-\u0246\u0248\u024A\u024C\u024E\u0294\u02B0-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0300-\u0370\u0372\u0374\u0376\u037A\u037F\u0386\u0388-\u038A\u038C\u038E-\u038F\u0391-\u03A1\u03A3-\u03AB\u03CF\u03D2-\u03D4\u03D8\u03DA\u03DC\u03DE\u03E0\u03E2\u03E4\u03E6\u03E8\u03EA\u03EC\u03EE\u03F4\u03F7\u03F9-\u03FA\u03FD-\u042F\u0460\u0462\u0464\u0466\u0468\u046A\u046C\u046E\u0470\u0472\u0474\u0476\u0478\u047A\u047C\u047E\u0480\u0483-\u048A\u048C\u048E\u0490\u0492\u0494\u0496\u0498\u049A\u049C\u049E\u04A0\u04A2\u04A4\u04A6\u04A8\u04AA\u04AC\u04AE\u04B0\u04B2\u04B4\u04B6\u04B8\u04BA\u04BC\u04BE\u04C0-\u04C1\u04C3\u04C5\u04C7\u04C9\u04CB\u04CD\u04D0\u04D2\u04D4\u04D6\u04D8\u04DA\u04DC\u04DE\u04E0\u04E2\u04E4\u04E6\u04E8\u04EA\u04EC\u04EE\u04F0\u04F2\u04F4\u04F6\u04F8\u04FA\u04FC\u04FE\u0500\u0502\u0504\u0506\u0508\u050A\u050C\u050E\u0510\u0512\u0514\u0516\u0518\u051A\u051C\u051E\u0520\u0522\u0524\u0526\u0528\u052A\u052C\u052E\u0531-\u0556\u0559\u0591-\u05BD\u05BF\u05C1-\u05C2\u05C4-\u05C5\u05C7\u05D0-\u05EA\u05EF-\u05F2\u0610-\u061A\u0620-\u065F\u066E-\u06D3\u06D5-\u06DC\u06DF-\u06E8\u06EA-\u06EF\u06FA-\u06FC\u06FF\u0710-\u074A\u074D-\u07B1\u07CA-\u07F5\u07FA\u07FD\u0800-\u082D\u0840-\u085B\u0860-\u086A\u0870-\u0887\u0889-\u088E\u0897-\u08E1\u08E3-\u0963\u0971-\u0983\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BC-\u09C4\u09C7-\u09C8\u09CB-\u09CE\u09D7\u09DC-\u09DD\u09DF-\u09E3\u09F0-\u09F1\u09FC\u09FE\u0A01-\u0A03\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33\u0A35-\u0A36\u0A38-\u0A39\u0A3C\u0A3E-\u0A42\u0A47-\u0A48\u0A4B-\u0A4D\u0A51\u0A59-\u0A5C\u0A5E\u0A70-\u0A75\u0A81-\u0A83\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABC-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD\u0AD0\u0AE0-\u0AE3\u0AF9-\u0AFF\u0B01-\u0B03\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32-\u0B33\u0B35-\u0B39\u0B3C-\u0B44\u0B47-\u0B48\u0B4B-\u0B4D\u0B55-\u0B57\u0B5C-\u0B5D\u0B5F-\u0B63\u0B71\u0B82-\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD0\u0BD7\u0C00-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3C-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55-\u0C56\u0C58-\u0C5A\u0C5D\u0C60-\u0C63\u0C80-\u0C83\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBC-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5-\u0CD6\u0CDD-\u0CDE\u0CE0-\u0CE3\u0CF1-\u0CF3\u0D00-\u0D0C\u0D0E-\u0D10\u0D12-\u0D44\u0D46-\u0D48\u0D4A-\u0D4E\u0D54-\u0D57\u0D5F-\u0D63\u0D7A-\u0D7F\u0D81-\u0D83\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2-\u0DF3\u0E01-\u0E3A\u0E40-\u0E4E\u0E81-\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EBD\u0EC0-\u0EC4\u0EC6\u0EC8-\u0ECE\u0EDC-\u0EDF\u0F00\u0F18-\u0F19\u0F35\u0F37\u0F39\u0F3E-\u0F47\u0F49-\u0F6C\u0F71-\u0F84\u0F86-\u0F97\u0F99-\u0FBC\u0FC6\u1000-\u103F\u1050-\u108F\u109A-\u109D\u10A0-\u10C5\u10C7\u10CD\u10FC\u1100-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u135D-\u135F\u1380-\u138F\u13A0-\u13F5\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16F1-\u16F8\u1700-\u1715\u171F-\u1734\u1740-\u1753\u1760-\u176C\u176E-\u1770\u1772-\u1773\u1780-\u17D3\u17D7\u17DC-\u17DD\u180B-\u180D\u180F\u1820-\u1878\u1880-\u18AA\u18B0-\u18F5\u1900-\u191E\u1920-\u192B\u1930-\u193B\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u1A00-\u1A1B\u1A20-\u1A5E\u1A60-\u1A7C\u1A7F\u1AA7\u1AB0-\u1ACE\u1B00-\u1B4C\u1B6B-\u1B73\u1B80-\u1BAF\u1BBA-\u1BF3\u1C00-\u1C37\u1C4D-\u1C4F\u1C5A-\u1C7D\u1C89\u1C90-\u1CBA\u1CBD-\u1CBF\u1CD0-\u1CD2\u1CD4-\u1CFA\u1D2C-\u1D6A\u1D78\u1D9B-\u1E00\u1E02\u1E04\u1E06\u1E08\u1E0A\u1E0C\u1E0E\u1E10\u1E12\u1E14\u1E16\u1E18\u1E1A\u1E1C\u1E1E\u1E20\u1E22\u1E24\u1E26\u1E28\u1E2A\u1E2C\u1E2E\u1E30\u1E32\u1E34\u1E36\u1E38\u1E3A\u1E3C\u1E3E\u1E40\u1E42\u1E44\u1E46\u1E48\u1E4A\u1E4C\u1E4E\u1E50\u1E52\u1E54\u1E56\u1E58\u1E5A\u1E5C\u1E5E\u1E60\u1E62\u1E64\u1E66\u1E68\u1E6A\u1E6C\u1E6E\u1E70\u1E72\u1E74\u1E76\u1E78\u1E7A\u1E7C\u1E7E\u1E80\u1E82\u1E84\u1E86\u1E88\u1E8A\u1E8C\u1E8E\u1E90\u1E92\u1E94\u1E9E\u1EA0\u1EA2\u1EA4\u1EA6\u1EA8\u1EAA\u1EAC\u1EAE\u1EB0\u1EB2\u1EB4\u1EB6\u1EB8\u1EBA\u1EBC\u1EBE\u1EC0\u1EC2\u1EC4\u1EC6\u1EC8\u1ECA\u1ECC\u1ECE\u1ED0\u1ED2\u1ED4\u1ED6\u1ED8\u1EDA\u1EDC\u1EDE\u1EE0\u1EE2\u1EE4\u1EE6\u1EE8\u1EEA\u1EEC\u1EEE\u1EF0\u1EF2\u1EF4\u1EF6\u1EF8\u1EFA\u1EFC\u1EFE\u1F08-\u1F0F\u1F18-\u1F1D\u1F28-\u1F2F\u1F38-\u1F3F\u1F48-\u1F4D\u1F59\u1F5B\u1F5D\u1F5F\u1F68-\u1F6F\u1F88-\u1F8F\u1F98-\u1F9F\u1FA8-\u1FAF\u1FB8-\u1FBC\u1FC8-\u1FCC\u1FD8-\u1FDB\u1FE8-\u1FEC\u1FF8-\u1FFC\u2071\u207F\u2090-\u209C\u20D0-\u20F0\u2102\u2107\u210B-\u210D\u2110-\u2112\u2115\u2119-\u211D\u2124\u2126\u2128\u212A-\u212D\u2130-\u2133\u2135-\u2138\u213E-\u213F\u2145\u2183\u2C00-\u2C2F\u2C60\u2C62-\u2C64\u2C67\u2C69\u2C6B\u2C6D-\u2C70\u2C72\u2C75\u2C7C-\u2C80\u2C82\u2C84\u2C86\u2C88\u2C8A\u2C8C\u2C8E\u2C90\u2C92\u2C94\u2C96\u2C98\u2C9A\u2C9C\u2C9E\u2CA0\u2CA2\u2CA4\u2CA6\u2CA8\u2CAA\u2CAC\u2CAE\u2CB0\u2CB2\u2CB4\u2CB6\u2CB8\u2CBA\u2CBC\u2CBE\u2CC0\u2CC2\u2CC4\u2CC6\u2CC8\u2CCA\u2CCC\u2CCE\u2CD0\u2CD2\u2CD4\u2CD6\u2CD8\u2CDA\u2CDC\u2CDE\u2CE0\u2CE2\u2CEB\u2CED\u2CEF-\u2CF2\u2D30-\u2D67\u2D6F\u2D7F-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2DE0-\u2DFF\u2E2F\u3005-\u3006\u302A-\u302F\u3031-\u3035\u303B-\u303C\u3041-\u3096\u3099-\u309A\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u31A0-\u31BF\u31F0-\u31FF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A-\uA62B\uA640\uA642\uA644\uA646\uA648\uA64A\uA64C\uA64E\uA650\uA652\uA654\uA656\uA658\uA65A\uA65C\uA65E\uA660\uA662\uA664\uA666\uA668\uA66A\uA66C\uA66E-\uA672\uA674-\uA67D\uA67F-\uA680\uA682\uA684\uA686\uA688\uA68A\uA68C\uA68E\uA690\uA692\uA694\uA696\uA698\uA69A\uA69C-\uA6E5\uA6F0-\uA6F1\uA717-\uA71F\uA722\uA724\uA726\uA728\uA72A\uA72C\uA72E\uA732\uA734\uA736\uA738\uA73A\uA73C\uA73E\uA740\uA742\uA744\uA746\uA748\uA74A\uA74C\uA74E\uA750\uA752\uA754\uA756\uA758\uA75A\uA75C\uA75E\uA760\uA762\uA764\uA766\uA768\uA76A\uA76C\uA76E\uA770\uA779\uA77B\uA77D-\uA77E\uA780\uA782\uA784\uA786\uA788\uA78B\uA78D\uA78F-\uA790\uA792\uA796\uA798\uA79A\uA79C\uA79E\uA7A0\uA7A2\uA7A4\uA7A6\uA7A8\uA7AA-\uA7AE\uA7B0-\uA7B4\uA7B6\uA7B8\uA7BA\uA7BC\uA7BE\uA7C0\uA7C2\uA7C4-\uA7C7\uA7C9\uA7CB-\uA7CC\uA7D0\uA7D6\uA7D8\uA7DA\uA7DC\uA7F2-\uA7F5\uA7F7-\uA7F9\uA7FB-\uA827\uA82C\uA840-\uA873\uA880-\uA8C5\uA8E0-\uA8F7\uA8FB\uA8FD-\uA8FF\uA90A-\uA92D\uA930-\uA953\uA960-\uA97C\uA980-\uA9C0\uA9CF\uA9E0-\uA9EF\uA9FA-\uA9FE\uAA00-\uAA36\uAA40-\uAA4D\uAA60-\uAA76\uAA7A-\uAAC2\uAADB-\uAADD\uAAE0-\uAAEF\uAAF2-\uAAF6\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB5C-\uAB5F\uAB69\uABC0-\uABEA\uABEC-\uABED\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB1D-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE00-\uFE0F\uFE20-\uFE2F\uFE70-\uFE74\uFE76-\uFEFC\uFF21-\uFF3A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC\u{10000}-\u{1000B}\u{1000D}-\u{10026}\u{10028}-\u{1003A}\u{1003C}-\u{1003D}\u{1003F}-\u{1004D}\u{10050}-\u{1005D}\u{10080}-\u{100FA}\u{101FD}\u{10280}-\u{1029C}\u{102A0}-\u{102D0}\u{102E0}\u{10300}-\u{1031F}\u{1032D}-\u{10340}\u{10342}-\u{10349}\u{10350}-\u{1037A}\u{10380}-\u{1039D}\u{103A0}-\u{103C3}\u{103C8}-\u{103CF}\u{10400}-\u{10427}\u{10450}-\u{1049D}\u{104B0}-\u{104D3}\u{10500}-\u{10527}\u{10530}-\u{10563}\u{10570}-\u{1057A}\u{1057C}-\u{1058A}\u{1058C}-\u{10592}\u{10594}-\u{10595}\u{105C0}-\u{105F3}\u{10600}-\u{10736}\u{10740}-\u{10755}\u{10760}-\u{10767}\u{10780}-\u{10785}\u{10787}-\u{107B0}\u{107B2}-\u{107BA}\u{10800}-\u{10805}\u{10808}\u{1080A}-\u{10835}\u{10837}-\u{10838}\u{1083C}\u{1083F}-\u{10855}\u{10860}-\u{10876}\u{10880}-\u{1089E}\u{108E0}-\u{108F2}\u{108F4}-\u{108F5}\u{10900}-\u{10915}\u{10920}-\u{10939}\u{10980}-\u{109B7}\u{109BE}-\u{109BF}\u{10A00}-\u{10A03}\u{10A05}-\u{10A06}\u{10A0C}-\u{10A13}\u{10A15}-\u{10A17}\u{10A19}-\u{10A35}\u{10A38}-\u{10A3A}\u{10A3F}\u{10A60}-\u{10A7C}\u{10A80}-\u{10A9C}\u{10AC0}-\u{10AC7}\u{10AC9}-\u{10AE6}\u{10B00}-\u{10B35}\u{10B40}-\u{10B55}\u{10B60}-\u{10B72}\u{10B80}-\u{10B91}\u{10C00}-\u{10C48}\u{10C80}-\u{10CB2}\u{10D00}-\u{10D27}\u{10D4A}-\u{10D65}\u{10D69}-\u{10D6D}\u{10D6F}\u{10E80}-\u{10EA9}\u{10EAB}-\u{10EAC}\u{10EB0}-\u{10EB1}\u{10EC2}-\u{10EC4}\u{10EFC}-\u{10F1C}\u{10F27}\u{10F30}-\u{10F50}\u{10F70}-\u{10F85}\u{10FB0}-\u{10FC4}\u{10FE0}-\u{10FF6}\u{11000}-\u{11046}\u{11070}-\u{11075}\u{1107F}-\u{110BA}\u{110C2}\u{110D0}-\u{110E8}\u{11100}-\u{11134}\u{11144}-\u{11147}\u{11150}-\u{11173}\u{11176}\u{11180}-\u{111C4}\u{111C9}-\u{111CC}\u{111CE}-\u{111CF}\u{111DA}\u{111DC}\u{11200}-\u{11211}\u{11213}-\u{11237}\u{1123E}-\u{11241}\u{11280}-\u{11286}\u{11288}\u{1128A}-\u{1128D}\u{1128F}-\u{1129D}\u{1129F}-\u{112A8}\u{112B0}-\u{112EA}\u{11300}-\u{11303}\u{11305}-\u{1130C}\u{1130F}-\u{11310}\u{11313}-\u{11328}\u{1132A}-\u{11330}\u{11332}-\u{11333}\u{11335}-\u{11339}\u{1133B}-\u{11344}\u{11347}-\u{11348}\u{1134B}-\u{1134D}\u{11350}\u{11357}\u{1135D}-\u{11363}\u{11366}-\u{1136C}\u{11370}-\u{11374}\u{11380}-\u{11389}\u{1138B}\u{1138E}\u{11390}-\u{113B5}\u{113B7}-\u{113C0}\u{113C2}\u{113C5}\u{113C7}-\u{113CA}\u{113CC}-\u{113D3}\u{113E1}-\u{113E2}\u{11400}-\u{1144A}\u{1145E}-\u{11461}\u{11480}-\u{114C5}\u{114C7}\u{11580}-\u{115B5}\u{115B8}-\u{115C0}\u{115D8}-\u{115DD}\u{11600}-\u{11640}\u{11644}\u{11680}-\u{116B8}\u{11700}-\u{1171A}\u{1171D}-\u{1172B}\u{11740}-\u{11746}\u{11800}-\u{1183A}\u{118A0}-\u{118BF}\u{118FF}-\u{11906}\u{11909}\u{1190C}-\u{11913}\u{11915}-\u{11916}\u{11918}-\u{11935}\u{11937}-\u{11938}\u{1193B}-\u{11943}\u{119A0}-\u{119A7}\u{119AA}-\u{119D7}\u{119DA}-\u{119E1}\u{119E3}-\u{119E4}\u{11A00}-\u{11A3E}\u{11A47}\u{11A50}-\u{11A99}\u{11A9D}\u{11AB0}-\u{11AF8}\u{11BC0}-\u{11BE0}\u{11C00}-\u{11C08}\u{11C0A}-\u{11C36}\u{11C38}-\u{11C40}\u{11C72}-\u{11C8F}\u{11C92}-\u{11CA7}\u{11CA9}-\u{11CB6}\u{11D00}-\u{11D06}\u{11D08}-\u{11D09}\u{11D0B}-\u{11D36}\u{11D3A}\u{11D3C}-\u{11D3D}\u{11D3F}-\u{11D47}\u{11D60}-\u{11D65}\u{11D67}-\u{11D68}\u{11D6A}-\u{11D8E}\u{11D90}-\u{11D91}\u{11D93}-\u{11D98}\u{11EE0}-\u{11EF6}\u{11F00}-\u{11F10}\u{11F12}-\u{11F3A}\u{11F3E}-\u{11F42}\u{11F5A}\u{11FB0}\u{12000}-\u{12399}\u{12480}-\u{12543}\u{12F90}-\u{12FF0}\u{13000}-\u{1342F}\u{13440}-\u{13455}\u{13460}-\u{143FA}\u{14400}-\u{14646}\u{16100}-\u{1612F}\u{16800}-\u{16A38}\u{16A40}-\u{16A5E}\u{16A70}-\u{16ABE}\u{16AD0}-\u{16AED}\u{16AF0}-\u{16AF4}\u{16B00}-\u{16B36}\u{16B40}-\u{16B43}\u{16B63}-\u{16B77}\u{16B7D}-\u{16B8F}\u{16D40}-\u{16D6C}\u{16E40}-\u{16E5F}\u{16F00}-\u{16F4A}\u{16F4F}-\u{16F87}\u{16F8F}-\u{16F9F}\u{16FE0}-\u{16FE1}\u{16FE3}-\u{16FE4}\u{16FF0}-\u{16FF1}\u{17000}-\u{187F7}\u{18800}-\u{18CD5}\u{18CFF}-\u{18D08}\u{1AFF0}-\u{1AFF3}\u{1AFF5}-\u{1AFFB}\u{1AFFD}-\u{1AFFE}\u{1B000}-\u{1B122}\u{1B132}\u{1B150}-\u{1B152}\u{1B155}\u{1B164}-\u{1B167}\u{1B170}-\u{1B2FB}\u{1BC00}-\u{1BC6A}\u{1BC70}-\u{1BC7C}\u{1BC80}-\u{1BC88}\u{1BC90}-\u{1BC99}\u{1BC9D}-\u{1BC9E}\u{1CF00}-\u{1CF2D}\u{1CF30}-\u{1CF46}\u{1D165}-\u{1D169}\u{1D16D}-\u{1D172}\u{1D17B}-\u{1D182}\u{1D185}-\u{1D18B}\u{1D1AA}-\u{1D1AD}\u{1D242}-\u{1D244}\u{1D400}-\u{1D419}\u{1D434}-\u{1D44D}\u{1D468}-\u{1D481}\u{1D49C}\u{1D49E}-\u{1D49F}\u{1D4A2}\u{1D4A5}-\u{1D4A6}\u{1D4A9}-\u{1D4AC}\u{1D4AE}-\u{1D4B5}\u{1D4D0}-\u{1D4E9}\u{1D504}-\u{1D505}\u{1D507}-\u{1D50A}\u{1D50D}-\u{1D514}\u{1D516}-\u{1D51C}\u{1D538}-\u{1D539}\u{1D53B}-\u{1D53E}\u{1D540}-\u{1D544}\u{1D546}\u{1D54A}-\u{1D550}\u{1D56C}-\u{1D585}\u{1D5A0}-\u{1D5B9}\u{1D5D4}-\u{1D5ED}\u{1D608}-\u{1D621}\u{1D63C}-\u{1D655}\u{1D670}-\u{1D689}\u{1D6A8}-\u{1D6C0}\u{1D6E2}-\u{1D6FA}\u{1D71C}-\u{1D734}\u{1D756}-\u{1D76E}\u{1D790}-\u{1D7A8}\u{1D7CA}\u{1DA00}-\u{1DA36}\u{1DA3B}-\u{1DA6C}\u{1DA75}\u{1DA84}\u{1DA9B}-\u{1DA9F}\u{1DAA1}-\u{1DAAF}\u{1DF0A}\u{1E000}-\u{1E006}\u{1E008}-\u{1E018}\u{1E01B}-\u{1E021}\u{1E023}-\u{1E024}\u{1E026}-\u{1E02A}\u{1E030}-\u{1E06D}\u{1E08F}\u{1E100}-\u{1E12C}\u{1E130}-\u{1E13D}\u{1E14E}\u{1E290}-\u{1E2AE}\u{1E2C0}-\u{1E2EF}\u{1E4D0}-\u{1E4EF}\u{1E5D0}-\u{1E5F0}\u{1E7E0}-\u{1E7E6}\u{1E7E8}-\u{1E7EB}\u{1E7ED}-\u{1E7EE}\u{1E7F0}-\u{1E7FE}\u{1E800}-\u{1E8C4}\u{1E8D0}-\u{1E8D6}\u{1E900}-\u{1E921}\u{1E944}-\u{1E94B}\u{1EE00}-\u{1EE03}\u{1EE05}-\u{1EE1F}\u{1EE21}-\u{1EE22}\u{1EE24}\u{1EE27}\u{1EE29}-\u{1EE32}\u{1EE34}-\u{1EE37}\u{1EE39}\u{1EE3B}\u{1EE42}\u{1EE47}\u{1EE49}\u{1EE4B}\u{1EE4D}-\u{1EE4F}\u{1EE51}-\u{1EE52}\u{1EE54}\u{1EE57}\u{1EE59}\u{1EE5B}\u{1EE5D}\u{1EE5F}\u{1EE61}-\u{1EE62}\u{1EE64}\u{1EE67}-\u{1EE6A}\u{1EE6C}-\u{1EE72}\u{1EE74}-\u{1EE77}\u{1EE79}-\u{1EE7C}\u{1EE7E}\u{1EE80}-\u{1EE89}\u{1EE8B}-\u{1EE9B}\u{1EEA1}-\u{1EEA3}\u{1EEA5}-\u{1EEA9}\u{1EEAB}-\u{1EEBB}\u{20000}-\u{2A6DF}\u{2A700}-\u{2B739}\u{2B740}-\u{2B81D}\u{2B820}-\u{2CEA1}\u{2CEB0}-\u{2EBE0}\u{2EBF0}-\u{2EE5D}\u{2F800}-\u{2FA1D}\u{30000}-\u{3134A}\u{31350}-\u{323AF}\u{E0100}-\u{E01EF}`;
|
|
375
|
+
var UNICODE_16_O200K_LOWER_RANGES = String.raw`\u0061-\u007A\u00AA\u00B5\u00BA\u00DF-\u00F6\u00F8-\u00FF\u0101\u0103\u0105\u0107\u0109\u010B\u010D\u010F\u0111\u0113\u0115\u0117\u0119\u011B\u011D\u011F\u0121\u0123\u0125\u0127\u0129\u012B\u012D\u012F\u0131\u0133\u0135\u0137-\u0138\u013A\u013C\u013E\u0140\u0142\u0144\u0146\u0148-\u0149\u014B\u014D\u014F\u0151\u0153\u0155\u0157\u0159\u015B\u015D\u015F\u0161\u0163\u0165\u0167\u0169\u016B\u016D\u016F\u0171\u0173\u0175\u0177\u017A\u017C\u017E-\u0180\u0183\u0185\u0188\u018C-\u018D\u0192\u0195\u0199-\u019B\u019E\u01A1\u01A3\u01A5\u01A8\u01AA-\u01AB\u01AD\u01B0\u01B4\u01B6\u01B9-\u01BB\u01BD-\u01C3\u01C6\u01C9\u01CC\u01CE\u01D0\u01D2\u01D4\u01D6\u01D8\u01DA\u01DC-\u01DD\u01DF\u01E1\u01E3\u01E5\u01E7\u01E9\u01EB\u01ED\u01EF-\u01F0\u01F3\u01F5\u01F9\u01FB\u01FD\u01FF\u0201\u0203\u0205\u0207\u0209\u020B\u020D\u020F\u0211\u0213\u0215\u0217\u0219\u021B\u021D\u021F\u0221\u0223\u0225\u0227\u0229\u022B\u022D\u022F\u0231\u0233-\u0239\u023C\u023F-\u0240\u0242\u0247\u0249\u024B\u024D\u024F-\u02C1\u02C6-\u02D1\u02E0-\u02E4\u02EC\u02EE\u0300-\u036F\u0371\u0373-\u0374\u0377\u037A-\u037D\u0390\u03AC-\u03CE\u03D0-\u03D1\u03D5-\u03D7\u03D9\u03DB\u03DD\u03DF\u03E1\u03E3\u03E5\u03E7\u03E9\u03EB\u03ED\u03EF-\u03F3\u03F5\u03F8\u03FB-\u03FC\u0430-\u045F\u0461\u0463\u0465\u0467\u0469\u046B\u046D\u046F\u0471\u0473\u0475\u0477\u0479\u047B\u047D\u047F\u0481\u0483-\u0489\u048B\u048D\u048F\u0491\u0493\u0495\u0497\u0499\u049B\u049D\u049F\u04A1\u04A3\u04A5\u04A7\u04A9\u04AB\u04AD\u04AF\u04B1\u04B3\u04B5\u04B7\u04B9\u04BB\u04BD\u04BF\u04C2\u04C4\u04C6\u04C8\u04CA\u04CC\u04CE-\u04CF\u04D1\u04D3\u04D5\u04D7\u04D9\u04DB\u04DD\u04DF\u04E1\u04E3\u04E5\u04E7\u04E9\u04EB\u04ED\u04EF\u04F1\u04F3\u04F5\u04F7\u04F9\u04FB\u04FD\u04FF\u0501\u0503\u0505\u0507\u0509\u050B\u050D\u050F\u0511\u0513\u0515\u0517\u0519\u051B\u051D\u051F\u0521\u0523\u0525\u0527\u0529\u052B\u052D\u052F\u0559\u0560-\u0588\u0591-\u05BD\u05BF\u05C1-\u05C2\u05C4-\u05C5\u05C7\u05D0-\u05EA\u05EF-\u05F2\u0610-\u061A\u0620-\u065F\u066E-\u06D3\u06D5-\u06DC\u06DF-\u06E8\u06EA-\u06EF\u06FA-\u06FC\u06FF\u0710-\u074A\u074D-\u07B1\u07CA-\u07F5\u07FA\u07FD\u0800-\u082D\u0840-\u085B\u0860-\u086A\u0870-\u0887\u0889-\u088E\u0897-\u08E1\u08E3-\u0963\u0971-\u0983\u0985-\u098C\u098F-\u0990\u0993-\u09A8\u09AA-\u09B0\u09B2\u09B6-\u09B9\u09BC-\u09C4\u09C7-\u09C8\u09CB-\u09CE\u09D7\u09DC-\u09DD\u09DF-\u09E3\u09F0-\u09F1\u09FC\u09FE\u0A01-\u0A03\u0A05-\u0A0A\u0A0F-\u0A10\u0A13-\u0A28\u0A2A-\u0A30\u0A32-\u0A33\u0A35-\u0A36\u0A38-\u0A39\u0A3C\u0A3E-\u0A42\u0A47-\u0A48\u0A4B-\u0A4D\u0A51\u0A59-\u0A5C\u0A5E\u0A70-\u0A75\u0A81-\u0A83\u0A85-\u0A8D\u0A8F-\u0A91\u0A93-\u0AA8\u0AAA-\u0AB0\u0AB2-\u0AB3\u0AB5-\u0AB9\u0ABC-\u0AC5\u0AC7-\u0AC9\u0ACB-\u0ACD\u0AD0\u0AE0-\u0AE3\u0AF9-\u0AFF\u0B01-\u0B03\u0B05-\u0B0C\u0B0F-\u0B10\u0B13-\u0B28\u0B2A-\u0B30\u0B32-\u0B33\u0B35-\u0B39\u0B3C-\u0B44\u0B47-\u0B48\u0B4B-\u0B4D\u0B55-\u0B57\u0B5C-\u0B5D\u0B5F-\u0B63\u0B71\u0B82-\u0B83\u0B85-\u0B8A\u0B8E-\u0B90\u0B92-\u0B95\u0B99-\u0B9A\u0B9C\u0B9E-\u0B9F\u0BA3-\u0BA4\u0BA8-\u0BAA\u0BAE-\u0BB9\u0BBE-\u0BC2\u0BC6-\u0BC8\u0BCA-\u0BCD\u0BD0\u0BD7\u0C00-\u0C0C\u0C0E-\u0C10\u0C12-\u0C28\u0C2A-\u0C39\u0C3C-\u0C44\u0C46-\u0C48\u0C4A-\u0C4D\u0C55-\u0C56\u0C58-\u0C5A\u0C5D\u0C60-\u0C63\u0C80-\u0C83\u0C85-\u0C8C\u0C8E-\u0C90\u0C92-\u0CA8\u0CAA-\u0CB3\u0CB5-\u0CB9\u0CBC-\u0CC4\u0CC6-\u0CC8\u0CCA-\u0CCD\u0CD5-\u0CD6\u0CDD-\u0CDE\u0CE0-\u0CE3\u0CF1-\u0CF3\u0D00-\u0D0C\u0D0E-\u0D10\u0D12-\u0D44\u0D46-\u0D48\u0D4A-\u0D4E\u0D54-\u0D57\u0D5F-\u0D63\u0D7A-\u0D7F\u0D81-\u0D83\u0D85-\u0D96\u0D9A-\u0DB1\u0DB3-\u0DBB\u0DBD\u0DC0-\u0DC6\u0DCA\u0DCF-\u0DD4\u0DD6\u0DD8-\u0DDF\u0DF2-\u0DF3\u0E01-\u0E3A\u0E40-\u0E4E\u0E81-\u0E82\u0E84\u0E86-\u0E8A\u0E8C-\u0EA3\u0EA5\u0EA7-\u0EBD\u0EC0-\u0EC4\u0EC6\u0EC8-\u0ECE\u0EDC-\u0EDF\u0F00\u0F18-\u0F19\u0F35\u0F37\u0F39\u0F3E-\u0F47\u0F49-\u0F6C\u0F71-\u0F84\u0F86-\u0F97\u0F99-\u0FBC\u0FC6\u1000-\u103F\u1050-\u108F\u109A-\u109D\u10D0-\u10FA\u10FC-\u1248\u124A-\u124D\u1250-\u1256\u1258\u125A-\u125D\u1260-\u1288\u128A-\u128D\u1290-\u12B0\u12B2-\u12B5\u12B8-\u12BE\u12C0\u12C2-\u12C5\u12C8-\u12D6\u12D8-\u1310\u1312-\u1315\u1318-\u135A\u135D-\u135F\u1380-\u138F\u13F8-\u13FD\u1401-\u166C\u166F-\u167F\u1681-\u169A\u16A0-\u16EA\u16F1-\u16F8\u1700-\u1715\u171F-\u1734\u1740-\u1753\u1760-\u176C\u176E-\u1770\u1772-\u1773\u1780-\u17D3\u17D7\u17DC-\u17DD\u180B-\u180D\u180F\u1820-\u1878\u1880-\u18AA\u18B0-\u18F5\u1900-\u191E\u1920-\u192B\u1930-\u193B\u1950-\u196D\u1970-\u1974\u1980-\u19AB\u19B0-\u19C9\u1A00-\u1A1B\u1A20-\u1A5E\u1A60-\u1A7C\u1A7F\u1AA7\u1AB0-\u1ACE\u1B00-\u1B4C\u1B6B-\u1B73\u1B80-\u1BAF\u1BBA-\u1BF3\u1C00-\u1C37\u1C4D-\u1C4F\u1C5A-\u1C7D\u1C80-\u1C88\u1C8A\u1CD0-\u1CD2\u1CD4-\u1CFA\u1D00-\u1DFF\u1E01\u1E03\u1E05\u1E07\u1E09\u1E0B\u1E0D\u1E0F\u1E11\u1E13\u1E15\u1E17\u1E19\u1E1B\u1E1D\u1E1F\u1E21\u1E23\u1E25\u1E27\u1E29\u1E2B\u1E2D\u1E2F\u1E31\u1E33\u1E35\u1E37\u1E39\u1E3B\u1E3D\u1E3F\u1E41\u1E43\u1E45\u1E47\u1E49\u1E4B\u1E4D\u1E4F\u1E51\u1E53\u1E55\u1E57\u1E59\u1E5B\u1E5D\u1E5F\u1E61\u1E63\u1E65\u1E67\u1E69\u1E6B\u1E6D\u1E6F\u1E71\u1E73\u1E75\u1E77\u1E79\u1E7B\u1E7D\u1E7F\u1E81\u1E83\u1E85\u1E87\u1E89\u1E8B\u1E8D\u1E8F\u1E91\u1E93\u1E95-\u1E9D\u1E9F\u1EA1\u1EA3\u1EA5\u1EA7\u1EA9\u1EAB\u1EAD\u1EAF\u1EB1\u1EB3\u1EB5\u1EB7\u1EB9\u1EBB\u1EBD\u1EBF\u1EC1\u1EC3\u1EC5\u1EC7\u1EC9\u1ECB\u1ECD\u1ECF\u1ED1\u1ED3\u1ED5\u1ED7\u1ED9\u1EDB\u1EDD\u1EDF\u1EE1\u1EE3\u1EE5\u1EE7\u1EE9\u1EEB\u1EED\u1EEF\u1EF1\u1EF3\u1EF5\u1EF7\u1EF9\u1EFB\u1EFD\u1EFF-\u1F07\u1F10-\u1F15\u1F20-\u1F27\u1F30-\u1F37\u1F40-\u1F45\u1F50-\u1F57\u1F60-\u1F67\u1F70-\u1F7D\u1F80-\u1F87\u1F90-\u1F97\u1FA0-\u1FA7\u1FB0-\u1FB4\u1FB6-\u1FB7\u1FBE\u1FC2-\u1FC4\u1FC6-\u1FC7\u1FD0-\u1FD3\u1FD6-\u1FD7\u1FE0-\u1FE7\u1FF2-\u1FF4\u1FF6-\u1FF7\u2071\u207F\u2090-\u209C\u20D0-\u20F0\u210A\u210E-\u210F\u2113\u212F\u2134-\u2139\u213C-\u213D\u2146-\u2149\u214E\u2184\u2C30-\u2C5F\u2C61\u2C65-\u2C66\u2C68\u2C6A\u2C6C\u2C71\u2C73-\u2C74\u2C76-\u2C7D\u2C81\u2C83\u2C85\u2C87\u2C89\u2C8B\u2C8D\u2C8F\u2C91\u2C93\u2C95\u2C97\u2C99\u2C9B\u2C9D\u2C9F\u2CA1\u2CA3\u2CA5\u2CA7\u2CA9\u2CAB\u2CAD\u2CAF\u2CB1\u2CB3\u2CB5\u2CB7\u2CB9\u2CBB\u2CBD\u2CBF\u2CC1\u2CC3\u2CC5\u2CC7\u2CC9\u2CCB\u2CCD\u2CCF\u2CD1\u2CD3\u2CD5\u2CD7\u2CD9\u2CDB\u2CDD\u2CDF\u2CE1\u2CE3-\u2CE4\u2CEC\u2CEE-\u2CF1\u2CF3\u2D00-\u2D25\u2D27\u2D2D\u2D30-\u2D67\u2D6F\u2D7F-\u2D96\u2DA0-\u2DA6\u2DA8-\u2DAE\u2DB0-\u2DB6\u2DB8-\u2DBE\u2DC0-\u2DC6\u2DC8-\u2DCE\u2DD0-\u2DD6\u2DD8-\u2DDE\u2DE0-\u2DFF\u2E2F\u3005-\u3006\u302A-\u302F\u3031-\u3035\u303B-\u303C\u3041-\u3096\u3099-\u309A\u309D-\u309F\u30A1-\u30FA\u30FC-\u30FF\u3105-\u312F\u3131-\u318E\u31A0-\u31BF\u31F0-\u31FF\u3400-\u4DBF\u4E00-\uA48C\uA4D0-\uA4FD\uA500-\uA60C\uA610-\uA61F\uA62A-\uA62B\uA641\uA643\uA645\uA647\uA649\uA64B\uA64D\uA64F\uA651\uA653\uA655\uA657\uA659\uA65B\uA65D\uA65F\uA661\uA663\uA665\uA667\uA669\uA66B\uA66D-\uA672\uA674-\uA67D\uA67F\uA681\uA683\uA685\uA687\uA689\uA68B\uA68D\uA68F\uA691\uA693\uA695\uA697\uA699\uA69B-\uA6E5\uA6F0-\uA6F1\uA717-\uA71F\uA723\uA725\uA727\uA729\uA72B\uA72D\uA72F-\uA731\uA733\uA735\uA737\uA739\uA73B\uA73D\uA73F\uA741\uA743\uA745\uA747\uA749\uA74B\uA74D\uA74F\uA751\uA753\uA755\uA757\uA759\uA75B\uA75D\uA75F\uA761\uA763\uA765\uA767\uA769\uA76B\uA76D\uA76F-\uA778\uA77A\uA77C\uA77F\uA781\uA783\uA785\uA787-\uA788\uA78C\uA78E-\uA78F\uA791\uA793-\uA795\uA797\uA799\uA79B\uA79D\uA79F\uA7A1\uA7A3\uA7A5\uA7A7\uA7A9\uA7AF\uA7B5\uA7B7\uA7B9\uA7BB\uA7BD\uA7BF\uA7C1\uA7C3\uA7C8\uA7CA\uA7CD\uA7D1\uA7D3\uA7D5\uA7D7\uA7D9\uA7DB\uA7F2-\uA7F4\uA7F6-\uA827\uA82C\uA840-\uA873\uA880-\uA8C5\uA8E0-\uA8F7\uA8FB\uA8FD-\uA8FF\uA90A-\uA92D\uA930-\uA953\uA960-\uA97C\uA980-\uA9C0\uA9CF\uA9E0-\uA9EF\uA9FA-\uA9FE\uAA00-\uAA36\uAA40-\uAA4D\uAA60-\uAA76\uAA7A-\uAAC2\uAADB-\uAADD\uAAE0-\uAAEF\uAAF2-\uAAF6\uAB01-\uAB06\uAB09-\uAB0E\uAB11-\uAB16\uAB20-\uAB26\uAB28-\uAB2E\uAB30-\uAB5A\uAB5C-\uAB69\uAB70-\uABEA\uABEC-\uABED\uAC00-\uD7A3\uD7B0-\uD7C6\uD7CB-\uD7FB\uF900-\uFA6D\uFA70-\uFAD9\uFB00-\uFB06\uFB13-\uFB17\uFB1D-\uFB28\uFB2A-\uFB36\uFB38-\uFB3C\uFB3E\uFB40-\uFB41\uFB43-\uFB44\uFB46-\uFBB1\uFBD3-\uFD3D\uFD50-\uFD8F\uFD92-\uFDC7\uFDF0-\uFDFB\uFE00-\uFE0F\uFE20-\uFE2F\uFE70-\uFE74\uFE76-\uFEFC\uFF41-\uFF5A\uFF66-\uFFBE\uFFC2-\uFFC7\uFFCA-\uFFCF\uFFD2-\uFFD7\uFFDA-\uFFDC\u{10000}-\u{1000B}\u{1000D}-\u{10026}\u{10028}-\u{1003A}\u{1003C}-\u{1003D}\u{1003F}-\u{1004D}\u{10050}-\u{1005D}\u{10080}-\u{100FA}\u{101FD}\u{10280}-\u{1029C}\u{102A0}-\u{102D0}\u{102E0}\u{10300}-\u{1031F}\u{1032D}-\u{10340}\u{10342}-\u{10349}\u{10350}-\u{1037A}\u{10380}-\u{1039D}\u{103A0}-\u{103C3}\u{103C8}-\u{103CF}\u{10428}-\u{1049D}\u{104D8}-\u{104FB}\u{10500}-\u{10527}\u{10530}-\u{10563}\u{10597}-\u{105A1}\u{105A3}-\u{105B1}\u{105B3}-\u{105B9}\u{105BB}-\u{105BC}\u{105C0}-\u{105F3}\u{10600}-\u{10736}\u{10740}-\u{10755}\u{10760}-\u{10767}\u{10780}-\u{10785}\u{10787}-\u{107B0}\u{107B2}-\u{107BA}\u{10800}-\u{10805}\u{10808}\u{1080A}-\u{10835}\u{10837}-\u{10838}\u{1083C}\u{1083F}-\u{10855}\u{10860}-\u{10876}\u{10880}-\u{1089E}\u{108E0}-\u{108F2}\u{108F4}-\u{108F5}\u{10900}-\u{10915}\u{10920}-\u{10939}\u{10980}-\u{109B7}\u{109BE}-\u{109BF}\u{10A00}-\u{10A03}\u{10A05}-\u{10A06}\u{10A0C}-\u{10A13}\u{10A15}-\u{10A17}\u{10A19}-\u{10A35}\u{10A38}-\u{10A3A}\u{10A3F}\u{10A60}-\u{10A7C}\u{10A80}-\u{10A9C}\u{10AC0}-\u{10AC7}\u{10AC9}-\u{10AE6}\u{10B00}-\u{10B35}\u{10B40}-\u{10B55}\u{10B60}-\u{10B72}\u{10B80}-\u{10B91}\u{10C00}-\u{10C48}\u{10CC0}-\u{10CF2}\u{10D00}-\u{10D27}\u{10D4A}-\u{10D4F}\u{10D69}-\u{10D6D}\u{10D6F}-\u{10D85}\u{10E80}-\u{10EA9}\u{10EAB}-\u{10EAC}\u{10EB0}-\u{10EB1}\u{10EC2}-\u{10EC4}\u{10EFC}-\u{10F1C}\u{10F27}\u{10F30}-\u{10F50}\u{10F70}-\u{10F85}\u{10FB0}-\u{10FC4}\u{10FE0}-\u{10FF6}\u{11000}-\u{11046}\u{11070}-\u{11075}\u{1107F}-\u{110BA}\u{110C2}\u{110D0}-\u{110E8}\u{11100}-\u{11134}\u{11144}-\u{11147}\u{11150}-\u{11173}\u{11176}\u{11180}-\u{111C4}\u{111C9}-\u{111CC}\u{111CE}-\u{111CF}\u{111DA}\u{111DC}\u{11200}-\u{11211}\u{11213}-\u{11237}\u{1123E}-\u{11241}\u{11280}-\u{11286}\u{11288}\u{1128A}-\u{1128D}\u{1128F}-\u{1129D}\u{1129F}-\u{112A8}\u{112B0}-\u{112EA}\u{11300}-\u{11303}\u{11305}-\u{1130C}\u{1130F}-\u{11310}\u{11313}-\u{11328}\u{1132A}-\u{11330}\u{11332}-\u{11333}\u{11335}-\u{11339}\u{1133B}-\u{11344}\u{11347}-\u{11348}\u{1134B}-\u{1134D}\u{11350}\u{11357}\u{1135D}-\u{11363}\u{11366}-\u{1136C}\u{11370}-\u{11374}\u{11380}-\u{11389}\u{1138B}\u{1138E}\u{11390}-\u{113B5}\u{113B7}-\u{113C0}\u{113C2}\u{113C5}\u{113C7}-\u{113CA}\u{113CC}-\u{113D3}\u{113E1}-\u{113E2}\u{11400}-\u{1144A}\u{1145E}-\u{11461}\u{11480}-\u{114C5}\u{114C7}\u{11580}-\u{115B5}\u{115B8}-\u{115C0}\u{115D8}-\u{115DD}\u{11600}-\u{11640}\u{11644}\u{11680}-\u{116B8}\u{11700}-\u{1171A}\u{1171D}-\u{1172B}\u{11740}-\u{11746}\u{11800}-\u{1183A}\u{118C0}-\u{118DF}\u{118FF}-\u{11906}\u{11909}\u{1190C}-\u{11913}\u{11915}-\u{11916}\u{11918}-\u{11935}\u{11937}-\u{11938}\u{1193B}-\u{11943}\u{119A0}-\u{119A7}\u{119AA}-\u{119D7}\u{119DA}-\u{119E1}\u{119E3}-\u{119E4}\u{11A00}-\u{11A3E}\u{11A47}\u{11A50}-\u{11A99}\u{11A9D}\u{11AB0}-\u{11AF8}\u{11BC0}-\u{11BE0}\u{11C00}-\u{11C08}\u{11C0A}-\u{11C36}\u{11C38}-\u{11C40}\u{11C72}-\u{11C8F}\u{11C92}-\u{11CA7}\u{11CA9}-\u{11CB6}\u{11D00}-\u{11D06}\u{11D08}-\u{11D09}\u{11D0B}-\u{11D36}\u{11D3A}\u{11D3C}-\u{11D3D}\u{11D3F}-\u{11D47}\u{11D60}-\u{11D65}\u{11D67}-\u{11D68}\u{11D6A}-\u{11D8E}\u{11D90}-\u{11D91}\u{11D93}-\u{11D98}\u{11EE0}-\u{11EF6}\u{11F00}-\u{11F10}\u{11F12}-\u{11F3A}\u{11F3E}-\u{11F42}\u{11F5A}\u{11FB0}\u{12000}-\u{12399}\u{12480}-\u{12543}\u{12F90}-\u{12FF0}\u{13000}-\u{1342F}\u{13440}-\u{13455}\u{13460}-\u{143FA}\u{14400}-\u{14646}\u{16100}-\u{1612F}\u{16800}-\u{16A38}\u{16A40}-\u{16A5E}\u{16A70}-\u{16ABE}\u{16AD0}-\u{16AED}\u{16AF0}-\u{16AF4}\u{16B00}-\u{16B36}\u{16B40}-\u{16B43}\u{16B63}-\u{16B77}\u{16B7D}-\u{16B8F}\u{16D40}-\u{16D6C}\u{16E60}-\u{16E7F}\u{16F00}-\u{16F4A}\u{16F4F}-\u{16F87}\u{16F8F}-\u{16F9F}\u{16FE0}-\u{16FE1}\u{16FE3}-\u{16FE4}\u{16FF0}-\u{16FF1}\u{17000}-\u{187F7}\u{18800}-\u{18CD5}\u{18CFF}-\u{18D08}\u{1AFF0}-\u{1AFF3}\u{1AFF5}-\u{1AFFB}\u{1AFFD}-\u{1AFFE}\u{1B000}-\u{1B122}\u{1B132}\u{1B150}-\u{1B152}\u{1B155}\u{1B164}-\u{1B167}\u{1B170}-\u{1B2FB}\u{1BC00}-\u{1BC6A}\u{1BC70}-\u{1BC7C}\u{1BC80}-\u{1BC88}\u{1BC90}-\u{1BC99}\u{1BC9D}-\u{1BC9E}\u{1CF00}-\u{1CF2D}\u{1CF30}-\u{1CF46}\u{1D165}-\u{1D169}\u{1D16D}-\u{1D172}\u{1D17B}-\u{1D182}\u{1D185}-\u{1D18B}\u{1D1AA}-\u{1D1AD}\u{1D242}-\u{1D244}\u{1D41A}-\u{1D433}\u{1D44E}-\u{1D454}\u{1D456}-\u{1D467}\u{1D482}-\u{1D49B}\u{1D4B6}-\u{1D4B9}\u{1D4BB}\u{1D4BD}-\u{1D4C3}\u{1D4C5}-\u{1D4CF}\u{1D4EA}-\u{1D503}\u{1D51E}-\u{1D537}\u{1D552}-\u{1D56B}\u{1D586}-\u{1D59F}\u{1D5BA}-\u{1D5D3}\u{1D5EE}-\u{1D607}\u{1D622}-\u{1D63B}\u{1D656}-\u{1D66F}\u{1D68A}-\u{1D6A5}\u{1D6C2}-\u{1D6DA}\u{1D6DC}-\u{1D6E1}\u{1D6FC}-\u{1D714}\u{1D716}-\u{1D71B}\u{1D736}-\u{1D74E}\u{1D750}-\u{1D755}\u{1D770}-\u{1D788}\u{1D78A}-\u{1D78F}\u{1D7AA}-\u{1D7C2}\u{1D7C4}-\u{1D7C9}\u{1D7CB}\u{1DA00}-\u{1DA36}\u{1DA3B}-\u{1DA6C}\u{1DA75}\u{1DA84}\u{1DA9B}-\u{1DA9F}\u{1DAA1}-\u{1DAAF}\u{1DF00}-\u{1DF1E}\u{1DF25}-\u{1DF2A}\u{1E000}-\u{1E006}\u{1E008}-\u{1E018}\u{1E01B}-\u{1E021}\u{1E023}-\u{1E024}\u{1E026}-\u{1E02A}\u{1E030}-\u{1E06D}\u{1E08F}\u{1E100}-\u{1E12C}\u{1E130}-\u{1E13D}\u{1E14E}\u{1E290}-\u{1E2AE}\u{1E2C0}-\u{1E2EF}\u{1E4D0}-\u{1E4EF}\u{1E5D0}-\u{1E5F0}\u{1E7E0}-\u{1E7E6}\u{1E7E8}-\u{1E7EB}\u{1E7ED}-\u{1E7EE}\u{1E7F0}-\u{1E7FE}\u{1E800}-\u{1E8C4}\u{1E8D0}-\u{1E8D6}\u{1E922}-\u{1E94B}\u{1EE00}-\u{1EE03}\u{1EE05}-\u{1EE1F}\u{1EE21}-\u{1EE22}\u{1EE24}\u{1EE27}\u{1EE29}-\u{1EE32}\u{1EE34}-\u{1EE37}\u{1EE39}\u{1EE3B}\u{1EE42}\u{1EE47}\u{1EE49}\u{1EE4B}\u{1EE4D}-\u{1EE4F}\u{1EE51}-\u{1EE52}\u{1EE54}\u{1EE57}\u{1EE59}\u{1EE5B}\u{1EE5D}\u{1EE5F}\u{1EE61}-\u{1EE62}\u{1EE64}\u{1EE67}-\u{1EE6A}\u{1EE6C}-\u{1EE72}\u{1EE74}-\u{1EE77}\u{1EE79}-\u{1EE7C}\u{1EE7E}\u{1EE80}-\u{1EE89}\u{1EE8B}-\u{1EE9B}\u{1EEA1}-\u{1EEA3}\u{1EEA5}-\u{1EEA9}\u{1EEAB}-\u{1EEBB}\u{20000}-\u{2A6DF}\u{2A700}-\u{2B739}\u{2B740}-\u{2B81D}\u{2B820}-\u{2CEA1}\u{2CEB0}-\u{2EBE0}\u{2EBF0}-\u{2EE5D}\u{2F800}-\u{2FA1D}\u{30000}-\u{3134A}\u{31350}-\u{323AF}\u{E0100}-\u{E01EF}`;
|
|
376
|
+
var UNICODE_17_TO_16_LETTER_EXCLUDE_RANGES = String.raw`\u088F\u0C5C\u0CDC\uA7CE-\uA7CF\uA7D2\uA7D4\uA7F1\u{10940}-\u{10959}\u{10EC5}-\u{10EC7}\u{11DB0}-\u{11DDB}\u{16EA0}-\u{16EB8}\u{16EBB}-\u{16ED3}\u{16FF2}-\u{16FF3}\u{187F8}-\u{187FF}\u{18D09}-\u{18D1E}\u{18D80}-\u{18DF2}\u{1E6C0}-\u{1E6DE}\u{1E6E0}-\u{1E6E2}\u{1E6E4}-\u{1E6E5}\u{1E6E7}-\u{1E6ED}\u{1E6F0}-\u{1E6F4}\u{1E6FE}-\u{1E6FF}\u{2B73A}-\u{2B73F}\u{2CEA2}-\u{2CEAD}\u{323B0}-\u{33479}`;
|
|
377
|
+
var UNICODE_17_TO_16_LETTER_INCLUDE_RANGES = String.raw``;
|
|
378
|
+
var UNICODE_17_TO_16_NUMBER_EXCLUDE_RANGES = String.raw`\u{11DE0}-\u{11DE9}\u{16FF4}-\u{16FF6}`;
|
|
379
|
+
var UNICODE_17_TO_16_NUMBER_INCLUDE_RANGES = String.raw``;
|
|
380
|
+
var UNICODE_17_TO_16_O200K_UPPER_EXCLUDE_RANGES = String.raw`\u0295\u088F\u0C5C\u0CDC\u1ACF-\u1ADD\u1AE0-\u1AEB\uA7CE\uA7D2\uA7D4\uA7F1\u{10940}-\u{10959}\u{10EC5}-\u{10EC7}\u{10EFA}-\u{10EFB}\u{11B60}-\u{11B67}\u{11DB0}-\u{11DDB}\u{16EA0}-\u{16EB8}\u{16FF2}-\u{16FF3}\u{187F8}-\u{187FF}\u{18D09}-\u{18D1E}\u{18D80}-\u{18DF2}\u{1E6C0}-\u{1E6DE}\u{1E6E0}-\u{1E6F5}\u{1E6FE}-\u{1E6FF}\u{2B73A}-\u{2B73F}\u{2CEA2}-\u{2CEAD}\u{323B0}-\u{33479}`;
|
|
381
|
+
var UNICODE_17_TO_16_O200K_UPPER_INCLUDE_RANGES = String.raw``;
|
|
382
|
+
var UNICODE_17_TO_16_O200K_LOWER_EXCLUDE_RANGES = String.raw`\u088F\u0C5C\u0CDC\u1ACF-\u1ADD\u1AE0-\u1AEB\uA7CF\uA7F1\u{10940}-\u{10959}\u{10EC5}-\u{10EC7}\u{10EFA}-\u{10EFB}\u{11B60}-\u{11B67}\u{11DB0}-\u{11DDB}\u{16EBB}-\u{16ED3}\u{16FF2}-\u{16FF3}\u{187F8}-\u{187FF}\u{18D09}-\u{18D1E}\u{18D80}-\u{18DF2}\u{1E6C0}-\u{1E6DE}\u{1E6E0}-\u{1E6F5}\u{1E6FE}-\u{1E6FF}\u{2B73A}-\u{2B73F}\u{2CEA2}-\u{2CEAD}\u{323B0}-\u{33479}`;
|
|
383
|
+
var UNICODE_17_TO_16_O200K_LOWER_INCLUDE_RANGES = String.raw``;
|
|
384
|
+
var UNICODE_17_TO_16_GUARD_RANGES = String.raw`\u0295\u088F\u0C5C\u0CDC\u1ACF-\u1ADD\u1AE0-\u1AEB\uA7CE-\uA7CF\uA7D2\uA7D4\uA7F1\uD802-\uD803\uD806-\uD807\uD81B\uD821\uD823\uD839\uD86D\uD873\uD888-\uD88D`;
|
|
385
|
+
|
|
386
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/constants.js
|
|
387
|
+
var nativeVersion = typeof process === "undefined" ? void 0 : process.versions?.unicode;
|
|
388
|
+
var unicode17Sets = (() => {
|
|
389
|
+
if (nativeVersion !== "17.0") return false;
|
|
390
|
+
try {
|
|
391
|
+
return new RegExp("[a--b]", "v").test("a");
|
|
392
|
+
} catch {
|
|
393
|
+
return false;
|
|
394
|
+
}
|
|
395
|
+
})();
|
|
396
|
+
var flags = unicode17Sets ? "gv" : "gu";
|
|
397
|
+
function category(native2, stable2, exclude, include) {
|
|
398
|
+
if (nativeVersion === "16.0") return native2;
|
|
399
|
+
if (!unicode17Sets) return stable2;
|
|
400
|
+
const corrected = exclude ? `[[${native2}]--[${exclude}]]` : `[${native2}]`;
|
|
401
|
+
return include ? `[${corrected}${include}]` : corrected;
|
|
402
|
+
}
|
|
403
|
+
var L = category(String.raw`\p{L}`, UNICODE_16_LETTER_RANGES, UNICODE_17_TO_16_LETTER_EXCLUDE_RANGES, UNICODE_17_TO_16_LETTER_INCLUDE_RANGES);
|
|
404
|
+
var N = category(String.raw`\p{N}`, UNICODE_16_NUMBER_RANGES, UNICODE_17_TO_16_NUMBER_EXCLUDE_RANGES, UNICODE_17_TO_16_NUMBER_INCLUDE_RANGES);
|
|
405
|
+
var UPPER = category(String.raw`\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}`, UNICODE_16_O200K_UPPER_RANGES, UNICODE_17_TO_16_O200K_UPPER_EXCLUDE_RANGES, UNICODE_17_TO_16_O200K_UPPER_INCLUDE_RANGES);
|
|
406
|
+
var LOWER = category(String.raw`\p{Ll}\p{Lm}\p{Lo}\p{M}`, UNICODE_16_O200K_LOWER_RANGES, UNICODE_17_TO_16_O200K_LOWER_EXCLUDE_RANGES, UNICODE_17_TO_16_O200K_LOWER_INCLUDE_RANGES);
|
|
407
|
+
var StableUnicodeRegExp = class extends RegExp {
|
|
408
|
+
stableRegex;
|
|
409
|
+
nativeRegex;
|
|
410
|
+
affected;
|
|
411
|
+
constructor(stablePattern, nativePattern, affected2) {
|
|
412
|
+
super(stablePattern, flags);
|
|
413
|
+
this.stableRegex = new RegExp(stablePattern, flags);
|
|
414
|
+
this.nativeRegex = new RegExp(nativePattern, "gu");
|
|
415
|
+
this.affected = affected2;
|
|
416
|
+
}
|
|
417
|
+
static get [Symbol.species]() {
|
|
418
|
+
return RegExp;
|
|
419
|
+
}
|
|
420
|
+
[Symbol.match](text3) {
|
|
421
|
+
const selected = this.affected.test(text3) ? this.stableRegex : this.nativeRegex;
|
|
422
|
+
selected.lastIndex = this.lastIndex;
|
|
423
|
+
const result = selected[Symbol.match](text3);
|
|
424
|
+
this.lastIndex = selected.lastIndex;
|
|
425
|
+
return result;
|
|
426
|
+
}
|
|
427
|
+
[Symbol.matchAll](text3) {
|
|
428
|
+
const selected = this.affected.test(text3) ? this.stableRegex : this.nativeRegex;
|
|
429
|
+
selected.lastIndex = this.lastIndex;
|
|
430
|
+
return selected[Symbol.matchAll](text3);
|
|
431
|
+
}
|
|
432
|
+
exec(text3) {
|
|
433
|
+
const selected = this.affected.test(text3) ? this.stableRegex : this.nativeRegex;
|
|
434
|
+
selected.lastIndex = this.lastIndex;
|
|
435
|
+
const result = selected.exec(text3);
|
|
436
|
+
this.lastIndex = selected.lastIndex;
|
|
437
|
+
return result;
|
|
438
|
+
}
|
|
439
|
+
};
|
|
440
|
+
var affected = unicode17Sets ? new RegExp(`[${UNICODE_17_TO_16_GUARD_RANGES}]`) : new RegExp("\\P{ASCII}", "u");
|
|
441
|
+
var CONTRACTION_SUFFIX_PATTERN = String.raw`'(?:[sSſ]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`;
|
|
442
|
+
var OPTIONAL_CONTRACTION_SUFFIX = String.raw`(?:${CONTRACTION_SUFFIX_PATTERN})?`;
|
|
443
|
+
function patterns(letter, number, upper, lower) {
|
|
444
|
+
return [
|
|
445
|
+
String.raw`'s|'t|'re|'ve|'m|'ll|'d| ?[${letter}]+| ?[${number}]+| ?[^\p{White_Space}${letter}${number}]+|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}+`,
|
|
446
|
+
String.raw`${CONTRACTION_SUFFIX_PATTERN}|[^\r\n${letter}${number}]?[${letter}]+|[${number}]{1,3}| ?[^\p{White_Space}${letter}${number}]+[\r\n]*|\p{White_Space}+$|\p{White_Space}*[\r\n]|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}`,
|
|
447
|
+
String.raw`[^\r\n${letter}${number}]?[${upper}]*[${lower}]+${OPTIONAL_CONTRACTION_SUFFIX}|[^\r\n${letter}${number}]?[${upper}]+[${lower}]*${OPTIONAL_CONTRACTION_SUFFIX}|[${number}]{1,3}| ?[^\p{White_Space}${letter}${number}]+[\r\n\/]*|\p{White_Space}*[\r\n]+|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}+`
|
|
448
|
+
];
|
|
449
|
+
}
|
|
450
|
+
var native = patterns(String.raw`\p{L}`, String.raw`\p{N}`, String.raw`\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}`, String.raw`\p{Ll}\p{Lm}\p{Lo}\p{M}`);
|
|
451
|
+
var stable = patterns(L, N, UPPER, LOWER);
|
|
452
|
+
var makeRegex = (index) => nativeVersion === "16.0" ? new RegExp(native[index], "gu") : new StableUnicodeRegExp(stable[index], native[index], affected);
|
|
453
|
+
var R50K_TOKEN_SPLIT_REGEX = makeRegex(0);
|
|
454
|
+
var CL100K_TOKEN_SPLIT_REGEX = makeRegex(1);
|
|
455
|
+
var O200K_TOKEN_SPLIT_REGEX = makeRegex(2);
|
|
456
|
+
|
|
457
|
+
// node_modules/gpt-tokenizer/esm/IncrementalTokenCounter.js
|
|
458
|
+
function upperBound(pieces, offset) {
|
|
459
|
+
let low = 0;
|
|
460
|
+
let high = pieces.length;
|
|
461
|
+
while (low < high) {
|
|
462
|
+
const mid = low + high >>> 1;
|
|
463
|
+
if (pieces[mid].end <= offset) low = mid + 1;
|
|
464
|
+
else high = mid;
|
|
465
|
+
}
|
|
466
|
+
return low;
|
|
467
|
+
}
|
|
468
|
+
function createCounter({ scan, canRestart, validate, restartMargin, canReuseSuffix }) {
|
|
469
|
+
let text3 = "";
|
|
470
|
+
const pieces = [];
|
|
471
|
+
let total = 0;
|
|
472
|
+
const applyEdit = (start, deleteCount, inserted) => {
|
|
473
|
+
if (!Number.isSafeInteger(start) || !Number.isSafeInteger(deleteCount) || start < 0 || deleteCount < 0 || start > text3.length || deleteCount > text3.length - start) throw new RangeError("The edit range must lie within the tracked text.");
|
|
474
|
+
if (deleteCount === 0 && inserted.length === 0) return total;
|
|
475
|
+
const next = text3.slice(0, start) + inserted + text3.slice(start + deleteCount);
|
|
476
|
+
validate(next, start, inserted.length);
|
|
477
|
+
let keep = upperBound(pieces, start - restartMargin);
|
|
478
|
+
while (keep > 0 && !canRestart(text3, pieces[keep - 1].end)) keep--;
|
|
479
|
+
const anchor = keep > 0 ? pieces[keep - 1].end : 0;
|
|
480
|
+
const delta = inserted.length - deleteCount;
|
|
481
|
+
const replacement = [];
|
|
482
|
+
let resume = pieces.length;
|
|
483
|
+
for (const piece of scan(next, anchor)) {
|
|
484
|
+
replacement.push(piece);
|
|
485
|
+
const oldEnd = piece.end - delta;
|
|
486
|
+
if (!canReuseSuffix || piece.end < start + inserted.length || oldEnd < start + deleteCount) continue;
|
|
487
|
+
const index = upperBound(pieces, oldEnd) - 1;
|
|
488
|
+
if (index >= keep && pieces[index].end === oldEnd) {
|
|
489
|
+
resume = index + 1;
|
|
490
|
+
break;
|
|
491
|
+
}
|
|
492
|
+
}
|
|
493
|
+
let nextTotal = total;
|
|
494
|
+
for (let i = keep; i < resume; i++) nextTotal -= pieces[i].count;
|
|
495
|
+
for (const piece of replacement) nextTotal += piece.count;
|
|
496
|
+
const oldLength = pieces.length;
|
|
497
|
+
const suffixStart = keep + replacement.length;
|
|
498
|
+
const nextLength = oldLength + replacement.length - (resume - keep);
|
|
499
|
+
if (nextLength > oldLength) pieces.length = nextLength;
|
|
500
|
+
if (suffixStart !== resume) pieces.copyWithin(suffixStart, resume, oldLength);
|
|
501
|
+
pieces.length = nextLength;
|
|
502
|
+
for (let i = 0; i < replacement.length; i++) pieces[keep + i] = replacement[i];
|
|
503
|
+
if (delta !== 0) for (let i = suffixStart; i < pieces.length; i++) pieces[i].end += delta;
|
|
504
|
+
text3 = next;
|
|
505
|
+
total = nextTotal;
|
|
506
|
+
return total;
|
|
507
|
+
};
|
|
508
|
+
return {
|
|
509
|
+
applyEdit,
|
|
510
|
+
append: (inserted) => applyEdit(text3.length, 0, inserted),
|
|
511
|
+
update(next) {
|
|
512
|
+
let start = 0;
|
|
513
|
+
const shared = Math.min(text3.length, next.length);
|
|
514
|
+
while (start < shared && text3.charCodeAt(start) === next.charCodeAt(start)) start++;
|
|
515
|
+
let oldEnd = text3.length;
|
|
516
|
+
let newEnd = next.length;
|
|
517
|
+
while (oldEnd > start && newEnd > start && text3.charCodeAt(oldEnd - 1) === next.charCodeAt(newEnd - 1)) {
|
|
518
|
+
oldEnd--;
|
|
519
|
+
newEnd--;
|
|
520
|
+
}
|
|
521
|
+
return applyEdit(start, oldEnd - start, next.slice(start, newEnd));
|
|
522
|
+
},
|
|
523
|
+
count: () => total,
|
|
524
|
+
reset() {
|
|
525
|
+
text3 = "";
|
|
526
|
+
pieces.length = 0;
|
|
527
|
+
total = 0;
|
|
528
|
+
}
|
|
529
|
+
};
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// node_modules/gpt-tokenizer/esm/BytePairEncodingCore.js
|
|
533
|
+
var MAX_BATCHED_SPLIT_LENGTH = 1048576;
|
|
534
|
+
function isAsciiString(input) {
|
|
535
|
+
for (let i = 0; i < input.length; i++) if (!isAscii(input.charCodeAt(i))) return false;
|
|
536
|
+
return true;
|
|
537
|
+
}
|
|
538
|
+
var decoder = new TextDecoder("utf8", { ignoreBOM: true });
|
|
539
|
+
function decodeShortUtf8(word, length) {
|
|
540
|
+
const a = word & 255, b = word >>> 8 & 255, c = word >>> 16 & 255, d = word >>> 24;
|
|
541
|
+
if (length === 1 && a < 128) return String.fromCharCode(a);
|
|
542
|
+
if (length === 2 && a >= 194 && a <= 223 && (b & 192) === 128) return String.fromCharCode((a & 31) << 6 | b & 63);
|
|
543
|
+
if (length === 3 && a >= 224 && a <= 239 && (b & 192) === 128 && (c & 192) === 128) {
|
|
544
|
+
const cp = (a & 15) << 12 | (b & 63) << 6 | c & 63;
|
|
545
|
+
if (cp >= 2048 && (cp < 55296 || cp > 57343)) return String.fromCharCode(cp);
|
|
546
|
+
}
|
|
547
|
+
if (length === 4 && a >= 240 && a <= 244 && (b & 192) === 128 && (c & 192) === 128 && (d & 192) === 128) {
|
|
548
|
+
const cp = (a & 7) << 18 | (b & 63) << 12 | (c & 63) << 6 | d & 63;
|
|
549
|
+
if (cp >= 65536 && cp <= 1114111) return String.fromCodePoint(cp);
|
|
550
|
+
}
|
|
551
|
+
const bytes = new Uint8Array(length);
|
|
552
|
+
for (let i = 0; i < length; i++) bytes[i] = word >>> i * 8 & 255;
|
|
553
|
+
return decoder.decode(bytes);
|
|
554
|
+
}
|
|
555
|
+
var BytePairEncodingCore = class {
|
|
556
|
+
mergeableBytePairRankCount;
|
|
557
|
+
/**
|
|
558
|
+
* an array where the index is the BPE rank,
|
|
559
|
+
* and the value is the string or the array of bytes that it decodes to
|
|
560
|
+
* it may contain holes if token is unused
|
|
561
|
+
*/
|
|
562
|
+
bytePairRankDecoder;
|
|
563
|
+
bytePairNonUtfRankDecoder = /* @__PURE__ */ new Map();
|
|
564
|
+
decodeByteTable;
|
|
565
|
+
bytePairNonUtfSortedEncoder;
|
|
566
|
+
bytePairNonUtfFirstByteOffsets = /* @__PURE__ */ new Uint32Array(257);
|
|
567
|
+
singleByteRanks = (/* @__PURE__ */ new Int32Array(256)).fill(-1);
|
|
568
|
+
/**
|
|
569
|
+
* a reverse map of the bytePairRankDecoder,
|
|
570
|
+
* where the key is the string and the value is the rank
|
|
571
|
+
* values that cannot be represented as a string are present in `bytePairNonUtfSortedEncoder`.
|
|
572
|
+
* Bounded merged-piece results share this map so warm encode/count needs one
|
|
573
|
+
* lookup. Their arrays must be excluded from vocabulary-rank lookups.
|
|
574
|
+
*/
|
|
575
|
+
bytePairStringRankEncoder;
|
|
576
|
+
tokenSplitRegex;
|
|
577
|
+
specialTokensEncoder;
|
|
578
|
+
specialTokensDecoder;
|
|
579
|
+
specialTokenPatternRegex;
|
|
580
|
+
textEncoder = new TextEncoder();
|
|
581
|
+
mergeCache;
|
|
582
|
+
mergeCacheKeys = [];
|
|
583
|
+
mergeCacheHead = 0;
|
|
584
|
+
mergeCacheSize;
|
|
585
|
+
constructor({ bytePairRankDecoder, specialTokensEncoder, tokenSplitRegex, mergeCacheSize = DEFAULT_MERGE_CACHE_SIZE }) {
|
|
586
|
+
this.bytePairRankDecoder = bytePairRankDecoder;
|
|
587
|
+
this.bytePairStringRankEncoder = /* @__PURE__ */ new Map();
|
|
588
|
+
this.mergeCacheSize = mergeCacheSize;
|
|
589
|
+
if (mergeCacheSize > 0) this.mergeCache = /* @__PURE__ */ new Map();
|
|
590
|
+
this.mergeableBytePairRankCount = Object.keys(bytePairRankDecoder).length;
|
|
591
|
+
const binaryLookup = [];
|
|
592
|
+
bytePairRankDecoder.forEach((value, rank) => {
|
|
593
|
+
if (typeof value === "string") {
|
|
594
|
+
this.bytePairStringRankEncoder.set(value, rank);
|
|
595
|
+
if (value.length === 1 && value.charCodeAt(0) <= 127) this.singleByteRanks[value.charCodeAt(0)] = rank;
|
|
596
|
+
return;
|
|
597
|
+
}
|
|
598
|
+
const byteArray = new Uint8Array(value);
|
|
599
|
+
binaryLookup.push([byteArray, rank]);
|
|
600
|
+
if (byteArray.length === 1) this.singleByteRanks[byteArray[0]] = rank;
|
|
601
|
+
this.bytePairNonUtfRankDecoder.set(rank, byteArray);
|
|
602
|
+
if (startsWithByteOrderMark(byteArray)) {
|
|
603
|
+
const asString = tryConvertToString(byteArray);
|
|
604
|
+
if (asString !== void 0) this.bytePairStringRankEncoder.set(asString, rank);
|
|
605
|
+
}
|
|
606
|
+
});
|
|
607
|
+
this.bytePairNonUtfSortedEncoder = binaryLookup.sort((a, b) => compareUint8Arrays(a[0], b[0]));
|
|
608
|
+
let offset = 0;
|
|
609
|
+
for (let firstByte = 0; firstByte <= 256; firstByte++) {
|
|
610
|
+
while (offset < binaryLookup.length && (binaryLookup[offset][0].length === 0 || binaryLookup[offset][0][0] < firstByte)) offset++;
|
|
611
|
+
this.bytePairNonUtfFirstByteOffsets[firstByte] = offset;
|
|
612
|
+
}
|
|
613
|
+
this.specialTokensEncoder = specialTokensEncoder ?? /* @__PURE__ */ new Map();
|
|
614
|
+
this.specialTokensDecoder = specialTokensEncoder ? new Map([...specialTokensEncoder].map(([key, value]) => [value, key])) : /* @__PURE__ */ new Map();
|
|
615
|
+
this.tokenSplitRegex = tokenSplitRegex;
|
|
616
|
+
const allSpecialTokensRegex = [...this.specialTokensEncoder.keys()].map(escapeRegExp).join("|");
|
|
617
|
+
try {
|
|
618
|
+
this.specialTokenPatternRegex = new RegExp(allSpecialTokensRegex, "g");
|
|
619
|
+
} catch {
|
|
620
|
+
throw new Error("Invalid regular expression pattern.");
|
|
621
|
+
}
|
|
622
|
+
}
|
|
623
|
+
setMergeCacheSize(newSize) {
|
|
624
|
+
if (this.mergeCacheSize === 0 && newSize > 0) this.mergeCache = /* @__PURE__ */ new Map();
|
|
625
|
+
this.mergeCacheSize = newSize;
|
|
626
|
+
if (this.mergeCache) {
|
|
627
|
+
const oldestKeys = this.mergeCache.keys();
|
|
628
|
+
while (this.mergeCache.size > newSize && this.mergeCache.size > 0) {
|
|
629
|
+
const key = oldestKeys.next().value;
|
|
630
|
+
this.mergeCache.delete(key);
|
|
631
|
+
this.bytePairStringRankEncoder.delete(key);
|
|
632
|
+
}
|
|
633
|
+
this.mergeCacheKeys = [...this.mergeCache.keys()];
|
|
634
|
+
this.mergeCacheHead = 0;
|
|
635
|
+
}
|
|
636
|
+
if (newSize === 0) {
|
|
637
|
+
this.mergeCache = void 0;
|
|
638
|
+
this.mergeCacheKeys = [];
|
|
639
|
+
this.mergeCacheHead = 0;
|
|
640
|
+
}
|
|
641
|
+
}
|
|
642
|
+
clearMergeCache() {
|
|
643
|
+
if (this.mergeCache) {
|
|
644
|
+
for (const key of this.mergeCache.keys()) this.bytePairStringRankEncoder.delete(key);
|
|
645
|
+
this.mergeCache.clear();
|
|
646
|
+
}
|
|
647
|
+
this.mergeCacheKeys = [];
|
|
648
|
+
this.mergeCacheHead = 0;
|
|
649
|
+
}
|
|
650
|
+
/** @internal Called with the encoding's already normalized special policy. */
|
|
651
|
+
createIncrementalCounter(allowedSpecial, validate) {
|
|
652
|
+
const knownPattern = [
|
|
653
|
+
R50K_TOKEN_SPLIT_REGEX,
|
|
654
|
+
CL100K_TOKEN_SPLIT_REGEX,
|
|
655
|
+
O200K_TOKEN_SPLIT_REGEX
|
|
656
|
+
].includes(this.tokenSplitRegex);
|
|
657
|
+
const maxSpecialLength = Math.max(0, ...Array.from(this.specialTokensEncoder.keys(), (token) => token.length));
|
|
658
|
+
return createCounter({
|
|
659
|
+
scan: (text3, start) => this.countNativePieces(text3, start, allowedSpecial),
|
|
660
|
+
canRestart: (text3, offset) => knownPattern && /^[ \t\r\n][A-Za-z0-9]/.test(text3.slice(offset, offset + 2)),
|
|
661
|
+
validate,
|
|
662
|
+
restartMargin: maxSpecialLength + 2,
|
|
663
|
+
canReuseSuffix: knownPattern
|
|
664
|
+
});
|
|
665
|
+
}
|
|
666
|
+
*countNativePieces(text3, start, allowedSpecial) {
|
|
667
|
+
while (start < text3.length) {
|
|
668
|
+
const special = this.findNextSpecialToken(text3, allowedSpecial, start);
|
|
669
|
+
const end = special?.[0] ?? text3.length;
|
|
670
|
+
const plain = text3.slice(start, end);
|
|
671
|
+
for (const match of plain.matchAll(this.tokenSplitRegex)) yield {
|
|
672
|
+
end: start + match.index + match[0].length,
|
|
673
|
+
count: this.countPiece(match[0])
|
|
674
|
+
};
|
|
675
|
+
if (!special) break;
|
|
676
|
+
start = end + special[1].length;
|
|
677
|
+
yield {
|
|
678
|
+
end: start,
|
|
679
|
+
count: 1
|
|
680
|
+
};
|
|
681
|
+
}
|
|
682
|
+
}
|
|
683
|
+
countPiece(piece) {
|
|
684
|
+
const token = this.bytePairStringRankEncoder.get(piece);
|
|
685
|
+
return typeof token === "number" ? 1 : (token ?? this.bytePairEncode(piece)).length;
|
|
686
|
+
}
|
|
687
|
+
*encodeNativeGenerator(text3, allowedSpecial) {
|
|
688
|
+
let startIndex = 0;
|
|
689
|
+
let lastTokenLength = 0;
|
|
690
|
+
while (true) {
|
|
691
|
+
const nextSpecialMatch = this.findNextSpecialToken(text3, allowedSpecial, startIndex);
|
|
692
|
+
const nextSpecialStartIndex = nextSpecialMatch?.[0];
|
|
693
|
+
const endIndex = nextSpecialStartIndex ?? text3.length;
|
|
694
|
+
const textBeforeSpecial = startIndex === 0 && endIndex === text3.length ? text3 : text3.slice(startIndex, endIndex);
|
|
695
|
+
for (const [match] of textBeforeSpecial.matchAll(this.tokenSplitRegex)) {
|
|
696
|
+
const token = this.getBpeRankFromString(match);
|
|
697
|
+
if (token !== void 0) {
|
|
698
|
+
lastTokenLength = 1;
|
|
699
|
+
yield [token];
|
|
700
|
+
continue;
|
|
701
|
+
}
|
|
702
|
+
const tokens = this.bytePairEncode(match);
|
|
703
|
+
lastTokenLength = tokens.length;
|
|
704
|
+
yield tokens.slice();
|
|
705
|
+
}
|
|
706
|
+
if (nextSpecialStartIndex !== void 0) {
|
|
707
|
+
const specialToken = nextSpecialMatch[1];
|
|
708
|
+
const specialTokenValue = this.specialTokensEncoder.get(specialToken);
|
|
709
|
+
if (specialTokenValue === void 0) throw new Error(`Special token "${specialToken}" is not in the special token encoder.`);
|
|
710
|
+
yield [specialTokenValue];
|
|
711
|
+
startIndex = nextSpecialStartIndex + specialToken.length;
|
|
712
|
+
lastTokenLength = 1;
|
|
713
|
+
} else break;
|
|
714
|
+
}
|
|
715
|
+
return lastTokenLength;
|
|
716
|
+
}
|
|
717
|
+
encodeNative(text3, allowedSpecial) {
|
|
718
|
+
if (!this.tokenSplitRegex.global) throw new TypeError("The token split regular expression must be global.");
|
|
719
|
+
let startIndex = 0;
|
|
720
|
+
const tokensArray = [];
|
|
721
|
+
while (true) {
|
|
722
|
+
const nextSpecialMatch = this.findNextSpecialToken(text3, allowedSpecial, startIndex);
|
|
723
|
+
const nextSpecialStartIndex = nextSpecialMatch?.[0];
|
|
724
|
+
const endIndex = nextSpecialStartIndex ?? text3.length;
|
|
725
|
+
const textBeforeSpecial = startIndex === 0 && endIndex === text3.length ? text3 : text3.slice(startIndex, endIndex);
|
|
726
|
+
if (textBeforeSpecial.length > MAX_BATCHED_SPLIT_LENGTH) for (const tokens of this.encodeNativeGenerator(textBeforeSpecial)) for (let i = 0; i < tokens.length; i++) tokensArray.push(tokens[i]);
|
|
727
|
+
else {
|
|
728
|
+
const pieces = textBeforeSpecial.match(this.tokenSplitRegex);
|
|
729
|
+
if (pieces) for (let i = 0; i < pieces.length; i++) {
|
|
730
|
+
const match = pieces[i];
|
|
731
|
+
const token = this.bytePairStringRankEncoder.get(match);
|
|
732
|
+
if (typeof token === "number") {
|
|
733
|
+
tokensArray.push(token);
|
|
734
|
+
continue;
|
|
735
|
+
}
|
|
736
|
+
const tokens = token ?? this.bytePairEncode(match);
|
|
737
|
+
for (let j = 0; j < tokens.length; j++) tokensArray.push(tokens[j]);
|
|
738
|
+
}
|
|
739
|
+
}
|
|
740
|
+
if (nextSpecialStartIndex !== void 0) {
|
|
741
|
+
const specialToken = nextSpecialMatch[1];
|
|
742
|
+
const specialTokenValue = this.specialTokensEncoder.get(specialToken);
|
|
743
|
+
if (specialTokenValue === void 0) throw new Error(`Special token "${specialToken}" is not in the special token encoder.`);
|
|
744
|
+
tokensArray.push(specialTokenValue);
|
|
745
|
+
startIndex = nextSpecialStartIndex + specialToken.length;
|
|
746
|
+
} else break;
|
|
747
|
+
}
|
|
748
|
+
return tokensArray;
|
|
749
|
+
}
|
|
750
|
+
countNative(text3, allowedSpecial) {
|
|
751
|
+
if (!this.tokenSplitRegex.global) throw new TypeError("The token split regular expression must be global.");
|
|
752
|
+
let startIndex = 0;
|
|
753
|
+
let tokensCount = 0;
|
|
754
|
+
while (true) {
|
|
755
|
+
const nextSpecialMatch = this.findNextSpecialToken(text3, allowedSpecial, startIndex);
|
|
756
|
+
const nextSpecialStartIndex = nextSpecialMatch?.[0];
|
|
757
|
+
const endIndex = nextSpecialStartIndex ?? text3.length;
|
|
758
|
+
const textBeforeSpecial = startIndex === 0 && endIndex === text3.length ? text3 : text3.slice(startIndex, endIndex);
|
|
759
|
+
if (textBeforeSpecial.length > MAX_BATCHED_SPLIT_LENGTH) for (const tokens of this.encodeNativeGenerator(textBeforeSpecial)) tokensCount += tokens.length;
|
|
760
|
+
else {
|
|
761
|
+
const pieces = textBeforeSpecial.match(this.tokenSplitRegex);
|
|
762
|
+
if (pieces) for (let i = 0; i < pieces.length; i++) {
|
|
763
|
+
const match = pieces[i];
|
|
764
|
+
tokensCount += this.countPiece(match);
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
if (nextSpecialStartIndex !== void 0) {
|
|
768
|
+
const specialToken = nextSpecialMatch[1];
|
|
769
|
+
if (this.specialTokensEncoder.get(specialToken) === void 0) throw new Error(`Special token "${specialToken}" is not in the special token encoder.`);
|
|
770
|
+
tokensCount++;
|
|
771
|
+
startIndex = nextSpecialStartIndex + specialToken.length;
|
|
772
|
+
} else break;
|
|
773
|
+
}
|
|
774
|
+
return tokensCount;
|
|
775
|
+
}
|
|
776
|
+
*decodeNativeGenerator(tokens) {
|
|
777
|
+
for (const token of tokens) {
|
|
778
|
+
const tokenBytes = this.tryDecodeToken(token);
|
|
779
|
+
if (tokenBytes === void 0) throw new Error(`Token ${token} is not in the byte pair encoder.`);
|
|
780
|
+
yield tokenBytes;
|
|
781
|
+
}
|
|
782
|
+
}
|
|
783
|
+
/** Cache only bytes actually visited; keep the vocabulary's string fast path. */
|
|
784
|
+
decodeByteHeavy(tokens) {
|
|
785
|
+
const table = this.decodeByteTable ??= {
|
|
786
|
+
bytes: /* @__PURE__ */ new Uint8Array(4096),
|
|
787
|
+
offsets: new Uint32Array(this.bytePairRankDecoder.length),
|
|
788
|
+
lengths: new Uint32Array(this.bytePairRankDecoder.length),
|
|
789
|
+
words: new Uint32Array(this.bytePairRankDecoder.length),
|
|
790
|
+
used: 0
|
|
791
|
+
};
|
|
792
|
+
let length = 0;
|
|
793
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
794
|
+
const token = tokens[index];
|
|
795
|
+
let size = table.lengths[token];
|
|
796
|
+
if (!size) {
|
|
797
|
+
const value = this.bytePairRankDecoder[token];
|
|
798
|
+
if (value === void 0 || !Number.isInteger(token) || token < 0) return void 0;
|
|
799
|
+
const bound = typeof value === "string" ? value.length * 3 : value.length;
|
|
800
|
+
if (table.used + bound > table.bytes.length) {
|
|
801
|
+
const next = new Uint8Array(Math.max(table.bytes.length * 2, table.used + bound));
|
|
802
|
+
next.set(table.bytes.subarray(0, table.used));
|
|
803
|
+
table.bytes = next;
|
|
804
|
+
}
|
|
805
|
+
table.offsets[token] = table.used;
|
|
806
|
+
if (typeof value === "string") size = this.textEncoder.encodeInto(value, table.bytes.subarray(table.used)).written;
|
|
807
|
+
else {
|
|
808
|
+
table.bytes.set(value, table.used);
|
|
809
|
+
size = value.length;
|
|
810
|
+
}
|
|
811
|
+
table.lengths[token] = size;
|
|
812
|
+
if (size <= 4) {
|
|
813
|
+
const at = table.used;
|
|
814
|
+
table.words[token] = table.bytes[at] | table.bytes[at + 1] << 8 | table.bytes[at + 2] << 16 | table.bytes[at + 3] << 24;
|
|
815
|
+
}
|
|
816
|
+
table.used += size;
|
|
817
|
+
}
|
|
818
|
+
length += size;
|
|
819
|
+
}
|
|
820
|
+
const { bytes, offsets, lengths, words } = table;
|
|
821
|
+
const output = new Uint8Array(length + 3);
|
|
822
|
+
const view = new DataView(output.buffer);
|
|
823
|
+
let position = 0;
|
|
824
|
+
for (let index = 0; index < tokens.length; index++) {
|
|
825
|
+
const token = tokens[index];
|
|
826
|
+
const size = lengths[token];
|
|
827
|
+
if (size > 0 && size <= 4) {
|
|
828
|
+
view.setUint32(position, words[token], true);
|
|
829
|
+
position += size;
|
|
830
|
+
} else {
|
|
831
|
+
const end = offsets[token] + size;
|
|
832
|
+
for (let start = offsets[token]; start < end; start++) output[position++] = bytes[start];
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
return decoder.decode(output.subarray(0, length));
|
|
836
|
+
}
|
|
837
|
+
decodeArrayByteRun(tokens, start) {
|
|
838
|
+
let index = start;
|
|
839
|
+
let word = 0;
|
|
840
|
+
let used = 0;
|
|
841
|
+
let buffer;
|
|
842
|
+
for (; index < tokens.length; index++) {
|
|
843
|
+
const token = tokens[index];
|
|
844
|
+
const value = this.tryDecodeToken(token);
|
|
845
|
+
if (value === void 0) throw new Error(`Token ${token} is not in the byte pair encoder.`);
|
|
846
|
+
if (typeof value === "string") {
|
|
847
|
+
if (value.length) break;
|
|
848
|
+
continue;
|
|
849
|
+
}
|
|
850
|
+
if (used + value.length <= 4) for (let offset = 0; offset < value.length; offset++) word |= value[offset] << used++ * 8;
|
|
851
|
+
else {
|
|
852
|
+
if (!buffer || used + value.length > buffer.length) {
|
|
853
|
+
const next = new Uint8Array(Math.max(1024, (buffer?.length ?? 0) * 2, used + value.length));
|
|
854
|
+
if (used > 4) next.set(buffer.subarray(0, used));
|
|
855
|
+
buffer = next;
|
|
856
|
+
}
|
|
857
|
+
if (used <= 4) for (let offset = 0; offset < used; offset++) buffer[offset] = word >>> offset * 8 & 255;
|
|
858
|
+
buffer.set(value, used);
|
|
859
|
+
used += value.length;
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
return {
|
|
863
|
+
index,
|
|
864
|
+
text: used <= 4 ? decodeShortUtf8(word, used) : decoder.decode(buffer.subarray(0, used))
|
|
865
|
+
};
|
|
866
|
+
}
|
|
867
|
+
decodeArray(tokens) {
|
|
868
|
+
let decoded = "";
|
|
869
|
+
for (let index = 0; index < tokens.length; ) {
|
|
870
|
+
const token = tokens[index];
|
|
871
|
+
const value = this.tryDecodeToken(token);
|
|
872
|
+
if (value === void 0) throw new Error(`Token ${token} is not in the byte pair encoder.`);
|
|
873
|
+
if (typeof value === "string") {
|
|
874
|
+
decoded += value;
|
|
875
|
+
index++;
|
|
876
|
+
} else {
|
|
877
|
+
const part = this.decodeArrayByteRun(tokens, index);
|
|
878
|
+
decoded += part.text;
|
|
879
|
+
index = part.index;
|
|
880
|
+
}
|
|
881
|
+
}
|
|
882
|
+
return decoded;
|
|
883
|
+
}
|
|
884
|
+
decodeNative(tokens) {
|
|
885
|
+
if ((Array.isArray(tokens) || tokens instanceof Uint32Array) && tokens.length >= 128) {
|
|
886
|
+
let binaryTokens = 0;
|
|
887
|
+
const sampleLength = Math.min(tokens.length, 128);
|
|
888
|
+
for (let index = 0; index < sampleLength; index++) if (typeof this.bytePairRankDecoder[tokens[index]] === "object") binaryTokens++;
|
|
889
|
+
if (binaryTokens >= sampleLength / 4) {
|
|
890
|
+
const decoded2 = this.decodeByteHeavy(tokens);
|
|
891
|
+
if (decoded2 !== void 0) return decoded2;
|
|
892
|
+
}
|
|
893
|
+
return this.decodeArray(tokens);
|
|
894
|
+
}
|
|
895
|
+
let decoded = "";
|
|
896
|
+
let word = 0;
|
|
897
|
+
let used = 0;
|
|
898
|
+
let buffer;
|
|
899
|
+
for (const token of tokens) {
|
|
900
|
+
const value = this.tryDecodeToken(token);
|
|
901
|
+
if (value === void 0) throw new Error(`Token ${token} is not in the byte pair encoder.`);
|
|
902
|
+
if (typeof value === "string") {
|
|
903
|
+
if (value.length === 0) continue;
|
|
904
|
+
if (used) {
|
|
905
|
+
decoded += used <= 4 ? decodeShortUtf8(word, used) : decoder.decode(buffer.subarray(0, used));
|
|
906
|
+
word = 0;
|
|
907
|
+
used = 0;
|
|
908
|
+
}
|
|
909
|
+
decoded += value;
|
|
910
|
+
} else if (used + value.length <= 4) for (let index = 0; index < value.length; index++) word |= value[index] << used++ * 8;
|
|
911
|
+
else {
|
|
912
|
+
if (!buffer || used + value.length > buffer.length) {
|
|
913
|
+
const next = new Uint8Array(Math.max(1024, (buffer?.length ?? 0) * 2, used + value.length));
|
|
914
|
+
if (used > 4) next.set(buffer.subarray(0, used));
|
|
915
|
+
buffer = next;
|
|
916
|
+
}
|
|
917
|
+
if (used <= 4) for (let index = 0; index < used; index++) buffer[index] = word >>> index * 8 & 255;
|
|
918
|
+
buffer.set(value, used);
|
|
919
|
+
used += value.length;
|
|
920
|
+
}
|
|
921
|
+
}
|
|
922
|
+
if (used) decoded += used <= 4 ? decodeShortUtf8(word, used) : decoder.decode(buffer.subarray(0, used));
|
|
923
|
+
return decoded;
|
|
924
|
+
}
|
|
925
|
+
async *decodeNativeAsyncIterable(tokens) {
|
|
926
|
+
for await (const token of tokens) {
|
|
927
|
+
const tokenBytesOrString = this.tryDecodeToken(token);
|
|
928
|
+
if (tokenBytesOrString === void 0) throw new Error(`Token ${token} is not in the byte pair encoder.`);
|
|
929
|
+
yield tokenBytesOrString;
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
getBpeRankFromString(key) {
|
|
933
|
+
const value = this.bytePairStringRankEncoder.get(key);
|
|
934
|
+
return typeof value === "number" ? value : void 0;
|
|
935
|
+
}
|
|
936
|
+
getBpeRankFromStringOrThrow(key) {
|
|
937
|
+
const value = this.getBpeRankFromString(key);
|
|
938
|
+
if (value === void 0) throw new Error(`The byte-pair encoding does not contain a value for: ${key}`);
|
|
939
|
+
return value;
|
|
940
|
+
}
|
|
941
|
+
getBpeRankFromBytes(key) {
|
|
942
|
+
if (key.length === 1) {
|
|
943
|
+
const rank = this.singleByteRanks[key[0]];
|
|
944
|
+
return rank < 0 ? void 0 : rank;
|
|
945
|
+
}
|
|
946
|
+
const keyAsString = tryConvertToString(key);
|
|
947
|
+
if (keyAsString !== void 0) return this.getBpeRankFromString(keyAsString);
|
|
948
|
+
const index = this.binarySearch(key);
|
|
949
|
+
if (index !== -1) return this.bytePairNonUtfSortedEncoder[index][1];
|
|
950
|
+
}
|
|
951
|
+
getBpeRankFromBytesOrThrow(key) {
|
|
952
|
+
const value = this.getBpeRankFromBytes(key);
|
|
953
|
+
if (value === void 0) throw new Error(`The byte-pair encoding does not contain a value for: ${key.toString()}`);
|
|
954
|
+
return value;
|
|
955
|
+
}
|
|
956
|
+
binarySearch(key) {
|
|
957
|
+
const firstByte = key[0];
|
|
958
|
+
let low = this.bytePairNonUtfFirstByteOffsets[firstByte];
|
|
959
|
+
let high = this.bytePairNonUtfFirstByteOffsets[firstByte + 1] - 1;
|
|
960
|
+
while (low <= high) {
|
|
961
|
+
const mid = low + high >>> 1;
|
|
962
|
+
const midKey = this.bytePairNonUtfSortedEncoder[mid][0];
|
|
963
|
+
let cmp = 0;
|
|
964
|
+
const maxLength = Math.min(midKey.length, key.length);
|
|
965
|
+
for (let i = 0; i < maxLength; i++) {
|
|
966
|
+
cmp = midKey[i] - key[i];
|
|
967
|
+
if (cmp !== 0) break;
|
|
968
|
+
}
|
|
969
|
+
if (cmp === 0) cmp = midKey.length - key.length;
|
|
970
|
+
if (cmp === 0) return mid;
|
|
971
|
+
if (cmp < 0) low = mid + 1;
|
|
972
|
+
else high = mid - 1;
|
|
973
|
+
}
|
|
974
|
+
return -1;
|
|
975
|
+
}
|
|
976
|
+
findNextSpecialToken(text3, allowedSpecial, startIndex) {
|
|
977
|
+
if (!allowedSpecial?.size) return;
|
|
978
|
+
let searchIndex = startIndex;
|
|
979
|
+
while (true) {
|
|
980
|
+
this.specialTokenPatternRegex.lastIndex = searchIndex;
|
|
981
|
+
const nextSpecialMatch = this.specialTokenPatternRegex.exec(text3);
|
|
982
|
+
if (!nextSpecialMatch) return;
|
|
983
|
+
const specialToken = nextSpecialMatch[0];
|
|
984
|
+
if (allowedSpecial.has(specialToken)) return [nextSpecialMatch.index, specialToken];
|
|
985
|
+
searchIndex = nextSpecialMatch.index + 1;
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
tryDecodeToken(tokenRank) {
|
|
989
|
+
const value = this.bytePairRankDecoder[tokenRank];
|
|
990
|
+
if (typeof value === "string") return value;
|
|
991
|
+
if (typeof value === "object") {
|
|
992
|
+
const fromBinary = this.bytePairNonUtfRankDecoder.get(tokenRank);
|
|
993
|
+
if (fromBinary) return fromBinary;
|
|
994
|
+
}
|
|
995
|
+
return this.specialTokensDecoder.get(tokenRank);
|
|
996
|
+
}
|
|
997
|
+
addToMergeCache(key, value) {
|
|
998
|
+
if (!this.mergeCache) return;
|
|
999
|
+
if (this.mergeCache.size > 0 && this.mergeCache.size >= this.mergeCacheSize) {
|
|
1000
|
+
const firstKey = this.mergeCacheKeys[this.mergeCacheHead];
|
|
1001
|
+
this.mergeCache.delete(firstKey);
|
|
1002
|
+
this.bytePairStringRankEncoder.delete(firstKey);
|
|
1003
|
+
this.mergeCacheKeys[this.mergeCacheHead] = key;
|
|
1004
|
+
this.mergeCacheHead = (this.mergeCacheHead + 1) % this.mergeCacheKeys.length;
|
|
1005
|
+
} else this.mergeCacheKeys.push(key);
|
|
1006
|
+
this.mergeCache.set(key, value);
|
|
1007
|
+
this.bytePairStringRankEncoder.set(key, value);
|
|
1008
|
+
}
|
|
1009
|
+
bytePairEncode(input) {
|
|
1010
|
+
if (input.length === 1 && isAscii(input.codePointAt(0))) return [this.getBpeRankFromStringOrThrow(input)];
|
|
1011
|
+
const cached = this.mergeCache?.get(input);
|
|
1012
|
+
if (cached !== void 0) return cached;
|
|
1013
|
+
const result = input.length <= 64 && isAsciiString(input) ? this.bytePairMergeAscii(input) : this.bytePairMerge(this.textEncoder.encode(input));
|
|
1014
|
+
this.addToMergeCache(input, result);
|
|
1015
|
+
return result;
|
|
1016
|
+
}
|
|
1017
|
+
bytePairMergeAscii(input) {
|
|
1018
|
+
const starts = [];
|
|
1019
|
+
const ranks = [];
|
|
1020
|
+
for (let i = 0; i <= input.length; i++) starts.push(i);
|
|
1021
|
+
const getPairRank = (index) => this.getBpeRankFromString(input.slice(starts[index], starts[index + 2])) ?? Number.POSITIVE_INFINITY;
|
|
1022
|
+
for (let i = 0; i < input.length - 1; i++) ranks[i] = getPairRank(i);
|
|
1023
|
+
while (starts.length > 2) {
|
|
1024
|
+
let bestRank = Number.POSITIVE_INFINITY;
|
|
1025
|
+
let bestIndex = -1;
|
|
1026
|
+
for (let i = 0; i < starts.length - 2; i++) if (ranks[i] < bestRank) {
|
|
1027
|
+
bestRank = ranks[i];
|
|
1028
|
+
bestIndex = i;
|
|
1029
|
+
}
|
|
1030
|
+
if (bestIndex === -1) break;
|
|
1031
|
+
starts.splice(bestIndex + 1, 1);
|
|
1032
|
+
ranks.splice(bestIndex + 1, 1);
|
|
1033
|
+
ranks[bestIndex] = bestIndex < starts.length - 2 ? getPairRank(bestIndex) : Number.POSITIVE_INFINITY;
|
|
1034
|
+
if (bestIndex > 0) ranks[bestIndex - 1] = getPairRank(bestIndex - 1);
|
|
1035
|
+
}
|
|
1036
|
+
const result = [];
|
|
1037
|
+
for (let i = 0; i < starts.length - 1; i++) result.push(this.getBpeRankFromStringOrThrow(input.slice(starts[i], starts[i + 1])));
|
|
1038
|
+
return result;
|
|
1039
|
+
}
|
|
1040
|
+
bytePairMerge(piece) {
|
|
1041
|
+
const { length } = piece;
|
|
1042
|
+
const next = new Int32Array(length + 1);
|
|
1043
|
+
const previous = new Int32Array(length + 1);
|
|
1044
|
+
const pairRanks = new Float64Array(length + 1);
|
|
1045
|
+
const live = new Uint8Array(length + 1);
|
|
1046
|
+
const getPairRank = (startIndex) => {
|
|
1047
|
+
const middle = next[startIndex];
|
|
1048
|
+
if (middle >= length) return Number.POSITIVE_INFINITY;
|
|
1049
|
+
return this.getBpeRankFromBytes(piece.subarray(startIndex, next[middle])) ?? Number.POSITIVE_INFINITY;
|
|
1050
|
+
};
|
|
1051
|
+
const heap = [];
|
|
1052
|
+
const isLower = (a, b) => heap[a] < heap[b] || heap[a] === heap[b] && heap[a + 1] < heap[b + 1];
|
|
1053
|
+
const swap = (a, b) => {
|
|
1054
|
+
const rank = heap[a];
|
|
1055
|
+
const startIndex = heap[a + 1];
|
|
1056
|
+
heap[a] = heap[b];
|
|
1057
|
+
heap[a + 1] = heap[b + 1];
|
|
1058
|
+
heap[b] = rank;
|
|
1059
|
+
heap[b + 1] = startIndex;
|
|
1060
|
+
};
|
|
1061
|
+
const push = (rank, startIndex) => {
|
|
1062
|
+
heap.push(rank, startIndex);
|
|
1063
|
+
let index = heap.length - 2;
|
|
1064
|
+
while (index > 0) {
|
|
1065
|
+
const parent = (index >> 1) - 1 >> 1 << 1;
|
|
1066
|
+
if (!isLower(index, parent)) break;
|
|
1067
|
+
swap(index, parent);
|
|
1068
|
+
index = parent;
|
|
1069
|
+
}
|
|
1070
|
+
};
|
|
1071
|
+
const pop = () => {
|
|
1072
|
+
const last = heap.length - 2;
|
|
1073
|
+
swap(0, last);
|
|
1074
|
+
const rank = heap[last];
|
|
1075
|
+
const startIndex = heap[last + 1];
|
|
1076
|
+
heap.length = last;
|
|
1077
|
+
let index = 0;
|
|
1078
|
+
while (true) {
|
|
1079
|
+
const left = index * 2 + 2;
|
|
1080
|
+
const right = left + 2;
|
|
1081
|
+
let lowest = index;
|
|
1082
|
+
if (left < heap.length && isLower(left, lowest)) lowest = left;
|
|
1083
|
+
if (right < heap.length && isLower(right, lowest)) lowest = right;
|
|
1084
|
+
if (lowest === index) break;
|
|
1085
|
+
swap(index, lowest);
|
|
1086
|
+
index = lowest;
|
|
1087
|
+
}
|
|
1088
|
+
return [rank, startIndex];
|
|
1089
|
+
};
|
|
1090
|
+
for (let i = 0; i <= length; i++) {
|
|
1091
|
+
next[i] = i + 1;
|
|
1092
|
+
previous[i] = i - 1;
|
|
1093
|
+
live[i] = 1;
|
|
1094
|
+
}
|
|
1095
|
+
for (let i = 0; i < length - 1; i++) {
|
|
1096
|
+
pairRanks[i] = getPairRank(i);
|
|
1097
|
+
if (pairRanks[i] !== Number.POSITIVE_INFINITY) push(pairRanks[i], i);
|
|
1098
|
+
}
|
|
1099
|
+
while (heap.length > 0) {
|
|
1100
|
+
const [rank, startIndex] = pop();
|
|
1101
|
+
if (!live[startIndex] || pairRanks[startIndex] !== rank) continue;
|
|
1102
|
+
const merged = next[startIndex];
|
|
1103
|
+
live[merged] = 0;
|
|
1104
|
+
next[startIndex] = next[merged];
|
|
1105
|
+
previous[next[merged]] = startIndex;
|
|
1106
|
+
pairRanks[startIndex] = getPairRank(startIndex);
|
|
1107
|
+
if (pairRanks[startIndex] !== Number.POSITIVE_INFINITY) push(pairRanks[startIndex], startIndex);
|
|
1108
|
+
const before = previous[startIndex];
|
|
1109
|
+
if (before >= 0) {
|
|
1110
|
+
pairRanks[before] = getPairRank(before);
|
|
1111
|
+
if (pairRanks[before] !== Number.POSITIVE_INFINITY) push(pairRanks[before], before);
|
|
1112
|
+
}
|
|
1113
|
+
}
|
|
1114
|
+
const output = [];
|
|
1115
|
+
for (let i = 0; i < length; i = next[i]) output.push(this.getBpeRankFromBytesOrThrow(piece.subarray(i, next[i])));
|
|
1116
|
+
return output;
|
|
1117
|
+
}
|
|
1118
|
+
};
|
|
1119
|
+
|
|
1120
|
+
// node_modules/gpt-tokenizer/esm/functionCalling.js
|
|
1121
|
+
var NEWLINE = "\n";
|
|
1122
|
+
function countMessageTokens(message, countStringTokens) {
|
|
1123
|
+
let tokens = 0;
|
|
1124
|
+
if (message.role) tokens += countStringTokens(message.role);
|
|
1125
|
+
if (message.content) tokens += countStringTokens(message.content);
|
|
1126
|
+
if (message.name) tokens += countStringTokens(message.name) + 1;
|
|
1127
|
+
if (message.function_call) {
|
|
1128
|
+
const { name, arguments: args } = message.function_call;
|
|
1129
|
+
if (name) tokens += countStringTokens(name);
|
|
1130
|
+
if (args) tokens += countStringTokens(args);
|
|
1131
|
+
tokens += 3;
|
|
1132
|
+
}
|
|
1133
|
+
tokens += 3;
|
|
1134
|
+
if (message.role === "function") tokens -= 2;
|
|
1135
|
+
return tokens;
|
|
1136
|
+
}
|
|
1137
|
+
function formatObjectProperties(obj, indent, formatType) {
|
|
1138
|
+
if (!obj.properties) return "";
|
|
1139
|
+
const lines = [];
|
|
1140
|
+
const requiredParams = new Set(obj.required ?? []);
|
|
1141
|
+
const indentString = " ".repeat(indent);
|
|
1142
|
+
for (const [name, param] of Object.entries(obj.properties)) {
|
|
1143
|
+
if (param.description && indent < 2) lines.push(`${indentString}// ${param.description}`);
|
|
1144
|
+
const isRequired = requiredParams.has(name);
|
|
1145
|
+
const formattedType = formatType(param, indent);
|
|
1146
|
+
lines.push(`${indentString}${name}${isRequired ? "" : "?"}: ${formattedType},`);
|
|
1147
|
+
}
|
|
1148
|
+
return lines.join("\n");
|
|
1149
|
+
}
|
|
1150
|
+
function formatFunctionType(param, indent) {
|
|
1151
|
+
switch (param.type) {
|
|
1152
|
+
case "string":
|
|
1153
|
+
return param.enum?.map((value) => JSON.stringify(value)).join(" | ") ?? "string";
|
|
1154
|
+
case "integer":
|
|
1155
|
+
case "number":
|
|
1156
|
+
return param.enum?.map((value) => `${value}`).join(" | ") ?? "number";
|
|
1157
|
+
case "boolean":
|
|
1158
|
+
return "boolean";
|
|
1159
|
+
case "null":
|
|
1160
|
+
return "null";
|
|
1161
|
+
case "array":
|
|
1162
|
+
return param.items ? `${formatFunctionType(param.items, indent)}[]` : "any[]";
|
|
1163
|
+
case "object":
|
|
1164
|
+
return `{
|
|
1165
|
+
${formatObjectProperties(param, indent + 2, formatFunctionType)}
|
|
1166
|
+
${" ".repeat(indent)}}`;
|
|
1167
|
+
default:
|
|
1168
|
+
return "any";
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
function formatFunctionDefinitions(functions) {
|
|
1172
|
+
const lines = ["namespace functions {", ""];
|
|
1173
|
+
for (const fn of functions) {
|
|
1174
|
+
if (fn.description) lines.push(`// ${fn.description}`);
|
|
1175
|
+
const { parameters } = fn;
|
|
1176
|
+
const properties = parameters?.properties;
|
|
1177
|
+
if (!parameters || !properties || Object.keys(properties).length === 0) lines.push(`type ${fn.name} = () => any;`);
|
|
1178
|
+
else {
|
|
1179
|
+
lines.push(`type ${fn.name} = (_: {`);
|
|
1180
|
+
const formattedProperties = formatObjectProperties(parameters, 0, formatFunctionType);
|
|
1181
|
+
if (formattedProperties.length > 0) lines.push(formattedProperties);
|
|
1182
|
+
lines.push("}) => any;");
|
|
1183
|
+
}
|
|
1184
|
+
lines.push("");
|
|
1185
|
+
}
|
|
1186
|
+
lines.push("} // namespace functions");
|
|
1187
|
+
return lines.join("\n");
|
|
1188
|
+
}
|
|
1189
|
+
function estimateTokensInFunctions(functions, countStringTokens) {
|
|
1190
|
+
let tokens = countStringTokens(formatFunctionDefinitions(functions));
|
|
1191
|
+
tokens += 9;
|
|
1192
|
+
return tokens;
|
|
1193
|
+
}
|
|
1194
|
+
function padSystemMessage(message, hasFunctions, isSystemPadded) {
|
|
1195
|
+
if (!hasFunctions || isSystemPadded || message.role !== "system") return message;
|
|
1196
|
+
if (!message.content || message.content.endsWith(NEWLINE)) return message;
|
|
1197
|
+
return {
|
|
1198
|
+
...message,
|
|
1199
|
+
content: `${message.content}${NEWLINE}`
|
|
1200
|
+
};
|
|
1201
|
+
}
|
|
1202
|
+
function computeChatCompletionTokenCount(request, countStringTokens) {
|
|
1203
|
+
const { messages, functions, function_call: functionCall } = request;
|
|
1204
|
+
const hasFunctions = Boolean(functions && functions.length > 0);
|
|
1205
|
+
let paddedSystem = false;
|
|
1206
|
+
let total = 0;
|
|
1207
|
+
for (const message of messages) {
|
|
1208
|
+
const messageToCount = padSystemMessage(message, hasFunctions, paddedSystem);
|
|
1209
|
+
if (messageToCount !== message && message.role === "system") paddedSystem = true;
|
|
1210
|
+
else if (message.role === "system" && hasFunctions && !paddedSystem) paddedSystem = true;
|
|
1211
|
+
total += countMessageTokens(messageToCount, countStringTokens);
|
|
1212
|
+
}
|
|
1213
|
+
total += 3;
|
|
1214
|
+
if (hasFunctions && functions) {
|
|
1215
|
+
total += estimateTokensInFunctions(functions, countStringTokens);
|
|
1216
|
+
if (messages.some((message) => message.role === "system")) total -= 4;
|
|
1217
|
+
}
|
|
1218
|
+
if (functionCall && functionCall !== "auto") {
|
|
1219
|
+
if (functionCall === "none") total += 1;
|
|
1220
|
+
else if (typeof functionCall === "object" && functionCall.name) total += countStringTokens(functionCall.name) + 4;
|
|
1221
|
+
}
|
|
1222
|
+
return total;
|
|
1223
|
+
}
|
|
1224
|
+
|
|
1225
|
+
// node_modules/gpt-tokenizer/esm/specialTokens.js
|
|
1226
|
+
var EndOfText = "<|endoftext|>";
|
|
1227
|
+
var FimPrefix = "<|fim_prefix|>";
|
|
1228
|
+
var FimMiddle = "<|fim_middle|>";
|
|
1229
|
+
var FimSuffix = "<|fim_suffix|>";
|
|
1230
|
+
var ImStart = "<|im_start|>";
|
|
1231
|
+
var ImEnd = "<|im_end|>";
|
|
1232
|
+
var ImSep = "<|im_sep|>";
|
|
1233
|
+
var EndOfPrompt = "<|endofprompt|>";
|
|
1234
|
+
var HarmonyStartOfText = "<|startoftext|>";
|
|
1235
|
+
var HarmonyStart = "<|start|>";
|
|
1236
|
+
var HarmonyEnd = "<|end|>";
|
|
1237
|
+
var HarmonyMessage = "<|message|>";
|
|
1238
|
+
var HarmonyChannel = "<|channel|>";
|
|
1239
|
+
var HarmonyReturn = "<|return|>";
|
|
1240
|
+
var HarmonyConstrain = "<|constrain|>";
|
|
1241
|
+
var HarmonyCall = "<|call|>";
|
|
1242
|
+
|
|
1243
|
+
// node_modules/gpt-tokenizer/esm/modelsChatEnabled.gen.js
|
|
1244
|
+
var chatEnabledModels = [
|
|
1245
|
+
"chat-latest",
|
|
1246
|
+
"chatgpt-4o-latest",
|
|
1247
|
+
"codex-mini-latest",
|
|
1248
|
+
"computer-use-preview",
|
|
1249
|
+
"computer-use-preview-2025-03-11",
|
|
1250
|
+
"daybreak-blue-latest",
|
|
1251
|
+
"daybreak-red-latest",
|
|
1252
|
+
"gpt-3.5",
|
|
1253
|
+
"gpt-3.5-0301",
|
|
1254
|
+
"gpt-3.5-turbo",
|
|
1255
|
+
"gpt-3.5-turbo-0125",
|
|
1256
|
+
"gpt-3.5-turbo-0613",
|
|
1257
|
+
"gpt-3.5-turbo-1106",
|
|
1258
|
+
"gpt-3.5-turbo-16k-0613",
|
|
1259
|
+
"gpt-3.5-turbo-instruct",
|
|
1260
|
+
"gpt-4",
|
|
1261
|
+
"gpt-4-0125-preview",
|
|
1262
|
+
"gpt-4-0314",
|
|
1263
|
+
"gpt-4-0613",
|
|
1264
|
+
"gpt-4-1106-preview",
|
|
1265
|
+
"gpt-4-1106-vision-preview",
|
|
1266
|
+
"gpt-4-32k",
|
|
1267
|
+
"gpt-4-turbo",
|
|
1268
|
+
"gpt-4-turbo-2024-04-09",
|
|
1269
|
+
"gpt-4-turbo-preview",
|
|
1270
|
+
"gpt-4.1",
|
|
1271
|
+
"gpt-4.1-2025-04-14",
|
|
1272
|
+
"gpt-4.1-mini",
|
|
1273
|
+
"gpt-4.1-mini-2025-04-14",
|
|
1274
|
+
"gpt-4.1-nano",
|
|
1275
|
+
"gpt-4.1-nano-2025-04-14",
|
|
1276
|
+
"gpt-4.5-preview",
|
|
1277
|
+
"gpt-4.5-preview-2025-02-27",
|
|
1278
|
+
"gpt-4o",
|
|
1279
|
+
"gpt-4o-2024-05-13",
|
|
1280
|
+
"gpt-4o-2024-08-06",
|
|
1281
|
+
"gpt-4o-2024-11-20",
|
|
1282
|
+
"gpt-4o-audio-preview",
|
|
1283
|
+
"gpt-4o-audio-preview-2024-10-01",
|
|
1284
|
+
"gpt-4o-audio-preview-2024-12-17",
|
|
1285
|
+
"gpt-4o-audio-preview-2025-06-03",
|
|
1286
|
+
"gpt-4o-mini",
|
|
1287
|
+
"gpt-4o-mini-2024-07-18",
|
|
1288
|
+
"gpt-4o-mini-audio-preview",
|
|
1289
|
+
"gpt-4o-mini-audio-preview-2024-12-17",
|
|
1290
|
+
"gpt-4o-mini-search-preview",
|
|
1291
|
+
"gpt-4o-mini-search-preview-2025-03-11",
|
|
1292
|
+
"gpt-4o-search-preview",
|
|
1293
|
+
"gpt-4o-search-preview-2025-03-11",
|
|
1294
|
+
"gpt-5",
|
|
1295
|
+
"gpt-5-2025-08-07",
|
|
1296
|
+
"gpt-5-chat-latest",
|
|
1297
|
+
"gpt-5-codex",
|
|
1298
|
+
"gpt-5-mini",
|
|
1299
|
+
"gpt-5-mini-2025-08-07",
|
|
1300
|
+
"gpt-5-nano",
|
|
1301
|
+
"gpt-5-nano-2025-08-07",
|
|
1302
|
+
"gpt-5-pro",
|
|
1303
|
+
"gpt-5-pro-2025-10-06",
|
|
1304
|
+
"gpt-5.1",
|
|
1305
|
+
"gpt-5.1-2025-11-13",
|
|
1306
|
+
"gpt-5.1-chat-latest",
|
|
1307
|
+
"gpt-5.1-codex",
|
|
1308
|
+
"gpt-5.1-codex-max",
|
|
1309
|
+
"gpt-5.1-codex-mini",
|
|
1310
|
+
"gpt-5.2",
|
|
1311
|
+
"gpt-5.2-2025-12-11",
|
|
1312
|
+
"gpt-5.2-chat-latest",
|
|
1313
|
+
"gpt-5.2-codex",
|
|
1314
|
+
"gpt-5.2-pro",
|
|
1315
|
+
"gpt-5.2-pro-2025-12-11",
|
|
1316
|
+
"gpt-5.3-chat-latest",
|
|
1317
|
+
"gpt-5.3-codex",
|
|
1318
|
+
"gpt-5.4",
|
|
1319
|
+
"gpt-5.4-2026-03-05",
|
|
1320
|
+
"gpt-5.4-mini",
|
|
1321
|
+
"gpt-5.4-mini-2026-03-17",
|
|
1322
|
+
"gpt-5.4-nano",
|
|
1323
|
+
"gpt-5.4-nano-2026-03-17",
|
|
1324
|
+
"gpt-5.4-pro",
|
|
1325
|
+
"gpt-5.4-pro-2026-03-05",
|
|
1326
|
+
"gpt-5.5",
|
|
1327
|
+
"gpt-5.5-2026-04-23",
|
|
1328
|
+
"gpt-5.5-pro",
|
|
1329
|
+
"gpt-5.5-pro-2026-04-23",
|
|
1330
|
+
"gpt-5.6-cyber",
|
|
1331
|
+
"gpt-5.6-luna",
|
|
1332
|
+
"gpt-5.6-sol",
|
|
1333
|
+
"gpt-5.6-terra",
|
|
1334
|
+
"gpt-audio",
|
|
1335
|
+
"gpt-audio-1.5",
|
|
1336
|
+
"gpt-audio-2025-08-28",
|
|
1337
|
+
"gpt-audio-mini",
|
|
1338
|
+
"gpt-audio-mini-2025-10-06",
|
|
1339
|
+
"gpt-audio-mini-2025-12-15",
|
|
1340
|
+
"gpt-image-1",
|
|
1341
|
+
"gpt-oss-120b",
|
|
1342
|
+
"gpt-oss-20b",
|
|
1343
|
+
"o1",
|
|
1344
|
+
"o1-2024-12-17",
|
|
1345
|
+
"o1-mini",
|
|
1346
|
+
"o1-mini-2024-09-12",
|
|
1347
|
+
"o1-preview",
|
|
1348
|
+
"o1-preview-2024-09-12",
|
|
1349
|
+
"o1-pro",
|
|
1350
|
+
"o1-pro-2025-03-19",
|
|
1351
|
+
"o3",
|
|
1352
|
+
"o3-2025-04-16",
|
|
1353
|
+
"o3-deep-research",
|
|
1354
|
+
"o3-deep-research-2025-06-26",
|
|
1355
|
+
"o3-mini",
|
|
1356
|
+
"o3-mini-2025-01-31",
|
|
1357
|
+
"o3-pro",
|
|
1358
|
+
"o3-pro-2025-06-10",
|
|
1359
|
+
"o4-mini",
|
|
1360
|
+
"o4-mini-2025-04-16",
|
|
1361
|
+
"o4-mini-deep-research",
|
|
1362
|
+
"o4-mini-deep-research-2025-06-26"
|
|
1363
|
+
];
|
|
1364
|
+
|
|
1365
|
+
// node_modules/gpt-tokenizer/esm/_virtual/_rolldown/runtime.js
|
|
1366
|
+
var __defProp2 = Object.defineProperty;
|
|
1367
|
+
var __exportAll = (all, no_symbols) => {
|
|
1368
|
+
let target = {};
|
|
1369
|
+
for (var name in all) __defProp2(target, name, {
|
|
1370
|
+
get: all[name],
|
|
1371
|
+
enumerable: true
|
|
1372
|
+
});
|
|
1373
|
+
if (!no_symbols) __defProp2(target, Symbol.toStringTag, { value: "Module" });
|
|
1374
|
+
return target;
|
|
1375
|
+
};
|
|
1376
|
+
|
|
1377
|
+
// node_modules/gpt-tokenizer/esm/modelsMap.js
|
|
1378
|
+
var modelsMap_exports = /* @__PURE__ */ __exportAll({
|
|
1379
|
+
cl100k_base: () => cl100k_base,
|
|
1380
|
+
gpt2: () => gpt2,
|
|
1381
|
+
o200k_base: () => o200k_base,
|
|
1382
|
+
o200k_harmony: () => o200k_harmony,
|
|
1383
|
+
p50k_base: () => p50k_base,
|
|
1384
|
+
p50k_edit: () => p50k_edit,
|
|
1385
|
+
r50k_base: () => r50k_base
|
|
1386
|
+
});
|
|
1387
|
+
var gpt2 = ["gpt2", "gpt-2"];
|
|
1388
|
+
var p50k_base = [
|
|
1389
|
+
"text-davinci-002",
|
|
1390
|
+
"text-davinci-003",
|
|
1391
|
+
"code-davinci-001",
|
|
1392
|
+
"code-davinci-002",
|
|
1393
|
+
"davinci-codex",
|
|
1394
|
+
"code-cushman-001",
|
|
1395
|
+
"code-cushman-002",
|
|
1396
|
+
"cushman-codex"
|
|
1397
|
+
];
|
|
1398
|
+
var r50k_base = [
|
|
1399
|
+
"text-ada-001",
|
|
1400
|
+
"text-babbage-001",
|
|
1401
|
+
"text-curie-001",
|
|
1402
|
+
"text-davinci-001",
|
|
1403
|
+
"ada",
|
|
1404
|
+
"babbage",
|
|
1405
|
+
"curie",
|
|
1406
|
+
"davinci",
|
|
1407
|
+
"code-search-ada-code-001",
|
|
1408
|
+
"code-search-ada-text-001",
|
|
1409
|
+
"text-similarity-ada-001",
|
|
1410
|
+
"text-search-ada-doc-001",
|
|
1411
|
+
"text-search-ada-query-001",
|
|
1412
|
+
"text-similarity-babbage-001",
|
|
1413
|
+
"text-search-babbage-doc-001",
|
|
1414
|
+
"text-search-babbage-query-001",
|
|
1415
|
+
"code-search-babbage-code-001",
|
|
1416
|
+
"code-search-babbage-text-001",
|
|
1417
|
+
"text-similarity-curie-001",
|
|
1418
|
+
"text-search-curie-doc-001",
|
|
1419
|
+
"text-search-curie-query-001",
|
|
1420
|
+
"text-similarity-davinci-001",
|
|
1421
|
+
"text-search-davinci-doc-001",
|
|
1422
|
+
"text-search-davinci-query-001"
|
|
1423
|
+
];
|
|
1424
|
+
var p50k_edit = ["code-davinci-edit-001", "text-davinci-edit-001"];
|
|
1425
|
+
var cl100k_base = [
|
|
1426
|
+
"gpt-3.5",
|
|
1427
|
+
"gpt-3.5-0301",
|
|
1428
|
+
"gpt-3.5-turbo",
|
|
1429
|
+
"gpt-3.5-turbo-0125",
|
|
1430
|
+
"gpt-3.5-turbo-0613",
|
|
1431
|
+
"gpt-3.5-turbo-1106",
|
|
1432
|
+
"gpt-3.5-turbo-16k-0613",
|
|
1433
|
+
"gpt-3.5-turbo-instruct",
|
|
1434
|
+
"gpt-4",
|
|
1435
|
+
"gpt-4-0125-preview",
|
|
1436
|
+
"gpt-4-0314",
|
|
1437
|
+
"gpt-4-0613",
|
|
1438
|
+
"gpt-4-1106-preview",
|
|
1439
|
+
"gpt-4-1106-vision-preview",
|
|
1440
|
+
"gpt-4-32k",
|
|
1441
|
+
"gpt-4-turbo",
|
|
1442
|
+
"gpt-4-turbo-2024-04-09",
|
|
1443
|
+
"gpt-4-turbo-preview",
|
|
1444
|
+
"text-embedding-3-large",
|
|
1445
|
+
"text-embedding-3-small",
|
|
1446
|
+
"text-embedding-ada-002",
|
|
1447
|
+
"babbage-002",
|
|
1448
|
+
"davinci-002"
|
|
1449
|
+
];
|
|
1450
|
+
var o200k_base = [];
|
|
1451
|
+
var o200k_harmony = ["gpt-oss-20b", "gpt-oss-120b"];
|
|
1452
|
+
|
|
1453
|
+
// node_modules/gpt-tokenizer/esm/mapping.js
|
|
1454
|
+
var modelToEncodingMap = Object.fromEntries(Object.entries(modelsMap_exports).flatMap(([encodingName, models]) => models.map((modelName) => [modelName, encodingName])));
|
|
1455
|
+
var gpt3params = {
|
|
1456
|
+
messageSeparator: "\n",
|
|
1457
|
+
roleSeparator: "\n"
|
|
1458
|
+
};
|
|
1459
|
+
var gpt4params = {
|
|
1460
|
+
messageSeparator: "",
|
|
1461
|
+
roleSeparator: ImSep
|
|
1462
|
+
};
|
|
1463
|
+
var chatModelParams = Object.fromEntries(chatEnabledModels.flatMap((modelName) => modelName.startsWith("gpt-3.5") ? [[modelName, gpt3params]] : [[modelName, gpt4params]]));
|
|
1464
|
+
|
|
1465
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/cl100k_base.js
|
|
1466
|
+
function Cl100KBase(bytePairRankDecoder) {
|
|
1467
|
+
return {
|
|
1468
|
+
tokenSplitRegex: CL100K_TOKEN_SPLIT_REGEX,
|
|
1469
|
+
bytePairRankDecoder,
|
|
1470
|
+
specialTokensEncoder: /* @__PURE__ */ new Map([
|
|
1471
|
+
[EndOfText, 100257],
|
|
1472
|
+
[FimPrefix, 100258],
|
|
1473
|
+
[FimMiddle, 100259],
|
|
1474
|
+
[FimSuffix, 100260],
|
|
1475
|
+
[ImStart, 100264],
|
|
1476
|
+
[ImEnd, 100265],
|
|
1477
|
+
[ImSep, 100266],
|
|
1478
|
+
[EndOfPrompt, 100276]
|
|
1479
|
+
])
|
|
1480
|
+
};
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/o200k_base.js
|
|
1484
|
+
var O200K_BASE_SPECIAL_TOKEN_ENTRIES = [
|
|
1485
|
+
[EndOfText, 199999],
|
|
1486
|
+
[FimPrefix, 2e5],
|
|
1487
|
+
[FimMiddle, 200001],
|
|
1488
|
+
[FimSuffix, 200002],
|
|
1489
|
+
[ImStart, 200003],
|
|
1490
|
+
[ImEnd, 200004],
|
|
1491
|
+
[ImSep, 200005],
|
|
1492
|
+
[EndOfPrompt, 200006]
|
|
1493
|
+
];
|
|
1494
|
+
var createO200KSpecialTokenMap = () => new Map(O200K_BASE_SPECIAL_TOKEN_ENTRIES);
|
|
1495
|
+
function O200KBase(bytePairRankDecoder) {
|
|
1496
|
+
return {
|
|
1497
|
+
tokenSplitRegex: O200K_TOKEN_SPLIT_REGEX,
|
|
1498
|
+
bytePairRankDecoder,
|
|
1499
|
+
specialTokensEncoder: createO200KSpecialTokenMap()
|
|
1500
|
+
};
|
|
1501
|
+
}
|
|
1502
|
+
|
|
1503
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/o200k_harmony.js
|
|
1504
|
+
var RESERVED_TOKEN_RANGE_START = 200013;
|
|
1505
|
+
var RESERVED_TOKEN_RANGE_END = 201088;
|
|
1506
|
+
var STATIC_SPECIAL_TOKEN_ENTRIES = [
|
|
1507
|
+
[HarmonyStartOfText, 199998],
|
|
1508
|
+
[EndOfText, 199999],
|
|
1509
|
+
["<|reserved_200000|>", 2e5],
|
|
1510
|
+
["<|reserved_200001|>", 200001],
|
|
1511
|
+
[HarmonyReturn, 200002],
|
|
1512
|
+
[HarmonyConstrain, 200003],
|
|
1513
|
+
["<|reserved_200004|>", 200004],
|
|
1514
|
+
[HarmonyChannel, 200005],
|
|
1515
|
+
[HarmonyStart, 200006],
|
|
1516
|
+
[HarmonyEnd, 200007],
|
|
1517
|
+
[HarmonyMessage, 200008],
|
|
1518
|
+
["<|reserved_200009|>", 200009],
|
|
1519
|
+
["<|reserved_200010|>", 200010],
|
|
1520
|
+
["<|reserved_200011|>", 200011],
|
|
1521
|
+
[HarmonyCall, 200012]
|
|
1522
|
+
];
|
|
1523
|
+
function O200KHarmony(bytePairRankDecoder) {
|
|
1524
|
+
const specialTokensEncoder = new Map(STATIC_SPECIAL_TOKEN_ENTRIES);
|
|
1525
|
+
for (let tokenId = RESERVED_TOKEN_RANGE_START; tokenId < RESERVED_TOKEN_RANGE_END; tokenId += 1) specialTokensEncoder.set(`<|reserved_${tokenId}|>`, tokenId);
|
|
1526
|
+
specialTokensEncoder.set(EndOfPrompt, 200018);
|
|
1527
|
+
return {
|
|
1528
|
+
tokenSplitRegex: O200K_TOKEN_SPLIT_REGEX,
|
|
1529
|
+
bytePairRankDecoder,
|
|
1530
|
+
specialTokensEncoder,
|
|
1531
|
+
chatFormatter: "harmony"
|
|
1532
|
+
};
|
|
1533
|
+
}
|
|
1534
|
+
|
|
1535
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/p50k_base.js
|
|
1536
|
+
function P50KBase(bytePairRankDecoder) {
|
|
1537
|
+
return {
|
|
1538
|
+
expectedVocabularySize: 50281,
|
|
1539
|
+
tokenSplitRegex: R50K_TOKEN_SPLIT_REGEX,
|
|
1540
|
+
bytePairRankDecoder,
|
|
1541
|
+
specialTokensEncoder: /* @__PURE__ */ new Map([[EndOfText, 50256]])
|
|
1542
|
+
};
|
|
1543
|
+
}
|
|
1544
|
+
|
|
1545
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/p50k_edit.js
|
|
1546
|
+
function P50KEdit(bytePairRankDecoder) {
|
|
1547
|
+
return {
|
|
1548
|
+
tokenSplitRegex: R50K_TOKEN_SPLIT_REGEX,
|
|
1549
|
+
bytePairRankDecoder,
|
|
1550
|
+
specialTokensEncoder: /* @__PURE__ */ new Map([
|
|
1551
|
+
[EndOfText, 50256],
|
|
1552
|
+
[FimPrefix, 50281],
|
|
1553
|
+
[FimMiddle, 50282],
|
|
1554
|
+
[FimSuffix, 50283]
|
|
1555
|
+
])
|
|
1556
|
+
};
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
// node_modules/gpt-tokenizer/esm/encodingParams/r50k_base.js
|
|
1560
|
+
function R50KBase(bytePairRankDecoder) {
|
|
1561
|
+
return {
|
|
1562
|
+
expectedVocabularySize: 50257,
|
|
1563
|
+
tokenSplitRegex: R50K_TOKEN_SPLIT_REGEX,
|
|
1564
|
+
bytePairRankDecoder,
|
|
1565
|
+
specialTokensEncoder: /* @__PURE__ */ new Map([[EndOfText, 50256]])
|
|
1566
|
+
};
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
// node_modules/gpt-tokenizer/esm/modelParams.js
|
|
1570
|
+
function getEncodingParams(encodingName, getMergeableRanks) {
|
|
1571
|
+
const mergeableBytePairRanks = getMergeableRanks(encodingName);
|
|
1572
|
+
switch (encodingName.toLowerCase()) {
|
|
1573
|
+
case "gpt2":
|
|
1574
|
+
case "r50k_base":
|
|
1575
|
+
return R50KBase(mergeableBytePairRanks);
|
|
1576
|
+
case "p50k_base":
|
|
1577
|
+
return P50KBase(mergeableBytePairRanks);
|
|
1578
|
+
case "p50k_edit":
|
|
1579
|
+
return P50KEdit(mergeableBytePairRanks);
|
|
1580
|
+
case "cl100k_base":
|
|
1581
|
+
return Cl100KBase(mergeableBytePairRanks);
|
|
1582
|
+
case "o200k_base":
|
|
1583
|
+
return O200KBase(mergeableBytePairRanks);
|
|
1584
|
+
case "o200k_harmony":
|
|
1585
|
+
return O200KHarmony(mergeableBytePairRanks);
|
|
1586
|
+
default:
|
|
1587
|
+
throw new Error(`Unknown encoding name: ${encodingName}`);
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
// node_modules/gpt-tokenizer/esm/GptEncoding.js
|
|
1592
|
+
var GptEncoding = class GptEncoding2 {
|
|
1593
|
+
static EndOfPrompt = EndOfPrompt;
|
|
1594
|
+
static EndOfText = EndOfText;
|
|
1595
|
+
static FimMiddle = FimMiddle;
|
|
1596
|
+
static FimPrefix = FimPrefix;
|
|
1597
|
+
static FimSuffix = FimSuffix;
|
|
1598
|
+
modelName;
|
|
1599
|
+
modelSpec;
|
|
1600
|
+
bytePairEncodingCoreProcessor;
|
|
1601
|
+
specialTokensEncoder;
|
|
1602
|
+
specialTokensSet;
|
|
1603
|
+
allSpecialTokenRegex;
|
|
1604
|
+
defaultSpecialTokenConfig;
|
|
1605
|
+
chatFormatter;
|
|
1606
|
+
countChatCompletionTokens;
|
|
1607
|
+
vocabularySize;
|
|
1608
|
+
constructor({ bytePairRankDecoder: mergeableBytePairRanks, specialTokensEncoder, expectedVocabularySize, modelName, modelSpec, chatFormatter, ...rest }) {
|
|
1609
|
+
this.specialTokensEncoder = specialTokensEncoder;
|
|
1610
|
+
this.specialTokensSet = new Set(this.specialTokensEncoder.keys());
|
|
1611
|
+
this.allSpecialTokenRegex = getSpecialTokenRegex(this.specialTokensSet);
|
|
1612
|
+
this.bytePairEncodingCoreProcessor = new BytePairEncodingCore({
|
|
1613
|
+
bytePairRankDecoder: mergeableBytePairRanks,
|
|
1614
|
+
specialTokensEncoder,
|
|
1615
|
+
...rest
|
|
1616
|
+
});
|
|
1617
|
+
this.defaultSpecialTokenConfig = this.processSpecialTokens();
|
|
1618
|
+
const maxTokenValue = Math.max(mergeableBytePairRanks.length - 1, getMaxValueFromMap(specialTokensEncoder));
|
|
1619
|
+
this.vocabularySize = this.bytePairEncodingCoreProcessor.mergeableBytePairRankCount + specialTokensEncoder.size;
|
|
1620
|
+
if (expectedVocabularySize !== void 0) {
|
|
1621
|
+
if (this.vocabularySize !== expectedVocabularySize) throw new Error("The number of mergeable tokens and special tokens must be equal to expectedVocabularySize.");
|
|
1622
|
+
if (maxTokenValue !== expectedVocabularySize - 1) throw new Error(`The model encodings are invalid. The maximum token value must be equal to expectedVocabularySize - 1. Currently ${maxTokenValue}, expected ${expectedVocabularySize - 1}`);
|
|
1623
|
+
}
|
|
1624
|
+
this.encode = this.encode.bind(this);
|
|
1625
|
+
this.decode = this.decode.bind(this);
|
|
1626
|
+
this.encodeGenerator = this.encodeGenerator.bind(this);
|
|
1627
|
+
this.decodeGenerator = this.decodeGenerator.bind(this);
|
|
1628
|
+
this.decodeAsyncGenerator = this.decodeAsyncGenerator.bind(this);
|
|
1629
|
+
this.decodeAsync = this.decodeAsync.bind(this);
|
|
1630
|
+
this.isWithinTokenLimit = this.isWithinTokenLimit.bind(this);
|
|
1631
|
+
this.encodeChat = this.encodeChat.bind(this);
|
|
1632
|
+
this.encodeChatGenerator = this.encodeChatGenerator.bind(this);
|
|
1633
|
+
this.countTokens = this.countTokens.bind(this);
|
|
1634
|
+
this.createIncrementalCounter = this.createIncrementalCounter.bind(this);
|
|
1635
|
+
this.setMergeCacheSize = this.setMergeCacheSize.bind(this);
|
|
1636
|
+
this.clearMergeCache = this.clearMergeCache.bind(this);
|
|
1637
|
+
this.estimateCost = this.estimateCost.bind(this);
|
|
1638
|
+
if (modelSpec?.supported_features?.includes("function_calling")) this.countChatCompletionTokens = this.countChatCompletionTokensInternal.bind(this);
|
|
1639
|
+
this.modelName = modelName;
|
|
1640
|
+
this.modelSpec = modelSpec;
|
|
1641
|
+
this.chatFormatter = chatFormatter ?? "chatml";
|
|
1642
|
+
}
|
|
1643
|
+
*encodeHarmonyChatGenerator(chat, encodeOptions) {
|
|
1644
|
+
const harmonyStart = this.specialTokensEncoder.get(HarmonyStart);
|
|
1645
|
+
const harmonyMessage = this.specialTokensEncoder.get(HarmonyMessage);
|
|
1646
|
+
const harmonyEnd = this.specialTokensEncoder.get(HarmonyEnd);
|
|
1647
|
+
const harmonyReturn = this.specialTokensEncoder.get(HarmonyReturn);
|
|
1648
|
+
const harmonyCall = this.specialTokensEncoder.get(HarmonyCall);
|
|
1649
|
+
const harmonyChannel = this.specialTokensEncoder.get(HarmonyChannel);
|
|
1650
|
+
const harmonyConstrain = this.specialTokensEncoder.get(HarmonyConstrain);
|
|
1651
|
+
if (harmonyStart === void 0 || harmonyMessage === void 0 || harmonyEnd === void 0 || harmonyReturn === void 0 || harmonyCall === void 0 || harmonyChannel === void 0 || harmonyConstrain === void 0) throw new Error("Harmony chat format requires dedicated special tokens.");
|
|
1652
|
+
const encodeHeaderText = (text3) => text3.length > 0 ? this.encode(text3) : [];
|
|
1653
|
+
const resolveTerminatorToken = (terminator) => {
|
|
1654
|
+
switch (terminator) {
|
|
1655
|
+
case "<|return|>":
|
|
1656
|
+
return harmonyReturn;
|
|
1657
|
+
case "<|call|>":
|
|
1658
|
+
return harmonyCall;
|
|
1659
|
+
default:
|
|
1660
|
+
return harmonyEnd;
|
|
1661
|
+
}
|
|
1662
|
+
};
|
|
1663
|
+
for (const message of chat) {
|
|
1664
|
+
if (message.content === void 0) throw new Error("Content must be defined for all messages.");
|
|
1665
|
+
const roleOrName = message.name ?? message.role ?? "user";
|
|
1666
|
+
yield [harmonyStart];
|
|
1667
|
+
yield encodeHeaderText(roleOrName);
|
|
1668
|
+
const recipientInRole = message.recipient && (message.recipientPlacement === "role" || !message.channel);
|
|
1669
|
+
const recipientInChannel = message.recipient && !recipientInRole;
|
|
1670
|
+
if (recipientInRole) yield encodeHeaderText(` to=${message.recipient}`);
|
|
1671
|
+
if (message.channel) {
|
|
1672
|
+
yield [harmonyChannel];
|
|
1673
|
+
yield encodeHeaderText(message.channel);
|
|
1674
|
+
if (recipientInChannel) yield encodeHeaderText(` to=${message.recipient}`);
|
|
1675
|
+
}
|
|
1676
|
+
if (message.constraint) {
|
|
1677
|
+
yield [harmonyConstrain];
|
|
1678
|
+
yield encodeHeaderText(message.constraint);
|
|
1679
|
+
}
|
|
1680
|
+
yield [harmonyMessage];
|
|
1681
|
+
yield* this.encodeGenerator(message.content, encodeOptions);
|
|
1682
|
+
yield [resolveTerminatorToken(message.terminator)];
|
|
1683
|
+
}
|
|
1684
|
+
const assistantPrime = encodeOptions?.primeWithAssistantResponse ?? "assistant";
|
|
1685
|
+
if (assistantPrime.length > 0) {
|
|
1686
|
+
yield [harmonyStart];
|
|
1687
|
+
yield encodeHeaderText(assistantPrime);
|
|
1688
|
+
}
|
|
1689
|
+
}
|
|
1690
|
+
static getEncodingApi(encodingName, getMergeableRanks) {
|
|
1691
|
+
const modelParams = getEncodingParams(encodingName, getMergeableRanks);
|
|
1692
|
+
return new GptEncoding2(modelParams);
|
|
1693
|
+
}
|
|
1694
|
+
static getEncodingApiForModel(modelName, getMergeableRanks, modelSpec) {
|
|
1695
|
+
const encodingName = modelToEncodingMap[modelName] ?? "o200k_base";
|
|
1696
|
+
const modelParams = getEncodingParams(encodingName, getMergeableRanks);
|
|
1697
|
+
return new GptEncoding2({
|
|
1698
|
+
...modelParams,
|
|
1699
|
+
modelName,
|
|
1700
|
+
modelSpec
|
|
1701
|
+
});
|
|
1702
|
+
}
|
|
1703
|
+
processSpecialTokens({ allowedSpecial, disallowedSpecial } = {}) {
|
|
1704
|
+
let regexPattern;
|
|
1705
|
+
if (allowedSpecial === "all" || allowedSpecial?.has("all")) {
|
|
1706
|
+
allowedSpecial = new Set(this.specialTokensSet);
|
|
1707
|
+
const allowedSpecialSet = allowedSpecial;
|
|
1708
|
+
if (disallowedSpecial === "all") throw new Error('allowedSpecial and disallowedSpecial cannot both be set to "all".');
|
|
1709
|
+
if (typeof disallowedSpecial === "object") disallowedSpecial.forEach((val) => allowedSpecialSet.delete(val));
|
|
1710
|
+
else disallowedSpecial = /* @__PURE__ */ new Set();
|
|
1711
|
+
}
|
|
1712
|
+
if (!disallowedSpecial || disallowedSpecial === "all" || disallowedSpecial.has("all")) {
|
|
1713
|
+
disallowedSpecial = new Set(this.specialTokensSet);
|
|
1714
|
+
const disallowedSpecialSet = disallowedSpecial;
|
|
1715
|
+
if (allowedSpecial?.size) {
|
|
1716
|
+
allowedSpecial.forEach((val) => disallowedSpecialSet.delete(val));
|
|
1717
|
+
disallowedSpecial.forEach((val) => allowedSpecial.delete(val));
|
|
1718
|
+
if (disallowedSpecial.size > 0) regexPattern = getSpecialTokenRegex(disallowedSpecial);
|
|
1719
|
+
} else regexPattern = this.allSpecialTokenRegex;
|
|
1720
|
+
}
|
|
1721
|
+
return {
|
|
1722
|
+
allowedSpecial,
|
|
1723
|
+
regexPattern
|
|
1724
|
+
};
|
|
1725
|
+
}
|
|
1726
|
+
encodeGenerator(lineToEncode, encodeOptions) {
|
|
1727
|
+
const specialTokenConfig = encodeOptions ? this.processSpecialTokens(encodeOptions) : this.defaultSpecialTokenConfig;
|
|
1728
|
+
if (specialTokenConfig.regexPattern) {
|
|
1729
|
+
const match = lineToEncode.match(specialTokenConfig.regexPattern);
|
|
1730
|
+
if (match !== null) throw new Error(`Disallowed special token found: ${match[0]}`);
|
|
1731
|
+
}
|
|
1732
|
+
return this.bytePairEncodingCoreProcessor.encodeNativeGenerator(lineToEncode, specialTokenConfig.allowedSpecial);
|
|
1733
|
+
}
|
|
1734
|
+
encode(lineToEncode, encodeOptions) {
|
|
1735
|
+
const specialTokenConfig = encodeOptions ? this.processSpecialTokens(encodeOptions) : this.defaultSpecialTokenConfig;
|
|
1736
|
+
if (specialTokenConfig.regexPattern) {
|
|
1737
|
+
const match = lineToEncode.match(specialTokenConfig.regexPattern);
|
|
1738
|
+
if (match !== null) throw new Error(`Disallowed special token found: ${match[0]}`);
|
|
1739
|
+
}
|
|
1740
|
+
return this.bytePairEncodingCoreProcessor.encodeNative(lineToEncode, specialTokenConfig.allowedSpecial);
|
|
1741
|
+
}
|
|
1742
|
+
/**
|
|
1743
|
+
* Progressively tokenizes an OpenAI chat.
|
|
1744
|
+
* Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
|
|
1745
|
+
* Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
|
|
1746
|
+
* Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
|
|
1747
|
+
* Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
|
|
1748
|
+
* @param encodeOptions Options controlling how special tokens are handled.
|
|
1749
|
+
*/
|
|
1750
|
+
*encodeChatGenerator(chat, model = this.modelName, encodeOptions) {
|
|
1751
|
+
if (!model) throw new Error("Model name must be provided either during initialization or passed in to the method.");
|
|
1752
|
+
const params = chatModelParams[model];
|
|
1753
|
+
if (!params) throw new Error(`Model '${model}' does not support chat.`);
|
|
1754
|
+
if (this.chatFormatter === "harmony") {
|
|
1755
|
+
yield* this.encodeHarmonyChatGenerator(chat, encodeOptions);
|
|
1756
|
+
return;
|
|
1757
|
+
}
|
|
1758
|
+
const chatStartToken = this.specialTokensEncoder.get(ImStart);
|
|
1759
|
+
const chatEndToken = this.specialTokensEncoder.get(ImEnd);
|
|
1760
|
+
if (chatStartToken === void 0 || chatEndToken === void 0) throw new Error(`Model '${model}' does not support chat.`);
|
|
1761
|
+
const allowedSpecial = /* @__PURE__ */ new Set([ImSep]);
|
|
1762
|
+
const { messageSeparator, roleSeparator } = params;
|
|
1763
|
+
const encodedMessageSeparator = messageSeparator.length > 0 ? this.encode(messageSeparator) : [];
|
|
1764
|
+
const encodedRoleSeparator = roleSeparator.length > 0 ? this.encode(roleSeparator, { allowedSpecial }) : [];
|
|
1765
|
+
const nameCache = /* @__PURE__ */ new Map();
|
|
1766
|
+
for (const { role = "system", name = role, content } of chat) {
|
|
1767
|
+
if (content === void 0) throw new Error("Content must be defined for all messages.");
|
|
1768
|
+
yield [chatStartToken];
|
|
1769
|
+
const encodedName = nameCache.get(name) ?? this.encode(name);
|
|
1770
|
+
nameCache.set(name, encodedName);
|
|
1771
|
+
yield encodedName;
|
|
1772
|
+
if (encodedRoleSeparator.length > 0) yield encodedRoleSeparator;
|
|
1773
|
+
yield* this.encodeGenerator(content, encodeOptions);
|
|
1774
|
+
yield [chatEndToken];
|
|
1775
|
+
yield encodedMessageSeparator;
|
|
1776
|
+
}
|
|
1777
|
+
const assistantPrime = encodeOptions?.primeWithAssistantResponse ?? "assistant";
|
|
1778
|
+
if (assistantPrime.length > 0) {
|
|
1779
|
+
yield [chatStartToken];
|
|
1780
|
+
yield* this.encodeGenerator(assistantPrime, encodeOptions);
|
|
1781
|
+
}
|
|
1782
|
+
if (encodedRoleSeparator.length > 0) yield encodedRoleSeparator;
|
|
1783
|
+
}
|
|
1784
|
+
/**
|
|
1785
|
+
* Encodes a chat into a single array of tokens.
|
|
1786
|
+
* Warning: gpt-3.5-turbo and gpt-4 chat format may change over time.
|
|
1787
|
+
* Returns tokens assuming the 'gpt-3.5-turbo-0301' / 'gpt-4-0314' format.
|
|
1788
|
+
* Based on OpenAI's guidelines: https://github.com/openai/openai-python/blob/main/chatml.md
|
|
1789
|
+
* Also mentioned in section 6 of this document: https://github.com/openai/openai-cookbook/blob/main/examples/How_to_count_tokens_with_tiktoken.ipynb
|
|
1790
|
+
* @param encodeOptions Options controlling how special tokens are handled.
|
|
1791
|
+
*/
|
|
1792
|
+
encodeChat(chat, model = this.modelName, encodeOptions) {
|
|
1793
|
+
return [...this.encodeChatGenerator(chat, model, encodeOptions)].flat();
|
|
1794
|
+
}
|
|
1795
|
+
/**
|
|
1796
|
+
* Checks whether the provided input stays within the provided token limit.
|
|
1797
|
+
* @param input The string or chat messages to evaluate.
|
|
1798
|
+
* @param tokenLimit The maximum allowed number of tokens.
|
|
1799
|
+
* @param encodeOptions Options controlling how special tokens are handled.
|
|
1800
|
+
* @returns {false | number} false if token limit is exceeded, otherwise the number of tokens
|
|
1801
|
+
*/
|
|
1802
|
+
isWithinTokenLimit(input, tokenLimit, encodeOptions) {
|
|
1803
|
+
const tokenGenerator = typeof input === "string" ? this.encodeGenerator(input, encodeOptions) : this.encodeChatGenerator(input, void 0, encodeOptions);
|
|
1804
|
+
let count = 0;
|
|
1805
|
+
for (const tokens of tokenGenerator) {
|
|
1806
|
+
count += tokens.length;
|
|
1807
|
+
if (count > tokenLimit) return false;
|
|
1808
|
+
}
|
|
1809
|
+
return count;
|
|
1810
|
+
}
|
|
1811
|
+
/**
|
|
1812
|
+
* Counts the number of tokens in the input.
|
|
1813
|
+
* @param input The string or chat messages to evaluate.
|
|
1814
|
+
* @param encodeOptions Options controlling how special tokens are handled.
|
|
1815
|
+
* @returns {number} The number of tokens.
|
|
1816
|
+
*/
|
|
1817
|
+
countTokens(input, encodeOptions) {
|
|
1818
|
+
if (typeof input === "string") {
|
|
1819
|
+
const specialTokenConfig = encodeOptions ? this.processSpecialTokens(encodeOptions) : this.defaultSpecialTokenConfig;
|
|
1820
|
+
if (specialTokenConfig.regexPattern) {
|
|
1821
|
+
const match = input.match(specialTokenConfig.regexPattern);
|
|
1822
|
+
if (match !== null) throw new Error(`Disallowed special token found: ${match[0]}`);
|
|
1823
|
+
}
|
|
1824
|
+
return this.bytePairEncodingCoreProcessor.countNative(input, specialTokenConfig.allowedSpecial);
|
|
1825
|
+
}
|
|
1826
|
+
const tokenGenerator = this.encodeChatGenerator(input, void 0, encodeOptions);
|
|
1827
|
+
let count = 0;
|
|
1828
|
+
for (const tokens of tokenGenerator) count += tokens.length;
|
|
1829
|
+
return count;
|
|
1830
|
+
}
|
|
1831
|
+
countStringTokens(text3) {
|
|
1832
|
+
if (!text3) return 0;
|
|
1833
|
+
return this.bytePairEncodingCoreProcessor.countNative(text3);
|
|
1834
|
+
}
|
|
1835
|
+
/** Create an independent counter sharing this encoding's BPE cache. */
|
|
1836
|
+
createIncrementalCounter(encodeOptions) {
|
|
1837
|
+
const snapshot = encodeOptions && {
|
|
1838
|
+
allowedSpecial: typeof encodeOptions.allowedSpecial === "object" ? new Set(encodeOptions.allowedSpecial) : encodeOptions.allowedSpecial,
|
|
1839
|
+
disallowedSpecial: typeof encodeOptions.disallowedSpecial === "object" ? new Set(encodeOptions.disallowedSpecial) : encodeOptions.disallowedSpecial
|
|
1840
|
+
};
|
|
1841
|
+
const { allowedSpecial, regexPattern } = snapshot ? this.processSpecialTokens(snapshot) : this.defaultSpecialTokenConfig;
|
|
1842
|
+
const maxSpecialLength = Math.max(0, ...Array.from(this.specialTokensSet, (token) => token.length));
|
|
1843
|
+
return this.bytePairEncodingCoreProcessor.createIncrementalCounter(allowedSpecial, (text3, start, insertedLength) => {
|
|
1844
|
+
const match = regexPattern && text3.slice(Math.max(0, start - maxSpecialLength), start + insertedLength + maxSpecialLength).match(regexPattern);
|
|
1845
|
+
if (match) throw new Error(`Disallowed special token found: ${match[0]}`);
|
|
1846
|
+
});
|
|
1847
|
+
}
|
|
1848
|
+
countChatCompletionTokensInternal(request) {
|
|
1849
|
+
return computeChatCompletionTokenCount(request, (text3) => this.countStringTokens(text3));
|
|
1850
|
+
}
|
|
1851
|
+
setMergeCacheSize(size) {
|
|
1852
|
+
this.bytePairEncodingCoreProcessor.setMergeCacheSize(size);
|
|
1853
|
+
}
|
|
1854
|
+
clearMergeCache() {
|
|
1855
|
+
this.bytePairEncodingCoreProcessor.clearMergeCache();
|
|
1856
|
+
}
|
|
1857
|
+
decode(inputTokensToDecode) {
|
|
1858
|
+
return this.bytePairEncodingCoreProcessor.decodeNative(inputTokensToDecode);
|
|
1859
|
+
}
|
|
1860
|
+
*decodeGenerator(inputTokensToDecode) {
|
|
1861
|
+
const decodedByteGenerator = this.bytePairEncodingCoreProcessor.decodeNativeGenerator(inputTokensToDecode);
|
|
1862
|
+
const decoder2 = new TextDecoder("utf8", { ignoreBOM: true });
|
|
1863
|
+
let hasPendingBytes = false;
|
|
1864
|
+
let buffer = "";
|
|
1865
|
+
for (const decodedPart of decodedByteGenerator) {
|
|
1866
|
+
if (typeof decodedPart === "string") {
|
|
1867
|
+
if (decodedPart.length === 0) continue;
|
|
1868
|
+
if (hasPendingBytes) {
|
|
1869
|
+
buffer += decoder2.decode();
|
|
1870
|
+
hasPendingBytes = false;
|
|
1871
|
+
}
|
|
1872
|
+
buffer += decodedPart;
|
|
1873
|
+
} else {
|
|
1874
|
+
buffer += decoder2.decode(decodedPart, { stream: true });
|
|
1875
|
+
hasPendingBytes = true;
|
|
1876
|
+
}
|
|
1877
|
+
if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) continue;
|
|
1878
|
+
else {
|
|
1879
|
+
yield buffer;
|
|
1880
|
+
buffer = "";
|
|
1881
|
+
}
|
|
1882
|
+
}
|
|
1883
|
+
buffer += decoder2.decode();
|
|
1884
|
+
if (buffer.length > 0) yield buffer;
|
|
1885
|
+
}
|
|
1886
|
+
async *decodeAsyncGenerator(inputTokensToDecode) {
|
|
1887
|
+
const decodedByteGenerator = this.bytePairEncodingCoreProcessor.decodeNativeAsyncIterable(inputTokensToDecode);
|
|
1888
|
+
const decoder2 = new TextDecoder("utf8", { ignoreBOM: true });
|
|
1889
|
+
let hasPendingBytes = false;
|
|
1890
|
+
let buffer = "";
|
|
1891
|
+
for await (const decodedPart of decodedByteGenerator) {
|
|
1892
|
+
if (typeof decodedPart === "string") {
|
|
1893
|
+
if (decodedPart.length === 0) continue;
|
|
1894
|
+
if (hasPendingBytes) {
|
|
1895
|
+
buffer += decoder2.decode();
|
|
1896
|
+
hasPendingBytes = false;
|
|
1897
|
+
}
|
|
1898
|
+
buffer += decodedPart;
|
|
1899
|
+
} else {
|
|
1900
|
+
buffer += decoder2.decode(decodedPart, { stream: true });
|
|
1901
|
+
hasPendingBytes = true;
|
|
1902
|
+
}
|
|
1903
|
+
if (buffer.length === 0 || endsWithIncompleteUtfPairSurrogate(buffer)) continue;
|
|
1904
|
+
else {
|
|
1905
|
+
yield buffer;
|
|
1906
|
+
buffer = "";
|
|
1907
|
+
}
|
|
1908
|
+
}
|
|
1909
|
+
buffer += decoder2.decode();
|
|
1910
|
+
if (buffer.length > 0) yield buffer;
|
|
1911
|
+
}
|
|
1912
|
+
async decodeAsync(inputTokensToDecode) {
|
|
1913
|
+
const decodedByteGenerator = this.bytePairEncodingCoreProcessor.decodeNativeAsyncIterable(inputTokensToDecode);
|
|
1914
|
+
const decoder2 = new TextDecoder("utf8", { ignoreBOM: true });
|
|
1915
|
+
let hasPendingBytes = false;
|
|
1916
|
+
let buffer = "";
|
|
1917
|
+
for await (const decodedPart of decodedByteGenerator) if (typeof decodedPart === "string") {
|
|
1918
|
+
if (decodedPart.length === 0) continue;
|
|
1919
|
+
if (hasPendingBytes) {
|
|
1920
|
+
buffer += decoder2.decode();
|
|
1921
|
+
hasPendingBytes = false;
|
|
1922
|
+
}
|
|
1923
|
+
buffer += decodedPart;
|
|
1924
|
+
} else {
|
|
1925
|
+
buffer += decoder2.decode(decodedPart, { stream: true });
|
|
1926
|
+
hasPendingBytes = true;
|
|
1927
|
+
}
|
|
1928
|
+
return buffer + decoder2.decode();
|
|
1929
|
+
}
|
|
1930
|
+
/**
|
|
1931
|
+
* Estimates the cost of processing a given token count using the model's pricing.
|
|
1932
|
+
*
|
|
1933
|
+
* @param tokenCount - The number of tokens to estimate cost for
|
|
1934
|
+
* @returns Cost estimate object with applicable price components (input, output, batchInput, batchOutput)
|
|
1935
|
+
*/
|
|
1936
|
+
estimateCost(tokenCount, modelSpec = this.modelSpec) {
|
|
1937
|
+
if (!modelSpec) throw new Error("Model spec must be provided either during initialization or passed in to the method.");
|
|
1938
|
+
if (!modelSpec.price_data) throw new Error(`No cost information available for model: ${modelSpec.name}`);
|
|
1939
|
+
const priceDataPerMillion = modelSpec.price_data;
|
|
1940
|
+
const result = {};
|
|
1941
|
+
const millionTokens = tokenCount / 1e6;
|
|
1942
|
+
if (priceDataPerMillion.main) result.main = {
|
|
1943
|
+
input: priceDataPerMillion.main.input && priceDataPerMillion.main.input * millionTokens,
|
|
1944
|
+
output: priceDataPerMillion.main.output && priceDataPerMillion.main.output * millionTokens,
|
|
1945
|
+
cached_input: priceDataPerMillion.main.cached_input && priceDataPerMillion.main.cached_input * millionTokens,
|
|
1946
|
+
cached_output: priceDataPerMillion.main.cached_output && priceDataPerMillion.main.cached_output * millionTokens
|
|
1947
|
+
};
|
|
1948
|
+
if (priceDataPerMillion.batch) result.batch = {
|
|
1949
|
+
input: priceDataPerMillion.batch.input && priceDataPerMillion.batch.input * millionTokens,
|
|
1950
|
+
output: priceDataPerMillion.batch.output && priceDataPerMillion.batch.output * millionTokens,
|
|
1951
|
+
cached_input: priceDataPerMillion.batch.cached_input && priceDataPerMillion.batch.cached_input * millionTokens,
|
|
1952
|
+
cached_output: priceDataPerMillion.batch.cached_output && priceDataPerMillion.batch.cached_output * millionTokens
|
|
1953
|
+
};
|
|
1954
|
+
return result;
|
|
1955
|
+
}
|
|
1956
|
+
};
|
|
1957
|
+
|
|
1958
|
+
// node_modules/gpt-tokenizer/esm/bpeRanks/cl100k_base.js
|
|
1959
|
+
var import_node_crypto = require("node:crypto");
|
|
1960
|
+
var import_node_fs = require("node:fs");
|
|
1961
|
+
var import_node_path = require("node:path");
|
|
1962
|
+
var import_node_zlib = require("node:zlib");
|
|
1963
|
+
function loadBudgetVocabulary() {
|
|
1964
|
+
const path = (0, import_node_path.join)(__dirname, "budgets-cl100k.json.gz");
|
|
1965
|
+
try {
|
|
1966
|
+
const bytes = (0, import_node_zlib.gunzipSync)((0, import_node_fs.readFileSync)(path), { maxOutputLength: 957029 });
|
|
1967
|
+
if ((0, import_node_crypto.createHash)("sha256").update(bytes).digest("hex") !== "b56372fb0b95dbd370f9de6f44b6c15006a8a36980fbc6c0568a0108971aea45") {
|
|
1968
|
+
throw new Error("Vocabulary checksum mismatch");
|
|
1969
|
+
}
|
|
1970
|
+
return JSON.parse(bytes.toString("utf8"));
|
|
1971
|
+
} catch (error) {
|
|
1972
|
+
throw new Error(`Budget tokenizer vocabulary missing or corrupt: ${path}`, { cause: error });
|
|
1973
|
+
}
|
|
1974
|
+
}
|
|
1975
|
+
|
|
1976
|
+
// node_modules/gpt-tokenizer/esm/encoding/cl100k_base.js
|
|
1977
|
+
var encoding;
|
|
1978
|
+
function countTokens(text3, options) {
|
|
1979
|
+
encoding ??= GptEncoding.getEncodingApi("cl100k_base", loadBudgetVocabulary);
|
|
1980
|
+
return encoding.countTokens(text3, options);
|
|
1981
|
+
}
|
|
1982
|
+
|
|
1983
|
+
// packages/coding-agent/src/core/budgets/policy.ts
|
|
1984
|
+
var MAX_DIFF_EDIT_LENGTH = 4096;
|
|
1985
|
+
var ORDINARY_TEXT = { allowedSpecial: /* @__PURE__ */ new Set(), disallowedSpecial: /* @__PURE__ */ new Set() };
|
|
1986
|
+
var DEFAULT_BUDGET_LIMITS = Object.freeze({
|
|
1987
|
+
maxFileTokens: 5e3,
|
|
1988
|
+
maxDocFileTokens: 1500,
|
|
1989
|
+
maxCodeTokens: null,
|
|
1990
|
+
maxDocTokens: null,
|
|
1991
|
+
maxOversizedGrowthTokens: 250
|
|
1992
|
+
});
|
|
1993
|
+
var LIMIT_KEYS = Object.keys(DEFAULT_BUDGET_LIMITS);
|
|
1994
|
+
function parseCount(value, key, minimum) {
|
|
1995
|
+
if (typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum) {
|
|
1996
|
+
const kind = minimum === 0 ? "a non-negative" : "a positive";
|
|
1997
|
+
throw new Error(`Invalid budget limit ${key}: expected ${kind} safe integer`);
|
|
1998
|
+
}
|
|
1999
|
+
return value;
|
|
2000
|
+
}
|
|
2001
|
+
function parseTaskLimit(value, key) {
|
|
2002
|
+
return value === null ? null : parseCount(value, key, 1);
|
|
2003
|
+
}
|
|
2004
|
+
function parseBudgetLimits(input) {
|
|
2005
|
+
if (typeof input !== "object" || input === null || Array.isArray(input)) {
|
|
2006
|
+
throw new Error("Budget limits must be an object");
|
|
2007
|
+
}
|
|
2008
|
+
for (const key of Object.keys(input)) {
|
|
2009
|
+
if (!LIMIT_KEYS.includes(key)) throw new Error(`Unknown budget limit: ${key}`);
|
|
2010
|
+
}
|
|
2011
|
+
return {
|
|
2012
|
+
maxFileTokens: "maxFileTokens" in input ? parseCount(input.maxFileTokens, "maxFileTokens", 1) : DEFAULT_BUDGET_LIMITS.maxFileTokens,
|
|
2013
|
+
maxDocFileTokens: "maxDocFileTokens" in input ? parseCount(input.maxDocFileTokens, "maxDocFileTokens", 1) : DEFAULT_BUDGET_LIMITS.maxDocFileTokens,
|
|
2014
|
+
maxCodeTokens: "maxCodeTokens" in input ? parseTaskLimit(input.maxCodeTokens, "maxCodeTokens") : DEFAULT_BUDGET_LIMITS.maxCodeTokens,
|
|
2015
|
+
maxDocTokens: "maxDocTokens" in input ? parseTaskLimit(input.maxDocTokens, "maxDocTokens") : DEFAULT_BUDGET_LIMITS.maxDocTokens,
|
|
2016
|
+
maxOversizedGrowthTokens: "maxOversizedGrowthTokens" in input ? parseCount(input.maxOversizedGrowthTokens, "maxOversizedGrowthTokens", 0) : DEFAULT_BUDGET_LIMITS.maxOversizedGrowthTokens
|
|
2017
|
+
};
|
|
2018
|
+
}
|
|
2019
|
+
function countTokens2(text3) {
|
|
2020
|
+
return countTokens(text3, ORDINARY_TEXT);
|
|
2021
|
+
}
|
|
2022
|
+
var DOCUMENTATION = /\.(?:md|mdx|markdown|rst|adoc|asciidoc|txt)$/i;
|
|
2023
|
+
var TEXT_CODE = /(?:^|\/)(?:CMakeLists|requirements[^/]*|constraints[^/]*)\.txt$/i;
|
|
2024
|
+
function budgetCategory(path) {
|
|
2025
|
+
return DOCUMENTATION.test(path) && !TEXT_CODE.test(path) ? "docs" : "code";
|
|
2026
|
+
}
|
|
2027
|
+
function fileCeiling(limit, baselineTokens, growthTokens) {
|
|
2028
|
+
return baselineTokens > limit ? baselineTokens + growthTokens : limit;
|
|
2029
|
+
}
|
|
2030
|
+
function measureTokens(before, after) {
|
|
2031
|
+
const baselineTokens = countTokens2(before ?? "");
|
|
2032
|
+
const fileTokens = countTokens2(after ?? "");
|
|
2033
|
+
let addedTokens = before ? 0 : fileTokens;
|
|
2034
|
+
let deletedTokens = after ? 0 : baselineTokens;
|
|
2035
|
+
if (before && after && before !== after) {
|
|
2036
|
+
const changes = diffLines(before, after, { maxEditLength: MAX_DIFF_EDIT_LENGTH });
|
|
2037
|
+
if (!changes) {
|
|
2038
|
+
addedTokens = fileTokens;
|
|
2039
|
+
deletedTokens = baselineTokens;
|
|
2040
|
+
} else {
|
|
2041
|
+
for (const change of changes) {
|
|
2042
|
+
if (change.added) addedTokens += countTokens2(change.value);
|
|
2043
|
+
if (change.removed) deletedTokens += countTokens2(change.value);
|
|
2044
|
+
}
|
|
2045
|
+
}
|
|
2046
|
+
}
|
|
2047
|
+
return { addedTokens, deletedTokens, fileTokens, baselineTokens };
|
|
2048
|
+
}
|
|
2049
|
+
function budgetChange(path, measured, limits) {
|
|
2050
|
+
const category2 = budgetCategory(path);
|
|
2051
|
+
const limit = category2 === "docs" ? limits.maxDocFileTokens : limits.maxFileTokens;
|
|
2052
|
+
const ceiling = fileCeiling(limit, measured.baselineTokens, limits.maxOversizedGrowthTokens);
|
|
2053
|
+
return {
|
|
2054
|
+
path,
|
|
2055
|
+
category: category2,
|
|
2056
|
+
addedTokens: measured.addedTokens,
|
|
2057
|
+
deletedTokens: measured.deletedTokens,
|
|
2058
|
+
fileTokens: measured.fileTokens,
|
|
2059
|
+
fileCeiling: ceiling,
|
|
2060
|
+
fileLimitExceeded: measured.fileTokens > ceiling
|
|
2061
|
+
};
|
|
2062
|
+
}
|
|
2063
|
+
|
|
2064
|
+
// packages/coding-agent/src/core/budgets/renames.ts
|
|
2065
|
+
var MIN_RENAME_SIMILARITY = 0.5;
|
|
2066
|
+
function lineProfile(text3) {
|
|
2067
|
+
const lines = /* @__PURE__ */ new Map();
|
|
2068
|
+
for (const line of text3.split("\n")) lines.set(line, (lines.get(line) ?? 0) + line.length + 1);
|
|
2069
|
+
return { lines, size: text3.length + 1 };
|
|
2070
|
+
}
|
|
2071
|
+
function sharedSuffix(left, right) {
|
|
2072
|
+
const leftParts = left.split("/");
|
|
2073
|
+
const rightParts = right.split("/");
|
|
2074
|
+
let shared = 0;
|
|
2075
|
+
while (shared < Math.min(leftParts.length, rightParts.length) && leftParts[leftParts.length - 1 - shared] === rightParts[rightParts.length - 1 - shared]) {
|
|
2076
|
+
shared++;
|
|
2077
|
+
}
|
|
2078
|
+
return shared;
|
|
2079
|
+
}
|
|
2080
|
+
function pairRenames(deleted, added, content) {
|
|
2081
|
+
const pairs = /* @__PURE__ */ new Map();
|
|
2082
|
+
const sources = new Map(deleted);
|
|
2083
|
+
const identical = /* @__PURE__ */ new Map();
|
|
2084
|
+
for (const [path, hash] of deleted) {
|
|
2085
|
+
const paths = identical.get(hash);
|
|
2086
|
+
if (paths) paths.push(path);
|
|
2087
|
+
else identical.set(hash, [path]);
|
|
2088
|
+
}
|
|
2089
|
+
for (const [path, hash] of added) {
|
|
2090
|
+
const source = identical.get(hash)?.find((candidate) => sources.has(candidate));
|
|
2091
|
+
if (source === void 0) continue;
|
|
2092
|
+
pairs.set(path, source);
|
|
2093
|
+
sources.delete(source);
|
|
2094
|
+
}
|
|
2095
|
+
const targets = added.filter(([path]) => !pairs.has(path));
|
|
2096
|
+
if (targets.length === 0 || sources.size === 0) return pairs;
|
|
2097
|
+
const sizes = /* @__PURE__ */ new Map();
|
|
2098
|
+
const holders = /* @__PURE__ */ new Map();
|
|
2099
|
+
for (const [source, sourceHash] of sources) {
|
|
2100
|
+
const { lines, size } = lineProfile(content(sourceHash));
|
|
2101
|
+
sizes.set(source, size);
|
|
2102
|
+
for (const [line, lineSize] of lines) {
|
|
2103
|
+
const entries = holders.get(line);
|
|
2104
|
+
if (entries) entries.push([source, lineSize]);
|
|
2105
|
+
else holders.set(line, [[source, lineSize]]);
|
|
2106
|
+
}
|
|
2107
|
+
}
|
|
2108
|
+
const matches = [];
|
|
2109
|
+
for (const [order, [path, hash]] of targets.entries()) {
|
|
2110
|
+
const { lines, size } = lineProfile(content(hash));
|
|
2111
|
+
const overlaps = /* @__PURE__ */ new Map();
|
|
2112
|
+
for (const [line, lineSize] of lines) {
|
|
2113
|
+
for (const [source, sourceSize] of holders.get(line) ?? []) {
|
|
2114
|
+
overlaps.set(source, (overlaps.get(source) ?? 0) + Math.min(lineSize, sourceSize));
|
|
2115
|
+
}
|
|
2116
|
+
}
|
|
2117
|
+
for (const [source, overlap] of overlaps) {
|
|
2118
|
+
const score = 2 * overlap / (size + (sizes.get(source) ?? 0));
|
|
2119
|
+
if (score < MIN_RENAME_SIMILARITY) continue;
|
|
2120
|
+
matches.push({ path, source, score, shared: sharedSuffix(path, source), order });
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
2123
|
+
matches.sort((left, right) => right.score - left.score || right.shared - left.shared || left.order - right.order);
|
|
2124
|
+
for (const { path, source } of matches) {
|
|
2125
|
+
if (pairs.has(path) || !sources.has(source)) continue;
|
|
2126
|
+
pairs.set(path, source);
|
|
2127
|
+
sources.delete(source);
|
|
2128
|
+
}
|
|
2129
|
+
return pairs;
|
|
2130
|
+
}
|
|
2131
|
+
|
|
2132
|
+
// packages/coding-agent/src/core/budgets/state.ts
|
|
2133
|
+
var import_node_crypto3 = require("node:crypto");
|
|
2134
|
+
var import_node_fs4 = require("node:fs");
|
|
2135
|
+
var import_node_path4 = require("node:path");
|
|
2136
|
+
|
|
2137
|
+
// packages/coding-agent/src/core/budgets/exclusions.ts
|
|
2138
|
+
var LOCKFILES = /* @__PURE__ */ new Set([
|
|
2139
|
+
"bun.lock",
|
|
2140
|
+
"bun.lockb",
|
|
2141
|
+
"cargo.lock",
|
|
2142
|
+
"composer.lock",
|
|
2143
|
+
"deno.lock",
|
|
2144
|
+
"flake.lock",
|
|
2145
|
+
"gemfile.lock",
|
|
2146
|
+
"go.sum",
|
|
2147
|
+
"go.work.sum",
|
|
2148
|
+
"gradle.lockfile",
|
|
2149
|
+
"mix.lock",
|
|
2150
|
+
"npm-shrinkwrap.json",
|
|
2151
|
+
"package-lock.json",
|
|
2152
|
+
"package.resolved",
|
|
2153
|
+
"packages.lock.json",
|
|
2154
|
+
"pdm.lock",
|
|
2155
|
+
"pipfile.lock",
|
|
2156
|
+
"pnpm-lock.yaml",
|
|
2157
|
+
"podfile.lock",
|
|
2158
|
+
"poetry.lock",
|
|
2159
|
+
"pubspec.lock",
|
|
2160
|
+
"uv.lock",
|
|
2161
|
+
"yarn.lock"
|
|
2162
|
+
]);
|
|
2163
|
+
var CREDENTIALS = /^(?:\.env(?:\..+)?|\.npmrc|\.pypirc|\.netrc|id_(?:rsa|dsa|ecdsa|ed25519)|.+\.(?:pem|key|p12|pfx))$/;
|
|
2164
|
+
function isAlwaysExcluded(path) {
|
|
2165
|
+
const name = path.slice(path.lastIndexOf("/") + 1).toLowerCase();
|
|
2166
|
+
return LOCKFILES.has(name) || name === "changelog.md" || CREDENTIALS.test(name);
|
|
2167
|
+
}
|
|
2168
|
+
function parseExcludes(value) {
|
|
2169
|
+
if (!Array.isArray(value)) throw new Error("Budget exclude must be an array of glob patterns");
|
|
2170
|
+
return value.map((pattern) => {
|
|
2171
|
+
if (typeof pattern !== "string" || !pattern.trim() || pattern.startsWith("!") || pattern.includes("\0")) {
|
|
2172
|
+
throw new Error(`Invalid budget exclude pattern: ${JSON.stringify(pattern)}`);
|
|
2173
|
+
}
|
|
2174
|
+
return pattern;
|
|
2175
|
+
});
|
|
2176
|
+
}
|
|
2177
|
+
function globSource(glob) {
|
|
2178
|
+
let source = "";
|
|
2179
|
+
for (let index = 0; index < glob.length; ) {
|
|
2180
|
+
if (glob.startsWith("**/", index)) {
|
|
2181
|
+
source += "(?:.*/)?";
|
|
2182
|
+
index += 3;
|
|
2183
|
+
} else if (glob.startsWith("**", index)) {
|
|
2184
|
+
source += ".*";
|
|
2185
|
+
index += 2;
|
|
2186
|
+
} else {
|
|
2187
|
+
const character = glob[index++];
|
|
2188
|
+
if (character === "*") source += "[^/]*";
|
|
2189
|
+
else if (character === "?") source += "[^/]";
|
|
2190
|
+
else source += character.replace(/[.+^${}()|[\]\\]/g, "\\$&");
|
|
2191
|
+
}
|
|
2192
|
+
}
|
|
2193
|
+
return source;
|
|
2194
|
+
}
|
|
2195
|
+
function excludeMatcher(patterns2) {
|
|
2196
|
+
const expressions = patterns2.map((pattern) => {
|
|
2197
|
+
const body = pattern.replace(/^\/+/, "").replace(/\/+$/, "");
|
|
2198
|
+
const anchored = pattern.replace(/\/+$/, "").includes("/");
|
|
2199
|
+
const below = pattern.endsWith("/") ? "/.*" : "(?:/.*)?";
|
|
2200
|
+
return new RegExp(`^${anchored ? "" : "(?:.*/)?"}${globSource(body)}${below}$`);
|
|
2201
|
+
});
|
|
2202
|
+
return (path) => expressions.some((expression) => expression.test(path));
|
|
2203
|
+
}
|
|
2204
|
+
|
|
2205
|
+
// packages/coding-agent/src/core/budgets/workspace.ts
|
|
2206
|
+
var import_node_child_process = require("node:child_process");
|
|
2207
|
+
var import_node_fs2 = require("node:fs");
|
|
2208
|
+
var import_node_path2 = require("node:path");
|
|
2209
|
+
var SOURCE_EXTENSIONS = new Set(
|
|
2210
|
+
"ts tsx mts cts js jsx mjs cjs py pyi rb rs go java kt kts scala c h cc cpp hpp cs fs fsx swift m mm php lua sh bash zsh fish ps1 r jl ex exs erl hrl hs clj cljs vue svelte astro html htm css scss sass less json jsonc jsonl yaml yml toml ini conf xml svg sql csv graphql gql proto md mdx markdown rst txt adoc asciidoc".split(
|
|
2211
|
+
" "
|
|
2212
|
+
)
|
|
2213
|
+
);
|
|
2214
|
+
var REPOSITORY_GIT_ENV = new Set(
|
|
2215
|
+
"GIT_ALTERNATE_OBJECT_DIRECTORIES GIT_CONFIG GIT_CONFIG_PARAMETERS GIT_CONFIG_COUNT GIT_OBJECT_DIRECTORY GIT_DIR GIT_WORK_TREE GIT_IMPLICIT_WORK_TREE GIT_GRAFT_FILE GIT_INDEX_FILE GIT_NO_REPLACE_OBJECTS GIT_REPLACE_REF_BASE GIT_PREFIX GIT_SHALLOW_FILE GIT_COMMON_DIR GIT_NAMESPACE GIT_CEILING_DIRECTORIES GIT_DISCOVERY_ACROSS_FILESYSTEM".split(
|
|
2216
|
+
" "
|
|
2217
|
+
)
|
|
2218
|
+
);
|
|
2219
|
+
function isMissingPath(error) {
|
|
2220
|
+
return error instanceof Error && "code" in error && (error.code === "ENOENT" || error.code === "ENOTDIR");
|
|
2221
|
+
}
|
|
2222
|
+
function budgetPathExists(path) {
|
|
2223
|
+
try {
|
|
2224
|
+
(0, import_node_fs2.lstatSync)(path);
|
|
2225
|
+
return true;
|
|
2226
|
+
} catch (error) {
|
|
2227
|
+
if (isMissingPath(error)) return false;
|
|
2228
|
+
throw error;
|
|
2229
|
+
}
|
|
2230
|
+
}
|
|
2231
|
+
function runGit(cwd, args, input) {
|
|
2232
|
+
const env = { ...process.env, GIT_OPTIONAL_LOCKS: "0", LC_ALL: "C" };
|
|
2233
|
+
for (const key of Object.keys(env)) {
|
|
2234
|
+
if (REPOSITORY_GIT_ENV.has(key) || /^GIT_CONFIG_(?:KEY|VALUE)_/.test(key)) delete env[key];
|
|
2235
|
+
}
|
|
2236
|
+
const result = (0, import_node_child_process.spawnSync)("git", ["-c", "core.fsmonitor=false", ...args], {
|
|
2237
|
+
cwd,
|
|
2238
|
+
encoding: "utf8",
|
|
2239
|
+
env,
|
|
2240
|
+
input,
|
|
2241
|
+
maxBuffer: 64 * 1024 * 1024,
|
|
2242
|
+
timeout: 1e4
|
|
2243
|
+
});
|
|
2244
|
+
if (result.error) throw result.error;
|
|
2245
|
+
return result;
|
|
2246
|
+
}
|
|
2247
|
+
function git(cwd, args, input) {
|
|
2248
|
+
const result = runGit(cwd, args, input);
|
|
2249
|
+
if (result.status !== 0) throw new Error(`Budget workspace inspection failed: ${result.stderr.trim()}`);
|
|
2250
|
+
return result.stdout;
|
|
2251
|
+
}
|
|
2252
|
+
function findBudgetWorkspace(cwd) {
|
|
2253
|
+
const directory = (0, import_node_fs2.realpathSync)(cwd);
|
|
2254
|
+
const result = runGit(directory, ["rev-parse", "--show-toplevel", "--git-common-dir"]);
|
|
2255
|
+
if (result.status !== 0) {
|
|
2256
|
+
if (result.stderr.includes("not a git repository")) return void 0;
|
|
2257
|
+
throw new Error(`Budget workspace inspection failed: ${result.stderr.trim()}`);
|
|
2258
|
+
}
|
|
2259
|
+
const [root, commonDir] = result.stdout.split(/\r?\n/);
|
|
2260
|
+
return { root: (0, import_node_fs2.realpathSync)(root), commonDir: (0, import_node_fs2.realpathSync)((0, import_node_path2.resolve)(directory, commonDir)) };
|
|
2261
|
+
}
|
|
2262
|
+
function gitCommonDirectory(dotGit) {
|
|
2263
|
+
if ((0, import_node_fs2.lstatSync)(dotGit).isDirectory()) return dotGit;
|
|
2264
|
+
const gitdir = /^gitdir: (.+)$/m.exec((0, import_node_fs2.readFileSync)(dotGit, "utf8"))?.[1]?.trim();
|
|
2265
|
+
if (!gitdir) return void 0;
|
|
2266
|
+
const directory = (0, import_node_path2.resolve)((0, import_node_path2.dirname)(dotGit), gitdir);
|
|
2267
|
+
const commondir = (0, import_node_path2.join)(directory, "commondir");
|
|
2268
|
+
return (0, import_node_fs2.existsSync)(commondir) ? (0, import_node_path2.resolve)(directory, (0, import_node_fs2.readFileSync)(commondir, "utf8").trim()) : directory;
|
|
2269
|
+
}
|
|
2270
|
+
function mayHaveBudgets(cwd, marker) {
|
|
2271
|
+
let start;
|
|
2272
|
+
try {
|
|
2273
|
+
start = (0, import_node_fs2.realpathSync)(cwd);
|
|
2274
|
+
} catch {
|
|
2275
|
+
return true;
|
|
2276
|
+
}
|
|
2277
|
+
for (let directory = start; ; directory = (0, import_node_path2.dirname)(directory)) {
|
|
2278
|
+
if ((0, import_node_fs2.existsSync)((0, import_node_path2.join)(directory, ".draht", "budgets.json"))) return true;
|
|
2279
|
+
const dotGit = (0, import_node_path2.join)(directory, ".git");
|
|
2280
|
+
if ((0, import_node_fs2.existsSync)(dotGit)) {
|
|
2281
|
+
try {
|
|
2282
|
+
const common = gitCommonDirectory(dotGit);
|
|
2283
|
+
return common === void 0 || (0, import_node_fs2.existsSync)((0, import_node_path2.join)(common, marker));
|
|
2284
|
+
} catch {
|
|
2285
|
+
return true;
|
|
2286
|
+
}
|
|
2287
|
+
}
|
|
2288
|
+
if ((0, import_node_path2.dirname)(directory) === directory) return false;
|
|
2289
|
+
}
|
|
2290
|
+
}
|
|
2291
|
+
function headCommit(workspace) {
|
|
2292
|
+
const result = runGit(workspace.root, ["rev-parse", "--verify", "--quiet", "HEAD^{commit}"]);
|
|
2293
|
+
return result.status === 0 ? result.stdout.trim() : null;
|
|
2294
|
+
}
|
|
2295
|
+
function generatedPaths(workspace, commit, paths) {
|
|
2296
|
+
const generated = /* @__PURE__ */ new Set();
|
|
2297
|
+
if (!commit || paths.length === 0) return generated;
|
|
2298
|
+
const fields = git(
|
|
2299
|
+
workspace.root,
|
|
2300
|
+
["check-attr", `--source=${commit}`, "-z", "--stdin", "linguist-generated"],
|
|
2301
|
+
`${paths.join("\0")}\0`
|
|
2302
|
+
).split("\0");
|
|
2303
|
+
for (let index = 0; index + 2 < fields.length; index += 3) {
|
|
2304
|
+
if (fields[index + 2] === "set" || fields[index + 2] === "true") generated.add(fields[index]);
|
|
2305
|
+
}
|
|
2306
|
+
return generated;
|
|
2307
|
+
}
|
|
2308
|
+
function isBudgetRelativePath(path) {
|
|
2309
|
+
return path.length > 0 && !(0, import_node_path2.isAbsolute)(path) && path.split(/[\\/]/).every((part) => part !== ".." && part !== ".git");
|
|
2310
|
+
}
|
|
2311
|
+
function listWorkspacePaths(workspace) {
|
|
2312
|
+
const paths = new Set(
|
|
2313
|
+
git(workspace.root, ["ls-files", "--cached", "--others", "--exclude-standard", "-z"]).split("\0").filter(Boolean)
|
|
2314
|
+
);
|
|
2315
|
+
for (const path of paths) {
|
|
2316
|
+
if (!isBudgetRelativePath(path)) throw new Error(`Invalid budget snapshot path: ${path}`);
|
|
2317
|
+
}
|
|
2318
|
+
return paths;
|
|
2319
|
+
}
|
|
2320
|
+
function symlinkCheck(root) {
|
|
2321
|
+
const directories = /* @__PURE__ */ new Map();
|
|
2322
|
+
const linked = (path) => {
|
|
2323
|
+
const parent = (0, import_node_path2.dirname)(path);
|
|
2324
|
+
let inherited = false;
|
|
2325
|
+
if (parent !== ".") {
|
|
2326
|
+
inherited = directories.get(parent) ?? linked(parent);
|
|
2327
|
+
directories.set(parent, inherited);
|
|
2328
|
+
}
|
|
2329
|
+
if (inherited) return true;
|
|
2330
|
+
try {
|
|
2331
|
+
return (0, import_node_fs2.lstatSync)((0, import_node_path2.join)(root, path)).isSymbolicLink();
|
|
2332
|
+
} catch (error) {
|
|
2333
|
+
if (isMissingPath(error)) return false;
|
|
2334
|
+
throw error;
|
|
2335
|
+
}
|
|
2336
|
+
};
|
|
2337
|
+
return linked;
|
|
2338
|
+
}
|
|
2339
|
+
function canonicalBudgetPath(cwd, path) {
|
|
2340
|
+
const missing = [];
|
|
2341
|
+
let existing = (0, import_node_path2.resolve)(cwd, path);
|
|
2342
|
+
while (!(0, import_node_fs2.existsSync)(existing) && (0, import_node_path2.dirname)(existing) !== existing) {
|
|
2343
|
+
missing.unshift((0, import_node_path2.basename)(existing));
|
|
2344
|
+
existing = (0, import_node_path2.dirname)(existing);
|
|
2345
|
+
}
|
|
2346
|
+
return (0, import_node_path2.join)((0, import_node_fs2.realpathSync)(existing), ...missing);
|
|
2347
|
+
}
|
|
2348
|
+
function inNestedRepository(workspace, path) {
|
|
2349
|
+
for (let parent = (0, import_node_path2.dirname)(path); parent !== "."; parent = (0, import_node_path2.dirname)(parent)) {
|
|
2350
|
+
if (budgetPathExists((0, import_node_path2.join)(workspace.root, parent, ".git"))) return true;
|
|
2351
|
+
}
|
|
2352
|
+
return false;
|
|
2353
|
+
}
|
|
2354
|
+
function budgetFilePath(workspace, target) {
|
|
2355
|
+
const file = (0, import_node_path2.relative)(workspace.root, target);
|
|
2356
|
+
if (!isBudgetRelativePath(file) || inNestedRepository(workspace, file)) return void 0;
|
|
2357
|
+
const ignored = runGit(workspace.root, ["check-ignore", "-q", "--", file]);
|
|
2358
|
+
if (ignored.status === 0) return void 0;
|
|
2359
|
+
if (ignored.status !== 1) throw new Error(`Budget ignore inspection failed: ${ignored.stderr.trim()}`);
|
|
2360
|
+
return file.split(import_node_path2.sep).join("/");
|
|
2361
|
+
}
|
|
2362
|
+
function pathsOutsideBudget(workspace, paths) {
|
|
2363
|
+
const nested = new Set(paths.filter((path) => inNestedRepository(workspace, path)));
|
|
2364
|
+
const others = paths.filter((path) => !nested.has(path));
|
|
2365
|
+
if (others.length === 0) return [...nested];
|
|
2366
|
+
const ignored = runGit(workspace.root, ["check-ignore", "-z", "--stdin"], `${others.join("\0")}\0`);
|
|
2367
|
+
if (ignored.status !== 0 && ignored.status !== 1) {
|
|
2368
|
+
throw new Error(`Budget ignore inspection failed: ${ignored.stderr.trim()}`);
|
|
2369
|
+
}
|
|
2370
|
+
return [...nested, ...ignored.stdout.split("\0").filter(Boolean)];
|
|
2371
|
+
}
|
|
2372
|
+
function isSourcePath(path) {
|
|
2373
|
+
return SOURCE_EXTENSIONS.has((0, import_node_path2.extname)(path).slice(1).toLowerCase()) || /^(dockerfile|makefile)$/i.test((0, import_node_path2.basename)(path));
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
// packages/coding-agent/src/core/budgets/locking.ts
|
|
2377
|
+
var import_node_crypto2 = require("node:crypto");
|
|
2378
|
+
var import_node_fs3 = require("node:fs");
|
|
2379
|
+
var import_node_path3 = require("node:path");
|
|
2380
|
+
var LOCK_WAIT_MS = 15e3;
|
|
2381
|
+
var ABANDONED_MS = 3e4;
|
|
2382
|
+
function hasCode(error, code) {
|
|
2383
|
+
return error instanceof Error && "code" in error && error.code === code;
|
|
2384
|
+
}
|
|
2385
|
+
var host;
|
|
2386
|
+
function hostIdentity() {
|
|
2387
|
+
const read = (value) => {
|
|
2388
|
+
try {
|
|
2389
|
+
return value().trim();
|
|
2390
|
+
} catch {
|
|
2391
|
+
return "";
|
|
2392
|
+
}
|
|
2393
|
+
};
|
|
2394
|
+
host ??= {
|
|
2395
|
+
boot: read(() => (0, import_node_fs3.readFileSync)("/proc/sys/kernel/random/boot_id", "utf8")),
|
|
2396
|
+
namespace: read(() => (0, import_node_fs3.readlinkSync)("/proc/self/ns/pid"))
|
|
2397
|
+
};
|
|
2398
|
+
return host;
|
|
2399
|
+
}
|
|
2400
|
+
function readOwner(lock) {
|
|
2401
|
+
try {
|
|
2402
|
+
return (0, import_node_fs3.readFileSync)((0, import_node_path3.join)(lock, "owner"), "utf8");
|
|
2403
|
+
} catch (error) {
|
|
2404
|
+
if (hasCode(error, "ENOENT")) return void 0;
|
|
2405
|
+
throw error;
|
|
2406
|
+
}
|
|
2407
|
+
}
|
|
2408
|
+
function parseOwner(owner) {
|
|
2409
|
+
try {
|
|
2410
|
+
const value = JSON.parse(owner);
|
|
2411
|
+
if (typeof value !== "object" || value === null || !("pid" in value) || !("boot" in value) || !("namespace" in value) || !("id" in value) || typeof value.pid !== "number" || !Number.isSafeInteger(value.pid) || value.pid < 1 || typeof value.boot !== "string" || typeof value.namespace !== "string" || typeof value.id !== "string") {
|
|
2412
|
+
return void 0;
|
|
2413
|
+
}
|
|
2414
|
+
return { pid: value.pid, boot: value.boot, namespace: value.namespace, id: value.id };
|
|
2415
|
+
} catch {
|
|
2416
|
+
return void 0;
|
|
2417
|
+
}
|
|
2418
|
+
}
|
|
2419
|
+
function ownerIsDead(owner) {
|
|
2420
|
+
const current = hostIdentity();
|
|
2421
|
+
if (owner.boot && current.boot && owner.boot !== current.boot) return true;
|
|
2422
|
+
if (owner.namespace !== current.namespace) return false;
|
|
2423
|
+
try {
|
|
2424
|
+
process.kill(owner.pid, 0);
|
|
2425
|
+
return false;
|
|
2426
|
+
} catch (error) {
|
|
2427
|
+
if (hasCode(error, "ESRCH")) return true;
|
|
2428
|
+
if (hasCode(error, "EPERM")) return false;
|
|
2429
|
+
throw error;
|
|
2430
|
+
}
|
|
2431
|
+
}
|
|
2432
|
+
function abandoned(path) {
|
|
2433
|
+
try {
|
|
2434
|
+
return Date.now() - (0, import_node_fs3.statSync)(path).mtimeMs > ABANDONED_MS;
|
|
2435
|
+
} catch (error) {
|
|
2436
|
+
if (hasCode(error, "ENOENT")) return false;
|
|
2437
|
+
throw error;
|
|
2438
|
+
}
|
|
2439
|
+
}
|
|
2440
|
+
function recoverDeadLock(lock) {
|
|
2441
|
+
const recovery = `${lock}.recovery`;
|
|
2442
|
+
try {
|
|
2443
|
+
(0, import_node_fs3.mkdirSync)(recovery);
|
|
2444
|
+
} catch (error) {
|
|
2445
|
+
if (!hasCode(error, "EEXIST")) throw error;
|
|
2446
|
+
if (abandoned(recovery)) (0, import_node_fs3.rmSync)(recovery, { recursive: true, force: true });
|
|
2447
|
+
return;
|
|
2448
|
+
}
|
|
2449
|
+
try {
|
|
2450
|
+
const owner = readOwner(lock);
|
|
2451
|
+
const recorded = owner === void 0 ? void 0 : parseOwner(owner);
|
|
2452
|
+
const dead = recorded ? ownerIsDead(recorded) : abandoned(lock);
|
|
2453
|
+
if (!dead || readOwner(lock) !== owner) return;
|
|
2454
|
+
const removed = `${lock}.removed-${(0, import_node_crypto2.randomUUID)()}`;
|
|
2455
|
+
(0, import_node_fs3.renameSync)(lock, removed);
|
|
2456
|
+
(0, import_node_fs3.rmSync)(removed, { recursive: true });
|
|
2457
|
+
} finally {
|
|
2458
|
+
(0, import_node_fs3.rmSync)(recovery, { recursive: true, force: true });
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
function withBudgetLock(lock, action) {
|
|
2462
|
+
const owner = JSON.stringify({ pid: process.pid, ...hostIdentity(), id: (0, import_node_crypto2.randomUUID)() });
|
|
2463
|
+
const deadline = Date.now() + LOCK_WAIT_MS;
|
|
2464
|
+
while (true) {
|
|
2465
|
+
try {
|
|
2466
|
+
(0, import_node_fs3.mkdirSync)(lock);
|
|
2467
|
+
} catch (error) {
|
|
2468
|
+
if (!hasCode(error, "EEXIST")) throw error;
|
|
2469
|
+
recoverDeadLock(lock);
|
|
2470
|
+
if (Date.now() >= deadline) throw new Error(`Budget task is locked: ${lock}`);
|
|
2471
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 25);
|
|
2472
|
+
continue;
|
|
2473
|
+
}
|
|
2474
|
+
try {
|
|
2475
|
+
(0, import_node_fs3.writeFileSync)((0, import_node_path3.join)(lock, "owner"), owner, { flag: "wx", mode: 384 });
|
|
2476
|
+
} catch (error) {
|
|
2477
|
+
(0, import_node_fs3.rmSync)(lock, { recursive: true });
|
|
2478
|
+
throw error;
|
|
2479
|
+
}
|
|
2480
|
+
break;
|
|
2481
|
+
}
|
|
2482
|
+
try {
|
|
2483
|
+
const result = action();
|
|
2484
|
+
if (readOwner(lock) !== owner) throw new Error("Budget task lock ownership changed");
|
|
2485
|
+
return result;
|
|
2486
|
+
} finally {
|
|
2487
|
+
if (readOwner(lock) === owner) (0, import_node_fs3.rmSync)(lock, { recursive: true });
|
|
2488
|
+
}
|
|
2489
|
+
}
|
|
2490
|
+
|
|
2491
|
+
// packages/coding-agent/src/core/budgets/state.ts
|
|
2492
|
+
var MAX_CONFIG_BYTES = 64 * 1024;
|
|
2493
|
+
var LEDGER_RESET_HINT = "the user must start a new task with the budget new command";
|
|
2494
|
+
var BUDGET_TASK_MARKER = "draht-budgets.task";
|
|
2495
|
+
function budgetLedger(commonDir) {
|
|
2496
|
+
return {
|
|
2497
|
+
directory: (0, import_node_path4.join)(commonDir, "draht-budgets"),
|
|
2498
|
+
marker: (0, import_node_path4.join)(commonDir, BUDGET_TASK_MARKER),
|
|
2499
|
+
lock: (0, import_node_path4.join)(commonDir, "draht-budgets.lock")
|
|
2500
|
+
};
|
|
2501
|
+
}
|
|
2502
|
+
function record(value) {
|
|
2503
|
+
return value !== null && typeof value === "object" && !Array.isArray(value);
|
|
2504
|
+
}
|
|
2505
|
+
function hasCode2(error, code) {
|
|
2506
|
+
return error instanceof Error && "code" in error && error.code === code;
|
|
2507
|
+
}
|
|
2508
|
+
function readRegularFile(path, maxBytes = Number.POSITIVE_INFINITY) {
|
|
2509
|
+
let descriptor;
|
|
2510
|
+
try {
|
|
2511
|
+
descriptor = (0, import_node_fs4.openSync)(path, import_node_fs4.constants.O_RDONLY | (import_node_fs4.constants.O_NOFOLLOW ?? 0) | (import_node_fs4.constants.O_NONBLOCK ?? 0));
|
|
2512
|
+
} catch (error) {
|
|
2513
|
+
if (hasCode2(error, "ELOOP")) throw new Error(`Budget file must not be a symlink: ${path}`);
|
|
2514
|
+
throw error;
|
|
2515
|
+
}
|
|
2516
|
+
try {
|
|
2517
|
+
const stat = (0, import_node_fs4.fstatSync)(descriptor);
|
|
2518
|
+
if (!stat.isFile()) throw new Error(`Budget file must be a regular file: ${path}`);
|
|
2519
|
+
if (stat.size > maxBytes) throw new Error(`Budget file exceeds ${maxBytes} bytes: ${path}`);
|
|
2520
|
+
return (0, import_node_fs4.readFileSync)(descriptor, "utf8");
|
|
2521
|
+
} finally {
|
|
2522
|
+
(0, import_node_fs4.closeSync)(descriptor);
|
|
2523
|
+
}
|
|
2524
|
+
}
|
|
2525
|
+
function readBudgetConfig(path) {
|
|
2526
|
+
const value = budgetPathExists(path) ? JSON.parse(readRegularFile(path, MAX_CONFIG_BYTES)) : {};
|
|
2527
|
+
if (!record(value)) throw new Error("Budget configuration must be an object");
|
|
2528
|
+
const { enabled = true, exclude = [], ...limits } = value;
|
|
2529
|
+
if (typeof enabled !== "boolean") throw new Error("Budget enabled must be a boolean");
|
|
2530
|
+
return { enabled, limits: parseBudgetLimits(limits), exclude: parseExcludes(exclude) };
|
|
2531
|
+
}
|
|
2532
|
+
function parseSnapshot(value) {
|
|
2533
|
+
if (!record(value)) throw new Error("Invalid budget snapshot");
|
|
2534
|
+
const entries = [];
|
|
2535
|
+
for (const [path, hash] of Object.entries(value)) {
|
|
2536
|
+
if (!isBudgetRelativePath(path) || typeof hash !== "string" || !/^[a-f0-9]{64}$/.test(hash)) {
|
|
2537
|
+
throw new Error("Invalid budget snapshot entry");
|
|
2538
|
+
}
|
|
2539
|
+
entries.push([path, hash]);
|
|
2540
|
+
}
|
|
2541
|
+
return Object.fromEntries(entries);
|
|
2542
|
+
}
|
|
2543
|
+
function parseCommit(value) {
|
|
2544
|
+
if (value === null) return null;
|
|
2545
|
+
if (typeof value !== "string" || !/^(?:[a-f0-9]{40}|[a-f0-9]{64})$/.test(value)) {
|
|
2546
|
+
throw new Error("Invalid budget worktree attributes commit");
|
|
2547
|
+
}
|
|
2548
|
+
return value;
|
|
2549
|
+
}
|
|
2550
|
+
function parseParticipant(value) {
|
|
2551
|
+
if (!record(value) || typeof value.root !== "string" || !(0, import_node_path4.isAbsolute)(value.root) || typeof value.removed !== "boolean") {
|
|
2552
|
+
throw new Error("Invalid budget worktree");
|
|
2553
|
+
}
|
|
2554
|
+
return {
|
|
2555
|
+
root: value.root,
|
|
2556
|
+
attributes: parseCommit(value.attributes),
|
|
2557
|
+
removed: value.removed,
|
|
2558
|
+
baseline: parseSnapshot(value.baseline),
|
|
2559
|
+
current: parseSnapshot(value.current)
|
|
2560
|
+
};
|
|
2561
|
+
}
|
|
2562
|
+
function readBudgetState(ledger) {
|
|
2563
|
+
const path = (0, import_node_path4.join)(ledger.directory, "task.json");
|
|
2564
|
+
const hasState = budgetPathExists(path);
|
|
2565
|
+
const hasMarker = budgetPathExists(ledger.marker);
|
|
2566
|
+
if (!hasState && !hasMarker) return void 0;
|
|
2567
|
+
if (!hasState || !hasMarker) throw new Error(`Budget task ledger is incomplete; ${LEDGER_RESET_HINT}`);
|
|
2568
|
+
const value = JSON.parse(readRegularFile(path));
|
|
2569
|
+
if (!record(value) || value.version !== 2 || typeof value.taskId !== "string" || !value.taskId || !Array.isArray(value.worktrees) || !record(value.limits) || Object.keys(value.limits).length !== Object.keys(DEFAULT_BUDGET_LIMITS).length) {
|
|
2570
|
+
throw new Error("Invalid budget task ledger");
|
|
2571
|
+
}
|
|
2572
|
+
if (readRegularFile(ledger.marker, 256) !== value.taskId) {
|
|
2573
|
+
throw new Error(`Budget task ledger does not match its task marker; ${LEDGER_RESET_HINT}`);
|
|
2574
|
+
}
|
|
2575
|
+
const worktrees = value.worktrees.map(parseParticipant);
|
|
2576
|
+
const active = worktrees.filter(({ removed }) => !removed).map(({ root }) => root);
|
|
2577
|
+
if (new Set(active).size !== active.length) throw new Error("Duplicate budget worktrees in ledger");
|
|
2578
|
+
return {
|
|
2579
|
+
version: 2,
|
|
2580
|
+
taskId: value.taskId,
|
|
2581
|
+
limits: parseBudgetLimits(value.limits),
|
|
2582
|
+
exclude: parseExcludes(value.exclude),
|
|
2583
|
+
worktrees
|
|
2584
|
+
};
|
|
2585
|
+
}
|
|
2586
|
+
function writeAtomically(path, content) {
|
|
2587
|
+
const temp = `${path}.${(0, import_node_crypto3.randomUUID)()}.tmp`;
|
|
2588
|
+
try {
|
|
2589
|
+
(0, import_node_fs4.writeFileSync)(temp, content, { mode: 384, flag: "wx" });
|
|
2590
|
+
(0, import_node_fs4.renameSync)(temp, path);
|
|
2591
|
+
} finally {
|
|
2592
|
+
(0, import_node_fs4.rmSync)(temp, { force: true });
|
|
2593
|
+
}
|
|
2594
|
+
}
|
|
2595
|
+
function writeBudgetState(ledger, state) {
|
|
2596
|
+
(0, import_node_fs4.mkdirSync)(ledger.directory, { recursive: true, mode: 448 });
|
|
2597
|
+
writeAtomically((0, import_node_path4.join)(ledger.directory, "task.json"), JSON.stringify(state));
|
|
2598
|
+
}
|
|
2599
|
+
function markBudgetTask(ledger, taskId) {
|
|
2600
|
+
writeAtomically(ledger.marker, taskId);
|
|
2601
|
+
}
|
|
2602
|
+
function clearBudgetTask(ledger) {
|
|
2603
|
+
(0, import_node_fs4.rmSync)(ledger.marker, { force: true });
|
|
2604
|
+
(0, import_node_fs4.rmSync)(ledger.directory, { recursive: true, force: true });
|
|
2605
|
+
}
|
|
2606
|
+
function budgetContentHash(content) {
|
|
2607
|
+
return (0, import_node_crypto3.createHash)("sha256").update(content).digest("hex");
|
|
2608
|
+
}
|
|
2609
|
+
function storeBudgetObject(directory, content) {
|
|
2610
|
+
const hash = budgetContentHash(content);
|
|
2611
|
+
const object = (0, import_node_path4.join)(directory, "objects", hash);
|
|
2612
|
+
if (!budgetPathExists(object)) {
|
|
2613
|
+
(0, import_node_fs4.mkdirSync)((0, import_node_path4.join)(directory, "objects"), { recursive: true, mode: 448 });
|
|
2614
|
+
writeAtomically(object, content);
|
|
2615
|
+
}
|
|
2616
|
+
return hash;
|
|
2617
|
+
}
|
|
2618
|
+
function readBudgetCache(directory, name, parse) {
|
|
2619
|
+
try {
|
|
2620
|
+
return parse(JSON.parse((0, import_node_fs4.readFileSync)((0, import_node_path4.join)(directory, "cache", `${name}.json`), "utf8")));
|
|
2621
|
+
} catch {
|
|
2622
|
+
return void 0;
|
|
2623
|
+
}
|
|
2624
|
+
}
|
|
2625
|
+
function writeBudgetCache(directory, name, value) {
|
|
2626
|
+
(0, import_node_fs4.mkdirSync)((0, import_node_path4.join)(directory, "cache"), { recursive: true, mode: 448 });
|
|
2627
|
+
writeAtomically((0, import_node_path4.join)(directory, "cache", `${name}.json`), JSON.stringify(value));
|
|
2628
|
+
}
|
|
2629
|
+
function removeBudgetCache(directory, name) {
|
|
2630
|
+
(0, import_node_fs4.rmSync)((0, import_node_path4.join)(directory, "cache", `${name}.json`), { force: true });
|
|
2631
|
+
}
|
|
2632
|
+
function readBudgetContent(directory, hash) {
|
|
2633
|
+
if (!hash) return null;
|
|
2634
|
+
const content = (0, import_node_fs4.readFileSync)((0, import_node_path4.join)(directory, "objects", hash), "utf8");
|
|
2635
|
+
if (budgetContentHash(content) !== hash) throw new Error(`Corrupt budget content ${hash}; ${LEDGER_RESET_HINT}`);
|
|
2636
|
+
return content;
|
|
2637
|
+
}
|
|
2638
|
+
|
|
2639
|
+
// packages/coding-agent/src/core/budgets/evaluation.ts
|
|
2640
|
+
var SUMMARY_ISSUES = 10;
|
|
2641
|
+
function exceededSummary(messages) {
|
|
2642
|
+
const hidden = messages.length - SUMMARY_ISSUES;
|
|
2643
|
+
return `Budget exceeded: ${messages.slice(0, SUMMARY_ISSUES).join("; ")}${hidden > 0 ? `; and ${hidden} more` : ""}`;
|
|
2644
|
+
}
|
|
2645
|
+
function own(snapshot, path) {
|
|
2646
|
+
return Object.hasOwn(snapshot, path) ? snapshot[path] : void 0;
|
|
2647
|
+
}
|
|
2648
|
+
function evaluateBudget(directory, state, options = {}) {
|
|
2649
|
+
const projected = options.projected ?? /* @__PURE__ */ new Map();
|
|
2650
|
+
const known = options.measures ?? /* @__PURE__ */ new Map();
|
|
2651
|
+
const measures = /* @__PURE__ */ new Map();
|
|
2652
|
+
let measured = false;
|
|
2653
|
+
const content = (hash) => hash && projected.has(hash) ? projected.get(hash) ?? null : readBudgetContent(directory, hash);
|
|
2654
|
+
const measure = (beforeHash, afterHash) => {
|
|
2655
|
+
const key = `${beforeHash ?? ""}:${afterHash ?? ""}`;
|
|
2656
|
+
let value = measures.get(key) ?? known.get(key);
|
|
2657
|
+
if (!value) {
|
|
2658
|
+
value = measureTokens(content(beforeHash), content(afterHash));
|
|
2659
|
+
measured = true;
|
|
2660
|
+
}
|
|
2661
|
+
measures.set(key, value);
|
|
2662
|
+
return value;
|
|
2663
|
+
};
|
|
2664
|
+
const usage = { codeTokens: 0, docTokens: 0 };
|
|
2665
|
+
const issues = [];
|
|
2666
|
+
const seen = /* @__PURE__ */ new Set();
|
|
2667
|
+
for (const participant of state.worktrees) {
|
|
2668
|
+
const elsewhere = options.currentRoot !== void 0 && participant.root !== options.currentRoot;
|
|
2669
|
+
const where = elsewhere ? ` in worktree ${participant.root}` : "";
|
|
2670
|
+
const paths = /* @__PURE__ */ new Set([...Object.keys(participant.baseline), ...Object.keys(participant.current)]);
|
|
2671
|
+
const deleted = [];
|
|
2672
|
+
const added = [];
|
|
2673
|
+
for (const path of paths) {
|
|
2674
|
+
const before = own(participant.baseline, path);
|
|
2675
|
+
const after = own(participant.current, path);
|
|
2676
|
+
if (before && !after) deleted.push([path, before]);
|
|
2677
|
+
if (after && !before) added.push([path, after]);
|
|
2678
|
+
}
|
|
2679
|
+
const renames = pairRenames(deleted, added, (hash) => content(hash) ?? "");
|
|
2680
|
+
const moved = new Set(renames.values());
|
|
2681
|
+
for (const path of paths) {
|
|
2682
|
+
if (moved.has(path)) continue;
|
|
2683
|
+
const beforeHash = own(participant.baseline, renames.get(path) ?? path);
|
|
2684
|
+
const afterHash = own(participant.current, path);
|
|
2685
|
+
if (beforeHash === afterHash) continue;
|
|
2686
|
+
const change = budgetChange(path, measure(beforeHash, afterHash), state.limits);
|
|
2687
|
+
if (change.fileLimitExceeded && !participant.removed) {
|
|
2688
|
+
const label = change.category === "docs" ? "documentation" : "code";
|
|
2689
|
+
issues.push({
|
|
2690
|
+
key: JSON.stringify([participant.root, path]),
|
|
2691
|
+
excess: change.fileTokens - change.fileCeiling,
|
|
2692
|
+
message: `${path}${where}: ${change.fileTokens} ${label} tokens exceed file ceiling ${change.fileCeiling}`
|
|
2693
|
+
});
|
|
2694
|
+
}
|
|
2695
|
+
const fingerprint = JSON.stringify([path, beforeHash, afterHash]);
|
|
2696
|
+
if (seen.has(fingerprint)) continue;
|
|
2697
|
+
seen.add(fingerprint);
|
|
2698
|
+
usage[change.category === "code" ? "codeTokens" : "docTokens"] += change.addedTokens + change.deletedTokens;
|
|
2699
|
+
}
|
|
2700
|
+
}
|
|
2701
|
+
for (const [key, label, used, limit] of [
|
|
2702
|
+
["code", "code", usage.codeTokens, state.limits.maxCodeTokens],
|
|
2703
|
+
["docs", "documentation", usage.docTokens, state.limits.maxDocTokens]
|
|
2704
|
+
]) {
|
|
2705
|
+
if (limit === null || used <= limit) continue;
|
|
2706
|
+
issues.push({ key, excess: used - limit, message: `${used} changed ${label} tokens exceed task limit ${limit}` });
|
|
2707
|
+
}
|
|
2708
|
+
const violations = issues.map(({ message }) => message);
|
|
2709
|
+
return {
|
|
2710
|
+
issues,
|
|
2711
|
+
measures,
|
|
2712
|
+
measured,
|
|
2713
|
+
report: {
|
|
2714
|
+
status: issues.length ? "failed" : "passed",
|
|
2715
|
+
taskId: state.taskId,
|
|
2716
|
+
summary: issues.length ? exceededSummary(violations) : budgetUsageSummary(usage, state.limits),
|
|
2717
|
+
violations,
|
|
2718
|
+
usage,
|
|
2719
|
+
limits: state.limits
|
|
2720
|
+
}
|
|
2721
|
+
};
|
|
2722
|
+
}
|
|
2723
|
+
function budgetUsageSummary(usage, limits) {
|
|
2724
|
+
const parts = [
|
|
2725
|
+
`File budgets (cl100k_base): code files <= ${limits.maxFileTokens} tokens`,
|
|
2726
|
+
`documentation files <= ${limits.maxDocFileTokens} tokens`,
|
|
2727
|
+
`files already over a ceiling may grow by ${limits.maxOversizedGrowthTokens} tokens`
|
|
2728
|
+
];
|
|
2729
|
+
if (limits.maxCodeTokens !== null) parts.push(`code ${usage.codeTokens}/${limits.maxCodeTokens} changed tokens`);
|
|
2730
|
+
if (limits.maxDocTokens !== null) {
|
|
2731
|
+
parts.push(`documentation ${usage.docTokens}/${limits.maxDocTokens} changed tokens`);
|
|
2732
|
+
}
|
|
2733
|
+
return parts.join("; ");
|
|
2734
|
+
}
|
|
2735
|
+
function worsenedIssues(current, proposed) {
|
|
2736
|
+
const existing = new Map(current.issues.map(({ key, excess }) => [key, excess]));
|
|
2737
|
+
return proposed.issues.filter(({ key, excess }) => excess > (existing.get(key) ?? 0));
|
|
2738
|
+
}
|
|
2739
|
+
|
|
2740
|
+
// packages/coding-agent/src/core/budgets/snapshot.ts
|
|
2741
|
+
var import_node_buffer = require("node:buffer");
|
|
2742
|
+
var import_node_fs5 = require("node:fs");
|
|
2743
|
+
var import_node_path5 = require("node:path");
|
|
2744
|
+
var TIMESTAMP_GRANULARITY_MS = 2e3;
|
|
2745
|
+
function parseStatCache(value) {
|
|
2746
|
+
if (typeof value !== "object" || value === null || !("startedAt" in value) || !("files" in value)) return void 0;
|
|
2747
|
+
if (!("excluded" in value) || typeof value.startedAt !== "number" || !Array.isArray(value.excluded))
|
|
2748
|
+
return void 0;
|
|
2749
|
+
const { files } = value;
|
|
2750
|
+
if (typeof files !== "object" || files === null) return void 0;
|
|
2751
|
+
const entries = [];
|
|
2752
|
+
for (const [path, entry] of Object.entries(files)) {
|
|
2753
|
+
if (!Array.isArray(entry) || entry.length !== 5 || !entry.slice(0, 4).every((field2) => typeof field2 === "number") || entry[4] !== null && (typeof entry[4] !== "string" || !/^[a-f0-9]{64}$/.test(entry[4]))) {
|
|
2754
|
+
return void 0;
|
|
2755
|
+
}
|
|
2756
|
+
entries.push([path, [entry[0], entry[1], entry[2], entry[3], entry[4]]]);
|
|
2757
|
+
}
|
|
2758
|
+
return {
|
|
2759
|
+
startedAt: value.startedAt,
|
|
2760
|
+
files: Object.fromEntries(entries),
|
|
2761
|
+
excluded: value.excluded.filter((path) => typeof path === "string")
|
|
2762
|
+
};
|
|
2763
|
+
}
|
|
2764
|
+
function regularFile(path) {
|
|
2765
|
+
try {
|
|
2766
|
+
const stat = (0, import_node_fs5.lstatSync)(path);
|
|
2767
|
+
return stat.isFile() ? stat : void 0;
|
|
2768
|
+
} catch (error) {
|
|
2769
|
+
if (isMissingPath(error)) return void 0;
|
|
2770
|
+
throw error;
|
|
2771
|
+
}
|
|
2772
|
+
}
|
|
2773
|
+
function snapshotWorkspace(options) {
|
|
2774
|
+
const { workspace, directory, cacheName } = options;
|
|
2775
|
+
const startedAt = Date.now();
|
|
2776
|
+
const previous = readBudgetCache(directory, cacheName, parseStatCache);
|
|
2777
|
+
const listed = listWorkspacePaths(workspace);
|
|
2778
|
+
const paths = [.../* @__PURE__ */ new Set([...listed, ...options.additionalPaths ?? []])];
|
|
2779
|
+
const known = /* @__PURE__ */ new Set([...previous?.excluded ?? [], ...Object.keys(previous?.files ?? {})]);
|
|
2780
|
+
const excluded = /* @__PURE__ */ new Set([
|
|
2781
|
+
...previous?.excluded ?? [],
|
|
2782
|
+
...options.excluded(paths.filter((path) => !known.has(path)))
|
|
2783
|
+
]);
|
|
2784
|
+
const linked = symlinkCheck(workspace.root);
|
|
2785
|
+
const files = [];
|
|
2786
|
+
const snapshot = [];
|
|
2787
|
+
const unmeasured = [];
|
|
2788
|
+
const vanished = [];
|
|
2789
|
+
let reread = false;
|
|
2790
|
+
for (const path of paths) {
|
|
2791
|
+
if (excluded.has(path) || linked(path)) continue;
|
|
2792
|
+
const absolutePath = (0, import_node_path5.join)(workspace.root, path);
|
|
2793
|
+
const stat = regularFile(absolutePath);
|
|
2794
|
+
if (!listed.has(path)) {
|
|
2795
|
+
if (stat) unmeasured.push(path);
|
|
2796
|
+
else vanished.push(path);
|
|
2797
|
+
continue;
|
|
2798
|
+
}
|
|
2799
|
+
if (!stat) continue;
|
|
2800
|
+
const cached = previous && Object.hasOwn(previous.files, path) ? previous.files[path] : void 0;
|
|
2801
|
+
const settled = previous && Math.max(stat.mtimeMs, stat.ctimeMs) < previous.startedAt - TIMESTAMP_GRANULARITY_MS;
|
|
2802
|
+
let hash;
|
|
2803
|
+
if (cached && settled && cached[0] === stat.size && cached[1] === stat.mtimeMs && cached[2] === stat.ctimeMs && cached[3] === stat.ino) {
|
|
2804
|
+
hash = cached[4];
|
|
2805
|
+
} else {
|
|
2806
|
+
const bytes = (0, import_node_fs5.readFileSync)(absolutePath);
|
|
2807
|
+
hash = (0, import_node_buffer.isUtf8)(bytes) || isSourcePath(path) ? storeBudgetObject(directory, bytes.toString("utf8")) : null;
|
|
2808
|
+
reread = true;
|
|
2809
|
+
}
|
|
2810
|
+
files.push([path, [stat.size, stat.mtimeMs, stat.ctimeMs, stat.ino, hash]]);
|
|
2811
|
+
if (hash) snapshot.push([path, hash]);
|
|
2812
|
+
}
|
|
2813
|
+
const next = {
|
|
2814
|
+
startedAt,
|
|
2815
|
+
files: Object.fromEntries(files),
|
|
2816
|
+
excluded: paths.filter((path) => excluded.has(path))
|
|
2817
|
+
};
|
|
2818
|
+
const same = previous && JSON.stringify(next.files) === JSON.stringify(previous.files) && JSON.stringify(next.excluded) === JSON.stringify(previous.excluded);
|
|
2819
|
+
if (reread || !same) writeBudgetCache(directory, cacheName, next);
|
|
2820
|
+
unmeasured.push(...pathsOutsideBudget(workspace, vanished));
|
|
2821
|
+
return { files: Object.fromEntries(snapshot), unmeasured };
|
|
2822
|
+
}
|
|
2823
|
+
|
|
2824
|
+
// packages/coding-agent/src/core/budgets/runtime.ts
|
|
2825
|
+
var NOT_CONFIGURED = "Budgets are not configured for this repository";
|
|
2826
|
+
var MEASURES_CACHE = "measures";
|
|
2827
|
+
function checkBudget(options) {
|
|
2828
|
+
return inspectBudget(options);
|
|
2829
|
+
}
|
|
2830
|
+
function auditBudget(options) {
|
|
2831
|
+
return inspectBudget(options);
|
|
2832
|
+
}
|
|
2833
|
+
function beginTask(options) {
|
|
2834
|
+
return inspectBudget(options);
|
|
2835
|
+
}
|
|
2836
|
+
function quietReport(status, summary) {
|
|
2837
|
+
return {
|
|
2838
|
+
status,
|
|
2839
|
+
summary,
|
|
2840
|
+
violations: status === "error" ? [summary] : [],
|
|
2841
|
+
usage: { codeTokens: 0, docTokens: 0 },
|
|
2842
|
+
limits: DEFAULT_BUDGET_LIMITS
|
|
2843
|
+
};
|
|
2844
|
+
}
|
|
2845
|
+
function configAbove(cwd) {
|
|
2846
|
+
let start = (0, import_node_path6.resolve)(cwd);
|
|
2847
|
+
try {
|
|
2848
|
+
start = (0, import_node_fs6.realpathSync)(cwd);
|
|
2849
|
+
} catch {
|
|
2850
|
+
}
|
|
2851
|
+
for (let directory = start; ; directory = (0, import_node_path6.dirname)(directory)) {
|
|
2852
|
+
if (budgetPathExists((0, import_node_path6.join)(directory, ".draht", "budgets.json"))) return true;
|
|
2853
|
+
if ((0, import_node_path6.dirname)(directory) === directory) return false;
|
|
2854
|
+
}
|
|
2855
|
+
}
|
|
2856
|
+
function locateBudget(cwd) {
|
|
2857
|
+
if (!mayHaveBudgets(cwd, BUDGET_TASK_MARKER)) return void 0;
|
|
2858
|
+
let workspace;
|
|
2859
|
+
try {
|
|
2860
|
+
workspace = findBudgetWorkspace(cwd);
|
|
2861
|
+
} catch (error) {
|
|
2862
|
+
if (configAbove(cwd)) throw error;
|
|
2863
|
+
return void 0;
|
|
2864
|
+
}
|
|
2865
|
+
if (!workspace) {
|
|
2866
|
+
if (configAbove(cwd)) throw new Error("Budgets require a Git working tree");
|
|
2867
|
+
return void 0;
|
|
2868
|
+
}
|
|
2869
|
+
const location = {
|
|
2870
|
+
workspace,
|
|
2871
|
+
ledger: budgetLedger(workspace.commonDir),
|
|
2872
|
+
configPath: (0, import_node_path6.join)(workspace.root, ".draht", "budgets.json")
|
|
2873
|
+
};
|
|
2874
|
+
return budgetPathExists(location.configPath) || budgetPathExists(location.ledger.marker) ? location : void 0;
|
|
2875
|
+
}
|
|
2876
|
+
function budgetConfigured(cwd) {
|
|
2877
|
+
try {
|
|
2878
|
+
return locateBudget(cwd) !== void 0;
|
|
2879
|
+
} catch {
|
|
2880
|
+
return true;
|
|
2881
|
+
}
|
|
2882
|
+
}
|
|
2883
|
+
function excludedPaths(workspace, state, attributes) {
|
|
2884
|
+
const matches = excludeMatcher(state.exclude);
|
|
2885
|
+
return (paths) => {
|
|
2886
|
+
const generated = generatedPaths(workspace, attributes, paths);
|
|
2887
|
+
return new Set(paths.filter((path) => isAlwaysExcluded(path) || matches(path) || generated.has(path)));
|
|
2888
|
+
};
|
|
2889
|
+
}
|
|
2890
|
+
function takeSnapshot(location, workspace, state, participant) {
|
|
2891
|
+
return snapshotWorkspace({
|
|
2892
|
+
workspace,
|
|
2893
|
+
directory: location.ledger.directory,
|
|
2894
|
+
cacheName: `files-${(0, import_node_crypto4.createHash)("sha256").update(workspace.root).digest("hex").slice(0, 32)}`,
|
|
2895
|
+
excluded: excludedPaths(workspace, state, participant.attributes),
|
|
2896
|
+
additionalPaths: [...Object.keys(participant.baseline), ...Object.keys(participant.current)]
|
|
2897
|
+
});
|
|
2898
|
+
}
|
|
2899
|
+
function startTask(location) {
|
|
2900
|
+
const config = budgetPathExists(location.configPath) ? readBudgetConfig(location.configPath) : void 0;
|
|
2901
|
+
clearBudgetTask(location.ledger);
|
|
2902
|
+
if (!config) return NOT_CONFIGURED;
|
|
2903
|
+
if (!config.enabled) return "Budgets are disabled by .draht/budgets.json";
|
|
2904
|
+
return { version: 2, taskId: (0, import_node_crypto4.randomUUID)(), limits: config.limits, exclude: config.exclude, worktrees: [] };
|
|
2905
|
+
}
|
|
2906
|
+
function enroll(location, state) {
|
|
2907
|
+
const { workspace } = location;
|
|
2908
|
+
if (state.worktrees.some(({ root, removed }) => root === workspace.root && !removed)) return void 0;
|
|
2909
|
+
const attributes = headCommit(workspace);
|
|
2910
|
+
const { files } = takeSnapshot(location, workspace, state, { attributes, baseline: {}, current: {} });
|
|
2911
|
+
const participant = { root: workspace.root, attributes, removed: false, baseline: files, current: { ...files } };
|
|
2912
|
+
state.worktrees.push(participant);
|
|
2913
|
+
return participant;
|
|
2914
|
+
}
|
|
2915
|
+
function sameSnapshot(left, right) {
|
|
2916
|
+
const paths = Object.keys(left);
|
|
2917
|
+
return paths.length === Object.keys(right).length && paths.every((path) => Object.hasOwn(right, path) && left[path] === right[path]);
|
|
2918
|
+
}
|
|
2919
|
+
function refreshWorktrees(location, state, enrolled) {
|
|
2920
|
+
let changed = false;
|
|
2921
|
+
for (const participant of state.worktrees) {
|
|
2922
|
+
if (participant.removed || participant === enrolled) continue;
|
|
2923
|
+
const workspace = participant.root === location.workspace.root ? location.workspace : budgetPathExists(participant.root) ? findBudgetWorkspace(participant.root) : void 0;
|
|
2924
|
+
if (workspace?.commonDir !== location.workspace.commonDir || workspace.root !== participant.root) {
|
|
2925
|
+
participant.removed = true;
|
|
2926
|
+
changed = true;
|
|
2927
|
+
continue;
|
|
2928
|
+
}
|
|
2929
|
+
const { files, unmeasured } = takeSnapshot(location, workspace, state, participant);
|
|
2930
|
+
const unchanged = unmeasured.flatMap(
|
|
2931
|
+
(path) => Object.hasOwn(participant.baseline, path) ? [[path, participant.baseline[path]]] : []
|
|
2932
|
+
);
|
|
2933
|
+
const current = unchanged.length ? Object.fromEntries([...Object.entries(files), ...unchanged]) : files;
|
|
2934
|
+
if (sameSnapshot(current, participant.current)) continue;
|
|
2935
|
+
participant.current = current;
|
|
2936
|
+
changed = true;
|
|
2937
|
+
}
|
|
2938
|
+
return changed;
|
|
2939
|
+
}
|
|
2940
|
+
function isLedgerFile(ledger, target) {
|
|
2941
|
+
return [ledger.directory, ledger.marker, ledger.lock].some(
|
|
2942
|
+
(path) => target === path || target.startsWith(`${path}${import_node_path6.sep}`)
|
|
2943
|
+
);
|
|
2944
|
+
}
|
|
2945
|
+
function projectBudget(location, state, cwd, proposed) {
|
|
2946
|
+
const participant = state.worktrees.find(({ root, removed }) => root === location.workspace.root && !removed);
|
|
2947
|
+
if (!participant) throw new Error("Budget worktree was not enrolled");
|
|
2948
|
+
const targets = /* @__PURE__ */ new Map();
|
|
2949
|
+
const ledgerEdits = [];
|
|
2950
|
+
for (const change of proposed) {
|
|
2951
|
+
const target = canonicalBudgetPath(cwd, change.path);
|
|
2952
|
+
if (isLedgerFile(location.ledger, target)) ledgerEdits.push(change.path);
|
|
2953
|
+
const path = budgetFilePath(location.workspace, target);
|
|
2954
|
+
if (path !== void 0) targets.set(path, change.content);
|
|
2955
|
+
}
|
|
2956
|
+
const excluded = excludedPaths(location.workspace, state, participant.attributes)([...targets.keys()]);
|
|
2957
|
+
const content = /* @__PURE__ */ new Map();
|
|
2958
|
+
for (const [path, text3] of targets) {
|
|
2959
|
+
if (excluded.has(path)) continue;
|
|
2960
|
+
if (text3 === null) {
|
|
2961
|
+
delete participant.current[path];
|
|
2962
|
+
continue;
|
|
2963
|
+
}
|
|
2964
|
+
const hash = budgetContentHash(text3);
|
|
2965
|
+
content.set(hash, text3);
|
|
2966
|
+
Object.defineProperty(participant.current, path, {
|
|
2967
|
+
value: hash,
|
|
2968
|
+
configurable: true,
|
|
2969
|
+
enumerable: true,
|
|
2970
|
+
writable: true
|
|
2971
|
+
});
|
|
2972
|
+
}
|
|
2973
|
+
return { content, ledgerEdits };
|
|
2974
|
+
}
|
|
2975
|
+
function parseMeasures(value) {
|
|
2976
|
+
if (typeof value !== "object" || value === null || Array.isArray(value)) return void 0;
|
|
2977
|
+
const measures = /* @__PURE__ */ new Map();
|
|
2978
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
2979
|
+
if (!Array.isArray(entry) || entry.length !== 4) return void 0;
|
|
2980
|
+
const [addedTokens, deletedTokens, fileTokens, baselineTokens] = entry;
|
|
2981
|
+
if (![addedTokens, deletedTokens, fileTokens, baselineTokens].every(Number.isSafeInteger)) return void 0;
|
|
2982
|
+
measures.set(key, { addedTokens, deletedTokens, fileTokens, baselineTokens });
|
|
2983
|
+
}
|
|
2984
|
+
return measures;
|
|
2985
|
+
}
|
|
2986
|
+
function saveMeasures(directory, ...evaluations) {
|
|
2987
|
+
if (!evaluations.some(({ measured }) => measured)) return;
|
|
2988
|
+
const entries = new Map(evaluations.flatMap(({ measures }) => [...measures]));
|
|
2989
|
+
writeBudgetCache(
|
|
2990
|
+
directory,
|
|
2991
|
+
MEASURES_CACHE,
|
|
2992
|
+
Object.fromEntries(
|
|
2993
|
+
Array.from(entries, ([key, value]) => [
|
|
2994
|
+
key,
|
|
2995
|
+
[value.addedTokens, value.deletedTokens, value.fileTokens, value.baselineTokens]
|
|
2996
|
+
])
|
|
2997
|
+
)
|
|
2998
|
+
);
|
|
2999
|
+
}
|
|
3000
|
+
function inspectBudget({
|
|
3001
|
+
cwd,
|
|
3002
|
+
newTask = false,
|
|
3003
|
+
proposed
|
|
3004
|
+
}) {
|
|
3005
|
+
try {
|
|
3006
|
+
const location = locateBudget(cwd);
|
|
3007
|
+
if (!location) return quietReport("disabled", NOT_CONFIGURED);
|
|
3008
|
+
return withBudgetLock(location.ledger.lock, () => {
|
|
3009
|
+
const existing = newTask ? void 0 : readBudgetState(location.ledger);
|
|
3010
|
+
const state = existing ?? startTask(location);
|
|
3011
|
+
if (typeof state === "string") return quietReport("disabled", state);
|
|
3012
|
+
const enrolled = enroll(location, state);
|
|
3013
|
+
if (refreshWorktrees(location, state, enrolled) || enrolled || !existing) {
|
|
3014
|
+
writeBudgetState(location.ledger, state);
|
|
3015
|
+
}
|
|
3016
|
+
if (!existing) markBudgetTask(location.ledger, state.taskId);
|
|
3017
|
+
const directory = location.ledger.directory;
|
|
3018
|
+
const currentRoot = location.workspace.root;
|
|
3019
|
+
const cached = readBudgetCache(directory, MEASURES_CACHE, parseMeasures);
|
|
3020
|
+
const actual = evaluateBudget(directory, state, { currentRoot, measures: cached });
|
|
3021
|
+
if (!proposed?.length) {
|
|
3022
|
+
saveMeasures(directory, actual);
|
|
3023
|
+
return actual.report;
|
|
3024
|
+
}
|
|
3025
|
+
const { content, ledgerEdits } = projectBudget(location, state, cwd, proposed);
|
|
3026
|
+
if (ledgerEdits.length) {
|
|
3027
|
+
saveMeasures(directory, actual);
|
|
3028
|
+
const violations = ledgerEdits.map((path) => `${path}: budget ledger files change only through a new task`);
|
|
3029
|
+
return { ...actual.report, status: "failed", violations, summary: exceededSummary(violations) };
|
|
3030
|
+
}
|
|
3031
|
+
const measures = new Map([...cached ?? [], ...actual.measures]);
|
|
3032
|
+
const projected = evaluateBudget(directory, state, { currentRoot, projected: content, measures });
|
|
3033
|
+
saveMeasures(directory, actual, projected);
|
|
3034
|
+
const worsened = worsenedIssues(actual, projected);
|
|
3035
|
+
if (worsened.length) {
|
|
3036
|
+
const violations = worsened.map(({ message }) => message);
|
|
3037
|
+
return { ...projected.report, status: "failed", violations, summary: exceededSummary(violations) };
|
|
3038
|
+
}
|
|
3039
|
+
return projected.report.status === "failed" ? { ...projected.report, status: "passed", summary: `Change accepted; ${projected.report.summary}` } : projected.report;
|
|
3040
|
+
});
|
|
3041
|
+
} catch (error) {
|
|
3042
|
+
return quietReport(
|
|
3043
|
+
"error",
|
|
3044
|
+
`Budget inspection failed: ${error instanceof Error ? error.message : String(error)}`
|
|
3045
|
+
);
|
|
3046
|
+
}
|
|
3047
|
+
}
|
|
3048
|
+
function stopCacheName(session) {
|
|
3049
|
+
return `stop-${(0, import_node_crypto4.createHash)("sha256").update(session).digest("hex").slice(0, 32)}`;
|
|
3050
|
+
}
|
|
3051
|
+
function parseStopFailures(value) {
|
|
3052
|
+
return typeof value === "object" && value !== null && "failures" in value && Number.isSafeInteger(value.failures) ? Number(value.failures) : void 0;
|
|
3053
|
+
}
|
|
3054
|
+
function recordStopFailure(cwd, session) {
|
|
3055
|
+
try {
|
|
3056
|
+
const location = locateBudget(cwd);
|
|
3057
|
+
if (!location) return 0;
|
|
3058
|
+
const name = stopCacheName(session);
|
|
3059
|
+
const failures = (readBudgetCache(location.ledger.directory, name, parseStopFailures) ?? 0) + 1;
|
|
3060
|
+
writeBudgetCache(location.ledger.directory, name, { failures });
|
|
3061
|
+
return failures;
|
|
3062
|
+
} catch {
|
|
3063
|
+
return Number.POSITIVE_INFINITY;
|
|
3064
|
+
}
|
|
3065
|
+
}
|
|
3066
|
+
function clearStopFailures(cwd, session) {
|
|
3067
|
+
try {
|
|
3068
|
+
const location = locateBudget(cwd);
|
|
3069
|
+
if (location) removeBudgetCache(location.ledger.directory, stopCacheName(session));
|
|
3070
|
+
} catch {
|
|
3071
|
+
}
|
|
3072
|
+
}
|
|
3073
|
+
|
|
3074
|
+
// packages/coding-agent/src/core/builtins/budget-guidance.ts
|
|
3075
|
+
var BUDGET_GUIDANCE = "Budgets are ceilings, not targets: write the smallest clear solution. Prefer clear names, cohesive functions, and focused modules. Keep documentation concise: explain contracts, setup, and non-obvious decisions; avoid restating code or writing progress diaries. Do not minify code or make arbitrary file splits just to fit a budget. Change .draht/budgets.json only when the user asks; edits apply only after the user starts a new budget task.";
|
|
3076
|
+
|
|
3077
|
+
// packages/coding-agent/src/core/budgets/tools.ts
|
|
3078
|
+
var import_node_path8 = require("node:path");
|
|
3079
|
+
|
|
3080
|
+
// packages/coding-agent/src/core/budgets/patch.ts
|
|
3081
|
+
var import_node_fs7 = require("node:fs");
|
|
3082
|
+
var import_node_path7 = require("node:path");
|
|
3083
|
+
function readEditableFile(path) {
|
|
3084
|
+
if (!(0, import_node_fs7.statSync)(path).isFile()) throw new Error(`Budget preview reads only regular files: ${path}`);
|
|
3085
|
+
return (0, import_node_fs7.readFileSync)(path, "utf8");
|
|
3086
|
+
}
|
|
3087
|
+
function locate(lines, match, start, atEnd) {
|
|
3088
|
+
const first = atEnd ? lines.length - match.length : start;
|
|
3089
|
+
const last = atEnd ? first : lines.length - match.length;
|
|
3090
|
+
for (const normalize of [(line) => line, (line) => line.trimEnd(), (line) => line.trim()]) {
|
|
3091
|
+
for (let index = Math.max(start, first); index <= last; index++) {
|
|
3092
|
+
if (match.every((line, offset) => normalize(lines[index + offset]) === normalize(line))) return index;
|
|
3093
|
+
}
|
|
3094
|
+
}
|
|
3095
|
+
throw new Error("Budget preview cannot locate a patch hunk; use an exact edit");
|
|
3096
|
+
}
|
|
3097
|
+
function update(content, patch) {
|
|
3098
|
+
const source = Array.from(content.matchAll(/([^\r\n]*)(\r\n|\r|\n|$)/g)).filter((match) => match[0].length > 0).map((match) => ({ text: match[1], ending: match[2] }));
|
|
3099
|
+
const lines = source.map((line) => line.text);
|
|
3100
|
+
const ending = source.find((line) => line.ending)?.ending ?? "\n";
|
|
3101
|
+
const replacements = [];
|
|
3102
|
+
let cursor = 0;
|
|
3103
|
+
let index = 0;
|
|
3104
|
+
while (index < patch.length) {
|
|
3105
|
+
const header = patch[index++];
|
|
3106
|
+
if (header !== "@@" && !header.startsWith("@@ ")) throw new Error("Budget preview expected a patch hunk header");
|
|
3107
|
+
const anchor = header.slice(3);
|
|
3108
|
+
if (anchor) cursor = locate(lines, [anchor], cursor, false) + 1;
|
|
3109
|
+
const before = [];
|
|
3110
|
+
const operations = [];
|
|
3111
|
+
let atEnd = false;
|
|
3112
|
+
while (index < patch.length && !patch[index].startsWith("@@")) {
|
|
3113
|
+
const line = patch[index++];
|
|
3114
|
+
if (line === "*** End of File") {
|
|
3115
|
+
atEnd = true;
|
|
3116
|
+
break;
|
|
3117
|
+
}
|
|
3118
|
+
const kind = line[0];
|
|
3119
|
+
if (kind !== " " && kind !== "+" && kind !== "-")
|
|
3120
|
+
throw new Error("Budget preview encountered an invalid patch line");
|
|
3121
|
+
const text3 = line.slice(1);
|
|
3122
|
+
if (kind !== "+") before.push(text3);
|
|
3123
|
+
operations.push({ kind, text: text3 });
|
|
3124
|
+
}
|
|
3125
|
+
const offset = before.length ? locate(lines, before, cursor, atEnd) : lines.length;
|
|
3126
|
+
const replacement = [];
|
|
3127
|
+
let original = offset;
|
|
3128
|
+
for (const operation of operations) {
|
|
3129
|
+
if (operation.kind === "+") replacement.push({ text: operation.text, ending });
|
|
3130
|
+
else {
|
|
3131
|
+
if (operation.kind === " ") replacement.push(source[original]);
|
|
3132
|
+
original++;
|
|
3133
|
+
}
|
|
3134
|
+
}
|
|
3135
|
+
replacements.push({ offset, length: before.length, lines: replacement });
|
|
3136
|
+
if (before.length) cursor = offset + before.length;
|
|
3137
|
+
}
|
|
3138
|
+
const result = [];
|
|
3139
|
+
let consumed = 0;
|
|
3140
|
+
for (const replacement of replacements.sort((a, b) => a.offset - b.offset)) {
|
|
3141
|
+
if (replacement.offset < consumed) throw new Error("Budget preview encountered overlapping patch hunks");
|
|
3142
|
+
while (consumed < replacement.offset) result.push(source[consumed++]);
|
|
3143
|
+
for (const line of replacement.lines) result.push(line);
|
|
3144
|
+
consumed += replacement.length;
|
|
3145
|
+
}
|
|
3146
|
+
while (consumed < source.length) result.push(source[consumed++]);
|
|
3147
|
+
return result.map((line) => line.text + (line.ending || ending)).join("");
|
|
3148
|
+
}
|
|
3149
|
+
function previewPatch(cwd, source) {
|
|
3150
|
+
const lines = source.trimEnd().split(/\r?\n/);
|
|
3151
|
+
if (lines.shift() !== "*** Begin Patch" || lines.pop() !== "*** End Patch") {
|
|
3152
|
+
throw new Error("Budget preview requires a complete apply_patch payload");
|
|
3153
|
+
}
|
|
3154
|
+
const changes = /* @__PURE__ */ new Map();
|
|
3155
|
+
let index = 0;
|
|
3156
|
+
while (index < lines.length) {
|
|
3157
|
+
const header = /^\*\*\* (Add|Update|Delete) File: (.+)$/.exec(lines[index++]);
|
|
3158
|
+
if (!header) throw new Error("Budget preview encountered an unsupported patch operation");
|
|
3159
|
+
const [, operation, name] = header;
|
|
3160
|
+
const path = (0, import_node_path7.resolve)(cwd, name);
|
|
3161
|
+
if (operation === "Delete") {
|
|
3162
|
+
changes.set(path, null);
|
|
3163
|
+
continue;
|
|
3164
|
+
}
|
|
3165
|
+
let destination = path;
|
|
3166
|
+
if (operation === "Update" && lines[index]?.startsWith("*** Move to: ")) {
|
|
3167
|
+
destination = (0, import_node_path7.resolve)(cwd, lines[index++].slice("*** Move to: ".length));
|
|
3168
|
+
}
|
|
3169
|
+
const body = [];
|
|
3170
|
+
while (index < lines.length && !/^\*\*\* (Add|Update|Delete) File: /.test(lines[index])) {
|
|
3171
|
+
body.push(lines[index++]);
|
|
3172
|
+
}
|
|
3173
|
+
if (operation === "Add") {
|
|
3174
|
+
if (body.some((line) => !line.startsWith("+"))) throw new Error("Invalid added-file patch");
|
|
3175
|
+
changes.set(path, `${body.map((line) => line.slice(1)).join("\n")}
|
|
3176
|
+
`);
|
|
3177
|
+
continue;
|
|
3178
|
+
}
|
|
3179
|
+
const original = changes.has(path) ? changes.get(path) : readEditableFile(path);
|
|
3180
|
+
if (original === null || original === void 0) throw new Error("Cannot patch a deleted file");
|
|
3181
|
+
if (destination !== path) changes.set(path, null);
|
|
3182
|
+
changes.set(destination, update(original, body));
|
|
3183
|
+
}
|
|
3184
|
+
return [...changes].map(([path, content]) => ({ path, content }));
|
|
3185
|
+
}
|
|
3186
|
+
|
|
3187
|
+
// packages/coding-agent/src/core/budgets/tools.ts
|
|
3188
|
+
function record2(value) {
|
|
3189
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Expected an edit object");
|
|
3190
|
+
return Object.fromEntries(Object.entries(value));
|
|
3191
|
+
}
|
|
3192
|
+
function text(value, name) {
|
|
3193
|
+
if (typeof value !== "string") throw new Error(`Missing ${name} in edit payload`);
|
|
3194
|
+
return value;
|
|
3195
|
+
}
|
|
3196
|
+
function replace(source, value) {
|
|
3197
|
+
const edit = record2(value);
|
|
3198
|
+
const before = text(edit.old_string ?? edit.oldText, "old_string");
|
|
3199
|
+
const after = text(edit.new_string ?? edit.newText, "new_string");
|
|
3200
|
+
if (!before || !source.includes(before)) throw new Error("Budget preview cannot locate the replacement text");
|
|
3201
|
+
if (edit.replace_all === true) return source.split(before).join(after);
|
|
3202
|
+
if (source.indexOf(before) !== source.lastIndexOf(before)) throw new Error("Budget preview: ambiguous replacement");
|
|
3203
|
+
return source.replace(before, () => after);
|
|
3204
|
+
}
|
|
3205
|
+
function previewTool(cwd, name, input) {
|
|
3206
|
+
const tool = name.split(/[.:]/).at(-1)?.toLowerCase();
|
|
3207
|
+
if (tool === "apply_patch") {
|
|
3208
|
+
const args2 = typeof input === "string" ? void 0 : record2(input);
|
|
3209
|
+
const value = args2 ? args2.command ?? args2.patch ?? args2.input : input;
|
|
3210
|
+
return previewPatch(cwd, text(value, "patch"));
|
|
3211
|
+
}
|
|
3212
|
+
if (tool !== "write" && tool !== "edit" && tool !== "multiedit") return void 0;
|
|
3213
|
+
const args = record2(input);
|
|
3214
|
+
const path = (0, import_node_path8.resolve)(cwd, text(args.file_path ?? args.path, "file_path"));
|
|
3215
|
+
if (tool === "write") return [{ path, content: text(args.content, "content") }];
|
|
3216
|
+
let content = readEditableFile(path);
|
|
3217
|
+
if (tool === "multiedit") {
|
|
3218
|
+
if (!Array.isArray(args.edits) || args.edits.length === 0) throw new Error("Missing edits in MultiEdit");
|
|
3219
|
+
for (const edit of args.edits) content = replace(content, edit);
|
|
3220
|
+
} else content = replace(content, args);
|
|
3221
|
+
return [{ path, content }];
|
|
3222
|
+
}
|
|
3223
|
+
|
|
3224
|
+
// packages/coding-agent/src/core/budgets/plugin.ts
|
|
3225
|
+
var READ_TOOLS = /* @__PURE__ */ new Set(["read", "grep", "find", "ls", "glob", "list_directory"]);
|
|
3226
|
+
var RESET_PROMPT = /^(?:\/(?:draht:)?budget|\$draht:budget)\s+new$/;
|
|
3227
|
+
var MAX_REPAIR_ATTEMPTS = 3;
|
|
3228
|
+
function previewEdit(cwd, name, args) {
|
|
3229
|
+
try {
|
|
3230
|
+
return previewTool(cwd, name, args);
|
|
3231
|
+
} catch {
|
|
3232
|
+
return void 0;
|
|
3233
|
+
}
|
|
3234
|
+
}
|
|
3235
|
+
function field(value, key) {
|
|
3236
|
+
return key in value ? Reflect.get(value, key) : void 0;
|
|
3237
|
+
}
|
|
3238
|
+
function text2(value, key) {
|
|
3239
|
+
const found = field(value, key);
|
|
3240
|
+
return typeof found === "string" ? found : "";
|
|
3241
|
+
}
|
|
3242
|
+
function runHook(event, input) {
|
|
3243
|
+
try {
|
|
3244
|
+
const value = JSON.parse(input);
|
|
3245
|
+
if (!value || typeof value !== "object" || Array.isArray(value)) throw new Error("Invalid budget hook payload");
|
|
3246
|
+
const cwd = text2(value, "cwd") || process.cwd();
|
|
3247
|
+
if (event === "SessionStart") {
|
|
3248
|
+
const report2 = beginTask({ cwd });
|
|
3249
|
+
if (report2.status === "error") process.stdout.write(`${report2.summary}
|
|
3250
|
+
`);
|
|
3251
|
+
else if (report2.status !== "disabled") process.stdout.write(`${report2.summary}
|
|
3252
|
+
${BUDGET_GUIDANCE}
|
|
3253
|
+
`);
|
|
3254
|
+
return;
|
|
3255
|
+
}
|
|
3256
|
+
if (event === "UserPromptSubmit") {
|
|
3257
|
+
const reset = RESET_PROMPT.test(text2(value, "prompt").trim());
|
|
3258
|
+
const report2 = beginTask({ cwd, newTask: reset });
|
|
3259
|
+
if (report2.status !== "disabled") clearStopFailures(cwd, text2(value, "session_id"));
|
|
3260
|
+
if (reset && report2.status !== "disabled") {
|
|
3261
|
+
process.stdout.write(`Budget task reset by the user's command. ${report2.summary}
|
|
3262
|
+
`);
|
|
3263
|
+
} else if (reset || report2.status === "failed" || report2.status === "error") {
|
|
3264
|
+
process.stdout.write(`${report2.summary}
|
|
3265
|
+
`);
|
|
3266
|
+
}
|
|
3267
|
+
return;
|
|
3268
|
+
}
|
|
3269
|
+
if (event !== "PreToolUse" && event !== "PostToolUse") throw new Error(`Unknown budget hook event: ${event}`);
|
|
3270
|
+
const name = text2(value, "tool_name");
|
|
3271
|
+
if (READ_TOOLS.has(name.split(/[.:]/).at(-1)?.toLowerCase() ?? "")) return;
|
|
3272
|
+
if (event === "PostToolUse") {
|
|
3273
|
+
const report2 = auditBudget({ cwd });
|
|
3274
|
+
if (report2.status === "failed") throw new Error(report2.summary);
|
|
3275
|
+
return;
|
|
3276
|
+
}
|
|
3277
|
+
if (!budgetConfigured(cwd)) return;
|
|
3278
|
+
const proposed = previewEdit(cwd, name, field(value, "tool_input") ?? field(value, "arguments") ?? {});
|
|
3279
|
+
if (!proposed) {
|
|
3280
|
+
beginTask({ cwd });
|
|
3281
|
+
return;
|
|
3282
|
+
}
|
|
3283
|
+
const report = checkBudget({ cwd, proposed });
|
|
3284
|
+
if (report.status === "failed") throw new Error(report.summary);
|
|
3285
|
+
} catch (error) {
|
|
3286
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}
|
|
3287
|
+
`);
|
|
3288
|
+
process.exitCode = 2;
|
|
3289
|
+
}
|
|
3290
|
+
}
|
|
3291
|
+
function stopBudget({ cwd, retry, session = "" }) {
|
|
3292
|
+
const report = auditBudget({ cwd });
|
|
3293
|
+
if (report.status === "failed") {
|
|
3294
|
+
const attempt = recordStopFailure(cwd, session);
|
|
3295
|
+
if (attempt > MAX_REPAIR_ATTEMPTS) {
|
|
3296
|
+
return {
|
|
3297
|
+
exitCode: 1,
|
|
3298
|
+
message: `Budget verification failed. Automatic repair stopped; this work is not within budget.
|
|
3299
|
+
${report.summary}`
|
|
3300
|
+
};
|
|
3301
|
+
}
|
|
3302
|
+
return {
|
|
3303
|
+
exitCode: 2,
|
|
3304
|
+
message: `Budget exceeded (repair ${attempt}/${MAX_REPAIR_ATTEMPTS}). Reduce the reported excess while keeping code readable and documentation concise.
|
|
3305
|
+
${report.summary}`
|
|
3306
|
+
};
|
|
3307
|
+
}
|
|
3308
|
+
clearStopFailures(cwd, session);
|
|
3309
|
+
if (report.status !== "error") return { exitCode: 0, message: "" };
|
|
3310
|
+
if (retry) return { exitCode: 1, message: report.summary };
|
|
3311
|
+
return {
|
|
3312
|
+
exitCode: 2,
|
|
3313
|
+
message: `${report.summary}
|
|
3314
|
+
Fix .draht/budgets.json if it is invalid; otherwise tell the user to start a new budget task.`
|
|
3315
|
+
};
|
|
3316
|
+
}
|
|
3317
|
+
|
|
3318
|
+
// packages/coding-agent/src/core/budgets/cli.ts
|
|
3319
|
+
var NEW_TASK_REFUSAL = "Only the user can start a new budget task: /budget new in draht, /draht:budget new in Claude Code, or $draht:budget new in Codex";
|
|
3320
|
+
function parseProposed(value) {
|
|
3321
|
+
if (value === void 0) return void 0;
|
|
3322
|
+
if (!Array.isArray(value)) throw new Error("Budget call proposed must be an array");
|
|
3323
|
+
return value.map((change) => {
|
|
3324
|
+
if (typeof change !== "object" || change === null || !("path" in change) || !("content" in change) || typeof change.path !== "string" || change.content !== null && typeof change.content !== "string") {
|
|
3325
|
+
throw new Error("Invalid budget call change");
|
|
3326
|
+
}
|
|
3327
|
+
return { path: change.path, content: change.content };
|
|
3328
|
+
});
|
|
3329
|
+
}
|
|
3330
|
+
function call(input) {
|
|
3331
|
+
const value = JSON.parse(input);
|
|
3332
|
+
if (typeof value !== "object" || value === null || !("method" in value) || !("request" in value)) {
|
|
3333
|
+
throw new Error("Invalid budget call");
|
|
3334
|
+
}
|
|
3335
|
+
const { method, request } = value;
|
|
3336
|
+
if (typeof request !== "object" || request === null || !("cwd" in request) || typeof request.cwd !== "string") {
|
|
3337
|
+
throw new Error("Invalid budget call request");
|
|
3338
|
+
}
|
|
3339
|
+
if ("newTask" in request) throw new Error(NEW_TASK_REFUSAL);
|
|
3340
|
+
const proposed = parseProposed("proposed" in request ? request.proposed : void 0);
|
|
3341
|
+
if (method === "beginTask") return beginTask({ cwd: request.cwd });
|
|
3342
|
+
if (method === "auditBudget") return auditBudget({ cwd: request.cwd });
|
|
3343
|
+
if (method === "checkBudget") return checkBudget({ cwd: request.cwd, proposed });
|
|
3344
|
+
throw new Error("Unknown budget call method");
|
|
3345
|
+
}
|
|
3346
|
+
if (typeof require !== "undefined" && require.main === module) {
|
|
3347
|
+
const [action = "status"] = process.argv.slice(2);
|
|
3348
|
+
try {
|
|
3349
|
+
if (action === "config") {
|
|
3350
|
+
const workspace = findBudgetWorkspace(process.cwd());
|
|
3351
|
+
if (!workspace) throw new Error("Budget configuration requires a Git working tree");
|
|
3352
|
+
const configPath = (0, import_node_path9.join)(workspace.root, ".draht", "budgets.json");
|
|
3353
|
+
const config = { configPath, exists: budgetPathExists(configPath), ...readBudgetConfig(configPath) };
|
|
3354
|
+
process.stdout.write(`${JSON.stringify(config, null, 2)}
|
|
3355
|
+
`);
|
|
3356
|
+
} else if (action === "status") {
|
|
3357
|
+
const result = auditBudget({ cwd: process.cwd() });
|
|
3358
|
+
(result.status === "error" ? process.stderr : process.stdout).write(`${result.summary}
|
|
3359
|
+
`);
|
|
3360
|
+
process.exitCode = result.status === "failed" || result.status === "error" ? 2 : 0;
|
|
3361
|
+
} else if (action === "call") {
|
|
3362
|
+
process.stdout.write(JSON.stringify(call((0, import_node_fs8.readFileSync)(0, "utf8"))));
|
|
3363
|
+
} else if (action === "new") {
|
|
3364
|
+
throw new Error(NEW_TASK_REFUSAL);
|
|
3365
|
+
} else {
|
|
3366
|
+
throw new Error("Usage: node budgets-runtime.cjs [config|status]");
|
|
3367
|
+
}
|
|
3368
|
+
} catch (error) {
|
|
3369
|
+
process.stderr.write(`${error instanceof Error ? error.message : String(error)}
|
|
3370
|
+
`);
|
|
3371
|
+
process.exitCode = 2;
|
|
3372
|
+
}
|
|
3373
|
+
}
|
|
3374
|
+
// Annotate the CommonJS export names for ESM import in node:
|
|
3375
|
+
0 && (module.exports = {
|
|
3376
|
+
auditBudget,
|
|
3377
|
+
beginTask,
|
|
3378
|
+
budgetConfigured,
|
|
3379
|
+
checkBudget,
|
|
3380
|
+
runHook,
|
|
3381
|
+
stopBudget
|
|
3382
|
+
});
|