bantamkit-mcp 0.29.2 → 0.31.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +83 -2
- package/assets/pricing/default.json +7 -0
- package/assets/schemas/shiftwork-checkpoint.json +12 -1
- package/assets/tools/memory_dream.json +20 -0
- package/assets/tools/repo_map.json +44 -0
- package/assets/tools/shiftwork_clock_out.json +1 -1
- package/assets/tools/token_ledger.json +40 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +135 -4
- package/dist/eventlog.js +6 -4
- package/dist/index.d.ts +5 -3
- package/dist/index.js +2 -1
- package/dist/mcp/identity.d.ts +80 -0
- package/dist/mcp/identity.js +466 -4
- package/dist/mcp/pyargs.d.ts +1 -1
- package/dist/mcp/pyargs.js +78 -0
- package/dist/mcp/server.d.ts +37 -6
- package/dist/mcp/server.js +209 -5
- package/dist/mcp/status.d.ts +54 -10
- package/dist/mcp/status.js +86 -10
- package/dist/memory/cli.js +1 -1
- package/dist/memory/component.d.ts +73 -2
- package/dist/memory/component.js +157 -5
- package/dist/memory/dream.d.ts +247 -0
- package/dist/memory/dream.js +903 -0
- package/dist/memory/store.d.ts +101 -1
- package/dist/memory/store.js +178 -4
- package/dist/pricing.d.ts +100 -0
- package/dist/pricing.js +357 -0
- package/dist/repomap.d.ts +247 -0
- package/dist/repomap.js +1040 -0
- package/dist/selfupdate.d.ts +320 -0
- package/dist/selfupdate.js +594 -0
- package/dist/shiftwork.d.ts +5 -3
- package/dist/shiftwork.js +82 -4
- package/dist/tokenledger.d.ts +91 -0
- package/dist/tokenledger.js +372 -0
- package/package.json +1 -1
package/dist/pricing.js
ADDED
|
@@ -0,0 +1,357 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Layer 4 (Policy/Profile): the price table, and the arithmetic that turns tokens into money.
|
|
3
|
+
*
|
|
4
|
+
* A port of `runtime-py/src/bantamkit/pricing.py`, arm for arm. The Python module is the
|
|
5
|
+
* reference and carries the full reasoning -- why the shipped table has NO rates, why the
|
|
6
|
+
* three properties are where they are, why money is an integer. Read it first; what follows
|
|
7
|
+
* here is only what this side does DIFFERENTLY, and every one of those is a mechanical
|
|
8
|
+
* consequence of the language rather than a decision.
|
|
9
|
+
*
|
|
10
|
+
* 1. `BigInt` FOR THE MULTIPLY, `number` ON THE WIRE. `count * rate` with both operands
|
|
11
|
+
* allowed up to 2**53-1 reaches 2**106, which a JS `number` rounds silently and Python
|
|
12
|
+
* does not. So the product, the half-up rounding and the running total are all `bigint`
|
|
13
|
+
* here, and only the final micro-USD -- bounded by the shared 2**53-1 ceiling, refused
|
|
14
|
+
* above it on BOTH sides with the same sentence -- becomes a `number`. Python needs none
|
|
15
|
+
* of this because its `int` is arbitrary precision; the ceiling exists so that the place
|
|
16
|
+
* where the two would part company is a refusal rather than a wrong answer.
|
|
17
|
+
*
|
|
18
|
+
* 2. `pyRepr` AND `cmpCodepoint`, NOT `JSON.stringify` AND `.sort()`. Every quoted name in
|
|
19
|
+
* a sentence goes through `pyRepr` because Python's `repr` is what the reference emits,
|
|
20
|
+
* and every "which offender is named first" goes through `cmpCodepoint` because
|
|
21
|
+
* `.sort()` is UTF-16 code-unit order and `sorted()` is code-point order. Both are
|
|
22
|
+
* already the port's shared primitives and both are already under conformance.
|
|
23
|
+
*
|
|
24
|
+
* 3. `JSON.parse` IS THE STRICTER READER AND PYTHON WAS BROUGHT TO IT. `json.loads` accepts
|
|
25
|
+
* the bare tokens `NaN` / `Infinity` / `-Infinity`; `JSON.parse` does not. The reference
|
|
26
|
+
* passes `parse_constant` to refuse them, so both sides answer
|
|
27
|
+
* `price table is not valid JSON: <path>` for the same bytes.
|
|
28
|
+
*
|
|
29
|
+
* 4. AN INTEGRAL FLOAT IS A COUNT. `JSON.parse('3000000.0')` is `3000000` and cannot be told
|
|
30
|
+
* from the integer; the reference therefore accepts an integral `float` and narrows it,
|
|
31
|
+
* rather than refusing a spelling this side cannot even see.
|
|
32
|
+
*
|
|
33
|
+
* `tools/conformance/suites/pricing.mjs` is the gate.
|
|
34
|
+
*/
|
|
35
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
36
|
+
import { join } from 'node:path';
|
|
37
|
+
import { assetsRoot } from './assets.js';
|
|
38
|
+
import { BantamError } from './errors.js';
|
|
39
|
+
import { cmpCodepoint, pyRepr } from './pysem.js';
|
|
40
|
+
export const PRICES_ENV = 'BANTAMKIT_PRICES';
|
|
41
|
+
export const PRICE_SCHEMA_VERSION = 1;
|
|
42
|
+
export const CURRENCY = 'USD';
|
|
43
|
+
export const RATE_UNIT = 'micro_usd_per_million_tokens';
|
|
44
|
+
/** A rate is micro-USD per this many tokens. */
|
|
45
|
+
export const TOKENS_PER_RATE_UNIT = 1000000;
|
|
46
|
+
/** Micro-USD in one USD. Also the rounding divisor, hence `HALF` below. */
|
|
47
|
+
export const MICROS_PER_UNIT = 1000000;
|
|
48
|
+
const MICROS_PER_UNIT_BIG = 1000000n;
|
|
49
|
+
const HALF_BIG = 500000n;
|
|
50
|
+
/** The largest integer both runtimes represent exactly. */
|
|
51
|
+
export const MAX_SAFE_INT = 2 ** 53 - 1;
|
|
52
|
+
const MAX_SAFE_INT_BIG = BigInt(Number.MAX_SAFE_INTEGER);
|
|
53
|
+
/**
|
|
54
|
+
* The four classes the host's own `usage` block distinguishes, in the order the ledger
|
|
55
|
+
* accumulates them. This tuple is the unit of pricing; see property 1 in the reference.
|
|
56
|
+
*/
|
|
57
|
+
export const TOKEN_CLASSES = [
|
|
58
|
+
'input_tokens',
|
|
59
|
+
'cache_creation_input_tokens',
|
|
60
|
+
'cache_read_input_tokens',
|
|
61
|
+
'output_tokens',
|
|
62
|
+
];
|
|
63
|
+
const CLASSES = TOKEN_CLASSES.join(', ');
|
|
64
|
+
const TABLE_KEYS = ['schema_version', 'currency', 'unit', 'note', 'rates'];
|
|
65
|
+
const RATE_META = ['recorded', 'source'];
|
|
66
|
+
/**
|
|
67
|
+
* The table on disk is malformed.
|
|
68
|
+
*
|
|
69
|
+
* This THROWS rather than returning `{unavailable}`, and the split is deliberate: a broken
|
|
70
|
+
* price table is an operator configuration fault that must stop and name itself, while an
|
|
71
|
+
* unpriced model is a normal, expected answer that a caller keeps working past.
|
|
72
|
+
*/
|
|
73
|
+
export class PriceTableError extends BantamError {
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* A non-negative integer both runtimes hold exactly, however JSON spelled it.
|
|
77
|
+
*
|
|
78
|
+
* `Number.isInteger` is false for `Infinity` and `NaN`, which is what makes `1e400` -- valid
|
|
79
|
+
* JSON here, `inf` after `json.loads` there -- refuse identically on both sides.
|
|
80
|
+
*/
|
|
81
|
+
function isCount(value) {
|
|
82
|
+
return typeof value === 'number' && Number.isInteger(value) && value >= 0 && value <= MAX_SAFE_INT;
|
|
83
|
+
}
|
|
84
|
+
/** The number behind an accepted count. Only ever called after `isCount`. `-0` normalises to `0`. */
|
|
85
|
+
function asCount(value) {
|
|
86
|
+
return value + 0;
|
|
87
|
+
}
|
|
88
|
+
/**
|
|
89
|
+
* Sorted by code point, so "which key is named first" is not a function of object order.
|
|
90
|
+
*
|
|
91
|
+
* `JSON.parse` hoists integer-like keys ahead of the rest and orders them numerically;
|
|
92
|
+
* `json.loads` preserves the file. Naming the code-point-least offender makes the two answer
|
|
93
|
+
* the same sentence for the same file.
|
|
94
|
+
*/
|
|
95
|
+
function sortedByCodePoint(names) {
|
|
96
|
+
return [...names].sort(cmpCodepoint);
|
|
97
|
+
}
|
|
98
|
+
/**
|
|
99
|
+
* `YYYY-MM-DD`, shape and range only -- deliberately not a calendar.
|
|
100
|
+
*
|
|
101
|
+
* A real calendar check would be `datetime.date` on the reference side, which this side has
|
|
102
|
+
* no equivalent of without writing a third leap-year rule. 2026-02-31 passes and that is a
|
|
103
|
+
* known, stated limit.
|
|
104
|
+
*/
|
|
105
|
+
function isDate(value) {
|
|
106
|
+
if (typeof value !== 'string' || value.length !== 10)
|
|
107
|
+
return false;
|
|
108
|
+
if (value[4] !== '-' || value[7] !== '-')
|
|
109
|
+
return false;
|
|
110
|
+
for (const part of [value.slice(0, 4), value.slice(5, 7), value.slice(8, 10)]) {
|
|
111
|
+
for (const ch of part) {
|
|
112
|
+
if (ch < '0' || ch > '9')
|
|
113
|
+
return false;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
const month = Number(value.slice(5, 7));
|
|
117
|
+
const day = Number(value.slice(8, 10));
|
|
118
|
+
return month >= 1 && month <= 12 && day >= 1 && day <= 31;
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* `$BANTAMKIT_PRICES` verbatim if set, else the shipped table in the asset pack.
|
|
122
|
+
*
|
|
123
|
+
* The override wins even when it is wrong -- same arm order and same reasoning as
|
|
124
|
+
* `assetsRoot()`: an override that points nowhere must fail naming itself rather than fall
|
|
125
|
+
* through to a table the operator did not ask for.
|
|
126
|
+
*/
|
|
127
|
+
export function priceTablePath() {
|
|
128
|
+
const env = process.env[PRICES_ENV];
|
|
129
|
+
if (env)
|
|
130
|
+
return env;
|
|
131
|
+
return join(assetsRoot(), 'pricing', 'default.json');
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Read and validate a price table. `path` is used VERBATIM in every message it appears in,
|
|
135
|
+
* never normalised, because `Path('./x')` is `x` on the reference side and `./x` here and the
|
|
136
|
+
* fault sentence would then differ over a path neither runtime chose.
|
|
137
|
+
*/
|
|
138
|
+
export function loadPriceTable(path) {
|
|
139
|
+
const label = path === undefined ? priceTablePath() : path;
|
|
140
|
+
if (!existsSync(label)) {
|
|
141
|
+
throw new PriceTableError(`price table not found: ${label}`);
|
|
142
|
+
}
|
|
143
|
+
let raw;
|
|
144
|
+
try {
|
|
145
|
+
// The decoder's own text is NOT quoted. `validate.mjs` covers the two decoders'
|
|
146
|
+
// sentences separately and they are not identical everywhere; quoting one here would
|
|
147
|
+
// make every malformed table a divergence, over bytes that name the same fault.
|
|
148
|
+
raw = decodePriceJson(readFileSync(label, 'utf8'));
|
|
149
|
+
}
|
|
150
|
+
catch {
|
|
151
|
+
throw new PriceTableError(`price table is not valid JSON: ${label}`);
|
|
152
|
+
}
|
|
153
|
+
return validatePriceTable(raw);
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* The decode both runtimes must agree on, in one place so the gate can reach it.
|
|
157
|
+
*
|
|
158
|
+
* This side is already the strict one -- `JSON.parse` rejects the bare `NaN` / `Infinity` /
|
|
159
|
+
* `-Infinity` tokens that `json.loads` accepts -- so the function is a plain `JSON.parse`
|
|
160
|
+
* and the reference is what had to move. It exists as a named arm anyway, because a rule
|
|
161
|
+
* that lives in only one of two runtimes is a rule the conformance harness cannot reach.
|
|
162
|
+
*/
|
|
163
|
+
export function decodePriceJson(text) {
|
|
164
|
+
return JSON.parse(text);
|
|
165
|
+
}
|
|
166
|
+
function isPlainObject(value) {
|
|
167
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* Every rule the table must satisfy, each with its own sentence.
|
|
171
|
+
*
|
|
172
|
+
* Returns a NORMALISED table: the top-level keys in a fixed order, rate names in code-point
|
|
173
|
+
* order, and every count narrowed. See the reference for the ONE thing normalisation cannot
|
|
174
|
+
* buy -- the iteration order of `rates` for a model name JavaScript reads as an array index.
|
|
175
|
+
*/
|
|
176
|
+
export function validatePriceTable(raw) {
|
|
177
|
+
if (!isPlainObject(raw)) {
|
|
178
|
+
throw new PriceTableError('price table must be a JSON object');
|
|
179
|
+
}
|
|
180
|
+
const unknown = sortedByCodePoint(Object.keys(raw).filter((k) => !TABLE_KEYS.includes(k)));
|
|
181
|
+
if (unknown.length > 0) {
|
|
182
|
+
throw new PriceTableError(`price table has unknown key ${pyRepr(unknown[0])}`);
|
|
183
|
+
}
|
|
184
|
+
// No parsed VALUE is ever interpolated into a sentence below -- only key names and
|
|
185
|
+
// integers this module computed. A parsed value would drag the two runtimes' number
|
|
186
|
+
// formatting into the message, which is a serialiser divergence wearing an error
|
|
187
|
+
// message's clothes.
|
|
188
|
+
const version = raw['schema_version'];
|
|
189
|
+
if (!isCount(version) || version !== PRICE_SCHEMA_VERSION) {
|
|
190
|
+
throw new PriceTableError('price table schema_version must be the integer 1');
|
|
191
|
+
}
|
|
192
|
+
if (raw['currency'] !== CURRENCY) {
|
|
193
|
+
throw new PriceTableError("price table currency must be the string 'USD'");
|
|
194
|
+
}
|
|
195
|
+
if (raw['unit'] !== RATE_UNIT) {
|
|
196
|
+
throw new PriceTableError("price table unit must be the string 'micro_usd_per_million_tokens'");
|
|
197
|
+
}
|
|
198
|
+
if ('note' in raw && typeof raw['note'] !== 'string') {
|
|
199
|
+
throw new PriceTableError('price table note must be a string');
|
|
200
|
+
}
|
|
201
|
+
const rates = raw['rates'];
|
|
202
|
+
if (!isPlainObject(rates)) {
|
|
203
|
+
throw new PriceTableError('price table rates must be a JSON object');
|
|
204
|
+
}
|
|
205
|
+
const built = {};
|
|
206
|
+
for (const model of sortedByCodePoint(Object.keys(rates))) {
|
|
207
|
+
built[model] = validateRate(model, rates[model]);
|
|
208
|
+
}
|
|
209
|
+
// Assigned in the reference's own key order -- `note`, when present, sits between `unit`
|
|
210
|
+
// and `rates` -- because insertion order is what `JSON.stringify` emits and the suite
|
|
211
|
+
// compares the serialised table.
|
|
212
|
+
const out = {};
|
|
213
|
+
out.schema_version = PRICE_SCHEMA_VERSION;
|
|
214
|
+
out.currency = CURRENCY;
|
|
215
|
+
out.unit = RATE_UNIT;
|
|
216
|
+
if ('note' in raw)
|
|
217
|
+
out.note = raw['note'];
|
|
218
|
+
out.rates = built;
|
|
219
|
+
return out;
|
|
220
|
+
}
|
|
221
|
+
function validateRate(model, rate) {
|
|
222
|
+
if (!isPlainObject(rate)) {
|
|
223
|
+
throw new PriceTableError(`price table rate ${pyRepr(model)} must be a JSON object`);
|
|
224
|
+
}
|
|
225
|
+
const known = [...RATE_META, ...TOKEN_CLASSES];
|
|
226
|
+
const unknown = sortedByCodePoint(Object.keys(rate).filter((k) => !known.includes(k)));
|
|
227
|
+
if (unknown.length > 0) {
|
|
228
|
+
throw new PriceTableError(`price table rate ${pyRepr(model)} has unknown key ${pyRepr(unknown[0])}`);
|
|
229
|
+
}
|
|
230
|
+
// Property 2, enforced where it cannot be skipped: at load, before any arithmetic.
|
|
231
|
+
if (!('recorded' in rate)) {
|
|
232
|
+
throw new PriceTableError(`price table rate ${pyRepr(model)} is missing 'recorded' -- a rate with no date is not a fact`);
|
|
233
|
+
}
|
|
234
|
+
if (!isDate(rate['recorded'])) {
|
|
235
|
+
throw new PriceTableError(`price table rate ${pyRepr(model)} has a 'recorded' that is not a YYYY-MM-DD date`);
|
|
236
|
+
}
|
|
237
|
+
if (!('source' in rate)) {
|
|
238
|
+
throw new PriceTableError(`price table rate ${pyRepr(model)} is missing 'source' -- a rate with no source cannot be re-derived`);
|
|
239
|
+
}
|
|
240
|
+
if (typeof rate['source'] !== 'string' || rate['source'] === '') {
|
|
241
|
+
throw new PriceTableError(`price table rate ${pyRepr(model)} has a 'source' that is not a non-empty string`);
|
|
242
|
+
}
|
|
243
|
+
for (const cls of TOKEN_CLASSES) {
|
|
244
|
+
if (cls in rate && !isCount(rate[cls])) {
|
|
245
|
+
throw new PriceTableError(`price table rate ${pyRepr(model)} prices ${pyRepr(cls)} with something that is not a ` +
|
|
246
|
+
`non-negative integer below 2**53`);
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
if (!TOKEN_CLASSES.some((cls) => cls in rate)) {
|
|
250
|
+
throw new PriceTableError(`price table rate ${pyRepr(model)} prices no token class at all`);
|
|
251
|
+
}
|
|
252
|
+
const narrowed = { ...rate };
|
|
253
|
+
for (const cls of TOKEN_CLASSES) {
|
|
254
|
+
if (cls in rate)
|
|
255
|
+
narrowed[cls] = asCount(rate[cls]);
|
|
256
|
+
}
|
|
257
|
+
return rateView(narrowed);
|
|
258
|
+
}
|
|
259
|
+
/** `build_identity`'s shape, on purpose: one key, and the key IS the refusal. */
|
|
260
|
+
function unavailable(reason) {
|
|
261
|
+
return { unavailable: reason };
|
|
262
|
+
}
|
|
263
|
+
/**
|
|
264
|
+
* Micro-USD as a fixed-6-decimal string, built from integers only.
|
|
265
|
+
*
|
|
266
|
+
* Non-negative input only; every producer in this module is a sum of non-negative products.
|
|
267
|
+
*/
|
|
268
|
+
export function formatMicros(micros) {
|
|
269
|
+
const n = BigInt(micros);
|
|
270
|
+
return `${n / MICROS_PER_UNIT_BIG}.${String(n % MICROS_PER_UNIT_BIG).padStart(6, '0')}`;
|
|
271
|
+
}
|
|
272
|
+
/**
|
|
273
|
+
* The cost of one `usage` block under one model's rate, or a named refusal.
|
|
274
|
+
*
|
|
275
|
+
* `table` must be a table `validatePriceTable` returned.
|
|
276
|
+
*
|
|
277
|
+
* THE ORDER OF THE CHECKS IS PART OF THE CONTRACT. The usage is validated BEFORE the model is
|
|
278
|
+
* looked up, so a caller who passed a malformed usage is told that, rather than being sent to
|
|
279
|
+
* edit a price table over a bug in the call.
|
|
280
|
+
*
|
|
281
|
+
* THE USAGE MUST CARRY ALL FOUR CLASSES AND NOTHING ELSE. See the reference: a token class
|
|
282
|
+
* silently ignored is money silently dropped, and a class silently defaulted to zero is money
|
|
283
|
+
* silently invented.
|
|
284
|
+
*/
|
|
285
|
+
export function priceTokens(table, model, usage) {
|
|
286
|
+
if (typeof model !== 'string') {
|
|
287
|
+
return unavailable('model must be a string');
|
|
288
|
+
}
|
|
289
|
+
if (!isPlainObject(usage)) {
|
|
290
|
+
return unavailable('usage must be a mapping of token class to count');
|
|
291
|
+
}
|
|
292
|
+
const extra = sortedByCodePoint(Object.keys(usage).filter((k) => !TOKEN_CLASSES.includes(k)));
|
|
293
|
+
if (extra.length > 0) {
|
|
294
|
+
return unavailable(`usage has unknown token class ${pyRepr(extra[0])} -- the price table prices exactly ${CLASSES}`);
|
|
295
|
+
}
|
|
296
|
+
for (const cls of TOKEN_CLASSES) {
|
|
297
|
+
if (!(cls in usage)) {
|
|
298
|
+
return unavailable(`usage is missing token class ${pyRepr(cls)} -- all four of ${CLASSES} must be ` +
|
|
299
|
+
`given, because a missing count is not a zero`);
|
|
300
|
+
}
|
|
301
|
+
if (!isCount(usage[cls])) {
|
|
302
|
+
return unavailable(`usage token class ${pyRepr(cls)} is not a non-negative integer below 2**53`);
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
const rate = table.rates?.[model];
|
|
306
|
+
if (rate === undefined) {
|
|
307
|
+
return unavailable(`no rate recorded for model ${pyRepr(model)} -- add one with its date and source, or ` +
|
|
308
|
+
`point ${PRICES_ENV} at a table that has it`);
|
|
309
|
+
}
|
|
310
|
+
const breakdown = {};
|
|
311
|
+
let total = 0n;
|
|
312
|
+
for (const cls of TOKEN_CLASSES) {
|
|
313
|
+
const count = asCount(usage[cls]);
|
|
314
|
+
let micros;
|
|
315
|
+
if (!(cls in rate)) {
|
|
316
|
+
// Zero tokens cost zero under ANY rate, so this arm invents nothing. Any other count
|
|
317
|
+
// with no rate is property 3 and is refused rather than silently dropped.
|
|
318
|
+
if (count !== 0) {
|
|
319
|
+
return unavailable(`rate for model ${pyRepr(model)} does not price ${pyRepr(cls)}, and ${count} such ` +
|
|
320
|
+
`tokens were used -- a missing rate is not a zero`);
|
|
321
|
+
}
|
|
322
|
+
micros = 0n;
|
|
323
|
+
}
|
|
324
|
+
else {
|
|
325
|
+
micros = (BigInt(count) * BigInt(rate[cls]) + HALF_BIG) / MICROS_PER_UNIT_BIG;
|
|
326
|
+
}
|
|
327
|
+
breakdown[cls] = Number(micros);
|
|
328
|
+
total += micros;
|
|
329
|
+
}
|
|
330
|
+
if (total > MAX_SAFE_INT_BIG) {
|
|
331
|
+
return unavailable(`cost for model ${pyRepr(model)} exceeds 2**53-1 micro-USD, which is the largest ` +
|
|
332
|
+
`integer both runtimes represent exactly`);
|
|
333
|
+
}
|
|
334
|
+
const micros = Number(total);
|
|
335
|
+
return {
|
|
336
|
+
model,
|
|
337
|
+
currency: CURRENCY,
|
|
338
|
+
micros,
|
|
339
|
+
amount: formatMicros(micros),
|
|
340
|
+
breakdown,
|
|
341
|
+
rate: rateView(rate),
|
|
342
|
+
};
|
|
343
|
+
}
|
|
344
|
+
/**
|
|
345
|
+
* The rate that produced a cost, echoed in a fixed key order so the answer is re-derivable.
|
|
346
|
+
*
|
|
347
|
+
* Fixed order rather than the file's own order, because the file's order is not stable across
|
|
348
|
+
* the two JSON parsers and this object is compared byte for byte by the conformance suite.
|
|
349
|
+
*/
|
|
350
|
+
function rateView(rate) {
|
|
351
|
+
const view = { recorded: rate['recorded'], source: rate['source'] };
|
|
352
|
+
for (const cls of TOKEN_CLASSES) {
|
|
353
|
+
if (cls in rate)
|
|
354
|
+
view[cls] = rate[cls];
|
|
355
|
+
}
|
|
356
|
+
return view;
|
|
357
|
+
}
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
export declare const DAMPING = 0.2;
|
|
2
|
+
export declare const ITERATIONS = 30;
|
|
3
|
+
export declare const MIN_REFERENCE_LENGTH = 3;
|
|
4
|
+
export declare const REFERENCE_DF_MAX_NUM = 1;
|
|
5
|
+
export declare const REFERENCE_DF_MAX_DEN = 8;
|
|
6
|
+
export declare const MAX_DEFINITIONS_PER_FILE = 4;
|
|
7
|
+
export declare const DEFAULT_BUDGET = 4000;
|
|
8
|
+
export declare const SCORE_SCALE = 1000000000;
|
|
9
|
+
export declare const MAX_FILE_BYTES = 1048576;
|
|
10
|
+
export declare const SKIP_DIRS: readonly string[];
|
|
11
|
+
/**
|
|
12
|
+
* Suffix -> dialect. Two dialects only: `python` and `ecma`. TypeScript and JavaScript
|
|
13
|
+
* share a scanner because their DECLARATION forms are the same set; the extra TypeScript
|
|
14
|
+
* keywords (`interface`, `type`, `enum`) are simply absent from a `.js` file.
|
|
15
|
+
*/
|
|
16
|
+
export declare const LANGUAGES: Readonly<Record<string, string>>;
|
|
17
|
+
export declare const OMIT_UNKNOWN_LANGUAGE = "unknown-language";
|
|
18
|
+
export declare const OMIT_UNREADABLE = "unreadable-bytes";
|
|
19
|
+
export declare const OMIT_SIZE_CAP = "size-cap";
|
|
20
|
+
export declare const OMIT_NO_DEFINITIONS = "no-definitions";
|
|
21
|
+
export declare const OMIT_UNREACHABLE = "unreachable";
|
|
22
|
+
export declare const OMIT_PER_FILE_CAP = "per-file-cap";
|
|
23
|
+
export declare const OMIT_BUDGET = "budget";
|
|
24
|
+
/**
|
|
25
|
+
* Fixed render order for the footer, so two runs of the same map cannot disagree about the
|
|
26
|
+
* order of their disclosures.
|
|
27
|
+
*/
|
|
28
|
+
export declare const OMISSION_ORDER: readonly string[];
|
|
29
|
+
/**
|
|
30
|
+
* One definition, at the line it was declared on. `line` is 1-based and counts lines of the
|
|
31
|
+
* ORIGINAL file, not of the comment-stripped view: a caller that jumps to it must land
|
|
32
|
+
* where the reader would.
|
|
33
|
+
*/
|
|
34
|
+
export interface Definition {
|
|
35
|
+
readonly path: string;
|
|
36
|
+
readonly line: number;
|
|
37
|
+
readonly kind: string;
|
|
38
|
+
readonly name: string;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* Something the tree holds that the map does not carry, as a COUNT and the fact behind it —
|
|
42
|
+
* `docread`'s `Omission` discipline, deliberately re-stated here rather than imported,
|
|
43
|
+
* because these subjects are this module's and a shared vocabulary would make a change to
|
|
44
|
+
* one reader a change to the other.
|
|
45
|
+
*
|
|
46
|
+
* Never an adjective. `subject` is one of the `OMIT_*` tokens; a renderer that meets a token
|
|
47
|
+
* it does not know must still print the count. A silent skip is how a reader lies about its
|
|
48
|
+
* coverage.
|
|
49
|
+
*/
|
|
50
|
+
export interface Omission {
|
|
51
|
+
readonly subject: string;
|
|
52
|
+
readonly count: number;
|
|
53
|
+
readonly size: number;
|
|
54
|
+
readonly facts: readonly (readonly [string, number])[];
|
|
55
|
+
}
|
|
56
|
+
export interface RankedFile {
|
|
57
|
+
readonly path: string;
|
|
58
|
+
readonly rankUnits: number;
|
|
59
|
+
readonly definitions: readonly Definition[];
|
|
60
|
+
/**
|
|
61
|
+
* The raw double. Reported for a caller that wants it; NEVER rendered, and never the sort
|
|
62
|
+
* key — see trap (8).
|
|
63
|
+
*/
|
|
64
|
+
readonly score: number;
|
|
65
|
+
}
|
|
66
|
+
export interface RepoMap {
|
|
67
|
+
readonly text: string;
|
|
68
|
+
readonly ranked: readonly RankedFile[];
|
|
69
|
+
readonly omissions: readonly Omission[];
|
|
70
|
+
readonly focus: readonly string[];
|
|
71
|
+
readonly nodes: number;
|
|
72
|
+
readonly edges: number;
|
|
73
|
+
readonly definitions: number;
|
|
74
|
+
readonly damping: number;
|
|
75
|
+
readonly iterations: number;
|
|
76
|
+
readonly budget: number;
|
|
77
|
+
readonly perFile: number;
|
|
78
|
+
readonly listingBytes: number;
|
|
79
|
+
readonly filesRendered: number;
|
|
80
|
+
readonly definitionsRendered: number;
|
|
81
|
+
}
|
|
82
|
+
export interface RepoMapOptions {
|
|
83
|
+
readonly focus?: readonly string[];
|
|
84
|
+
readonly budget?: number;
|
|
85
|
+
readonly files?: readonly string[] | null;
|
|
86
|
+
readonly damping?: number;
|
|
87
|
+
readonly iterations?: number;
|
|
88
|
+
readonly perFile?: number;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* The dialect for a path, or `null` when this scanner has no reader for it.
|
|
92
|
+
*
|
|
93
|
+
* Dispatch is on SUFFIX, and that is a deliberate departure from `docread`, which sniffs
|
|
94
|
+
* because a suffix is measurably a lie about a document's container. A source file is
|
|
95
|
+
* different: `.py` and `.ts` name a GRAMMAR, and there is no magic number for a grammar. A
|
|
96
|
+
* file whose suffix is unknown is reported, never guessed at.
|
|
97
|
+
*/
|
|
98
|
+
export declare function languageOf(path: string): string | null;
|
|
99
|
+
/**
|
|
100
|
+
* The file's lines with comments and docstrings blanked, one output line per input line so
|
|
101
|
+
* line numbers survive.
|
|
102
|
+
*
|
|
103
|
+
* Line-oriented and stateful across lines, because both traps that matter span lines: a
|
|
104
|
+
* Python triple-quoted docstring and an ECMAScript block comment. What is removed:
|
|
105
|
+
*
|
|
106
|
+
* - `python` — `'''`/`"""` blocks (the delimiter that opens is the only one that can close),
|
|
107
|
+
* and everything from an unquoted `#` to end of line.
|
|
108
|
+
* - `ecma` — block comments, and everything from `//` to end of line.
|
|
109
|
+
*
|
|
110
|
+
* What is NOT removed, and is declared rather than hidden: the contents of a SINGLE-line
|
|
111
|
+
* string literal. Stripping those would lose the real references inside an f-string, and
|
|
112
|
+
* keeping them costs a definition-shaped line inside a one-line string being read as a
|
|
113
|
+
* definition. Backslash escapes are NOT honoured: a `"""` inside a string toggles the state
|
|
114
|
+
* machine. Both runtimes get that wrong identically, which is the requirement.
|
|
115
|
+
*/
|
|
116
|
+
export declare function stripNoncode(text: string, language: string): string[];
|
|
117
|
+
/**
|
|
118
|
+
* Every definition in `text`, in source order.
|
|
119
|
+
*
|
|
120
|
+
* `python` — `def` and `class` at ANY indentation, so a method counts. Methods are the
|
|
121
|
+
* signal that made the difference in measurement: a focus on `memory/dream.py` reaches
|
|
122
|
+
* `memory/store.py` through `_write_fact`, `_fact_path` and `_check_index_budget`, none of
|
|
123
|
+
* which is top-level. Plus a bare `NAME =` or `NAME: T =` at column 0, which is how this
|
|
124
|
+
* repository declares its constants.
|
|
125
|
+
*
|
|
126
|
+
* `ecma` — `function class interface type const let var enum` at COLUMN 0 only, optionally
|
|
127
|
+
* behind `export` and `default` and `async`. Restricting to column 0 was the second-largest
|
|
128
|
+
* measured improvement in this module (recall@10 0.361 -> 0.410 on its own): an indented
|
|
129
|
+
* `const` is a local, and `runtime-ts/src/docread.ts` was contributing 452 of them.
|
|
130
|
+
*
|
|
131
|
+
* The asymmetry is real and is not an oversight: an ECMAScript class member cannot be told
|
|
132
|
+
* from a call at the same indentation without tracking braces, and a scanner that guessed
|
|
133
|
+
* would invent definitions. Python's `def` keyword makes the same question free.
|
|
134
|
+
*/
|
|
135
|
+
export declare function scanDefinitions(text: string, language: string, path?: string): Definition[];
|
|
136
|
+
/**
|
|
137
|
+
* Every identifier of at least `MIN_REFERENCE_LENGTH` characters outside a comment.
|
|
138
|
+
*
|
|
139
|
+
* A SET, not a count. The edge weight below is "how many DISTINCT names of B does A
|
|
140
|
+
* mention", so a file that repeats one name a hundred times does not out-weigh a file that
|
|
141
|
+
* mentions ten. That is the choice the ambiguity filter and `MIN_REFERENCE_LENGTH` were
|
|
142
|
+
* tuned against and it keeps every weight an exact integer.
|
|
143
|
+
*/
|
|
144
|
+
export declare function referenceNames(text: string, language: string): Set<string>;
|
|
145
|
+
/**
|
|
146
|
+
* Every path under `root`, relative, POSIX, code-point sorted.
|
|
147
|
+
*
|
|
148
|
+
* DIRECTORY SYMLINKS ARE NOT FOLLOWED, AND ON THIS SIDE THAT IS REAL LOGIC RATHER THAN A
|
|
149
|
+
* FLAG. `os.walk(followlinks=False)` gives CPython the whole rule for free, and J45-9's
|
|
150
|
+
* mutation 24 — delete the explicit `islink` filter — SURVIVED there for exactly that
|
|
151
|
+
* reason. Node has no equivalent, and the Dirent that looks like the answer is the wrong
|
|
152
|
+
* one: `readdirSync(..., {withFileTypes: true})` reports from **lstat**, so a symlink to a
|
|
153
|
+
* directory answers `isDirectory() === false` and a naive port hands it to the scanner as a
|
|
154
|
+
* source FILE. CPython does the opposite — `os.scandir`'s `entry.is_dir()` FOLLOWS, so the
|
|
155
|
+
* link lands in `dirs`, and `followlinks=False` then declines to descend, so it appears in
|
|
156
|
+
* the output neither as a file nor as a directory.
|
|
157
|
+
*
|
|
158
|
+
* Measured on node v25.2.1 and CPython 3.12.13 over one tree holding `real/`,
|
|
159
|
+
* `linkdir -> real`, `linkfile.py -> plain.py` and a dangling `dangling.py`: CPython's walk
|
|
160
|
+
* yields `dirs=['linkdir', 'real'] files=['dangling.py', 'linkfile.py', 'plain.py']` and
|
|
161
|
+
* `walk_sources` returns `['dangling.py', 'linkfile.py', 'plain.py', 'real/a.py']`. So the
|
|
162
|
+
* three cases are separated by a FOLLOW-`stat`, not by the Dirent:
|
|
163
|
+
*
|
|
164
|
+
* - symlink whose target is a directory -> a directory, and not descended into: gone.
|
|
165
|
+
* - symlink whose target is a file -> a file: kept.
|
|
166
|
+
* - dangling symlink -> `stat` throws, CPython's `is_dir()` raises and
|
|
167
|
+
* is caught as False: a file, kept, and it will
|
|
168
|
+
* be counted as `unreadable-bytes` downstream.
|
|
169
|
+
*
|
|
170
|
+
* Two reasons the rule exists and both are measured elsewhere in this repository: a symlink
|
|
171
|
+
* cycle makes the walk non-terminating, and a dangling directory symlink is one error shape
|
|
172
|
+
* on POSIX and two on Windows (`reference-windows-dangling-symlink-two-shapes`).
|
|
173
|
+
*
|
|
174
|
+
* A directory whose listing throws is skipped whole, silently, because that is what
|
|
175
|
+
* `os.walk`'s default `onerror=None` does.
|
|
176
|
+
*/
|
|
177
|
+
export declare function walkSources(root: string): string[];
|
|
178
|
+
/**
|
|
179
|
+
* `source -> {target: weight}`, weight being the number of surviving names the source
|
|
180
|
+
* mentions that ONLY the target defines.
|
|
181
|
+
*
|
|
182
|
+
* Weights are exact integers, so `out[i]` in `pagerank` is an exact integer too and the only
|
|
183
|
+
* inexact step in the whole pass is the one division.
|
|
184
|
+
*/
|
|
185
|
+
export declare function buildGraph(definitions: Map<string, readonly Definition[]>, references: Map<string, ReadonlySet<string>>): Map<string, Map<string, number>>;
|
|
186
|
+
/**
|
|
187
|
+
* At most `limit` of a file's definitions, chosen by kind then reach, returned in SOURCE
|
|
188
|
+
* order.
|
|
189
|
+
*
|
|
190
|
+
* The key is `(kind rank, -reference count, line, name)`, and every part of it is an integer
|
|
191
|
+
* or a code-point comparison, so the choice is TOTAL — which is why this port does not lean
|
|
192
|
+
* on `Array.prototype.sort` being stable. `references` is the RAW count from `nameIndex`;
|
|
193
|
+
* see there for why it is not the filtered one.
|
|
194
|
+
*
|
|
195
|
+
* **This ordering is argued, not measured, and the difference matters.** Every other
|
|
196
|
+
* constant in this module beat a named alternative on a yardstick. This one cannot: the
|
|
197
|
+
* yardstick is "does the focus file's own import appear in the rendered map", which is a
|
|
198
|
+
* question about which FILES are listed, and selection only decides which lines appear UNDER
|
|
199
|
+
* a file already listed. Measured, it moves that number by zero. So it rests on a stated
|
|
200
|
+
* principle instead — name the file's subject before its details — and a later unit with a
|
|
201
|
+
* definition-level yardstick is entitled to overturn it. What it is NOT is source order,
|
|
202
|
+
* which put `VALID_TYPES`, `DURABLE_TYPES`, `NAME_RE` and `DUPLICATE_JACCARD` at the head of
|
|
203
|
+
* `memory/store.py`'s block.
|
|
204
|
+
*
|
|
205
|
+
* The selected definitions are re-sorted into SOURCE order for rendering: a block whose line
|
|
206
|
+
* numbers ran backwards would be harder to read for no gain, and the order a block is chosen
|
|
207
|
+
* in is not the order it is read in.
|
|
208
|
+
*/
|
|
209
|
+
export declare function selectDefinitions(definitions: readonly Definition[], references: Map<string, number>, limit: number): readonly Definition[];
|
|
210
|
+
/**
|
|
211
|
+
* Personalised PageRank, bit-identical to `repomap.py` by construction.
|
|
212
|
+
*
|
|
213
|
+
* The port contract, in the order the loop performs it:
|
|
214
|
+
*
|
|
215
|
+
* 1. `nodes` is already sorted; index by position in it.
|
|
216
|
+
* 2. `out[i]` is the exact integer sum of node i's out-weights.
|
|
217
|
+
* 3. Each node's out-edges are visited in sorted TARGET order.
|
|
218
|
+
* 4. The personalisation vector is `1/len(personal)` on each focus node and 0 elsewhere;
|
|
219
|
+
* with no focus it is `1/n` on every node. The INITIAL vector is that same vector.
|
|
220
|
+
* 5. Per round: zero `next`; walk sources in node order; a node with no out-edges adds its
|
|
221
|
+
* whole score to `dangling`; otherwise each out-edge adds `score[i] * weight / out[i]`,
|
|
222
|
+
* evaluated strictly left to right.
|
|
223
|
+
* 6. `tele = damping * dangling + (1.0 - damping)`, then
|
|
224
|
+
* `next[i] = next[i] * damping + tele * personal[i]`.
|
|
225
|
+
* 7. Exactly `iterations` rounds. No early stop, no convergence test — a branch on a float
|
|
226
|
+
* is one more thing for a port to get wrong, and the contraction bound (the linear part
|
|
227
|
+
* is `damping * column-stochastic`, so the residual after 30 rounds is under
|
|
228
|
+
* `2 * 0.2 ** 30` = 2.1e-21) makes the test unnecessary.
|
|
229
|
+
*
|
|
230
|
+
* THERE IS NO `reduce` HERE AND THERE MUST NOT BE ONE: every accumulation is a written-out
|
|
231
|
+
* loop in the same order as CPython's, because CPython's `sum()` is Neumaier-compensated and
|
|
232
|
+
* nothing in JavaScript is (trap 7). The Python side is held to the same rule by
|
|
233
|
+
* `test_no_float_summation_or_rounding_in_the_source`, which reads that function's body.
|
|
234
|
+
*/
|
|
235
|
+
export declare function pagerank(nodes: readonly string[], edges: Map<string, Map<string, number>>, personal: readonly string[], damping?: number, iterations?: number): number[];
|
|
236
|
+
/**
|
|
237
|
+
* Scan `root`, rank its files against `focus`, and render a listing within `budget`.
|
|
238
|
+
*
|
|
239
|
+
* `focus` is the current unit's files, relative to `root` and POSIX-separated; a focus entry
|
|
240
|
+
* that is not a scanned source is ignored for the personalisation vector but is still
|
|
241
|
+
* excluded from the listing, because a caller naming a file has it open already. An empty
|
|
242
|
+
* focus gives the uniform vector — plain centrality over the whole tree.
|
|
243
|
+
*
|
|
244
|
+
* `budget` is UTF-8 BYTES of the listing. See the header: this runtime has no model
|
|
245
|
+
* tokenizer and will not pretend to one.
|
|
246
|
+
*/
|
|
247
|
+
export declare function repoMap(root: string, options?: RepoMapOptions): RepoMap;
|