@giovannijecha/jecode 0.8.6 → 0.8.7

package/dist/context/manager.js

@@ -0,0 +1,67 @@
+// One automatic-compaction lifecycle, independent of terminal and persistence.
+import { automaticCompactionKey } from "./automatic.js";
+import { compactContext } from "./compactor.js";
+import { isContextOverflow } from "./policy.js";
+export function contextManager(options) {
+    let anchor;
+    return {
+        get anchor() {
+            return anchor;
+        },
+        async compact(canonical, projected, request) {
+            const force = request.reason === "overflow";
+            if (force && (request.error === undefined || !isContextOverflow(request.error)))
+                return undefined;
+            if (!force && request.inputTokens < request.policy.triggerTokens)
+                return undefined;
+            const nodeId = options.nodeId();
+            const attempt = {
+                key: automaticCompactionKey(options.provider.id, options.model, nodeId, canonical.length),
+                scope: `${options.provider.id}\0${options.model}\0${options.identity.conversationId}\0` +
+                    `${request.policy.windowTokens}\0${request.policy.triggerTokens}`,
+                reason: request.reason,
+                inputTokens: request.inputTokens,
+                retryGrowthTokens: Math.max(1_024, Math.min(request.policy.minimumPrefixTokens, request.policy.requestLimitTokens - 256 - request.inputTokens)),
+            };
+            if (!options.gate.allows(attempt))
+                return undefined;
+            let started = false;
+            const result = await compactContext({
+                reason: request.reason,
+                onDiagnostic: options.onDiagnostic,
+                provider: options.provider,
+                model: options.model,
+                effort: options.effort,
+                context: projected,
+                turn: canonical.slice(options.historyStart),
+                nodeId,
+                coveredMessages: anchor?.messageCount ?? 0,
+                lastInputTokens: 0,
+                estimatedInputTokens: request.inputTokens,
+                force,
+                policy: request.policy,
+                signal: options.signal,
+                requestEnvelope: {
+                    system: options.system,
+                    tools: options.tools,
+                    maxOutputTokens: options.maxOutputTokens,
+                },
+                requestIdentity: options.identity,
+                onBegin() {
+                    started = true;
+                    options.onStatus(true);
+                },
+                onEnd() { options.onStatus(false); },
+                onUsage: options.onUsage,
+            });
+            if (result === undefined) {
+                if (started && options.signal?.aborted !== true)
+                    options.gate.failed(attempt);
+                return undefined;
+            }
+            options.gate.succeeded(attempt);
+            anchor = result.anchor;
+            return result.messages;
+        },
+    };
+}

package/dist/context/manual.js

@@ -7,9 +7,8 @@ import { recordAuxiliaryUsage } from "../usage.js";
 import { toolSpecs } from "../tools/index.js";
 import { resolveContextPolicy } from "./capacity.js";
 import { compactContext } from "./compactor.js";
-import { estimateRequestInputTokensResponsive } from "./budget.js";
-import { estimateTokensResponsive, planCompaction } from "./policy.js";
-import { projectToolResultsNewest, toolResultProjectionBudget } from "./request-projection.js";
+import { measureInput, messageCounter } from "./measurement.js";
+import { planCompaction } from "./policy.js";
 import { requestIdentityForSession } from "../request-identity.js";
 const MIN_PREFIX_TOKENS = 512;
 export async function compactSession(session, options = {}) {
@@ -29,9 +28,11 @@ export async function compactSession(session, options = {}) {
         onStatus: (status) => options.onStatus?.(status),
     });
     const specs = toolSpecs(session.tools);
-    const estimatedInputTokens = await estimateRequestInputTokensResponsive({
+    const estimatedInputTokens = await measureInput(session.provider, {
+        model: session.model,
+        effort: session.config.effort,
         system: session.system,
-        messages: projectToolResultsNewest(context, toolResultProjectionBudget(policy)).messages,
+        messages: context,
         tools: specs,
     }, options.signal);
     if (estimatedInputTokens < MIN_PREFIX_TOKENS) {
@@ -41,9 +42,10 @@ export async function compactSession(session, options = {}) {
     const coveredMessages = active.context?.throughNodeId === active.id
         ? active.context.messageCount
         : 0;
-    const plan = await planCompaction(context, active.messages, coveredMessages, session.usage.lastInputTokens, true, policy, estimatedInputTokens, options.signal);
+    const countMessages = await messageCounter(session.provider, session.model, session.config.effort, options.signal);
+    const plan = await planCompaction(context, active.messages, coveredMessages, 0, true, policy, estimatedInputTokens, options.signal, countMessages);
     if (plan === undefined ||
-        await estimateTokensResponsive(plan.prefix, options.signal) < MIN_PREFIX_TOKENS) {
+        await countMessages(plan.prefix, options.signal) < MIN_PREFIX_TOKENS) {
         options.onStatus?.();
         return "unchanged";
     }
@@ -55,7 +57,7 @@ export async function compactSession(session, options = {}) {
         turn: active.messages,
         nodeId: active.id,
         coveredMessages,
-        lastInputTokens: session.usage.lastInputTokens,
+        lastInputTokens: 0,
         estimatedInputTokens,
         precomputedPlan: plan,
         signal: options.signal,
@@ -70,11 +72,10 @@ export async function compactSession(session, options = {}) {
         requestIdentity: requestIdentityForSession(session),
         onBegin: () => options.onStatus?.("Compacting"),
         onEnd: () => options.onStatus?.(),
+        onUsage: (usage) => recordAuxiliaryUsage(session.usage, usage),
     });
     if (result === undefined)
         throw new Error("context could not be compacted");
-    if (result.usage !== undefined)
-        recordAuxiliaryUsage(session.usage, result.usage);
     const next = session.conversation.commit({
         nodeId: active.id,
         parentId: active.parentId,

package/dist/context/measurement.js

@@ -0,0 +1,75 @@
+// Local input estimates anchored only to an unchanged, provider-measured prefix.
+import { createHash } from "node:crypto";
+import { estimateRequestInputTokensResponsive } from "./budget.js";
+/** A response's usage is evidence about that exact request, never later history. */
+export function inputMeter(provider) {
+    let baseline;
+    let epoch = 0;
+    return {
+        async measure(input, signal) {
+            const measuredEpoch = epoch;
+            const estimatedTokens = await measureInput(provider, input, signal);
+            const envelope = fingerprint({
+                provider: provider.id,
+                model: input.model,
+                effort: input.effort,
+                system: input.system,
+                tools: input.tools,
+            });
+            const messages = [];
+            for (const message of input.messages) {
+                signal?.throwIfAborted();
+                // Including raw and usage is deliberately conservative: even a change
+                // to an opaque reserve must invalidate the old provider observation.
+                messages.push(fingerprint(message));
+                await new Promise((resolve) => setImmediate(resolve));
+            }
+            const previous = baseline?.measurement;
+            const anchored = measuredEpoch === epoch && previous !== undefined && previous.envelope === envelope &&
+                previous.messages.length <= messages.length &&
+                previous.messages.every((value, index) => value === messages[index]);
+            const inputTokens = anchored && baseline !== undefined
+                ? baseline.reported + Math.max(0, estimatedTokens - baseline.measurement.estimatedTokens)
+                : estimatedTokens;
+            if (!anchored && measuredEpoch === epoch)
+                baseline = undefined;
+            return Object.freeze({
+                epoch: measuredEpoch,
+                inputTokens,
+                estimatedTokens,
+                source: anchored ? "provider-prefix" : "estimate",
+                envelope,
+                messages: Object.freeze(messages),
+            });
+        },
+        observe(measurement, reported) {
+            if (measurement.epoch !== epoch || reported === undefined || !Number.isSafeInteger(reported) || reported <= 0)
+                return;
+            baseline = { measurement, reported };
+        },
+        reset() {
+            epoch++;
+            baseline = undefined;
+        },
+    };
+}
+export async function measureInput(provider, input, signal) {
+    signal?.throwIfAborted();
+    const tokens = provider.measureInput === undefined
+        ? await estimateRequestInputTokensResponsive(input, signal)
+        : await provider.measureInput(input, signal);
+    signal?.throwIfAborted();
+    if (!Number.isSafeInteger(tokens) || tokens <= 0) {
+        throw new Error("provider returned an invalid input token estimate");
+    }
+    return tokens;
+}
+/** Retention must include opaque replay reserves, excluding the fixed envelope. */
+export async function messageCounter(provider, model, effort, signal) {
+    const input = { model, effort, system: "", messages: [], tools: [] };
+    const overhead = await measureInput(provider, input, signal);
+    return async (messages, currentSignal = signal) => Math.max(1, await measureInput(provider, { ...input, messages: [...messages] }, currentSignal) - overhead);
+}
+function fingerprint(value) {
+    return createHash("sha256").update(JSON.stringify(value)).digest("hex");
+}

package/dist/context/policy.js

@@ -6,13 +6,13 @@ export const MAX_COMPACTION_PERCENT = 95;
 export const FALLBACK_CONTEXT_WINDOW_TOKENS = 200_000;
 export const REQUEST_ESTIMATE_HEADROOM_PERCENT = 5;
 export const MIN_REQUEST_OUTPUT_TOKENS = 256;
-export async function planCompaction(context, turn, coveredMessages, lastInputTokens, force, policy, estimatedInputTokens, signal) {
+export async function planCompaction(context, turn, coveredMessages, lastInputTokens, force, policy, estimatedInputTokens, signal, countMessages = estimateTokensResponsive) {
     if (!validPolicy(policy) || coveredMessages < 0 || coveredMessages > turn.length)
         return undefined;
     if (estimatedInputTokens !== undefined &&
         (!Number.isSafeInteger(estimatedInputTokens) || estimatedInputTokens <= 0))
         return undefined;
-    const estimated = estimatedInputTokens ?? await estimateTokensResponsive(context, signal);
+    const estimated = estimatedInputTokens ?? await countMessages(context, signal);
     if (!force &&
         (estimated < policy.targetTokens || Math.max(estimated, lastInputTokens) < policy.triggerTokens))
         return undefined;
@@ -23,13 +23,13 @@ export async function planCompaction(context, turn, coveredMessages, lastInputTo
     if (!await sameMessages(context.slice(contextPrefix), turn.slice(coveredMessages), signal))
         return undefined;
     const targetTokens = force
-        ? Math.min(policy.targetTokens, Math.max(512, Math.floor(estimated / 4)))
+        ? Math.min(policy.targetTokens, Math.max(512, Math.floor(estimated / 2)))
         : policy.targetTokens;
     const recentTokens = Math.min(policy.recentTokens, Math.max(256, Math.floor(targetTokens / 2)));
-    const recent = await recentBoundary(turn, coveredMessages, recentTokens, signal);
+    const recent = await recentBoundary(turn, coveredMessages, recentTokens, signal, countMessages);
     let boundary = recent.boundary;
     let tail = turn.slice(boundary);
-    if (turn.length > 1 && recent.tokens > targetTokens) {
+    if (turn.length > 1 && recent.tokens > policy.requestLimitTokens) {
         boundary = turn.length;
         tail = [];
     }
@@ -37,7 +37,7 @@ export async function planCompaction(context, turn, coveredMessages, lastInputTo
     if (prefixEnd <= 0 || prefixEnd > context.length)
         return undefined;
     const prefix = context.slice(0, prefixEnd);
-    if (!force && await estimateTokensResponsive(prefix, signal) < policy.minimumPrefixTokens) {
+    if (!force && await countMessages(prefix, signal) < policy.minimumPrefixTokens) {
         return undefined;
     }
     return {
@@ -73,10 +73,13 @@ export function policyForContextWindow(context, compactionPercent) {
     });
 }
 export function estimateTokens(messages) {
-    return estimateSerializedTokens(messages) + messages.length * 8;
+    return estimateSerializedTokens(normalized(messages)) + messages.length * 8;
 }
 export async function estimateTokensResponsive(messages, signal) {
-    return await estimateSerializedTokensResponsive(messages, signal) + messages.length * 8;
+    return await estimateSerializedTokensResponsive(normalized(messages), signal) + messages.length * 8;
+}
+function normalized(messages) {
+    return messages.map(({ role, content }) => ({ role, content }));
 }
 export function isContextOverflow(error) {
     const candidate = error;
@@ -85,7 +88,7 @@ export function isContextOverflow(error) {
     const detail = `${candidate.message}\n${candidate.body ?? ""}`;
     return /context_length_exceeded|maximum context length|context window|prompt is too long|input (?:is )?too (?:long|large)|(?:input|prompt|context).{0,80}(?:exceed|maximum|max tokens)/i.test(detail);
 }
-async function recentBoundary(turn, coveredMessages, recentTokens, signal) {
+async function recentBoundary(turn, coveredMessages, recentTokens, signal, countMessages) {
     const minimum = Math.max(coveredMessages, minimumRecentBoundary(turn));
     const candidates = [];
     for (let candidate = coveredMessages; candidate <= minimum; candidate++) {
@@ -96,7 +99,7 @@ async function recentBoundary(turn, coveredMessages, recentTokens, signal) {
     const estimateAt = (candidate) => {
         let estimate = cache.get(candidate);
         if (estimate === undefined) {
-            estimate = estimateTokensResponsive(turn.slice(candidate), signal);
+            estimate = countMessages(turn.slice(candidate), signal);
             cache.set(candidate, estimate);
         }
         return estimate;

package/dist/context/request-observation.js

@@ -0,0 +1,46 @@
+import { publishDiagnostic } from "./diagnostics.js";
+export async function observePreparation(policy, reason, signal, prepare) {
+    const started = performance.now();
+    try {
+        return await prepare();
+    }
+    catch (error) {
+        publishDiagnostic({ kind: "preparation", reason, outcome: signal?.aborted ? "cancelled" : "failed",
+            elapsedMs: Math.round(performance.now() - started), windowTokens: policy.windowTokens,
+            triggerTokens: policy.triggerTokens, requestLimitTokens: policy.requestLimitTokens });
+        throw error;
+    }
+}
+export async function sendObserved(provider, request, measurement, policy, preparationMs, clippedResults) {
+    const started = performance.now();
+    let firstEventMs;
+    let response;
+    let outcome = "failed";
+    try {
+        response = await provider.send({ ...request, onStream(event) {
+                firstEventMs ??= Math.round(performance.now() - started);
+                request.onStream?.(event);
+            } });
+        request.signal?.throwIfAborted();
+        outcome = "completed";
+        return response;
+    }
+    catch (error) {
+        if (request.signal?.aborted)
+            outcome = "cancelled";
+        throw error;
+    }
+    finally {
+        publishDiagnostic({
+            kind: "request", source: measurement.source, outcome,
+            tokenization: provider.inputTokenization?.(request.model) ?? "heuristic",
+            estimatedTokens: measurement.estimatedTokens, inputTokens: measurement.inputTokens,
+            windowTokens: policy.windowTokens, triggerTokens: policy.triggerTokens,
+            requestLimitTokens: policy.requestLimitTokens, outputBudgetTokens: request.maxTokens,
+            preparationMs: Math.round(preparationMs), providerMs: Math.round(performance.now() - started),
+            clippedResults,
+            ...(firstEventMs === undefined ? {} : { firstEventMs }),
+            ...(response?.usage === undefined ? {} : { reportedInputTokens: response.usage.inputTokens }),
+        });
+    }
+}

package/dist/context/request-projection.js

@@ -3,43 +3,17 @@
 import { leadingText, trailingText } from "../text-boundary.js";
 export const TOOL_RESULT_CLIP_MARKER = "[tool output clipped]";
 const FAIR_CONTENT_CODE_UNITS = 256;
-const APPEND_STABLE_RESULT_CODE_UNITS = 16_384;
-const MAX_TOOL_RESULT_PROJECTION_CODE_UNITS = 256_000;
+const MAX_TOOL_RESULT_PROJECTION_CODE_UNITS = 1_000_000;
 export function toolResultProjectionBudget(policy) {
-    return Math.min(MAX_TOOL_RESULT_PROJECTION_CODE_UNITS, policy.targetTokens);
-}
-/**
- * Keep historical result excerpts byte-for-byte stable as new results append.
- * Semantic compaction is allowed to establish a new prefix before the safety
- * fallback below redistributes an exhausted aggregate budget.
- */
-export function projectToolResults(source, requestedCodeUnits) {
-    const results = toolResults(source, requestedCodeUnits);
-    if (results.length === 0) {
-        return { messages: [...source], clippedResults: 0, outputCodeUnits: 0, saturated: false };
-    }
-    const allocations = [];
-    let remaining = requestedCodeUnits;
-    let saturated = false;
-    for (const result of results) {
-        const desired = Math.min(result.output.length, APPEND_STABLE_RESULT_CODE_UNITS);
-        if (desired <= remaining) {
-            allocations.push(desired);
-            remaining -= desired;
-            continue;
-        }
-        saturated = true;
-        const excerpt = remaining >= TOOL_RESULT_CLIP_MARKER.length ? remaining : 0;
-        allocations.push(Math.min(result.output.length, excerpt));
-        remaining -= excerpt;
-    }
-    return projectAllocations(source, allocations, saturated);
+    // Character allocation is only a starting point. The actual provider input
+    // is measured again before it can be sent, including Unicode and framing.
+    return Math.min(MAX_TOOL_RESULT_PROJECTION_CODE_UNITS, policy.targetTokens * 3);
 }
 /** Prefer recent evidence when compaction cannot restore a stable prefix. */
 export function projectToolResultsNewest(source, requestedCodeUnits) {
     const results = toolResults(source, requestedCodeUnits);
     if (results.length === 0) {
-        return { messages: [...source], clippedResults: 0, outputCodeUnits: 0, saturated: false };
+        return { messages: [...source], clippedResults: 0, outputCodeUnits: 0 };
     }
     const allocations = results.map(() => 0);
     let remaining = requestedCodeUnits;
@@ -58,7 +32,7 @@ export function projectToolResultsNewest(source, requestedCodeUnits) {
         allocations[index] = allocations[index] + extra;
         remaining -= extra;
     }
-    return projectAllocations(source, allocations, true);
+    return projectAllocations(source, allocations);
 }
 function toolResults(source, requestedCodeUnits) {
     if (!Number.isSafeInteger(requestedCodeUnits) || requestedCodeUnits < 0) {
@@ -66,7 +40,7 @@ function toolResults(source, requestedCodeUnits) {
     }
     return source.flatMap((message) => message.content.filter((block) => block.kind === "tool_result"));
 }
-function projectAllocations(source, allocations, saturated) {
+function projectAllocations(source, allocations) {
     let resultIndex = 0;
     let clippedResults = 0;
     let outputCodeUnits = 0;
@@ -86,7 +60,7 @@ function projectAllocations(source, allocations, saturated) {
         });
         return changed ? { ...message, content } : message;
     });
-    return { messages, clippedResults, outputCodeUnits, saturated };
+    return { messages, clippedResults, outputCodeUnits };
 }
 function allocateFairExcerpt(results, allocations, available) {
     const needs = results.map((result, index) => {

package/dist/context/request.js

@@ -0,0 +1,22 @@
+// Complete ordinary input; bounded tool projection is an emergency fallback.
+import { MIN_REQUEST_OUTPUT_TOKENS } from "./policy.js";
+import { projectToolResultsNewest, toolResultProjectionBudget } from "./request-projection.js";
+/** Called after optional semantic compaction, immediately before sending. */
+export async function fitRequestInput(input, meter, policy, measurement, signal) {
+    const limit = policy.requestLimitTokens - MIN_REQUEST_OUTPUT_TOKENS;
+    if (measurement.inputTokens <= limit) {
+        return { messages: [...input.messages], measurement, clippedResults: 0 };
+    }
+    let budget = toolResultProjectionBudget(policy);
+    for (;;) {
+        signal?.throwIfAborted();
+        const projection = projectToolResultsNewest(input.messages, budget);
+        const next = await meter.measure({ ...input, messages: projection.messages }, signal);
+        if (next.inputTokens <= limit || budget === 0 || projection.clippedResults === 0) {
+            return { messages: projection.messages, measurement: next, clippedResults: projection.clippedResults };
+        }
+        // At most logarithmically many bounded attempts, then the existing request
+        // guard reports an irreducible prompt/schema instead of a provider call.
+        budget = budget < 256 ? 0 : Math.floor(budget / 2);
+    }
+}

package/dist/context/tokenizer/bpe.js

@@ -0,0 +1,68 @@
+export function countPiece(bytes, ranks) {
+    const text = bytes.toString("latin1");
+    if (ranks.has(text))
+        return 1;
+    const next = Int32Array.from({ length: bytes.length }, (_, i) => i + 1);
+    const previous = Int32Array.from({ length: bytes.length }, (_, i) => i - 1);
+    const heap = [];
+    let count = bytes.length;
+    const offer = (left) => {
+        if (left < 0 || left >= bytes.length)
+            return;
+        const right = next[left];
+        if (right < 0 || right >= bytes.length)
+            return;
+        const end = next[right];
+        const rank = ranks.get(text.slice(left, end));
+        if (rank !== undefined)
+            push(heap, { rank, left, right, end });
+    };
+    for (let i = 0; i + 1 < bytes.length; i++)
+        offer(i);
+    while (heap.length > 0) {
+        const pair = pop(heap);
+        if (next[pair.left] !== pair.right || next[pair.right] !== pair.end)
+            continue;
+        next[pair.left] = pair.end;
+        next[pair.right] = -1;
+        if (pair.end < bytes.length)
+            previous[pair.end] = pair.left;
+        count--;
+        offer(previous[pair.left]);
+        offer(pair.left);
+    }
+    return count;
+}
+function before(a, b) {
+    return a.rank < b.rank || (a.rank === b.rank && a.left < b.left);
+}
+function push(heap, pair) {
+    let index = heap.length;
+    heap.push(pair);
+    while (index > 0) {
+        const parent = (index - 1) >>> 1;
+        if (!before(pair, heap[parent]))
+            break;
+        heap[index] = heap[parent];
+        index = parent;
+    }
+    heap[index] = pair;
+}
+function pop(heap) {
+    const first = heap[0];
+    const tail = heap.pop();
+    if (heap.length === 0)
+        return first;
+    let index = 0;
+    while (index * 2 + 1 < heap.length) {
+        let child = index * 2 + 1;
+        if (child + 1 < heap.length && before(heap[child + 1], heap[child]))
+            child++;
+        if (!before(heap[child], tail))
+            break;
+        heap[index] = heap[child];
+        index = child;
+    }
+    heap[index] = tail;
+    return first;
+}

package/dist/context/tokenizer/o200k.js

@@ -0,0 +1,54 @@
+// Owned o200k_base ordinary-text counter. Special-looking user text stays literal.
+import { createHash } from "node:crypto";
+import { countPiece } from "./bpe.js";
+import { vocabulary } from "./vocabulary.js";
+// OpenAI tiktoken's o200k_base split, with scoped ASCII case-insensitivity
+// expanded for JavaScript. Vocabulary and expression attribution: assets/tokenizers/LICENSE.
+const pattern = /[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?:'[sStTmMdD]|'[rR][eE]|'[vV][eE]|'[lL][lL])?|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?:'[sStTmMdD]|'[rR][eE]|'[vV][eE]|'[lL][lL])?|\p{N}{1,3}| ?[^\p{White_Space}\p{L}\p{N}]+[\r\n/]*|\p{White_Space}*[\r\n]+|\p{White_Space}+(?!\P{White_Space})|\p{White_Space}+/gu;
+const cache = new Map();
+const CHUNK = 8_192;
+export async function countO200k(text, signal) {
+    signal?.throwIfAborted();
+    const key = createHash("sha256").update(text).digest("hex");
+    const cached = cache.get(key);
+    if (cached !== undefined)
+        return cached;
+    const ranks = await vocabulary();
+    signal?.throwIfAborted();
+    let count = 0;
+    // Repetitive logs or minified data must not repeat the same merge work.
+    // Keep text only within this call, bounded independently of input size.
+    const chunks = new Map();
+    let chunkUnits = 0;
+    for (let start = 0; start < text.length;) {
+        let end = Math.min(start + CHUNK, text.length);
+        if (end < text.length && /[\uD800-\uDBFF]/u.test(text[end - 1]))
+            end--;
+        const chunk = text.slice(start, end);
+        let tokens = chunks.get(chunk);
+        if (tokens === undefined) {
+            tokens = 0;
+            for (const match of chunk.matchAll(pattern))
+                tokens += countPiece(Buffer.from(match[0], "utf8"), ranks);
+            while (chunkUnits + chunk.length > 131_072) {
+                const oldest = chunks.keys().next().value;
+                chunks.delete(oldest);
+                chunkUnits -= oldest.length;
+            }
+            chunks.set(chunk, tokens);
+            chunkUnits += chunk.length;
+        }
+        count += tokens;
+        // Bounded slices may split a token/pre-token. Reserve boundary room; this
+        // is an input estimate, not a claim of exact whole-request tokenization.
+        if (end < text.length)
+            count += 8;
+        start = end;
+        await new Promise((resolve) => setImmediate(resolve));
+        signal?.throwIfAborted();
+    }
+    if (cache.size >= 2_048)
+        cache.delete(cache.keys().next().value);
+    cache.set(key, count);
+    return count;
+}

package/dist/context/tokenizer/vocabulary.js

@@ -0,0 +1,34 @@
+// Pinned data, loaded once on demand. No network, executable data, or user cache.
+import { createHash } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import { promisify } from "node:util";
+import { gunzip } from "node:zlib";
+let loading;
+const unpack = promisify(gunzip);
+const digest = "446a9538cb6c348e3516120d7c08b09f57c36495e2acfffe59a5bf8b0cfb1a2d";
+export function vocabulary() {
+    loading ??= load();
+    return loading;
+}
+async function load() {
+    const compressed = await readFile(new URL("../../../assets/tokenizers/o200k-base.tiktoken.gz", import.meta.url));
+    if (compressed.length > 2_000_000)
+        throw new Error("token vocabulary exceeds its size limit");
+    const data = await unpack(compressed, { maxOutputLength: 4_000_000 });
+    if (createHash("sha256").update(data).digest("hex") !== digest) {
+        throw new Error("token vocabulary checksum mismatch");
+    }
+    const ranks = new Map();
+    const lines = data.toString("ascii").trimEnd().split("\n");
+    for (let index = 0; index < lines.length; index++) {
+        const [encoded, rank] = lines[index].split(" ");
+        if (encoded === undefined || Number(rank) !== index)
+            throw new Error("invalid token vocabulary");
+        ranks.set(Buffer.from(encoded, "base64").toString("latin1"), index);
+        if (index % 2_048 === 0)
+            await new Promise((resolve) => setImmediate(resolve));
+    }
+    if (ranks.size !== 199_998)
+        throw new Error("incomplete token vocabulary");
+    return ranks;
+}