context-lens 0.6.0 → 0.6.2

package/dist/core/source.js.map

@@ -1 +1 @@
-{"version":3,"file":"source.js","sourceRoot":"","sources":["../../src/core/source.ts"],"names":[],"mappings":"AAMA;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IAClD,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE;IACpE,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE;IAC5D,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;IAC1D,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE;CACpE,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IAClD,EAAE,OAAO,EAAE,qCAAqC,EAAE,MAAM,EAAE,OAAO,EAAE;IACnE,EAAE,OAAO,EAAE,qBAAqB,EAAE,MAAM,EAAE,QAAQ,EAAE;IACpD,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,EAAE,MAAM,EAAE;IACpD,EAAE,OAAO,EAAE,6CAA6C,EAAE,MAAM,EAAE,IAAI,EAAE;CACzE,CAAC;AAEF;;;;;;;;GAQG;AACH,MAAM,UAAU,YAAY,CAC1B,WAAwB,EACxB,MAAqB,EACrB,OAAgC;IAEhC,IAAI,MAAM,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC;IAElD,iCAAiC;IACjC,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,IACE,GAAG,CAAC,OAAO,YAAY,MAAM;gBAC3B,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;gBACvB,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAC7B,CAAC;gBACD,OAAO,GAAG,CAAC,MAAM,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,MAAM,UAAU,GAAG,CAAC,WAAW,CAAC,aAAa,IAAI,EAAE,CAAC;SACjD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;SACvB,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACpC,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,OAAO,GAAG,CAAC,MAAM,CAAC;IAC1D,CAAC;IACD,OAAO,MAAM,IAAI,SAAS,CAAC;AAC7B,CAAC"}
+{"version":3,"file":"source.js","sourceRoot":"","sources":["../../src/core/source.ts"],"names":[],"mappings":"AAMA;;;;GAIG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IAClD,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,eAAe,EAAE,MAAM,EAAE,QAAQ,EAAE;IACpE,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE;IAC5D,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE;IAC1D,EAAE,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,QAAQ,EAAE;CACpE,CAAC;AAEF;;GAEG;AACH,MAAM,CAAC,MAAM,iBAAiB,GAAsB;IAClD,EAAE,OAAO,EAAE,qCAAqC,EAAE,MAAM,EAAE,OAAO,EAAE;IACnE,EAAE,OAAO,EAAE,qBAAqB,EAAE,MAAM,EAAE,QAAQ,EAAE;IACpD,EAAE,OAAO,EAAE,uBAAuB,EAAE,MAAM,EAAE,MAAM,EAAE;IACpD,EAAE,OAAO,EAAE,6CAA6C,EAAE,MAAM,EAAE,IAAI,EAAE;CACzE,CAAC;AAEF;;;;;;;;GAQG;AACH,oEAAoE;AACpE,mEAAmE;AACnE,MAAM,CAAC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IACpC,WAAW;IACX,QAAQ;IACR,QAAQ;IACR,SAAS;CACV,CAAC,CAAC;AAEH,MAAM,UAAU,YAAY,CAC1B,WAAwB,EACxB,MAAqB,EACrB,OAAgC;IAEhC,IAAI,MAAM,IAAI,MAAM,KAAK,SAAS,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;QAC/D,OAAO,MAAM,CAAC;IAEhB,iCAAiC;IACjC,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;YACpC,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAChC,IAAI,CAAC,GAAG;gBAAE,SAAS;YACnB,IACE,GAAG,CAAC,OAAO,YAAY,MAAM;gBAC3B,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC;gBACvB,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAC7B,CAAC;gBACD,OAAO,GAAG,CAAC,MAAM,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IAED,wCAAwC;IACxC,MAAM,UAAU,GAAG,CAAC,WAAW,CAAC,aAAa,IAAI,EAAE,CAAC;SACjD,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,EAAE,CAAC,OAAO,CAAC;SACvB,IAAI,CAAC,IAAI,CAAC,CAAC;IACd,KAAK,MAAM,GAAG,IAAI,iBAAiB,EAAE,CAAC;QACpC,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC;YAAE,OAAO,GAAG,CAAC,MAAM,CAAC;IAC1D,CAAC;IACD,OAAO,MAAM,IAAI,SAAS,CAAC;AAC7B,CAAC"}

package/dist/core/tokenizer.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Tiktoken-based tokenizer with lazy loading and model-to-encoding mapping.
+ *
+ * Encoders are cached per encoding name (not per model) since many models
+ * share the same encoding. Falls back to chars/4 for unknown encodings.
+ */
+/**
+ * Initialize the tokenizer by preloading the most commonly used encodings.
+ *
+ * Call this once at analysis server startup. Non-blocking: if loading fails,
+ * the tokenizer falls back to chars/4 estimation.
+ */
+export declare function initTokenizer(): Promise<void>;
+/**
+ * Count tokens for a string using the appropriate encoding for the model.
+ *
+ * If the tokenizer hasn't been initialized yet (or loading failed), falls
+ * back to chars/4. The model parameter is optional; when omitted, uses
+ * cl100k_base.
+ *
+ * This is synchronous when encoders are preloaded (the normal case after
+ * initTokenizer()). The encoder lookup is a Map.get(), and encoding is
+ * a pure CPU operation.
+ */
+export declare function countTokens(text: string, model?: string): number;
+/**
+ * Check if the tokenizer is ready (at least one encoding loaded).
+ */
+export declare function isTokenizerReady(): boolean;
+//# sourceMappingURL=tokenizer.d.ts.map

package/dist/core/tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.d.ts","sourceRoot":"","sources":["../../src/core/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAwGH;;;;;GAKG;AACH,wBAAsB,aAAa,IAAI,OAAO,CAAC,IAAI,CAAC,CAWnD;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAiBhE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,IAAI,OAAO,CAE1C"}

package/dist/core/tokenizer.js

@@ -0,0 +1,149 @@
+/**
+ * Tiktoken-based tokenizer with lazy loading and model-to-encoding mapping.
+ *
+ * Encoders are cached per encoding name (not per model) since many models
+ * share the same encoding. Falls back to chars/4 for unknown encodings.
+ */
+/**
+ * Map model name prefixes to tiktoken encoding names.
+ *
+ * Order: most specific first (longest prefix match wins).
+ *
+ * - cl100k_base: GPT-4, GPT-3.5-turbo, Claude (approximation)
+ * - o200k_base: GPT-4o, GPT-4.1, GPT-5, o1, o3, o4
+ *
+ * Anthropic hasn't published their tokenizer, but cl100k_base is a
+ * reasonable approximation (~5-10% off). Still much better than chars/4.
+ *
+ * Gemini models also use a proprietary tokenizer; cl100k_base is used
+ * as a best-effort approximation.
+ */
+const MODEL_ENCODING_MAP = [
+    // OpenAI: o200k_base models
+    ["gpt-5", "o200k_base"],
+    ["gpt-4.1", "o200k_base"],
+    ["gpt-4o", "o200k_base"],
+    ["o4-", "o200k_base"],
+    ["o3-", "o200k_base"],
+    ["o3", "o200k_base"],
+    ["o1-", "o200k_base"],
+    ["o1", "o200k_base"],
+    // OpenAI: cl100k_base models
+    ["gpt-4-turbo", "cl100k_base"],
+    ["gpt-4", "cl100k_base"],
+    ["gpt-3.5", "cl100k_base"],
+    // Anthropic: cl100k_base as approximation
+    ["claude-", "cl100k_base"],
+    // Gemini: cl100k_base as approximation
+    ["gemini-", "cl100k_base"],
+];
+/** Cached encoder instances keyed by encoding name. */
+const encoderCache = new Map();
+/** In-flight loading promises to avoid duplicate loads. */
+const loadingPromises = new Map();
+/** Whether the tokenizer has been initialized (at least one encoding loaded). */
+let initialized = false;
+/**
+ * Resolve a model string to an encoding name.
+ */
+function getEncodingForModel(model) {
+    const lower = model.toLowerCase();
+    for (const [prefix, encoding] of MODEL_ENCODING_MAP) {
+        if (lower.includes(prefix))
+            return encoding;
+    }
+    // Default fallback: cl100k_base (most widely applicable)
+    return "cl100k_base";
+}
+/**
+ * Dynamically import the rank data for an encoding name.
+ * Returns the Tiktoken constructor args.
+ */
+async function loadEncoding(encoding) {
+    const { Tiktoken } = await import("js-tiktoken/lite");
+    let ranks;
+    switch (encoding) {
+        case "o200k_base":
+            ranks = (await import("js-tiktoken/ranks/o200k_base")).default;
+            break;
+        case "cl100k_base":
+            ranks = (await import("js-tiktoken/ranks/cl100k_base")).default;
+            break;
+        default:
+            // Fallback to cl100k_base for unknown encodings
+            ranks = (await import("js-tiktoken/ranks/cl100k_base")).default;
+            break;
+    }
+    return new Tiktoken(ranks);
+}
+/**
+ * Get or lazily load an encoder for the given encoding name.
+ */
+async function getEncoder(encoding) {
+    const cached = encoderCache.get(encoding);
+    if (cached)
+        return cached;
+    // Deduplicate concurrent loads
+    let promise = loadingPromises.get(encoding);
+    if (!promise) {
+        promise = loadEncoding(encoding).then((enc) => {
+            encoderCache.set(encoding, enc);
+            loadingPromises.delete(encoding);
+            initialized = true;
+            return enc;
+        });
+        loadingPromises.set(encoding, promise);
+    }
+    return promise;
+}
+/**
+ * Initialize the tokenizer by preloading the most commonly used encodings.
+ *
+ * Call this once at analysis server startup. Non-blocking: if loading fails,
+ * the tokenizer falls back to chars/4 estimation.
+ */
+export async function initTokenizer() {
+    try {
+        // Preload both encodings in parallel
+        await Promise.all([getEncoder("cl100k_base"), getEncoder("o200k_base")]);
+        console.log("🔤 Tokenizer initialized (cl100k_base + o200k_base)");
+    }
+    catch (err) {
+        console.warn("Tokenizer init failed, using estimation fallback:", err instanceof Error ? err.message : String(err));
+    }
+}
+/**
+ * Count tokens for a string using the appropriate encoding for the model.
+ *
+ * If the tokenizer hasn't been initialized yet (or loading failed), falls
+ * back to chars/4. The model parameter is optional; when omitted, uses
+ * cl100k_base.
+ *
+ * This is synchronous when encoders are preloaded (the normal case after
+ * initTokenizer()). The encoder lookup is a Map.get(), and encoding is
+ * a pure CPU operation.
+ */
+export function countTokens(text, model) {
+    if (!text)
+        return 0;
+    const encoding = model ? getEncodingForModel(model) : "cl100k_base";
+    const encoder = encoderCache.get(encoding);
+    if (encoder) {
+        try {
+            return encoder.encode(text).length;
+        }
+        catch {
+            // Encoding failure (unlikely): fall back to estimation
+            return Math.ceil(text.length / 4);
+        }
+    }
+    // Encoder not loaded yet: fall back to chars/4
+    return Math.ceil(text.length / 4);
+}
+/**
+ * Check if the tokenizer is ready (at least one encoding loaded).
+ */
+export function isTokenizerReady() {
+    return initialized;
+}
+//# sourceMappingURL=tokenizer.js.map

package/dist/core/tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokenizer.js","sourceRoot":"","sources":["../../src/core/tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH;;;;;;;;;;;;;GAaG;AACH,MAAM,kBAAkB,GAAuB;IAC7C,4BAA4B;IAC5B,CAAC,OAAO,EAAE,YAAY,CAAC;IACvB,CAAC,SAAS,EAAE,YAAY,CAAC;IACzB,CAAC,QAAQ,EAAE,YAAY,CAAC;IACxB,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,CAAC,IAAI,EAAE,YAAY,CAAC;IACpB,CAAC,KAAK,EAAE,YAAY,CAAC;IACrB,CAAC,IAAI,EAAE,YAAY,CAAC;IACpB,6BAA6B;IAC7B,CAAC,aAAa,EAAE,aAAa,CAAC;IAC9B,CAAC,OAAO,EAAE,aAAa,CAAC;IACxB,CAAC,SAAS,EAAE,aAAa,CAAC;IAC1B,0CAA0C;IAC1C,CAAC,SAAS,EAAE,aAAa,CAAC;IAC1B,uCAAuC;IACvC,CAAC,SAAS,EAAE,aAAa,CAAC;CAC3B,CAAC;AAEF,uDAAuD;AACvD,MAAM,YAAY,GAAG,IAAI,GAAG,EAAoB,CAAC;AAEjD,2DAA2D;AAC3D,MAAM,eAAe,GAAG,IAAI,GAAG,EAA6B,CAAC;AAE7D,iFAAiF;AACjF,IAAI,WAAW,GAAG,KAAK,CAAC;AAExB;;GAEG;AACH,SAAS,mBAAmB,CAAC,KAAa;IACxC,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,KAAK,MAAM,CAAC,MAAM,EAAE,QAAQ,CAAC,IAAI,kBAAkB,EAAE,CAAC;QACpD,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC;YAAE,OAAO,QAAQ,CAAC;IAC9C,CAAC;IACD,yDAAyD;IACzD,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,KAAK,UAAU,YAAY,CAAC,QAAgB;IAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACtD,IAAI,KAAU,CAAC;IAEf,QAAQ,QAAQ,EAAE,CAAC;QACjB,KAAK,YAAY;YACf,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAAC,CAAC,OAAO,CAAC;YAC/D,MAAM;QACR,KAAK,aAAa;YAChB,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC,CAAC,OAAO,CAAC;YAChE,MAAM;QACR;YACE,gDAAgD;YAChD,KAAK,GAAG,CAAC,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAC,CAAC,OAAO,CAAC;YAChE,MAAM;IACV,CAAC;IAED,OAAO,IAAI,QAAQ,CAAC,KAAK,CAAC,CAAC;AAC7B,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,UAAU,CAAC,QAAgB;IACxC,MAAM,MAAM,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,MAAM;QAAE,OAAO,MAAM,CAAC;IAE1B,+BAA+B;IAC/B,IAAI,OAAO,GAAG,eAAe,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAC5C,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,OAAO,GAAG,YAAY,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE;YAC5C,YAAY,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;YAChC,eAAe,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACjC,WAAW,GAAG,IAAI,CAAC;YACnB,OAAO,GAAG,CAAC;QACb,CAAC,CAAC,CAAC;QACH,eAAe,CAAC,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACzC,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa;IACjC,IAAI,CAAC;QACH,qCAAqC;QACrC,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,qDAAqD,CAAC,CAAC;IACrE,CAAC;IAAC,OAAO,GAAY,EAAE,CAAC;QACtB,OAAO,CAAC,IAAI,CACV,mDAAmD,EACnD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CACjD,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,KAAc;IACtD,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;IACpE,MAAM,OAAO,GAAG,YAAY,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAE3C,IAAI,OAAO,EAAE,CAAC;QACZ,IAAI,CAAC;YACH,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QACrC,CAAC;QAAC,MAAM,CAAC;YACP,uDAAuD;YACvD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACpC,CAAC;IACH,CAAC;IAED,+CAA+C;IAC/C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB;IAC9B,OAAO,WAAW,CAAC;AACrB,CAAC"}

package/dist/core/tokens.d.ts

@@ -1,13 +1,19 @@
 /**
- * Lightweight token estimator used throughout Context Lens.
+ * Token estimator used throughout Context Lens.
  *
- * Approximates tokens as `ceil(chars / 4)`. For image content blocks,
- * uses a fixed per-image estimate instead of stringifying base64 data.
+ * When the tiktoken-based tokenizer is initialized (analysis server),
+ * uses accurate BPE tokenization for the given model. Falls back to
+ * `ceil(chars / 4)` when the tokenizer hasn't loaded yet or for
+ * unknown models.
+ *
+ * For image content blocks, uses a fixed per-image estimate instead
+ * of stringifying base64 data.
  *
  * @param text - Value to estimate tokens for. Objects are stringified as JSON.
- * @returns Estimated token count (>= 0).
+ * @param model - Optional model name for encoding selection.
+ * @returns Token count (>= 0).
  */
-export declare function estimateTokens(text: unknown): number;
+export declare function estimateTokens(text: unknown, model?: string): number;
 /**
  * Rescale all token sub-fields in a ContextInfo so they are internally
  * consistent with an authoritative total (typically from API usage data).

package/dist/core/tokens.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../../src/core/tokens.ts"],"names":[],"mappings":"AA8DA;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,OAAO,GAAG,MAAM,CAqBpD;AAiCD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAClC,EAAE,EAAE;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAChC,EACD,aAAa,EAAE,MAAM,GACpB,IAAI,CAyCN"}
+{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../../src/core/tokens.ts"],"names":[],"mappings":"AAgEA;;;;;;;;;;;;;;GAcG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,MAAM,CAqBpE;AAiCD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAClC,EAAE,EAAE;IACF,YAAY,EAAE,MAAM,CAAC;IACrB,WAAW,EAAE,MAAM,CAAC;IACpB,cAAc,EAAE,MAAM,CAAC;IACvB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE;QAAE,MAAM,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;CAChC,EACD,aAAa,EAAE,MAAM,GACpB,IAAI,CAyCN"}

package/dist/core/tokens.js

@@ -1,3 +1,4 @@
+import { countTokens } from "./tokenizer.js";
 /**
  * Approximate token cost for a single image.
  *
@@ -56,20 +57,26 @@ function stripImageData(val) {
     return obj;
 }
 /**
- * Lightweight token estimator used throughout Context Lens.
+ * Token estimator used throughout Context Lens.
  *
- * Approximates tokens as `ceil(chars / 4)`. For image content blocks,
- * uses a fixed per-image estimate instead of stringifying base64 data.
+ * When the tiktoken-based tokenizer is initialized (analysis server),
+ * uses accurate BPE tokenization for the given model. Falls back to
+ * `ceil(chars / 4)` when the tokenizer hasn't loaded yet or for
+ * unknown models.
+ *
+ * For image content blocks, uses a fixed per-image estimate instead
+ * of stringifying base64 data.
  *
  * @param text - Value to estimate tokens for. Objects are stringified as JSON.
- * @returns Estimated token count (>= 0).
+ * @param model - Optional model name for encoding selection.
+ * @returns Token count (>= 0).
  */
-export function estimateTokens(text) {
+export function estimateTokens(text, model) {
     if (!text)
         return 0;
     // Fast path: plain strings never contain image data
     if (typeof text === "string") {
-        return Math.ceil(text.length / 4);
+        return countTokens(text, model);
     }
     // Single image block
     if (isImageBlock(text)) {
@@ -78,7 +85,7 @@ export function estimateTokens(text) {
     // Object/array: strip image data, then stringify the rest
     const cleaned = stripImageData(text);
     const s = JSON.stringify(cleaned);
-    const baseTokens = Math.ceil(s.length / 4);
+    const baseTokens = countTokens(s, model);
     // Count image blocks and add fixed estimate for each
     const imageCount = countImages(text);
     return baseTokens + imageCount * IMAGE_TOKEN_ESTIMATE;

package/dist/core/tokens.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/core/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,MAAM,oBAAoB,GAAG,KAAK,CAAC;AAEnC;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,GAAY;IAChC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxE,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAClE,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,GAAY;IAClC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IAEhD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACjC,CAAC;IAED,gDAAgD;IAChD,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO;YACL,IAAI,EAAG,GAA+B,CAAC,IAAI,IAAI,OAAO;YACtD,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,GAA8B,CAAC;IAE3C,sEAAsE;IACtE,IAAI,GAAG,CAAC,IAAI,KAAK,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7D,OAAO,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;IAC9D,CAAC;IAED,kDAAkD;IAClD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;IAC1D,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,IAAa;IAC1C,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,oDAAoD;IACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,qBAAqB;IACrB,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED,0DAA0D;IAC1D,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAE3C,qDAAqD;IACrD,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,UAAU,GAAG,UAAU,GAAG,oBAAoB,CAAC;AACxD,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IAE9C,IAAI,YAAY,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;YACvB,KAAK,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,GAA8B,CAAC;IAE3C,6CAA6C;IAC7C,IAAI,GAAG,CAAC,IAAI,KAAK,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7D,OAAO,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,qBAAqB;IACrB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAClC,EAMC,EACD,aAAqB;IAErB,MAAM,SAAS,GAAG,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,cAAc,CAAC;IACvE,IAAI,SAAS,KAAK,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QAC3C,EAAE,CAAC,YAAY,GAAG,CAAC,CAAC;QACpB,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC;QACnB,EAAE,CAAC,cAAc,GAAG,CAAC,CAAC;QACtB,EAAE,CAAC,WAAW,GAAG,aAAa,CAAC;QAC/B,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;YAC9B,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,OAAO;IACT,CAAC;IACD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,EAAE,CAAC,WAAW,GAAG,aAAa,CAAC;QAC/B,OAAO;IACT,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,GAAG,SAAS,CAAC;IACxC,EAAE,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC;IACtD,EAAE,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC,CAAC;IACpD,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC9B,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IAC9C,CAAC;IACD,EAAE,CAAC,cAAc,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAClE,EAAE,CAAC,WAAW,GAAG,aAAa,CAAC;IAC/B,iDAAiD;IACjD,gEAAgE;IAChE,uCAAuC;IACvC,MAAM,QAAQ,GACZ,aAAa,GAAG,CAAC,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC;IACzE,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,EAAE,CAAC,cAAc,IAAI,QAAQ,CAAC;QAC9B,0EAA0E;QAC1E,oDAAoD;QACpD,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,IAAI,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM;oBAAE,MAAM,GAAG,CAAC,CAAC;YACrE,CAAC;YACD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/core/tokens.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAE7C;;;;;;;GAOG;AACH,MAAM,oBAAoB,GAAG,KAAK,CAAC;AAEnC;;;;;;GAMG;AACH,SAAS,YAAY,CAAC,GAAY;IAChC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACxE,MAAM,GAAG,GAAG,GAA8B,CAAC;IAC3C,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,IAAI,GAAG,CAAC,IAAI,KAAK,WAAW;QAAE,OAAO,IAAI,CAAC;IAClE,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,GAAY;IAClC,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,GAAG,CAAC;IAEhD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,OAAO,GAAG,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;IACjC,CAAC;IAED,gDAAgD;IAChD,IAAI,YAAY,CAAC,GAAG,CAAC,EAAE,CAAC;QACtB,OAAO;YACL,IAAI,EAAG,GAA+B,CAAC,IAAI,IAAI,OAAO;YACtD,MAAM,EAAE,IAAI;SACb,CAAC;IACJ,CAAC;IAED,MAAM,GAAG,GAAG,GAA8B,CAAC;IAE3C,sEAAsE;IACtE,IAAI,GAAG,CAAC,IAAI,KAAK,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7D,OAAO,EAAE,GAAG,GAAG,EAAE,OAAO,EAAE,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;IAC9D,CAAC;IAED,kDAAkD;IAClD,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,EAAE,GAAG,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,CAAC;IAC1D,CAAC;IAED,OAAO,GAAG,CAAC;AACb,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,cAAc,CAAC,IAAa,EAAE,KAAc;IAC1D,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,oDAAoD;IACpD,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,OAAO,WAAW,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAClC,CAAC;IAED,qBAAqB;IACrB,IAAI,YAAY,CAAC,IAAI,CAAC,EAAE,CAAC;QACvB,OAAO,oBAAoB,CAAC;IAC9B,CAAC;IAED,0DAA0D;IAC1D,MAAM,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,UAAU,GAAG,WAAW,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;IAEzC,qDAAqD;IACrD,MAAM,UAAU,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACrC,OAAO,UAAU,GAAG,UAAU,GAAG,oBAAoB,CAAC;AACxD,CAAC;AAED;;GAEG;AACH,SAAS,WAAW,CAAC,GAAY;IAC/B,IAAI,CAAC,GAAG,IAAI,OAAO,GAAG,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IAE9C,IAAI,YAAY,CAAC,GAAG,CAAC;QAAE,OAAO,CAAC,CAAC;IAEhC,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,KAAK,MAAM,IAAI,IAAI,GAAG,EAAE,CAAC;YACvB,KAAK,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC;QAC7B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,MAAM,GAAG,GAAG,GAA8B,CAAC;IAE3C,6CAA6C;IAC7C,IAAI,GAAG,CAAC,IAAI,KAAK,aAAa,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,CAAC;QAC7D,OAAO,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAClC,CAAC;IAED,qBAAqB;IACrB,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;QAC7B,OAAO,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAChC,CAAC;IAED,OAAO,CAAC,CAAC;AACX,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,oBAAoB,CAClC,EAMC,EACD,aAAqB;IAErB,MAAM,SAAS,GAAG,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,cAAc,CAAC;IACvE,IAAI,SAAS,KAAK,CAAC,IAAI,aAAa,KAAK,CAAC,EAAE,CAAC;QAC3C,EAAE,CAAC,YAAY,GAAG,CAAC,CAAC;QACpB,EAAE,CAAC,WAAW,GAAG,CAAC,CAAC;QACnB,EAAE,CAAC,cAAc,GAAG,CAAC,CAAC;QACtB,EAAE,CAAC,WAAW,GAAG,aAAa,CAAC;QAC/B,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;YAC9B,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;QACjB,CAAC;QACD,OAAO;IACT,CAAC;IACD,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,EAAE,CAAC,WAAW,GAAG,aAAa,CAAC;QAC/B,OAAO;IACT,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,GAAG,SAAS,CAAC;IACxC,EAAE,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,GAAG,KAAK,CAAC,CAAC;IACtD,EAAE,CAAC,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,WAAW,GAAG,KAAK,CAAC,CAAC;IACpD,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,QAAQ,EAAE,CAAC;QAC9B,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,KAAK,CAAC,CAAC;IAC9C,CAAC;IACD,EAAE,CAAC,cAAc,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAClE,EAAE,CAAC,WAAW,GAAG,aAAa,CAAC;IAC/B,iDAAiD;IACjD,gEAAgE;IAChE,uCAAuC;IACvC,MAAM,QAAQ,GACZ,aAAa,GAAG,CAAC,EAAE,CAAC,YAAY,GAAG,EAAE,CAAC,WAAW,GAAG,EAAE,CAAC,cAAc,CAAC,CAAC;IACzE,IAAI,QAAQ,KAAK,CAAC,EAAE,CAAC;QACnB,EAAE,CAAC,cAAc,IAAI,QAAQ,CAAC;QAC9B,0EAA0E;QAC1E,oDAAoD;QACpD,IAAI,EAAE,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC3B,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC5C,IAAI,EAAE,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM;oBAAE,MAAM,GAAG,CAAC,CAAC;YACrE,CAAC;YACD,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,QAAQ,CAAC;QACzC,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/core.d.ts

@@ -14,6 +14,7 @@ export type { AgentPathStep, AnalyzeOptions, CacheStats, CompactionEvent, Growth
 export { analyzeSession, buildAgentPaths, findCompactions, findGrowthBlocks, identifyUserTurns, } from "./core/session-analysis.js";
 export type { FormatOptions } from "./core/session-format.js";
 export { fmtCost, fmtDuration, fmtTokens, formatSessionAnalysis, shortModel, } from "./core/session-format.js";
-export { detectSource } from "./core/source.js";
+export { detectSource, PROVIDER_NAMES } from "./core/source.js";
+export { initTokenizer, isTokenizerReady } from "./core/tokenizer.js";
 export { estimateTokens, rescaleContextTokens } from "./core/tokens.js";
 //# sourceMappingURL=core.d.ts.map

package/dist/core.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../src/core.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,wBAAwB,EACxB,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,uBAAuB,GACxB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EACL,YAAY,EACZ,eAAe,GAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EACV,aAAa,EACb,cAAc,EACd,UAAU,EACV,eAAe,EACf,WAAW,EACX,eAAe,EACf,WAAW,EACX,QAAQ,GACT,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,cAAc,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,4BAA4B,CAAC;AACpC,YAAY,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,OAAO,EACP,WAAW,EACX,SAAS,EACT,qBAAqB,EACrB,UAAU,GACX,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"core.d.ts","sourceRoot":"","sources":["../src/core.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,wBAAwB,EACxB,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,uBAAuB,GACxB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EACL,YAAY,EACZ,eAAe,GAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,YAAY,EACV,aAAa,EACb,cAAc,EACd,UAAU,EACV,eAAe,EACf,WAAW,EACX,eAAe,EACf,WAAW,EACX,QAAQ,GACT,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,cAAc,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,4BAA4B,CAAC;AACpC,YAAY,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EACL,OAAO,EACP,WAAW,EACX,SAAS,EACT,qBAAqB,EACrB,UAAU,GACX,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/core.js

@@ -12,6 +12,7 @@ export { detectApiFormat, detectProvider, extractSource, resolveTargetUrl, } fro
 export { scanSecurity } from "./core/security.js";
 export { analyzeSession, buildAgentPaths, findCompactions, findGrowthBlocks, identifyUserTurns, } from "./core/session-analysis.js";
 export { fmtCost, fmtDuration, fmtTokens, formatSessionAnalysis, shortModel, } from "./core/session-format.js";
-export { detectSource } from "./core/source.js";
+export { detectSource, PROVIDER_NAMES } from "./core/source.js";
+export { initTokenizer, isTokenizerReady } from "./core/tokenizer.js";
 export { estimateTokens, rescaleContextTokens } from "./core/tokens.js";
 //# sourceMappingURL=core.js.map

package/dist/core.js.map

@@ -1 +1 @@
-{"version":3,"file":"core.js","sourceRoot":"","sources":["../src/core.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,wBAAwB,EACxB,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,uBAAuB,GACxB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EACL,YAAY,EACZ,eAAe,GAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAWlD,OAAO,EACL,cAAc,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EACL,OAAO,EACP,WAAW,EACX,SAAS,EACT,qBAAqB,EACrB,UAAU,GACX,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC"}
+{"version":3,"file":"core.js","sourceRoot":"","sources":["../src/core.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,wBAAwB,EACxB,mBAAmB,EACnB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,uBAAuB,GACxB,MAAM,wBAAwB,CAAC;AAEhC,OAAO,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EACL,YAAY,EACZ,eAAe,GAChB,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,OAAO,EACL,eAAe,EACf,cAAc,EACd,aAAa,EACb,gBAAgB,GACjB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAWlD,OAAO,EACL,cAAc,EACd,eAAe,EACf,eAAe,EACf,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,4BAA4B,CAAC;AAEpC,OAAO,EACL,OAAO,EACP,WAAW,EACX,SAAS,EACT,qBAAqB,EACrB,UAAU,GACX,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAChE,OAAO,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AACtE,OAAO,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC"}

package/dist/http/headers.d.ts

@@ -1,25 +1,17 @@
 /**
- * Shared header redaction utilities.
+ * Header redaction utilities for captures and LHAR export.
  *
- * Context Lens captures and exports some headers for debugging and provenance,
- * but must never persist secrets (API keys, cookies, auth challenges, etc.).
- *
- * Keep this as the single source of truth to avoid drift between "capture" and "export".
+ * `selectHeaders` and `SENSITIVE_HEADERS` are re-exported from @contextio/core
+ * (single source of truth). `redactHeaders` is context-lens-specific: it
+ * strips sensitive headers from an already-string-valued map (used in LHAR
+ * export where multi-value arrays are not possible).
  */
-/** Case-insensitive set of header names that must never be persisted/exported. */
-export declare const SENSITIVE_HEADERS: Set<string>;
+export { SENSITIVE_HEADERS, selectHeaders } from "@contextio/core";
 /**
- * Remove sensitive headers from a header map.
+ * Remove sensitive headers from a string-valued header map.
  *
- * @param headers - Header map (string -> string)
- * @returns A new object with sensitive headers removed.
+ * Unlike `selectHeaders`, this does not filter non-string values —
+ * it assumes all values are already strings (as in LHAR export).
  */
 export declare function redactHeaders(headers: Record<string, string>): Record<string, string>;
-/**
- * Select a safe subset of request/response headers for capture.
- *
- * - Drops sensitive headers (see `SENSITIVE_HEADERS`)
- * - Keeps only string-valued headers (Node can represent multi-valued headers as arrays)
- */
-export declare function selectHeaders(headers: Record<string, any>): Record<string, string>;
 //# sourceMappingURL=headers.d.ts.map

package/dist/http/headers.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../src/http/headers.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,kFAAkF;AAClF,eAAO,MAAM,iBAAiB,aAY5B,CAAC;AAEH;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOxB;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAC3B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOxB"}
+{"version":3,"file":"headers.d.ts","sourceRoot":"","sources":["../../src/http/headers.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAInE;;;;;GAKG;AACH,wBAAgB,aAAa,CAC3B,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,GAC9B,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAOxB"}

package/dist/http/headers.js

@@ -1,30 +1,18 @@
 /**
- * Shared header redaction utilities.
+ * Header redaction utilities for captures and LHAR export.
  *
- * Context Lens captures and exports some headers for debugging and provenance,
- * but must never persist secrets (API keys, cookies, auth challenges, etc.).
- *
- * Keep this as the single source of truth to avoid drift between "capture" and "export".
+ * `selectHeaders` and `SENSITIVE_HEADERS` are re-exported from @contextio/core
+ * (single source of truth). `redactHeaders` is context-lens-specific: it
+ * strips sensitive headers from an already-string-valued map (used in LHAR
+ * export where multi-value arrays are not possible).
  */
-/** Case-insensitive set of header names that must never be persisted/exported. */
-export const SENSITIVE_HEADERS = new Set([
-    "authorization",
-    "x-api-key",
-    "cookie",
-    "set-cookie",
-    "x-target-url",
-    "proxy-authorization",
-    "x-auth-token",
-    "x-forwarded-authorization",
-    "www-authenticate",
-    "proxy-authenticate",
-    "x-goog-api-key",
-]);
+export { SENSITIVE_HEADERS, selectHeaders } from "@contextio/core";
+import { SENSITIVE_HEADERS } from "@contextio/core";
 /**
- * Remove sensitive headers from a header map.
+ * Remove sensitive headers from a string-valued header map.
  *
- * @param headers - Header map (string -> string)
- * @returns A new object with sensitive headers removed.
+ * Unlike `selectHeaders`, this does not filter non-string values —
+ * it assumes all values are already strings (as in LHAR export).
  */
 export function redactHeaders(headers) {
     const result = {};
@@ -35,20 +23,4 @@ export function redactHeaders(headers) {
     }
     return result;
 }
-/**
- * Select a safe subset of request/response headers for capture.
- *
- * - Drops sensitive headers (see `SENSITIVE_HEADERS`)
- * - Keeps only string-valued headers (Node can represent multi-valued headers as arrays)
- */
-export function selectHeaders(headers) {
-    const result = {};
-    for (const [key, val] of Object.entries(headers)) {
-        if (SENSITIVE_HEADERS.has(key.toLowerCase()))
-            continue;
-        if (typeof val === "string")
-            result[key] = val;
-    }
-    return result;
-}
 //# sourceMappingURL=headers.js.map

package/dist/http/headers.js.map

@@ -1 +1 @@
-{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../src/http/headers.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,kFAAkF;AAClF,MAAM,CAAC,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC;IACvC,eAAe;IACf,WAAW;IACX,QAAQ;IACR,YAAY;IACZ,cAAc;IACd,qBAAqB;IACrB,cAAc;IACd,2BAA2B;IAC3B,kBAAkB;IAClB,oBAAoB;IACpB,gBAAgB;CACjB,CAAC,CAAC;AAEH;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,OAA+B;IAE/B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACjD,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACvD,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,OAA4B;IAE5B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACjD,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACvD,IAAI,OAAO,GAAG,KAAK,QAAQ;YAAE,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACjD,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
+{"version":3,"file":"headers.js","sourceRoot":"","sources":["../../src/http/headers.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEnE,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAC;AAEpD;;;;;GAKG;AACH,MAAM,UAAU,aAAa,CAC3B,OAA+B;IAE/B,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,KAAK,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QACjD,IAAI,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC;YAAE,SAAS;QACvD,MAAM,CAAC,GAAG,CAAC,GAAG,GAAG,CAAC;IACpB,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}

package/dist/lhar/composition.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../../src/lhar/composition.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,GACvC,gBAAgB,EAAE,CAwEpB;AA8KD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,gBAAgB,EAAE,EAC/B,aAAa,EAAE,MAAM,GACpB,IAAI,CAgCN"}
+{"version":3,"file":"composition.d.ts","sourceRoot":"","sources":["../../src/lhar/composition.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAEV,gBAAgB,EACjB,MAAM,4BAA4B,CAAC;AACpC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAE/C,wBAAgB,kBAAkB,CAChC,WAAW,EAAE,WAAW,EACxB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,GAAG,SAAS,GACvC,gBAAgB,EAAE,CA4EpB;AAiLD;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CAClC,WAAW,EAAE,gBAAgB,EAAE,EAC/B,aAAa,EAAE,MAAM,GACpB,IAAI,CAgCN"}