logicstamp-context 0.8.5 → 0.8.7

package/dist/utils/tokens.js

@@ -2,143 +2,125 @@
  * Token estimation utilities for GPT and Claude models
  * Uses character-based approximations by default
  *
- * For accurate token counts: These are included as optional dependencies in package.json.
- * npm automatically attempts to install them when installing logicstamp-context.
- * If installation succeeds, accurate token counts are used.
- * If installation fails or is skipped, falls back to character-based estimation.
- * - @dqbd/tiktoken for GPT models (gpt-4o encoding)
- * - @anthropic-ai/tokenizer for Claude (claude-3-5-sonnet-20241022 encoding)
+ * Optional tokenizers (package.json optionalDependencies; npm tries to install them):
+ * - @dqbd/tiktoken — GPT-4o encoding; counts align with that tiktoken model when load succeeds.
+ * - @anthropic-ai/tokenizer — Anthropic documents this as intended for older models; for Claude 3+
+ *   it is only a rough approximation vs API tokenization. Prefer `usage` in API responses for billing.
+ * If installation fails or is skipped, falls back to character-based estimation (~4 chars/token GPT,
+ * ~4.5 Claude).
  */
-// Lazy-loaded tokenizers (only loaded if available)
-// Note: Tokenizers are intentionally cached as singletons - they're expensive to load
-// Use clearTokenizerCache() to free memory if needed in long-running processes
-let tiktokenEncoder = null;
-let anthropicTokenizer = null;
-let tiktokenLoaded = false;
-let anthropicLoaded = false;
-/**
- * Clear tokenizer caches to free memory
- * Note: Tokenizers will be reloaded on next use (expensive operation)
- * Only call this in long-running processes that need to minimize memory
- */
-export function clearTokenizerCache() {
-    // Free the encoder if it has a free() method (tiktoken does)
-    if (tiktokenEncoder && typeof tiktokenEncoder.free === 'function') {
-        try {
-            tiktokenEncoder.free();
-        }
-        catch {
-            // Ignore errors during cleanup
-        }
-    }
+/** Mutable tokenizer state lives on this class instead of module-level lets. */
+export class TokenizerRuntime {
     tiktokenEncoder = null;
     anthropicTokenizer = null;
     tiktokenLoaded = false;
     anthropicLoaded = false;
-}
-/**
- * Try to load tiktoken encoder (lazy, only once)
- */
-async function loadTiktoken() {
-    if (tiktokenLoaded) {
-        return tiktokenEncoder !== null;
+    /**
+     * Free tokenizer memory. Tokenizers reload on next use (expensive).
+     * Use in long-running processes when you need to minimize memory.
+     */
+    clear() {
+        const enc = this.tiktokenEncoder;
+        if (enc && typeof enc.free === 'function') {
+            try {
+                enc.free();
+            }
+            catch {
+                // Ignore errors during cleanup
+            }
+        }
+        this.tiktokenEncoder = null;
+        this.anthropicTokenizer = null;
+        this.tiktokenLoaded = false;
+        this.anthropicLoaded = false;
     }
-    tiktokenLoaded = true;
-    try {
-        const tiktoken = await import('@dqbd/tiktoken');
-        // Try encoding_for_model first (preferred API)
-        if (typeof tiktoken.encoding_for_model === 'function') {
-            tiktokenEncoder = tiktoken.encoding_for_model('gpt-4o');
-            return true;
+    async loadTiktoken() {
+        if (this.tiktokenLoaded) {
+            return this.tiktokenEncoder !== null;
         }
-        // Fallback to get_encoding if available
-        if (typeof tiktoken.get_encoding === 'function') {
-            tiktokenEncoder = tiktoken.get_encoding('cl100k_base');
-            return true;
+        this.tiktokenLoaded = true;
+        try {
+            const tiktoken = await import('@dqbd/tiktoken');
+            if (typeof tiktoken.encoding_for_model === 'function') {
+                this.tiktokenEncoder = tiktoken.encoding_for_model('gpt-4o');
+                return true;
+            }
+            if (typeof tiktoken.get_encoding === 'function') {
+                this.tiktokenEncoder = tiktoken.get_encoding('cl100k_base');
+                return true;
+            }
+            return false;
         }
-        // If neither method exists, tokenizer not usable
-        return false;
-    }
-    catch (error) {
-        // tiktoken not installed - use fallback
-        return false;
-    }
-}
-/**
- * Try to load Anthropic tokenizer (lazy, only once)
- */
-async function loadAnthropicTokenizer() {
-    if (anthropicLoaded) {
-        return anthropicTokenizer !== null;
-    }
-    anthropicLoaded = true;
-    try {
-        const tokenizer = await import('@anthropic-ai/tokenizer');
-        // @anthropic-ai/tokenizer exports countTokens as a named export
-        if (typeof tokenizer.countTokens === 'function') {
-            anthropicTokenizer = tokenizer;
-            return true;
+        catch {
+            return false;
         }
-        // If countTokens not available, tokenizer not usable
-        return false;
     }
-    catch (error) {
-        // tokenizer not installed - use fallback
-        return false;
-    }
-}
-/**
- * Estimate GPT-4 tokens
- * Uses @dqbd/tiktoken if available, otherwise falls back to character-based approximation
- */
-export async function estimateGPT4Tokens(text) {
-    const hasTiktoken = await loadTiktoken();
-    if (hasTiktoken && tiktokenEncoder) {
+    async loadAnthropicTokenizer() {
+        if (this.anthropicLoaded) {
+            return this.anthropicTokenizer !== null;
+        }
+        this.anthropicLoaded = true;
         try {
-            return tiktokenEncoder.encode(text).length;
+            const tokenizer = await import('@anthropic-ai/tokenizer');
+            if (typeof tokenizer.countTokens === 'function') {
+                this.anthropicTokenizer = tokenizer;
+                return true;
+            }
+            return false;
         }
-        catch (error) {
-            // Fall through to character-based estimation
+        catch {
+            return false;
         }
     }
-    // Fallback: character-based approximation
-    // GPT-4 typically uses ~4 characters per token for code/JSON
-    return Math.ceil(text.length / 4);
-}
-/**
- * Estimate Claude tokens
- * Uses @anthropic-ai/tokenizer if available, otherwise falls back to character-based approximation
- */
-export async function estimateClaudeTokens(text) {
-    const hasTokenizer = await loadAnthropicTokenizer();
-    if (hasTokenizer && anthropicTokenizer) {
-        try {
-            // @anthropic-ai/tokenizer exports countTokens as a named export
-            if (typeof anthropicTokenizer.countTokens === 'function') {
-                return anthropicTokenizer.countTokens(text);
+    async estimateGPT4Tokens(text) {
+        const hasTiktoken = await this.loadTiktoken();
+        const encoder = this.tiktokenEncoder;
+        if (hasTiktoken && encoder && typeof encoder.encode === 'function') {
+            try {
+                return encoder.encode(text).length;
+            }
+            catch {
+                // Fall through to character-based estimation
             }
         }
-        catch (error) {
-            // Fall through to character-based estimation
+        return Math.ceil(text.length / 4);
+    }
+    async estimateClaudeTokens(text) {
+        const hasTokenizer = await this.loadAnthropicTokenizer();
+        if (hasTokenizer && this.anthropicTokenizer?.countTokens) {
+            try {
+                return this.anthropicTokenizer.countTokens(text);
+            }
+            catch {
+                // Fall through to character-based estimation
+            }
         }
+        return Math.ceil(text.length / 4.5);
+    }
+    async getTokenizerStatus() {
+        const gpt4 = await this.loadTiktoken();
+        const claude = await this.loadAnthropicTokenizer();
+        return { gpt4, claude };
     }
-    // Fallback: character-based approximation
-    // Claude typically uses ~4.5 characters per token for code/JSON
-    return Math.ceil(text.length / 4.5);
 }
-/**
- * Format token count with commas for readability
- */
+const defaultTokenizerRuntime = new TokenizerRuntime();
+/** @internal For tests or alternate lifetimes; CLI uses {@link defaultTokenizerRuntime}. */
+export function createTokenizerRuntime() {
+    return new TokenizerRuntime();
+}
+export function clearTokenizerCache() {
+    defaultTokenizerRuntime.clear();
+}
+export async function estimateGPT4Tokens(text) {
+    return defaultTokenizerRuntime.estimateGPT4Tokens(text);
+}
+export async function estimateClaudeTokens(text) {
+    return defaultTokenizerRuntime.estimateClaudeTokens(text);
+}
 export function formatTokenCount(count) {
     return count.toLocaleString();
 }
-/**
- * Check if tokenizers are available
- * Returns status for both GPT-4 and Claude tokenizers
- */
 export async function getTokenizerStatus() {
-    const gpt4 = await loadTiktoken();
-    const claude = await loadAnthropicTokenizer();
-    return { gpt4, claude };
+    return defaultTokenizerRuntime.getTokenizerStatus();
 }
 //# sourceMappingURL=tokens.js.map

package/dist/utils/tokens.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/utils/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,oDAAoD;AACpD,sFAAsF;AACtF,+EAA+E;AAC/E,IAAI,eAAe,GAAQ,IAAI,CAAC;AAChC,IAAI,kBAAkB,GAAQ,IAAI,CAAC;AACnC,IAAI,cAAc,GAAG,KAAK,CAAC;AAC3B,IAAI,eAAe,GAAG,KAAK,CAAC;AAE5B;;;;GAIG;AACH,MAAM,UAAU,mBAAmB;IACjC,6DAA6D;IAC7D,IAAI,eAAe,IAAI,OAAO,eAAe,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;QAClE,IAAI,CAAC;YACH,eAAe,CAAC,IAAI,EAAE,CAAC;QACzB,CAAC;QAAC,MAAM,CAAC;YACP,+BAA+B;QACjC,CAAC;IACH,CAAC;IACD,eAAe,GAAG,IAAI,CAAC;IACvB,kBAAkB,GAAG,IAAI,CAAC;IAC1B,cAAc,GAAG,KAAK,CAAC;IACvB,eAAe,GAAG,KAAK,CAAC;AAC1B,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,YAAY;IACzB,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,eAAe,KAAK,IAAI,CAAC;IAClC,CAAC;IACD,cAAc,GAAG,IAAI,CAAC;IAEtB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAChD,+CAA+C;QAC/C,IAAI,OAAO,QAAQ,CAAC,kBAAkB,KAAK,UAAU,EAAE,CAAC;YACtD,eAAe,GAAG,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACxD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,wCAAwC;QACxC,IAAI,OAAO,QAAQ,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YAChD,eAAe,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,iDAAiD;QACjD,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wCAAwC;QACxC,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,sBAAsB;IACnC,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,kBAAkB,KAAK,IAAI,CAAC;IACrC,CAAC;IACD,eAAe,GAAG,IAAI,CAAC;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC;QAC1D,gEAAgE;QAChE,IAAI,OAAO,SAAS,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;YAChD,kBAAkB,GAAG,SAAS,CAAC;YAC/B,OAAO,IAAI,CAAC;QACd,CAAC;QACD,qDAAqD;QACrD,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,yCAAyC;QACzC,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACnD,MAAM,WAAW,GAAG,MAAM,YAAY,EAAE,CAAC;IAEzC,IAAI,WAAW,IAAI,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,OAAO,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,6CAA6C;QAC/C,CAAC;IACH,CAAC;IAED,0CAA0C;IAC1C,6DAA6D;IAC7D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY;IACrD,MAAM,YAAY,GAAG,MAAM,sBAAsB,EAAE,CAAC;IAEpD,IAAI,YAAY,IAAI,kBAAkB,EAAE,CAAC;QACvC,IAAI,CAAC;YACH,gEAAgE;YAChE,IAAI,OAAO,kBAAkB,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;gBACzD,OAAO,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9C,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,6CAA6C;QAC/C,CAAC;IACH,CAAC;IAED,0CAA0C;IAC1C,gEAAgE;IAChE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,MAAM,IAAI,GAAG,MAAM,YAAY,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,MAAM,sBAAsB,EAAE,CAAC;IAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC1B,CAAC"}
+{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/utils/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,gFAAgF;AAChF,MAAM,OAAO,gBAAgB;IACnB,eAAe,GAAY,IAAI,CAAC;IAChC,kBAAkB,GAEf,IAAI,CAAC;IACR,cAAc,GAAG,KAAK,CAAC;IACvB,eAAe,GAAG,KAAK,CAAC;IAEhC;;;OAGG;IACH,KAAK;QACH,MAAM,GAAG,GAAG,IAAI,CAAC,eAA+C,CAAC;QACjE,IAAI,GAAG,IAAI,OAAO,GAAG,CAAC,IAAI,KAAK,UAAU,EAAE,CAAC;YAC1C,IAAI,CAAC;gBACH,GAAG,CAAC,IAAI,EAAE,CAAC;YACb,CAAC;YAAC,MAAM,CAAC;gBACP,+BAA+B;YACjC,CAAC;QACH,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAC5B,IAAI,CAAC,kBAAkB,GAAG,IAAI,CAAC;QAC/B,IAAI,CAAC,cAAc,GAAG,KAAK,CAAC;QAC5B,IAAI,CAAC,eAAe,GAAG,KAAK,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,YAAY;QACxB,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;YACxB,OAAO,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC;QACvC,CAAC;QACD,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;QAE3B,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;YAChD,IAAI,OAAO,QAAQ,CAAC,kBAAkB,KAAK,UAAU,EAAE,CAAC;gBACtD,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;gBAC7D,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,OAAO,QAAQ,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;gBAChD,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;gBAC5D,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,OAAO,IAAI,CAAC,kBAAkB,KAAK,IAAI,CAAC;QAC1C,CAAC;QACD,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;QAE5B,IAAI,CAAC;YACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC;YAC1D,IAAI,OAAO,SAAS,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;gBAChD,IAAI,CAAC,kBAAkB,GAAG,SAAS,CAAC;gBACpC,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,KAAK,CAAC;QACf,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,KAAK,CAAC;QACf,CAAC;IACH,CAAC;IAED,KAAK,CAAC,kBAAkB,CAAC,IAAY;QACnC,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QAC9C,MAAM,OAAO,GAAG,IAAI,CAAC,eAEb,CAAC;QAET,IAAI,WAAW,IAAI,OAAO,IAAI,OAAO,OAAO,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YACnE,IAAI,CAAC;gBACH,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;YACrC,CAAC;YAAC,MAAM,CAAC;gBACP,6CAA6C;YAC/C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACpC,CAAC;IAED,KAAK,CAAC,oBAAoB,CAAC,IAAY;QACrC,MAAM,YAAY,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QAEzD,IAAI,YAAY,IAAI,IAAI,CAAC,kBAAkB,EAAE,WAAW,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,OAAO,IAAI,CAAC,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACnD,CAAC;YAAC,MAAM,CAAC;gBACP,6CAA6C;YAC/C,CAAC;QACH,CAAC;QAED,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;IACtC,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACnD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAC1B,CAAC;CACF;AAED,MAAM,uBAAuB,GAAG,IAAI,gBAAgB,EAAE,CAAC;AAEvD,4FAA4F;AAC5F,MAAM,UAAU,sBAAsB;IACpC,OAAO,IAAI,gBAAgB,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,uBAAuB,CAAC,KAAK,EAAE,CAAC;AAClC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACnD,OAAO,uBAAuB,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY;IACrD,OAAO,uBAAuB,CAAC,oBAAoB,CAAC,IAAI,CAAC,CAAC;AAC5D,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;AAChC,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB;IAItC,OAAO,uBAAuB,CAAC,kBAAkB,EAAE,CAAC;AACtD,CAAC"}

package/package.json

@@ -1,16 +1,19 @@
 {
   "name": "logicstamp-context",
-  "version": "0.8.5",
+  "version": "0.8.7",
   "description": "The context compiler for TypeScript. Compile your codebase into deterministic architectural contracts and dependency graphs for AI coding workflows.",
   "type": "module",
   "main": "./dist/index.js",
   "types": "./dist/index.d.ts",
   "exports": {
-    ".": "./dist/index.js"
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
   },
   "bin": {
-    "logicstamp-context": "dist/cli/stamp.js",
-    "stamp": "dist/cli/stamp.js"
+    "logicstamp-context": "./dist/cli/stamp.js",
+    "stamp": "./dist/cli/stamp.js"
   },
   "files": [
     "dist",
@@ -21,6 +24,9 @@
   ],
   "scripts": {
     "build": "tsc",
+    "lint": "biome lint src tests vitest.config.ts",
+    "lint:fix": "biome lint --write src tests vitest.config.ts",
+    "format": "biome format --write src tests biome.json package.json tsconfig.json vitest.config.ts",
     "prepublishOnly": "npm run build",
     "test": "vitest run",
     "test:watch": "vitest",
@@ -73,6 +79,7 @@
     "@dqbd/tiktoken": "^1.0.17"
   },
   "devDependencies": {
+    "@biomejs/biome": "^2.4.11",
     "@types/node": "^20.19.27",
     "@vitest/coverage-v8": "^4.0.16",
     "@vitest/ui": "^4.0.16",