logicstamp-context 0.1.0 → 0.2.0

package/dist/utils/tokens.js

@@ -1,23 +1,116 @@
 /**
  * Token estimation utilities for GPT and Claude models
- * Uses character-based approximations
+ * Uses character-based approximations by default
  *
- * For production use, consider installing:
- * - @dqbd/tiktoken for GPT models
- * - @anthropic-ai/tokenizer for Claude
+ * For accurate token counts, optionally install:
+ * - @dqbd/tiktoken for GPT models (gpt-4o encoding)
+ * - @anthropic-ai/tokenizer for Claude (claude-3-5-sonnet-20241022 encoding)
  */
+// Lazy-loaded tokenizers (only loaded if available)
+let tiktokenEncoder = null;
+let anthropicTokenizer = null;
+let tiktokenLoaded = false;
+let anthropicLoaded = false;
 /**
- * Estimate GPT-4 tokens using character-based approximation
- * GPT-4 typically uses ~4 characters per token for code/JSON
+ * Try to load tiktoken encoder (lazy, only once)
  */
-export function estimateGPT4Tokens(text) {
+async function loadTiktoken() {
+    if (tiktokenLoaded) {
+        return tiktokenEncoder !== null;
+    }
+    tiktokenLoaded = true;
+    try {
+        const tiktoken = await import('@dqbd/tiktoken');
+        // Try encoding_for_model first (preferred API)
+        if (typeof tiktoken.encoding_for_model === 'function') {
+            tiktokenEncoder = tiktoken.encoding_for_model('gpt-4o');
+            return true;
+        }
+        // Fallback to get_encoding if available
+        if (typeof tiktoken.get_encoding === 'function') {
+            tiktokenEncoder = tiktoken.get_encoding('cl100k_base');
+            return true;
+        }
+        // If neither method exists, tokenizer not usable
+        return false;
+    }
+    catch (error) {
+        // tiktoken not installed - use fallback
+        return false;
+    }
+}
+/**
+ * Try to load Anthropic tokenizer (lazy, only once)
+ */
+async function loadAnthropicTokenizer() {
+    if (anthropicLoaded) {
+        return anthropicTokenizer !== null;
+    }
+    anthropicLoaded = true;
+    try {
+        const tokenizer = await import('@anthropic-ai/tokenizer');
+        // Check if it's a default export or named export
+        const tokenizerModule = tokenizer.default || tokenizer;
+        // Try to get tokenizer for claude-3-5-sonnet-20241022 (latest stable)
+        if (typeof tokenizerModule === 'function') {
+            anthropicTokenizer = tokenizerModule;
+            return true;
+        }
+        // Check for countTokens method directly
+        if (typeof tokenizerModule.countTokens === 'function') {
+            anthropicTokenizer = tokenizerModule;
+            return true;
+        }
+        // If no usable methods, tokenizer not usable
+        return false;
+    }
+    catch (error) {
+        // tokenizer not installed - use fallback
+        return false;
+    }
+}
+/**
+ * Estimate GPT-4 tokens
+ * Uses @dqbd/tiktoken if available, otherwise falls back to character-based approximation
+ */
+export async function estimateGPT4Tokens(text) {
+    const hasTiktoken = await loadTiktoken();
+    if (hasTiktoken && tiktokenEncoder) {
+        try {
+            return tiktokenEncoder.encode(text).length;
+        }
+        catch (error) {
+            // Fall through to character-based estimation
+        }
+    }
+    // Fallback: character-based approximation
+    // GPT-4 typically uses ~4 characters per token for code/JSON
     return Math.ceil(text.length / 4);
 }
 /**
- * Estimate Claude tokens using character-based approximation
- * Claude typically uses ~4.5 characters per token for code/JSON
+ * Estimate Claude tokens
+ * Uses @anthropic-ai/tokenizer if available, otherwise falls back to character-based approximation
  */
-export function estimateClaudeTokens(text) {
+export async function estimateClaudeTokens(text) {
+    const hasTokenizer = await loadAnthropicTokenizer();
+    if (hasTokenizer && anthropicTokenizer) {
+        try {
+            // Try countTokens method first (most common API)
+            if (typeof anthropicTokenizer.countTokens === 'function') {
+                return anthropicTokenizer.countTokens(text);
+            }
+            // Try as a function that takes text and returns count
+            if (typeof anthropicTokenizer === 'function') {
+                const result = anthropicTokenizer(text);
+                return typeof result === 'number' ? result : 0;
+            }
+        }
+        catch (error) {
+            // Fall through to character-based estimation
+        }
+    }
+    // Fallback: character-based approximation
+    // Claude typically uses ~4.5 characters per token for code/JSON
     return Math.ceil(text.length / 4.5);
 }
 /**
@@ -26,4 +119,13 @@ export function estimateClaudeTokens(text) {
 export function formatTokenCount(count) {
     return count.toLocaleString();
 }
+/**
+ * Check if tokenizers are available
+ * Returns status for both GPT-4 and Claude tokenizers
+ */
+export async function getTokenizerStatus() {
+    const gpt4 = await loadTiktoken();
+    const claude = await loadAnthropicTokenizer();
+    return { gpt4, claude };
+}
 //# sourceMappingURL=tokens.js.map

package/dist/utils/tokens.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/utils/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH;;;GAGG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,IAAY;IAC/C,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;AAChC,CAAC"}
+{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../../src/utils/tokens.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,oDAAoD;AACpD,IAAI,eAAe,GAAQ,IAAI,CAAC;AAChC,IAAI,kBAAkB,GAAQ,IAAI,CAAC;AACnC,IAAI,cAAc,GAAG,KAAK,CAAC;AAC3B,IAAI,eAAe,GAAG,KAAK,CAAC;AAE5B;;GAEG;AACH,KAAK,UAAU,YAAY;IACzB,IAAI,cAAc,EAAE,CAAC;QACnB,OAAO,eAAe,KAAK,IAAI,CAAC;IAClC,CAAC;IACD,cAAc,GAAG,IAAI,CAAC;IAEtB,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,gBAAgB,CAAC,CAAC;QAChD,+CAA+C;QAC/C,IAAI,OAAO,QAAQ,CAAC,kBAAkB,KAAK,UAAU,EAAE,CAAC;YACtD,eAAe,GAAG,QAAQ,CAAC,kBAAkB,CAAC,QAAQ,CAAC,CAAC;YACxD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,wCAAwC;QACxC,IAAI,OAAO,QAAQ,CAAC,YAAY,KAAK,UAAU,EAAE,CAAC;YAChD,eAAe,GAAG,QAAQ,CAAC,YAAY,CAAC,aAAa,CAAC,CAAC;YACvD,OAAO,IAAI,CAAC;QACd,CAAC;QACD,iDAAiD;QACjD,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,wCAAwC;QACxC,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;GAEG;AACH,KAAK,UAAU,sBAAsB;IACnC,IAAI,eAAe,EAAE,CAAC;QACpB,OAAO,kBAAkB,KAAK,IAAI,CAAC;IACrC,CAAC;IACD,eAAe,GAAG,IAAI,CAAC;IAEvB,IAAI,CAAC;QACH,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC,yBAAyB,CAAC,CAAC;QAC1D,iDAAiD;QACjD,MAAM,eAAe,GAAG,SAAS,CAAC,OAAO,IAAI,SAAS,CAAC;QAEvD,sEAAsE;QACtE,IAAI,OAAO,eAAe,KAAK,UAAU,EAAE,CAAC;YAC1C,kBAAkB,GAAG,eAAe,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,wCAAwC;QACxC,IAAI,OAAO,eAAe,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;YACtD,kBAAkB,GAAG,eAAe,CAAC;YACrC,OAAO,IAAI,CAAC;QACd,CAAC;QACD,6CAA6C;QAC7C,OAAO,KAAK,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,yCAAyC;QACzC,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,IAAY;IACnD,MAAM,WAAW,GAAG,MAAM,YAAY,EAAE,CAAC;IAEzC,IAAI,WAAW,IAAI,eAAe,EAAE,CAAC;QACnC,IAAI,CAAC;YACH,OAAO,eAAe,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;QAC7C,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,6CAA6C;QAC/C,CAAC;IACH,CAAC;IAED,0CAA0C;IAC1C,6DAA6D;IAC7D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,IAAY;IACrD,MAAM,YAAY,GAAG,MAAM,sBAAsB,EAAE,CAAC;IAEpD,IAAI,YAAY,IAAI,kBAAkB,EAAE,CAAC;QACvC,IAAI,CAAC;YACH,iDAAiD;YACjD,IAAI,OAAO,kBAAkB,CAAC,WAAW,KAAK,UAAU,EAAE,CAAC;gBACzD,OAAO,kBAAkB,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YAC9C,CAAC;YACD,sDAAsD;YACtD,IAAI,OAAO,kBAAkB,KAAK,UAAU,EAAE,CAAC;gBAC7C,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACxC,OAAO,OAAO,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,6CAA6C;QAC/C,CAAC;IACH,CAAC;IAED,0CAA0C;IAC1C,gEAAgE;IAChE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;AACtC,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAa;IAC5C,OAAO,KAAK,CAAC,cAAc,EAAE,CAAC;AAChC,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,kBAAkB;IACtC,MAAM,IAAI,GAAG,MAAM,YAAY,EAAE,CAAC;IAClC,MAAM,MAAM,GAAG,MAAM,sBAAsB,EAAE,CAAC;IAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC1B,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "logicstamp-context",
-  "version": "0.1.0",
+  "version": "0.2.0",
   "description": "Generate AI-friendly context bundles from React codebases with LogicStamp contracts",
   "type": "module",
   "main": "./dist/index.js",
@@ -57,6 +57,8 @@
     "ts-morph": "^21.0.1"
   },
   "devDependencies": {
+    "@anthropic-ai/tokenizer": "^0.0.4",
+    "@dqbd/tiktoken": "^1.0.17",
     "@types/node": "^20.11.5",
     "@vitest/coverage-v8": "^4.0.8",
     "@vitest/ui": "^4.0.8",

package/schema/logicstamp.context.schema.json

@@ -195,6 +195,10 @@
         "links": {
           "$ref": "#/definitions/ContractLinks"
         },
+        "style": {
+          "$ref": "#/definitions/StyleMetadata",
+          "description": "Optional style metadata (only when --include-style is used)"
+        },
         "semanticHash": {
           "type": "string",
           "pattern": "^uif:[a-f0-9]{24}$",
@@ -425,6 +429,290 @@
         }
       },
       "additionalProperties": false
+    },
+    "StyleMetadata": {
+      "type": "object",
+      "properties": {
+        "styleSources": {
+          "$ref": "#/definitions/StyleSources"
+        },
+        "layout": {
+          "$ref": "#/definitions/LayoutMetadata"
+        },
+        "visual": {
+          "$ref": "#/definitions/VisualMetadata"
+        },
+        "animation": {
+          "$ref": "#/definitions/AnimationMetadata"
+        },
+        "pageLayout": {
+          "$ref": "#/definitions/PageLayoutMetadata"
+        }
+      },
+      "additionalProperties": false
+    },
+    "StyleSources": {
+      "type": "object",
+      "properties": {
+        "tailwind": {
+          "type": "object",
+          "properties": {
+            "categories": {
+              "type": "object",
+              "additionalProperties": {
+                "type": "array",
+                "items": {
+                  "type": "string"
+                }
+              },
+              "description": "Tailwind classes categorized by type (layout, spacing, colors, typography, borders, effects)"
+            },
+            "breakpoints": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Responsive breakpoints used (sm, md, lg, xl, 2xl)"
+            },
+            "classCount": {
+              "type": "number",
+              "description": "Total number of Tailwind classes detected"
+            }
+          },
+          "additionalProperties": false
+        },
+        "scssModule": {
+          "type": "string",
+          "description": "Path to imported SCSS module file"
+        },
+        "scssDetails": {
+          "type": "object",
+          "properties": {
+            "selectors": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "CSS selectors found in the SCSS file"
+            },
+            "properties": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "CSS properties used"
+            },
+            "features": {
+              "type": "object",
+              "properties": {
+                "variables": {
+                  "type": "boolean",
+                  "description": "Uses SCSS variables"
+                },
+                "nesting": {
+                  "type": "boolean",
+                  "description": "Uses SCSS nesting"
+                },
+                "mixins": {
+                  "type": "boolean",
+                  "description": "Uses SCSS mixins"
+                }
+              },
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        },
+        "cssModule": {
+          "type": "string",
+          "description": "Path to imported CSS module file"
+        },
+        "cssDetails": {
+          "type": "object",
+          "properties": {
+            "selectors": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            },
+            "properties": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              }
+            }
+          },
+          "additionalProperties": false
+        },
+        "inlineStyles": {
+          "type": "boolean",
+          "description": "Uses inline styles (style={{...}})"
+        },
+        "styledComponents": {
+          "type": "object",
+          "properties": {
+            "components": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Styled component names (e.g., ['div', 'Button'])"
+            },
+            "usesTheme": {
+              "type": "boolean",
+              "description": "Uses theme"
+            },
+            "usesCssProp": {
+              "type": "boolean",
+              "description": "Uses css prop"
+            }
+          },
+          "additionalProperties": false
+        },
+        "motion": {
+          "type": "object",
+          "properties": {
+            "components": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Motion components (motion.div, motion.button, etc.)"
+            },
+            "variants": {
+              "type": "array",
+              "items": {
+                "type": "string"
+              },
+              "description": "Variant names used"
+            },
+            "features": {
+              "type": "object",
+              "properties": {
+                "gestures": {
+                  "type": "boolean",
+                  "description": "Uses gesture handlers (whileHover, whileTap, etc.)"
+                },
+                "layoutAnimations": {
+                  "type": "boolean",
+                  "description": "Uses layout animations (layout prop)"
+                },
+                "viewportAnimations": {
+                  "type": "boolean",
+                  "description": "Uses viewport-triggered animations (useInView, viewport)"
+                }
+              },
+              "required": ["gestures", "layoutAnimations", "viewportAnimations"],
+              "additionalProperties": false
+            }
+          },
+          "required": ["features"],
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "LayoutMetadata": {
+      "type": "object",
+      "properties": {
+        "type": {
+          "type": "string",
+          "enum": ["flex", "grid", "relative", "absolute"],
+          "description": "Layout type"
+        },
+        "cols": {
+          "type": "string",
+          "description": "Grid column pattern (e.g., 'grid-cols-2 md:grid-cols-3')"
+        },
+        "hasHeroPattern": {
+          "type": "boolean",
+          "description": "Indicates hero section pattern (large text + CTA buttons)"
+        },
+        "hasFeatureCards": {
+          "type": "boolean",
+          "description": "Indicates feature card grid pattern"
+        },
+        "sections": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Identified page sections"
+        }
+      },
+      "additionalProperties": false
+    },
+    "VisualMetadata": {
+      "type": "object",
+      "properties": {
+        "colors": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Color utility classes (sorted, limited to top 10)"
+        },
+        "spacing": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Spacing utility classes (sorted, limited to top 10)"
+        },
+        "radius": {
+          "type": "string",
+          "description": "Most common border radius pattern"
+        },
+        "typography": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Typography classes (sorted, limited to top 10)"
+        }
+      },
+      "additionalProperties": false
+    },
+    "AnimationMetadata": {
+      "type": "object",
+      "properties": {
+        "type": {
+          "type": "string",
+          "description": "Animation type (e.g., 'fade-in', 'slide')"
+        },
+        "library": {
+          "type": "string",
+          "enum": ["framer-motion", "css"],
+          "description": "Animation library"
+        },
+        "trigger": {
+          "type": "string",
+          "description": "Trigger type (e.g., 'inView', 'hover', 'click')"
+        }
+      },
+      "additionalProperties": false
+    },
+    "PageLayoutMetadata": {
+      "type": "object",
+      "properties": {
+        "pageRole": {
+          "type": "string",
+          "description": "Page role (e.g., 'landing', 'dashboard')"
+        },
+        "sections": {
+          "type": "array",
+          "items": {
+            "type": "string"
+          },
+          "description": "Page sections"
+        },
+        "ctaCount": {
+          "type": "number",
+          "description": "Number of call-to-action elements"
+        }
+      },
+      "additionalProperties": false
     }
   }
 }