@craftpipe/contextpack 1.0.0

package/lib/tokenizer.js

@@ -0,0 +1,291 @@
+'use strict';
+
+/**
+ * lib/tokenizer.js
+ * Estimates token count using character-based approximation or tiktoken-js;
+ * calculates total bundle token size; estimates token savings compared to raw file inclusion
+ */
+
+/**
+ * Average characters per token for GPT-style models (empirically ~4 chars/token)
+ */
+const CHARS_PER_TOKEN = 4;
+
+/**
+ * Overhead ratio for structured formats (JSON/Markdown adds keys, brackets, etc.)
+ */
+const JSON_OVERHEAD_RATIO = 1.15;
+const MARKDOWN_OVERHEAD_RATIO = 1.08;
+
+/**
+ * Try to load tiktoken-js for more accurate token estimation.
+ * Falls back to character-based approximation if unavailable.
+ */
+let tiktokenEncoder = null;
+
+(function tryLoadTiktoken() {
+  try {
+    // Attempt to load tiktoken or tiktoken-lite if available
+    const tiktoken = require('tiktoken');
+    if (tiktoken && typeof tiktoken.get_encoding === 'function') {
+      tiktokenEncoder = tiktoken.get_encoding('cl100k_base');
+    }
+  } catch (_e1) {
+    try {
+      const tiktoken = require('@dqbd/tiktoken');
+      if (tiktoken && typeof tiktoken.get_encoding === 'function') {
+        tiktokenEncoder = tiktoken.get_encoding('cl100k_base');
+      }
+    } catch (_e2) {
+      // Neither tiktoken package is available; use character-based approximation
+      tiktokenEncoder = null;
+    }
+  }
+})();
+
+/**
+ * Estimate the number of tokens in a string using tiktoken if available,
+ * otherwise fall back to character-based approximation.
+ *
+ * @param {string} text - The text to estimate tokens for
+ * @param {object} [options] - Optional configuration
+ * @param {boolean} [options.forceCharApprox=false] - Force character-based approximation even if tiktoken is available
+ * @returns {number} Estimated token count (integer >= 0)
+ */
+function estimateTokens(text, options) {
+  const opts = options || {};
+  const { forceCharApprox = false } = opts;
+
+  if (text === null || text === undefined) {
+    return 0;
+  }
+
+  const str = typeof text === 'string' ? text : String(text);
+
+  if (str.length === 0) {
+    return 0;
+  }
+
+  // Use tiktoken if available and not forced to use char approximation
+  if (!forceCharApprox && tiktokenEncoder !== null) {
+    try {
+      const encoded = tiktokenEncoder.encode(str);
+      const count = encoded ? encoded.length : 0;
+      // Free the encoded buffer if it has a free method (some tiktoken versions)
+      if (encoded && typeof encoded.free === 'function') {
+        encoded.free();
+      }
+      return count;
+    } catch (_err) {
+      // Fall through to character-based approximation
+    }
+  }
+
+  // Character-based approximation: ~4 characters per token
+  return Math.ceil(str.length / CHARS_PER_TOKEN);
+}
+
+/**
+ * Calculate the total token size of a bundle object.
+ * Accepts a bundle as produced by bundler.js (with fileSummaries, symbolIndex,
+ * dependencyMap, metadata) or any serializable object.
+ *
+ * @param {object} bundle - The bundle object to measure
+ * @param {object} [options] - Optional configuration
+ * @param {string} [options.format='json'] - Output format: 'json' or 'markdown'
+ * @param {boolean} [options.forceCharApprox=false] - Force character-based approximation
+ * @returns {object} Result with { totalTokens, breakdown, format }
+ */
+function calculateBundleSize(bundle, options) {
+  const opts = options || {};
+  const { format = 'json', forceCharApprox = false } = opts;
+
+  const emptyResult = {
+    totalTokens: 0,
+    breakdown: {
+      metadata: 0,
+      fileSummaries: 0,
+      symbolIndex: 0,
+      dependencyMap: 0,
+    },
+    format,
+    method: tiktokenEncoder && !forceCharApprox ? 'tiktoken' : 'char-approx',
+  };
+
+  if (bundle === null || bundle === undefined) {
+    return emptyResult;
+  }
+
+  if (typeof bundle !== 'object') {
+    // Treat as raw string
+    const tokens = estimateTokens(String(bundle), { forceCharApprox });
+    return {
+      totalTokens: tokens,
+      breakdown: {
+        metadata: 0,
+        fileSummaries: 0,
+        symbolIndex: 0,
+        dependencyMap: 0,
+        raw: tokens,
+      },
+      format,
+      method: tiktokenEncoder && !forceCharApprox ? 'tiktoken' : 'char-approx',
+    };
+  }
+
+  const overhead = format === 'markdown' ? MARKDOWN_OVERHEAD_RATIO : JSON_OVERHEAD_RATIO;
+
+  /**
+   * Safely serialize a value to a string for token counting
+   * @param {*} val
+   * @returns {string}
+   */
+  function safeSerialize(val) {
+    if (val === null || val === undefined) return '';
+    if (typeof val === 'string') return val;
+    try {
+      return JSON.stringify(val) || '';
+    } catch (_e) {
+      return String(val);
+    }
+  }
+
+  // Break down token counts by bundle section
+  const metadataStr = safeSerialize(bundle.metadata);
+  const fileSummariesStr = safeSerialize(bundle.fileSummaries);
+  const symbolIndexStr = safeSerialize(bundle.symbolIndex);
+  const dependencyMapStr = safeSerialize(bundle.dependencyMap);
+
+  const metadataTokens = Math.ceil(estimateTokens(metadataStr, { forceCharApprox }) * overhead);
+  const fileSummariesTokens = Math.ceil(estimateTokens(fileSummariesStr, { forceCharApprox }) * overhead);
+  const symbolIndexTokens = Math.ceil(estimateTokens(symbolIndexStr, { forceCharApprox }) * overhead);
+  const dependencyMapTokens = Math.ceil(estimateTokens(dependencyMapStr, { forceCharApprox }) * overhead);
+
+  // If the bundle has sections we recognize, use the breakdown
+  const hasKnownSections = (
+    bundle.metadata !== undefined ||
+    bundle.fileSummaries !== undefined ||
+    bundle.symbolIndex !== undefined ||
+    bundle.dependencyMap !== undefined
+  );
+
+  let totalTokens;
+  let breakdown;
+
+  if (hasKnownSections) {
+    breakdown = {
+      metadata: metadataTokens,
+      fileSummaries: fileSummariesTokens,
+      symbolIndex: symbolIndexTokens,
+      dependencyMap: dependencyMapTokens,
+    };
+    totalTokens = metadataTokens + fileSummariesTokens + symbolIndexTokens + dependencyMapTokens;
+  } else {
+    // Unknown bundle shape — serialize the whole thing
+    const wholeStr = safeSerialize(bundle);
+    totalTokens = Math.ceil(estimateTokens(wholeStr, { forceCharApprox }) * overhead);
+    breakdown = {
+      metadata: 0,
+      fileSummaries: 0,
+      symbolIndex: 0,
+      dependencyMap: 0,
+      total: totalTokens,
+    };
+  }
+
+  return {
+    totalTokens,
+    breakdown,
+    format,
+    method: tiktokenEncoder && !forceCharApprox ? 'tiktoken' : 'char-approx',
+  };
+}
+
+/**
+ * Estimate token savings achieved by using a bundle compared to including
+ * raw file contents directly.
+ *
+ * @param {object} bundle - The context bundle produced by bundler.js
+ * @param {object} [options] - Optional configuration
+ * @param {Array<object>} [options.rawFiles] - Array of raw file objects with { content, size } or { path, size }
+ * @param {number} [options.rawTokenCount] - Pre-computed raw token count (overrides rawFiles calculation)
+ * @param {string} [options.format='json'] - Bundle format for size calculation: 'json' or 'markdown'
+ * @param {boolean} [options.forceCharApprox=false] - Force character-based approximation
+ * @returns {object} Savings report with { bundleTokens, rawTokens, savedTokens, savingsPercent, method }
+ */
+function estimateSavings(bundle, options) {
+  const opts = options || {};
+  const {
+    rawFiles,
+    rawTokenCount,
+    format = 'json',
+    forceCharApprox = false,
+  } = opts;
+
+  const noSavingsResult = {
+    bundleTokens: 0,
+    rawTokens: 0,
+    savedTokens: 0,
+    savingsPercent: 0,
+    method: tiktokenEncoder && !forceCharApprox ? 'tiktoken' : 'char-approx',
+  };
+
+  if (bundle === null || bundle === undefined) {
+    return noSavingsResult;
+  }
+
+  // Calculate bundle token size
+  const bundleSizeResult = calculateBundleSize(bundle, { format, forceCharApprox });
+  const bundleTokens = bundleSizeResult.totalTokens || 0;
+
+  // Calculate raw token count
+  let rawTokens = 0;
+
+  if (typeof rawTokenCount === 'number' && rawTokenCount >= 0) {
+    // Use pre-computed value if provided
+    rawTokens = rawTokenCount;
+  } else if (Array.isArray(rawFiles) && rawFiles.length > 0) {
+    // Sum tokens from each raw file
+    for (const file of rawFiles) {
+      if (!file || typeof file !== 'object') continue;
+
+      if (typeof file.content === 'string') {
+        // Use actual content if available
+        rawTokens += estimateTokens(file.content, { forceCharApprox });
+      } else if (typeof file.size === 'number' && file.size > 0) {
+        // Fall back to size-based approximation
+        rawTokens += Math.ceil(file.size / CHARS_PER_TOKEN);
+      }
+    }
+  } else if (bundle.metadata && typeof bundle.metadata.totalSize === 'number') {
+    // Last resort: use bundle metadata totalSize to estimate raw tokens
+    rawTokens = Math.ceil(bundle.metadata.totalSize / CHARS_PER_TOKEN);
+  } else if (Array.isArray(bundle.fileSummaries) && bundle.fileSummaries.length > 0) {
+    // Estimate from file summaries if we have size info
+    for (const summary of bundle.fileSummaries) {
+      if (!summary || typeof summary !== 'object') continue;
+      if (typeof summary.size === 'number' && summary.size > 0) {
+        rawTokens += Math.ceil(summary.size / CHARS_PER_TOKEN);
+      }
+    }
+  }
+
+  const savedTokens = Math.max(0, rawTokens - bundleTokens);
+  const savingsPercent = rawTokens > 0
+    ? Math.round((savedTokens / rawTokens) * 10000) / 100
+    : 0;
+
+  return {
+    bundleTokens,
+    rawTokens,
+    savedTokens,
+    savingsPercent,
+    method: tiktokenEncoder && !forceCharApprox ? 'tiktoken' : 'char-approx',
+  };
+}
+
+module.exports = {
+  estimateTokens,
+  calculateBundleSize,
+  estimateSavings,
+};