claude-brain 0.30.2 → 0.30.3

package/src/intelligence/tokenizer.ts

@@ -1,118 +1,118 @@
-/**
- * Tokenizer — SLM Upgrade Phase 4B
- * GPT-2 BPE tokenizer for ONNX model inference.
- *
- * Strategy:
- *   1. Try to dynamically import `tiktoken` (JS package)
- *   2. Fall back to a simple whitespace tokenizer with hash-based IDs
- *
- * The tokenizer pads/truncates sequences to a fixed length and
- * returns both input_ids and attention_mask arrays.
- */
-
-import type { Logger } from 'pino'
-
-export interface TokenizerOutput {
-  inputIds: number[]
-  attentionMask: number[]
-}
-
-export interface Tokenizer {
-  encode(text: string, maxLength: number): TokenizerOutput
-  decode(tokenIds: number[]): string
-}
-
-/** Singleton cache */
-let cachedTokenizer: Tokenizer | null = null
-
-/**
- * Get a tokenizer instance. Tries tiktoken first, falls back to hash-based.
- */
-export async function getTokenizer(logger?: Logger): Promise<Tokenizer> {
-  if (cachedTokenizer) return cachedTokenizer
-
-  // Try tiktoken (JS binding for GPT-2 BPE)
-  try {
-    const tiktoken = await import('tiktoken')
-    const enc = tiktoken.encoding_for_model('gpt2')
-
-    cachedTokenizer = {
-      encode(text: string, maxLength: number): TokenizerOutput {
-        const tokens = Array.from(enc.encode(text))
-
-        // Truncate if necessary
-        const truncated = tokens.slice(0, maxLength)
-
-        // Pad to maxLength
-        const inputIds = new Array(maxLength).fill(0)
-        const attentionMask = new Array(maxLength).fill(0)
-
-        for (let i = 0; i < truncated.length; i++) {
-          inputIds[i] = truncated[i]
-          attentionMask[i] = 1
-        }
-
-        return { inputIds, attentionMask }
-      },
-      decode(tokenIds: number[]): string {
-        // Filter out padding (0) tokens
-        const filtered = tokenIds.filter(id => id !== 0)
-        return new TextDecoder().decode(enc.decode(new Uint32Array(filtered)))
-      }
-    }
-
-    logger?.debug('Using tiktoken GPT-2 tokenizer')
-    return cachedTokenizer
-  } catch {
-    // tiktoken not available
-  }
-
-  // Fallback: simple whitespace tokenizer with hash-based IDs
-  logger?.warn('tiktoken not available — using fallback hash-based tokenizer (reduced accuracy)')
-
-  cachedTokenizer = {
-    encode(text: string, maxLength: number): TokenizerOutput {
-      // Split on whitespace and punctuation, filter empties
-      const tokens = text
-        .toLowerCase()
-        .split(/(\s+|[.,!?;:'"()\[\]{}<>\/\\@#$%^&*+=~`|_-]+)/)
-        .filter(t => t.trim().length > 0)
-
-      // Hash each token to a stable ID in [1, 50256] range (GPT-2 vocab size)
-      const VOCAB_SIZE = 50256
-      const tokenIds = tokens.map(t => {
-        let hash = 5381
-        for (let i = 0; i < t.length; i++) {
-          hash = ((hash << 5) + hash + t.charCodeAt(i)) & 0x7fffffff
-        }
-        return (hash % (VOCAB_SIZE - 1)) + 1 // avoid 0 (used for padding)
-      })
-
-      // Truncate
-      const truncated = tokenIds.slice(0, maxLength)
-
-      // Pad to maxLength
-      const inputIds = new Array(maxLength).fill(0)
-      const attentionMask = new Array(maxLength).fill(0)
-
-      for (let i = 0; i < truncated.length; i++) {
-        inputIds[i] = truncated[i]
-        attentionMask[i] = 1
-      }
-
-      return { inputIds, attentionMask }
-    },
-    decode(_tokenIds: number[]): string {
-      // Hash-based tokenizer is one-way; decode is not possible.
-      // Compression will fall back to returning original text.
-      return ''
-    }
-  }
-
-  return cachedTokenizer
-}
-
-/** Reset cached tokenizer (for testing) */
-export function _resetTokenizerForTesting(): void {
-  cachedTokenizer = null
-}
+/**
+ * Tokenizer — SLM Upgrade Phase 4B
+ * GPT-2 BPE tokenizer for ONNX model inference.
+ *
+ * Strategy:
+ *   1. Try to dynamically import `tiktoken` (JS package)
+ *   2. Fall back to a simple whitespace tokenizer with hash-based IDs
+ *
+ * The tokenizer pads/truncates sequences to a fixed length and
+ * returns both input_ids and attention_mask arrays.
+ */
+
+import type { Logger } from 'pino'
+
+export interface TokenizerOutput {
+  inputIds: number[]
+  attentionMask: number[]
+}
+
+export interface Tokenizer {
+  encode(text: string, maxLength: number): TokenizerOutput
+  decode(tokenIds: number[]): string
+}
+
+/** Singleton cache */
+let cachedTokenizer: Tokenizer | null = null
+
+/**
+ * Get a tokenizer instance. Tries tiktoken first, falls back to hash-based.
+ */
+export async function getTokenizer(logger?: Logger): Promise<Tokenizer> {
+  if (cachedTokenizer) return cachedTokenizer
+
+  // Try tiktoken (JS binding for GPT-2 BPE)
+  try {
+    const tiktoken = await import('tiktoken')
+    const enc = tiktoken.encoding_for_model('gpt2')
+
+    cachedTokenizer = {
+      encode(text: string, maxLength: number): TokenizerOutput {
+        const tokens = Array.from(enc.encode(text))
+
+        // Truncate if necessary
+        const truncated = tokens.slice(0, maxLength)
+
+        // Pad to maxLength
+        const inputIds = new Array(maxLength).fill(0)
+        const attentionMask = new Array(maxLength).fill(0)
+
+        for (let i = 0; i < truncated.length; i++) {
+          inputIds[i] = truncated[i]
+          attentionMask[i] = 1
+        }
+
+        return { inputIds, attentionMask }
+      },
+      decode(tokenIds: number[]): string {
+        // Filter out padding (0) tokens
+        const filtered = tokenIds.filter(id => id !== 0)
+        return new TextDecoder().decode(enc.decode(new Uint32Array(filtered)))
+      }
+    }
+
+    logger?.debug('Using tiktoken GPT-2 tokenizer')
+    return cachedTokenizer
+  } catch {
+    // tiktoken not available
+  }
+
+  // Fallback: simple whitespace tokenizer with hash-based IDs
+  logger?.warn('tiktoken not available — using fallback hash-based tokenizer (reduced accuracy)')
+
+  cachedTokenizer = {
+    encode(text: string, maxLength: number): TokenizerOutput {
+      // Split on whitespace and punctuation, filter empties
+      const tokens = text
+        .toLowerCase()
+        .split(/(\s+|[.,!?;:'"()\[\]{}<>\/\\@#$%^&*+=~`|_-]+)/)
+        .filter(t => t.trim().length > 0)
+
+      // Hash each token to a stable ID in [1, 50256] range (GPT-2 vocab size)
+      const VOCAB_SIZE = 50256
+      const tokenIds = tokens.map(t => {
+        let hash = 5381
+        for (let i = 0; i < t.length; i++) {
+          hash = ((hash << 5) + hash + t.charCodeAt(i)) & 0x7fffffff
+        }
+        return (hash % (VOCAB_SIZE - 1)) + 1 // avoid 0 (used for padding)
+      })
+
+      // Truncate
+      const truncated = tokenIds.slice(0, maxLength)
+
+      // Pad to maxLength
+      const inputIds = new Array(maxLength).fill(0)
+      const attentionMask = new Array(maxLength).fill(0)
+
+      for (let i = 0; i < truncated.length; i++) {
+        inputIds[i] = truncated[i]
+        attentionMask[i] = 1
+      }
+
+      return { inputIds, attentionMask }
+    },
+    decode(_tokenIds: number[]): string {
+      // Hash-based tokenizer is one-way; decode is not possible.
+      // Compression will fall back to returning original text.
+      return ''
+    }
+  }
+
+  return cachedTokenizer
+}
+
+/** Reset cached tokenizer (for testing) */
+export function _resetTokenizerForTesting(): void {
+  cachedTokenizer = null
+}