bpe-lite 0.1.0 → 0.1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bpe-lite",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Offline BPE tokenizer for OpenAI, Anthropic, and Gemini — zero dependencies",
   "main": "src/index.js",
   "type": "commonjs",

package/src/index.js

@@ -2,6 +2,7 @@
 
 const path = require('path');
 const fs = require('fs');
+const zlib = require('zlib');
 const { Tokenizer } = require('./tokenizer');
 
 const VOCABS_DIR = path.join(__dirname, '..', 'vocabs');
@@ -12,15 +13,21 @@ const _cache = {};
 function loadTokenizer(provider) {
   if (_cache[provider]) return _cache[provider];
 
-  const filePath = path.join(VOCABS_DIR, `${provider}.json`);
-  if (!fs.existsSync(filePath)) {
+  const gzPath   = path.join(VOCABS_DIR, `${provider}.json.gz`);
+  const jsonPath  = path.join(VOCABS_DIR, `${provider}.json`);
+
+  let data;
+  if (fs.existsSync(gzPath)) {
+    data = JSON.parse(zlib.gunzipSync(fs.readFileSync(gzPath)).toString('utf8'));
+  } else if (fs.existsSync(jsonPath)) {
+    data = JSON.parse(fs.readFileSync(jsonPath, 'utf8'));
+  } else {
     throw new Error(
-      `Vocab file not found for provider "${provider}": ${filePath}\n` +
+      `Vocab file not found for provider "${provider}".\n` +
       'Run "node scripts/build-vocabs.js" to build vocab files.'
     );
   }
 
-  const data = JSON.parse(fs.readFileSync(filePath, 'utf8'));
   _cache[provider] = new Tokenizer(data);
   return _cache[provider];
 }