@nanocollective/nanocoder 1.18.0 → 1.19.0

package/dist/services/file-snapshot.spec.js

@@ -0,0 +1,285 @@
+import { existsSync } from 'fs';
+import * as path from 'path';
+import test from 'ava';
+import * as fs from 'fs/promises';
+import { FileSnapshotService } from './file-snapshot.js';
+// Helper to create a temporary directory for tests
+async function createTempDir() {
+    const tempDir = path.join(process.cwd(), '.test-temp', `snapshot-test-${Date.now()}-${Math.random().toString(36).slice(2)}`);
+    await fs.mkdir(tempDir, { recursive: true });
+    return tempDir;
+}
+// Helper to clean up temp directory
+async function cleanupTempDir(dir) {
+    try {
+        await fs.rm(dir, { recursive: true, force: true });
+    }
+    catch {
+        // Ignore cleanup errors
+    }
+}
+// Helper to create a test file
+async function createTestFile(dir, relativePath, content) {
+    const fullPath = path.join(dir, relativePath);
+    await fs.mkdir(path.dirname(fullPath), { recursive: true });
+    await fs.writeFile(fullPath, content, 'utf-8');
+    return fullPath;
+}
+test.serial('FileSnapshotService captures single file', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        await createTestFile(tempDir, 'test.txt', 'Hello, World!');
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = await service.captureFiles(['test.txt']);
+        t.is(snapshots.size, 1);
+        t.is(snapshots.get('test.txt'), 'Hello, World!');
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService captures multiple files', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        await createTestFile(tempDir, 'file1.txt', 'Content 1');
+        await createTestFile(tempDir, 'file2.txt', 'Content 2');
+        await createTestFile(tempDir, 'file3.txt', 'Content 3');
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = await service.captureFiles([
+            'file1.txt',
+            'file2.txt',
+            'file3.txt',
+        ]);
+        t.is(snapshots.size, 3);
+        t.is(snapshots.get('file1.txt'), 'Content 1');
+        t.is(snapshots.get('file2.txt'), 'Content 2');
+        t.is(snapshots.get('file3.txt'), 'Content 3');
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService captures files in subdirectories', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        await createTestFile(tempDir, 'src/index.ts', 'export {};');
+        await createTestFile(tempDir, 'src/utils/helper.ts', 'export function help() {}');
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = await service.captureFiles([
+            'src/index.ts',
+            'src/utils/helper.ts',
+        ]);
+        t.is(snapshots.size, 2);
+        t.is(snapshots.get('src/index.ts'), 'export {};');
+        t.is(snapshots.get('src/utils/helper.ts'), 'export function help() {}');
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService handles non-existent files gracefully', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        await createTestFile(tempDir, 'exists.txt', 'I exist');
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = await service.captureFiles([
+            'exists.txt',
+            'does-not-exist.txt',
+        ]);
+        // Should only capture the existing file
+        t.is(snapshots.size, 1);
+        t.is(snapshots.get('exists.txt'), 'I exist');
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService restores files', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = new Map();
+        snapshots.set('restored.txt', 'Restored content');
+        await service.restoreFiles(snapshots);
+        const restoredPath = path.join(tempDir, 'restored.txt');
+        t.true(existsSync(restoredPath));
+        const content = await fs.readFile(restoredPath, 'utf-8');
+        t.is(content, 'Restored content');
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService restores files in subdirectories', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = new Map();
+        snapshots.set('deep/nested/file.txt', 'Nested content');
+        await service.restoreFiles(snapshots);
+        const restoredPath = path.join(tempDir, 'deep/nested/file.txt');
+        t.true(existsSync(restoredPath));
+        const content = await fs.readFile(restoredPath, 'utf-8');
+        t.is(content, 'Nested content');
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService restores multiple files', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = new Map();
+        snapshots.set('file1.txt', 'Content 1');
+        snapshots.set('file2.txt', 'Content 2');
+        snapshots.set('subdir/file3.txt', 'Content 3');
+        await service.restoreFiles(snapshots);
+        t.true(existsSync(path.join(tempDir, 'file1.txt')));
+        t.true(existsSync(path.join(tempDir, 'file2.txt')));
+        t.true(existsSync(path.join(tempDir, 'subdir/file3.txt')));
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService overwrites existing files on restore', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        await createTestFile(tempDir, 'existing.txt', 'Old content');
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = new Map();
+        snapshots.set('existing.txt', 'New content');
+        await service.restoreFiles(snapshots);
+        const content = await fs.readFile(path.join(tempDir, 'existing.txt'), 'utf-8');
+        t.is(content, 'New content');
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService getSnapshotSize calculates correct size', async (t) => {
+    const service = new FileSnapshotService(process.cwd());
+    const snapshots = new Map();
+    snapshots.set('file1.txt', 'Hello'); // 5 bytes
+    snapshots.set('file2.txt', 'World!'); // 6 bytes
+    const size = service.getSnapshotSize(snapshots);
+    t.is(size, 11);
+});
+test.serial('FileSnapshotService getSnapshotSize handles empty snapshots', async (t) => {
+    const service = new FileSnapshotService(process.cwd());
+    const snapshots = new Map();
+    const size = service.getSnapshotSize(snapshots);
+    t.is(size, 0);
+});
+test.serial('FileSnapshotService getSnapshotSize handles unicode content', async (t) => {
+    const service = new FileSnapshotService(process.cwd());
+    const snapshots = new Map();
+    snapshots.set('unicode.txt', '日本語'); // 9 bytes in UTF-8
+    const size = service.getSnapshotSize(snapshots);
+    t.is(size, 9);
+});
+test.serial('FileSnapshotService validateRestorePath validates writable paths', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = new Map();
+        snapshots.set('new-file.txt', 'Content');
+        const result = await service.validateRestorePath(snapshots);
+        t.true(result.valid);
+        t.is(result.errors.length, 0);
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService validateRestorePath validates existing writable files', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        await createTestFile(tempDir, 'writable.txt', 'Original');
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = new Map();
+        snapshots.set('writable.txt', 'New content');
+        const result = await service.validateRestorePath(snapshots);
+        t.true(result.valid);
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService getModifiedFiles returns array', async (t) => {
+    // Note: This test runs in a git directory, so it may return files.
+    // The important thing is that it returns an array and doesn't throw.
+    const tempDir = await createTempDir();
+    try {
+        const service = new FileSnapshotService(tempDir);
+        const files = service.getModifiedFiles();
+        t.true(Array.isArray(files));
+        // Files should all be strings
+        t.true(files.every(f => typeof f === 'string'));
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService captures empty files', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        await createTestFile(tempDir, 'empty.txt', '');
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = await service.captureFiles(['empty.txt']);
+        t.is(snapshots.size, 1);
+        t.is(snapshots.get('empty.txt'), '');
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService captures files with special characters in content', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        const specialContent = 'Special chars: \n\t\r "quotes" & <tags> 日本語';
+        await createTestFile(tempDir, 'special.txt', specialContent);
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = await service.captureFiles(['special.txt']);
+        t.is(snapshots.get('special.txt'), specialContent);
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService uses relative paths in snapshots', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        await createTestFile(tempDir, 'src/file.ts', 'content');
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = await service.captureFiles(['src/file.ts']);
+        // Should use relative path, not absolute
+        const keys = Array.from(snapshots.keys());
+        t.is(keys.length, 1);
+        t.is(keys[0], 'src/file.ts');
+        t.false(keys[0].startsWith('/'));
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService handles large files', async (t) => {
+    const tempDir = await createTempDir();
+    try {
+        const largeContent = 'x'.repeat(100000); // 100KB
+        await createTestFile(tempDir, 'large.txt', largeContent);
+        const service = new FileSnapshotService(tempDir);
+        const snapshots = await service.captureFiles(['large.txt']);
+        t.is(snapshots.get('large.txt')?.length, 100000);
+    }
+    finally {
+        await cleanupTempDir(tempDir);
+    }
+});
+test.serial('FileSnapshotService defaults to current working directory', t => {
+    const service = new FileSnapshotService();
+    // Just verify it doesn't throw
+    t.truthy(service);
+});
+//# sourceMappingURL=file-snapshot.spec.js.map

package/dist/services/file-snapshot.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"file-snapshot.spec.js","sourceRoot":"","sources":["../../source/services/file-snapshot.spec.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAC,MAAM,IAAI,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,MAAM,CAAC;AAC7B,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,KAAK,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EAAC,mBAAmB,EAAC,MAAM,iBAAiB,CAAC;AAEpD,mDAAmD;AACnD,KAAK,UAAU,aAAa;IAC3B,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CACxB,OAAO,CAAC,GAAG,EAAE,EACb,YAAY,EACZ,iBAAiB,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CACpE,CAAC;IACF,MAAM,EAAE,CAAC,KAAK,CAAC,OAAO,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,CAAC,CAAC;IAC3C,OAAO,OAAO,CAAC;AAChB,CAAC;AAED,oCAAoC;AACpC,KAAK,UAAU,cAAc,CAAC,GAAW;IACxC,IAAI,CAAC;QACJ,MAAM,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAC,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAC,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACR,wBAAwB;IACzB,CAAC;AACF,CAAC;AAED,+BAA+B;AAC/B,KAAK,UAAU,cAAc,CAC5B,GAAW,EACX,YAAoB,EACpB,OAAe;IAEf,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;IAC9C,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAC,SAAS,EAAE,IAAI,EAAC,CAAC,CAAC;IAC1D,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;IAC/C,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,IAAI,CAAC,MAAM,CAAC,0CAA0C,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IACjE,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC,OAAO,EAAE,UAAU,EAAE,eAAe,CAAC,CAAC;QAE3D,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAE3D,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,eAAe,CAAC,CAAC;IAClD,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CAAC,6CAA6C,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IACpE,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QACxD,MAAM,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QACxD,MAAM,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,WAAW,CAAC,CAAC;QAExD,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC;YAC5C,WAAW;YACX,WAAW;YACX,WAAW;SACX,CAAC,CAAC;QAEH,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QAC9C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;QAC9C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,WAAW,CAAC,CAAC;IAC/C,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CAAC,sDAAsD,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IAC7E,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,YAAY,CAAC,CAAC;QAC5D,MAAM,cAAc,CACnB,OAAO,EACP,qBAAqB,EACrB,2BAA2B,CAC3B,CAAC;QAEF,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC;YAC5C,cAAc;YACd,qBAAqB;SACrB,CAAC,CAAC;QAEH,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC,CAAC;QAClD,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,qBAAqB,CAAC,EAAE,2BAA2B,CAAC,CAAC;IACzE,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CACV,2DAA2D,EAC3D,KAAK,EAAC,CAAC,EAAC,EAAE;IACT,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC,OAAO,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;QAEvD,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC;YAC5C,YAAY;YACZ,oBAAoB;SACpB,CAAC,CAAC;QAEH,wCAAwC;QACxC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,YAAY,CAAC,EAAE,SAAS,CAAC,CAAC;IAC9C,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CACD,CAAC;AAEF,IAAI,CAAC,MAAM,CAAC,oCAAoC,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IAC3D,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,SAAS,CAAC,GAAG,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;QAElD,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAEtC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;QACxD,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;IACnC,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CAAC,sDAAsD,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IAC7E,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,SAAS,CAAC,GAAG,CAAC,sBAAsB,EAAE,gBAAgB,CAAC,CAAC;QAExD,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAEtC,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,sBAAsB,CAAC,CAAC;QAChE,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC,CAAC;QACjC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC,CAAC;QACzD,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,gBAAgB,CAAC,CAAC;IACjC,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CAAC,6CAA6C,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IACpE,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,SAAS,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACxC,SAAS,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QACxC,SAAS,CAAC,GAAG,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAE/C,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAEtC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAC5D,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CACV,0DAA0D,EAC1D,KAAK,EAAC,CAAC,EAAC,EAAE;IACT,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,aAAa,CAAC,CAAC;QAE7D,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,SAAS,CAAC,GAAG,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;QAE7C,MAAM,OAAO,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC;QAEtC,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAChC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,cAAc,CAAC,EAClC,OAAO,CACP,CAAC;QACF,CAAC,CAAC,EAAE,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;IAC9B,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CACD,CAAC;AAEF,IAAI,CAAC,MAAM,CACV,6DAA6D,EAC7D,KAAK,EAAC,CAAC,EAAC,EAAE;IACT,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,SAAS,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC,UAAU;IAC/C,SAAS,CAAC,GAAG,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC,UAAU;IAEhD,MAAM,IAAI,GAAG,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IAEhD,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;AAChB,CAAC,CACD,CAAC;AAEF,IAAI,CAAC,MAAM,CACV,6DAA6D,EAC7D,KAAK,EAAC,CAAC,EAAC,EAAE;IACT,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE5C,MAAM,IAAI,GAAG,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IAEhD,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AACf,CAAC,CACD,CAAC;AAEF,IAAI,CAAC,MAAM,CACV,6DAA6D,EAC7D,KAAK,EAAC,CAAC,EAAC,EAAE;IACT,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC;IACvD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC5C,SAAS,CAAC,GAAG,CAAC,aAAa,EAAE,KAAK,CAAC,CAAC,CAAC,mBAAmB;IAExD,MAAM,IAAI,GAAG,OAAO,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;IAEhD,CAAC,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;AACf,CAAC,CACD,CAAC;AAEF,IAAI,CAAC,MAAM,CACV,kEAAkE,EAClE,KAAK,EAAC,CAAC,EAAC,EAAE;IACT,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,SAAS,CAAC,GAAG,CAAC,cAAc,EAAE,SAAS,CAAC,CAAC;QAEzC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAE5D,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACrB,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IAC/B,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CACD,CAAC;AAEF,IAAI,CAAC,MAAM,CACV,2EAA2E,EAC3E,KAAK,EAAC,CAAC,EAAC,EAAE;IACT,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC,OAAO,EAAE,cAAc,EAAE,UAAU,CAAC,CAAC;QAE1D,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAC5C,SAAS,CAAC,GAAG,CAAC,cAAc,EAAE,aAAa,CAAC,CAAC;QAE7C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,mBAAmB,CAAC,SAAS,CAAC,CAAC;QAE5D,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACtB,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CACD,CAAC;AAEF,IAAI,CAAC,MAAM,CAAC,oDAAoD,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IAC3E,mEAAmE;IACnE,qEAAqE;IACrE,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,KAAK,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAC;QAEzC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QAC7B,8BAA8B;QAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC,CAAC,CAAC;IACjD,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CAAC,0CAA0C,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IACjE,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,EAAE,CAAC,CAAC;QAE/C,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QAE5D,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QACxB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC;IACtC,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CACV,uEAAuE,EACvE,KAAK,EAAC,CAAC,EAAC,EAAE;IACT,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,cAAc,GAAG,6CAA6C,CAAC;QACrE,MAAM,cAAc,CAAC,OAAO,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;QAE7D,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;QAE9D,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,aAAa,CAAC,EAAE,cAAc,CAAC,CAAC;IACpD,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CACD,CAAC;AAEF,IAAI,CAAC,MAAM,CAAC,sDAAsD,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IAC7E,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,cAAc,CAAC,OAAO,EAAE,aAAa,EAAE,SAAS,CAAC,CAAC;QAExD,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC;QAE9D,yCAAyC;QACzC,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,CAAC,CAAC;QAC1C,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;QACrB,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;QAC7B,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IAClC,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CAAC,yCAAyC,EAAE,KAAK,EAAC,CAAC,EAAC,EAAE;IAChE,MAAM,OAAO,GAAG,MAAM,aAAa,EAAE,CAAC;IACtC,IAAI,CAAC;QACJ,MAAM,YAAY,GAAG,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ;QACjD,MAAM,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,YAAY,CAAC,CAAC;QAEzD,MAAM,OAAO,GAAG,IAAI,mBAAmB,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,YAAY,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;QAE5D,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,GAAG,CAAC,WAAW,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;IAClD,CAAC;YAAS,CAAC;QACV,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC;IAC/B,CAAC;AACF,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,MAAM,CAAC,2DAA2D,EAAE,CAAC,CAAC,EAAE;IAC5E,MAAM,OAAO,GAAG,IAAI,mBAAmB,EAAE,CAAC;IAC1C,+BAA+B;IAC/B,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC"}

package/dist/test-utils/render-with-theme.d.ts

@@ -0,0 +1,7 @@
+import { render } from 'ink-testing-library';
+import React from 'react';
+/**
+ * Render a component wrapped with ThemeContext for testing
+ */
+export declare function renderWithTheme(element: React.ReactElement): ReturnType<typeof render>;
+//# sourceMappingURL=render-with-theme.d.ts.map

package/dist/test-utils/render-with-theme.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"render-with-theme.d.ts","sourceRoot":"","sources":["../../source/test-utils/render-with-theme.tsx"],"names":[],"mappings":"AAEA,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAC3C,OAAO,KAAK,MAAM,OAAO,CAAC;AAqC1B;;GAEG;AACH,wBAAgB,eAAe,CAC9B,OAAO,EAAE,KAAK,CAAC,YAAY,GACzB,UAAU,CAAC,OAAO,MAAM,CAAC,CAE3B"}

package/dist/test-utils/render-with-theme.js

@@ -0,0 +1,38 @@
+import { jsx as _jsx } from "react/jsx-runtime";
+import { ThemeContext } from '../hooks/useTheme.js';
+import { render } from 'ink-testing-library';
+// Default test colors that match the structure used in the app
+const testColors = {
+    primary: 'blue',
+    secondary: 'gray',
+    white: 'white',
+    black: 'black',
+    info: 'cyan',
+    warning: 'yellow',
+    error: 'red',
+    success: 'green',
+    tool: 'magenta',
+    diffAdded: 'green',
+    diffRemoved: 'red',
+    diffAddedText: 'white',
+    diffRemovedText: 'white',
+};
+// Test theme context value
+const testThemeContext = {
+    currentTheme: 'tokyo-night',
+    colors: testColors,
+    setCurrentTheme: () => { },
+};
+/**
+ * Wrapper component that provides ThemeContext for tests
+ */
+function TestThemeProvider({ children }) {
+    return (_jsx(ThemeContext.Provider, { value: testThemeContext, children: children }));
+}
+/**
+ * Render a component wrapped with ThemeContext for testing
+ */
+export function renderWithTheme(element) {
+    return render(_jsx(TestThemeProvider, { children: element }));
+}
+//# sourceMappingURL=render-with-theme.js.map

package/dist/test-utils/render-with-theme.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"render-with-theme.js","sourceRoot":"","sources":["../../source/test-utils/render-with-theme.tsx"],"names":[],"mappings":";AAAA,OAAO,EAAC,YAAY,EAAC,MAAM,kBAAkB,CAAC;AAE9C,OAAO,EAAC,MAAM,EAAC,MAAM,qBAAqB,CAAC;AAG3C,+DAA+D;AAC/D,MAAM,UAAU,GAAW;IAC1B,OAAO,EAAE,MAAM;IACf,SAAS,EAAE,MAAM;IACjB,KAAK,EAAE,OAAO;IACd,KAAK,EAAE,OAAO;IACd,IAAI,EAAE,MAAM;IACZ,OAAO,EAAE,QAAQ;IACjB,KAAK,EAAE,KAAK;IACZ,OAAO,EAAE,OAAO;IAChB,IAAI,EAAE,SAAS;IACf,SAAS,EAAE,OAAO;IAClB,WAAW,EAAE,KAAK;IAClB,aAAa,EAAE,OAAO;IACtB,eAAe,EAAE,OAAO;CACxB,CAAC;AAEF,2BAA2B;AAC3B,MAAM,gBAAgB,GAAG;IACxB,YAAY,EAAE,aAA4B;IAC1C,MAAM,EAAE,UAAU;IAClB,eAAe,EAAE,GAAG,EAAE,GAAE,CAAC;CACzB,CAAC;AAEF;;GAEG;AACH,SAAS,iBAAiB,CAAC,EAAC,QAAQ,EAA8B;IACjE,OAAO,CACN,KAAC,YAAY,CAAC,QAAQ,IAAC,KAAK,EAAE,gBAAgB,YAC5C,QAAQ,GACc,CACxB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC9B,OAA2B;IAE3B,OAAO,MAAM,CAAC,KAAC,iBAAiB,cAAE,OAAO,GAAqB,CAAC,CAAC;AACjE,CAAC"}

package/dist/tokenization/tokenizer-factory.js

@@ -2,10 +2,10 @@
  * Tokenizer factory
  * Creates appropriate tokenizer based on provider and model
  */
-import { OpenAITokenizer } from './tokenizers/openai-tokenizer.js';
 import { AnthropicTokenizer } from './tokenizers/anthropic-tokenizer.js';
-import { LlamaTokenizer } from './tokenizers/llama-tokenizer.js';
 import { FallbackTokenizer } from './tokenizers/fallback-tokenizer.js';
+import { LlamaTokenizer } from './tokenizers/llama-tokenizer.js';
+import { OpenAITokenizer } from './tokenizers/openai-tokenizer.js';
 /**
  * Detect provider from model ID or provider name
  */

package/dist/tokenization/tokenizer-factory.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer-factory.js","sourceRoot":"","sources":["../../source/tokenization/tokenizer-factory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAC,eAAe,EAAC,MAAM,kCAAkC,CAAC;AACjE,OAAO,EAAC,kBAAkB,EAAC,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAC,cAAc,EAAC,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAErE;;GAEG;AACH,SAAS,cAAc,CACtB,YAAoB,EACpB,OAAe;IAEf,MAAM,aAAa,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;IACjD,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAEzC,sBAAsB;IACtB,IAAI,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACtC,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,IAAI,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7E,OAAO,WAAW,CAAC;IACpB,CAAC;IAED,qCAAqC;IACrC,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjE,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,OAAO,WAAW,CAAC;IACpB,CAAC;IAED,IACC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC5B,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC9B,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC3B,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC5B,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC;QAC1B,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC;QAChC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC;QAC/B,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,EAC7B,CAAC;QACF,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,sEAAsE;IACtE,IACC,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAChC,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC;QACnC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,EAC9B,CAAC;QACF,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,OAAO,UAAU,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC9B,YAAoB,EACpB,OAAe;IAEf,uDAAuD;IACvD,MAAM,iBAAiB,GACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACvD,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtB,CAAC,CAAC,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAC;IAEjE,QAAQ,QAAQ,EAAE,CAAC;QAClB,KAAK,QAAQ;YACZ,OAAO,IAAI,eAAe,CAAC,iBAAiB,CAAC,CAAC;QAE/C,KAAK,WAAW;YACf,OAAO,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;QAElD,KAAK,OAAO;YACX,OAAO,IAAI,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAE9C,KAAK,UAAU,CAAC;QAChB;YACC,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CACzC,QAA2B,EAC3B,OAAgB;IAEhB,QAAQ,QAAQ,EAAE,CAAC;QAClB,KAAK,QAAQ;YACZ,OAAO,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;QAErC,KAAK,WAAW;YACf,OAAO,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAExC,KAAK,OAAO;YACX,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;QAEpC,KAAK,MAAM;YACV,4CAA4C;YAC5C,IAAI,OAAO,EAAE,CAAC;gBACb,OAAO,eAAe,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACrC,CAAC;YAED,OAAO,IAAI,iBAAiB,EAAE,CAAC;QAEhC,KAAK,UAAU,CAAC;QAChB;YACC,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;AACF,CAAC"}
+{"version":3,"file":"tokenizer-factory.js","sourceRoot":"","sources":["../../source/tokenization/tokenizer-factory.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAC,kBAAkB,EAAC,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AACrE,OAAO,EAAC,cAAc,EAAC,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EAAC,eAAe,EAAC,MAAM,kCAAkC,CAAC;AAEjE;;GAEG;AACH,SAAS,cAAc,CACtB,YAAoB,EACpB,OAAe;IAEf,MAAM,aAAa,GAAG,YAAY,CAAC,WAAW,EAAE,CAAC;IACjD,MAAM,UAAU,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAEzC,sBAAsB;IACtB,IAAI,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACtC,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,IAAI,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC7E,OAAO,WAAW,CAAC;IACpB,CAAC;IAED,qCAAqC;IACrC,IAAI,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACjE,OAAO,QAAQ,CAAC;IACjB,CAAC;IAED,IAAI,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;QACnC,OAAO,WAAW,CAAC;IACpB,CAAC;IAED,IACC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC5B,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC;QAC9B,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC3B,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC;QAC5B,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC;QAC1B,UAAU,CAAC,QAAQ,CAAC,WAAW,CAAC;QAChC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC;QAC/B,UAAU,CAAC,QAAQ,CAAC,SAAS,CAAC,EAC7B,CAAC;QACF,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,sEAAsE;IACtE,IACC,aAAa,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAChC,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC;QACnC,aAAa,CAAC,QAAQ,CAAC,OAAO,CAAC,EAC9B,CAAC;QACF,OAAO,OAAO,CAAC;IAChB,CAAC;IAED,OAAO,UAAU,CAAC;AACnB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,eAAe,CAC9B,YAAoB,EACpB,OAAe;IAEf,uDAAuD;IACvD,MAAM,iBAAiB,GACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACvD,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QACtB,CAAC,CAAC,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,cAAc,CAAC,YAAY,EAAE,iBAAiB,CAAC,CAAC;IAEjE,QAAQ,QAAQ,EAAE,CAAC;QAClB,KAAK,QAAQ;YACZ,OAAO,IAAI,eAAe,CAAC,iBAAiB,CAAC,CAAC;QAE/C,KAAK,WAAW;YACf,OAAO,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;QAElD,KAAK,OAAO;YACX,OAAO,IAAI,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAE9C,KAAK,UAAU,CAAC;QAChB;YACC,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,0BAA0B,CACzC,QAA2B,EAC3B,OAAgB;IAEhB,QAAQ,QAAQ,EAAE,CAAC;QAClB,KAAK,QAAQ;YACZ,OAAO,IAAI,eAAe,CAAC,OAAO,CAAC,CAAC;QAErC,KAAK,WAAW;YACf,OAAO,IAAI,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAExC,KAAK,OAAO;YACX,OAAO,IAAI,cAAc,CAAC,OAAO,CAAC,CAAC;QAEpC,KAAK,MAAM;YACV,4CAA4C;YAC5C,IAAI,OAAO,EAAE,CAAC;gBACb,OAAO,eAAe,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;YACrC,CAAC;YAED,OAAO,IAAI,iBAAiB,EAAE,CAAC;QAEhC,KAAK,UAAU,CAAC;QAChB;YACC,OAAO,IAAI,iBAAiB,EAAE,CAAC;IACjC,CAAC;AACF,CAAC"}

package/dist/tokenization/tokenizer-factory.spec.js

@@ -3,10 +3,10 @@
  */
 import test from 'ava';
 import { createTokenizer, createTokenizerForProvider, } from './tokenizer-factory.js';
-import { OpenAITokenizer } from './tokenizers/openai-tokenizer.js';
 import { AnthropicTokenizer } from './tokenizers/anthropic-tokenizer.js';
-import { LlamaTokenizer } from './tokenizers/llama-tokenizer.js';
 import { FallbackTokenizer } from './tokenizers/fallback-tokenizer.js';
+import { LlamaTokenizer } from './tokenizers/llama-tokenizer.js';
+import { OpenAITokenizer } from './tokenizers/openai-tokenizer.js';
 console.log(`\ntokenizer-factory.spec.ts`);
 // Test createTokenizer with OpenAI provider detection
 test('createTokenizer detects OpenAI from provider name', t => {

package/dist/tokenization/tokenizer-factory.spec.js.map

@@ -1 +1 @@
-{"version":3,"file":"tokenizer-factory.spec.js","sourceRoot":"","sources":["../../source/tokenization/tokenizer-factory.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EACN,eAAe,EACf,0BAA0B,GAC1B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,eAAe,EAAC,MAAM,kCAAkC,CAAC;AACjE,OAAO,EAAC,kBAAkB,EAAC,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAC,cAAc,EAAC,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AAErE,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;AAE3C,sDAAsD;AACtD,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,qBAAqB,CAAC,CAAC;AAClD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oEAAoE,EAAE,CAAC,CAAC,EAAE;IAC9E,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,yDAAyD;AACzD,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0EAA0E,EAAE,CAAC,CAAC,EAAE;IACpF,MAAM,SAAS,GAAG,eAAe,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,qDAAqD;AACrD,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC1D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC1D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yDAAyD,EAAE,CAAC,CAAC,EAAE;IACnE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IAClE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC3D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,qCAAqC;AACrC,IAAI,CAAC,gEAAgE,EAAE,CAAC,CAAC,EAAE;IAC1E,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,8BAA8B;AAC9B,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;IAC5C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAC5C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;IAC5C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAC5C,CAAC,CAAC,CAAC;AAEH,0BAA0B;AAC1B,IAAI,CAAC,4DAA4D,EAAE,CAAC,CAAC,EAAE;IACtE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yDAAyD,EAAE,CAAC,CAAC,EAAE;IACnE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,kCAAkC;AAClC,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IAClE,MAAM,SAAS,GAAG,0BAA0B,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IACtE,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACjE,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACzD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4DAA4D,EAAE,CAAC,CAAC,EAAE;IACtE,MAAM,SAAS,GAAG,0BAA0B,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+DAA+D,EAAE,CAAC,CAAC,EAAE;IACzE,MAAM,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACvD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC"}
+{"version":3,"file":"tokenizer-factory.spec.js","sourceRoot":"","sources":["../../source/tokenization/tokenizer-factory.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EACN,eAAe,EACf,0BAA0B,GAC1B,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAC,kBAAkB,EAAC,MAAM,qCAAqC,CAAC;AACvE,OAAO,EAAC,iBAAiB,EAAC,MAAM,oCAAoC,CAAC;AACrE,OAAO,EAAC,cAAc,EAAC,MAAM,iCAAiC,CAAC;AAC/D,OAAO,EAAC,eAAe,EAAC,MAAM,kCAAkC,CAAC;AAEjE,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;AAE3C,sDAAsD;AACtD,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;IAC7C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,qBAAqB,CAAC,CAAC;AAClD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oEAAoE,EAAE,CAAC,CAAC,EAAE;IAC9E,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,yDAAyD;AACzD,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0EAA0E,EAAE,CAAC,CAAC,EAAE;IACpF,MAAM,SAAS,GAAG,eAAe,CAAC,iBAAiB,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,qDAAqD;AACrD,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC1D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC1D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACxD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yDAAyD,EAAE,CAAC,CAAC,EAAE;IACnE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IAClE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAC;IAC5D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,eAAe,CAAC,WAAW,EAAE,cAAc,CAAC,CAAC;IAC/D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,eAAe,CAAC,OAAO,EAAE,cAAc,CAAC,CAAC;IAC3D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,qCAAqC;AACrC,IAAI,CAAC,gEAAgE,EAAE,CAAC,CAAC,EAAE;IAC1E,MAAM,SAAS,GAAG,eAAe,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,8BAA8B;AAC9B,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;IAC5C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAC5C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,eAAe,CAAC,CAAC;IAC7D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;IAC5C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,eAAe,CAAC,CAAC;AAC5C,CAAC,CAAC,CAAC;AAEH,0BAA0B;AAC1B,IAAI,CAAC,4DAA4D,EAAE,CAAC,CAAC,EAAE;IACtE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACrD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yDAAyD,EAAE,CAAC,CAAC,EAAE;IACnE,MAAM,SAAS,GAAG,eAAe,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,kCAAkC;AAClC,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,0BAA0B,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAChE,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IAClE,MAAM,SAAS,GAAG,0BAA0B,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC;IACtE,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,kBAAkB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,0BAA0B,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IACjE,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,cAAc,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,0BAA0B,CAAC,UAAU,CAAC,CAAC;IACzD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4DAA4D,EAAE,CAAC,CAAC,EAAE;IACtE,MAAM,SAAS,GAAG,0BAA0B,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+DAA+D,EAAE,CAAC,CAAC,EAAE;IACzE,MAAM,SAAS,GAAG,0BAA0B,CAAC,MAAM,CAAC,CAAC;IACrD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,iBAAiB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACvD,CAAC,CAAC,IAAI,CAAC,SAAS,YAAY,eAAe,CAAC,CAAC;AAC9C,CAAC,CAAC,CAAC"}

package/dist/tokenization/tokenizers/anthropic-tokenizer.d.ts

@@ -2,8 +2,8 @@
  * Anthropic tokenizer for Claude models
  * Uses @anthropic-ai/tokenizer package
  */
-import type { Tokenizer } from '../../types/tokenization.js';
 import type { Message } from '../../types/core.js';
+import type { Tokenizer } from '../../types/tokenization.js';
 /**
  * Anthropic tokenizer for Claude models
  */

package/dist/tokenization/tokenizers/anthropic-tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"anthropic-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACxD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C;;GAEG;AACH,qBAAa,kBAAmB,YAAW,SAAS;IACnD,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAI5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAS5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;CAGjB"}
+{"version":3,"file":"anthropic-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAExD;;GAEG;AACH,qBAAa,kBAAmB,YAAW,SAAS;IACnD,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAI5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAS5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;CAGjB"}

package/dist/tokenization/tokenizers/anthropic-tokenizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"anthropic-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAC,WAAW,IAAI,oBAAoB,EAAC,MAAM,yBAAyB,CAAC;AAI5E;;GAEG;AACH,MAAM,OAAO,kBAAkB;IACtB,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,UAAU,CAAC;IACxC,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACR,+DAA+D;YAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,0DAA0D;QAC1D,8CAA8C;QAC9C,MAAM,eAAe,GAAG,CAAC,CAAC;QAE1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,aAAa,IAAI,CAAC,SAAS,EAAE,CAAC;IACtC,CAAC;CACD"}
+{"version":3,"file":"anthropic-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAC,WAAW,IAAI,oBAAoB,EAAC,MAAM,yBAAyB,CAAC;AAG5E;;GAEG;AACH,MAAM,OAAO,kBAAkB;IACtB,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,UAAU,CAAC;IACxC,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,OAAO,oBAAoB,CAAC,IAAI,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACR,+DAA+D;YAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,0DAA0D;QAC1D,8CAA8C;QAC9C,MAAM,eAAe,GAAG,CAAC,CAAC;QAE1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,aAAa,IAAI,CAAC,SAAS,EAAE,CAAC;IACtC,CAAC;CACD"}

package/dist/tokenization/tokenizers/anthropic-tokenizer.spec.js.map

@@ -1 +1 @@
-{"version":3,"file":"anthropic-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,kBAAkB,EAAC,MAAM,0BAA0B,CAAC;AAG5D,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;AAE7C,IAAI,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE;IAClD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,uCAAuC;IACvC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE;IAClD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iEAAiE,EAAE,CAAC,CAAC,EAAE;IAC3E,MAAM,SAAS,GAAG,IAAI,kBAAkB,EAAE,CAAC;IAE3C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;IAE5D,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,2BAA2B,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iDAAiD,EAAE,CAAC,CAAC,EAAE;IAC3D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,CAAC,CAAC,EAAE;IACpD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,2BAA2B,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,0BAA0B,CAAC,CAAC;AACvD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,CAAC,CAAC,EAAE;IACpD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAE1D,iDAAiD;IACjD,6CAA6C;IAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAE9C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC"}
+{"version":3,"file":"anthropic-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/anthropic-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,kBAAkB,EAAC,MAAM,0BAA0B,CAAC;AAE5D,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;AAE7C,IAAI,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE;IAClD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,uCAAuC;IACvC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE;IAClD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iEAAiE,EAAE,CAAC,CAAC,EAAE;IAC3E,MAAM,SAAS,GAAG,IAAI,kBAAkB,EAAE,CAAC;IAE3C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;IAE5D,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,2BAA2B,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iDAAiD,EAAE,CAAC,CAAC,EAAE;IAC3D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,mDAAmD,EAAE,CAAC,CAAC,EAAE;IAC7D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wDAAwD,EAAE,CAAC,CAAC,EAAE;IAClE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,CAAC,CAAC,EAAE;IACpD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,iBAAiB,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,2BAA2B,CAAC,CAAC;AACxD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,gBAAgB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,0BAA0B,CAAC,CAAC;AACvD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0CAA0C,EAAE,CAAC,CAAC,EAAE;IACpD,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAE1D,iDAAiD;IACjD,6CAA6C;IAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAE9C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,kBAAkB,CAAC,eAAe,CAAC,CAAC;IAC1D,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC"}

package/dist/tokenization/tokenizers/fallback-tokenizer.d.ts

@@ -1,5 +1,5 @@
-import type { Tokenizer } from '../../types/tokenization.js';
 import type { Message } from '../../types/core.js';
+import type { Tokenizer } from '../../types/tokenization.js';
 /**
  * Fallback tokenizer for unsupported models
  * Uses a simple character-based estimation (4 chars per token)

package/dist/tokenization/tokenizers/fallback-tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"fallback-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/fallback-tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACxD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,SAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAK;IAErC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAI5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,IAAI,MAAM;CAGjB"}
+{"version":3,"file":"fallback-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/fallback-tokenizer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAC1C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAExD;;;GAGG;AACH,qBAAa,iBAAkB,YAAW,SAAS;IAClD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAK;IAErC,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAI5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAQrC,OAAO,IAAI,MAAM;CAGjB"}

package/dist/tokenization/tokenizers/fallback-tokenizer.spec.js.map

@@ -1 +1 @@
-{"version":3,"file":"fallback-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/fallback-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAG1D,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;AAE5C,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,gFAAgF;IAChF,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE;IAClD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,gCAAgC;IAChC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,4BAA4B;IAC5B,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,CAAC,CAAC,EAAE;IACtD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,2CAA2C;IAC3C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAEnC,2CAA2C;IAC3C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,UAAU,CAAC,CAAC;AACvC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,+CAA+C;IAC/C,8BAA8B;IAC9B,mBAAmB;IACnB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,uDAAuD;IACvD,oCAAoC;IACpC,oBAAoB;IACpB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,wDAAwD;IACxD,iCAAiC;IACjC,mBAAmB;IACnB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,8BAA8B;IAC9B,kBAAkB;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,qDAAqD;IACrD,8BAA8B;IAC9B,kBAAkB;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,6DAA6D;IAC7D,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;IACrB,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,qBAAqB,GAAG,kBAAkB,CAAC,CAAC;AACzD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iDAAiD,EAAE,CAAC,CAAC,EAAE;IAC3D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,4CAA4C;IAC5C,8BAA8B;IAC9B,mBAAmB;IACnB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,CAAC,CAAC,EAAE;IACtD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAEpC,yCAAyC;IACzC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAEvC,qBAAqB;IACrB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAE3C,uBAAuB;IACvB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iCAAiC,EAAE,CAAC,CAAC,EAAE;IAC3C,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,6CAA6C,CAAC;IAE3D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEtC,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,OAAO,EAAE,aAAa;KACX,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC"}
+{"version":3,"file":"fallback-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/fallback-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,iBAAiB,EAAC,MAAM,yBAAyB,CAAC;AAE1D,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;AAE5C,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,gFAAgF;IAChF,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,wCAAwC,EAAE,CAAC,CAAC,EAAE;IAClD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,gCAAgC;IAChC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,4BAA4B;IAC5B,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAClC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IACtC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,cAAc,CAAC,EAAE,CAAC,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,CAAC,CAAC,EAAE;IACtD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,2CAA2C;IAC3C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;IAEnC,2CAA2C;IAC3C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAE1C,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,UAAU,CAAC,CAAC;AACvC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,+CAA+C;IAC/C,8BAA8B;IAC9B,mBAAmB;IACnB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,uDAAuD;IACvD,oCAAoC;IACpC,oBAAoB;IACpB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;AACjB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,wDAAwD;IACxD,iCAAiC;IACjC,mBAAmB;IACnB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qDAAqD,EAAE,CAAC,CAAC,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,8BAA8B;IAC9B,kBAAkB;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uDAAuD,EAAE,CAAC,CAAC,EAAE;IACjE,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,qDAAqD;IACrD,8BAA8B;IAC9B,kBAAkB;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,6DAA6D;IAC7D,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;AACzC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;IACrB,MAAM,qBAAqB,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAC7D,MAAM,kBAAkB,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACxD,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,qBAAqB,GAAG,kBAAkB,CAAC,CAAC;AACzD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iDAAiD,EAAE,CAAC,CAAC,EAAE;IAC3D,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,4CAA4C;IAC5C,8BAA8B;IAC9B,mBAAmB;IACnB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,4CAA4C,EAAE,CAAC,CAAC,EAAE;IACtD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAEpC,yCAAyC;IACzC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAEvC,qBAAqB;IACrB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAE3C,uBAAuB;IACvB,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,iCAAiC,EAAE,CAAC,CAAC,EAAE;IAC3C,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,IAAI,GAAG,6CAA6C,CAAC;IAE3D,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEtC,CAAC,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,iBAAiB,EAAE,CAAC;IAC1C,MAAM,OAAO,GAAY;QACxB,OAAO,EAAE,aAAa;KACX,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC"}

package/dist/tokenization/tokenizers/llama-tokenizer.d.ts

@@ -2,8 +2,8 @@
  * Llama tokenizer for local models
  * Uses llama-tokenizer-js package
  */
-import type { Tokenizer } from '../../types/tokenization.js';
 import type { Message } from '../../types/core.js';
+import type { Tokenizer } from '../../types/tokenization.js';
 export declare class LlamaTokenizer implements Tokenizer {
     private modelName;
     constructor(modelId?: string);

package/dist/tokenization/tokenizers/llama-tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"llama-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACxD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,qBAAa,cAAe,YAAW,SAAS;IAC/C,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAI5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAU5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;CAGjB"}
+{"version":3,"file":"llama-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAExD,qBAAa,cAAe,YAAW,SAAS;IAC/C,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAI5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAU5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;CAGjB"}

package/dist/tokenization/tokenizers/llama-tokenizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"llama-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,cAAc,MAAM,oBAAoB,CAAC;AAIhD,MAAM,OAAO,cAAc;IAClB,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,OAAO,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,OAAO,MAAM,CAAC,MAAM,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACR,+DAA+D;YAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,0EAA0E;QAC1E,8CAA8C;QAC9C,MAAM,eAAe,GAAG,CAAC,CAAC;QAE1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;IAClC,CAAC;CACD"}
+{"version":3,"file":"llama-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,cAAc,MAAM,oBAAoB,CAAC;AAGhD,MAAM,OAAO,cAAc;IAClB,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,OAAO,CAAC;IACrC,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC3C,OAAO,MAAM,CAAC,MAAM,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACR,+DAA+D;YAC/D,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,0EAA0E;QAC1E,8CAA8C;QAC9C,MAAM,eAAe,GAAG,CAAC,CAAC;QAE1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,SAAS,IAAI,CAAC,SAAS,EAAE,CAAC;IAClC,CAAC;CACD"}

package/dist/tokenization/tokenizers/llama-tokenizer.spec.js.map

@@ -1 +1 @@
-{"version":3,"file":"llama-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAGpD,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;AAEzC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,uCAAuC;IACvC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;IAEvC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC;AAC1C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,aAAa,CAAC,CAAC;IAEpD,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,mBAAmB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,6CAA6C,EAAE,CAAC,CAAC,EAAE;IACvD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC;AAC/C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,gBAAgB,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IAEnD,iDAAiD;IACjD,6CAA6C;IAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAE9C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,oBAAoB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,0BAA0B,CAAC,CAAC;AACvD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC"}
+{"version":3,"file":"llama-tokenizer.spec.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/llama-tokenizer.spec.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,IAAI,MAAM,KAAK,CAAC;AACvB,OAAO,EAAC,cAAc,EAAC,MAAM,sBAAsB,CAAC;AAEpD,OAAO,CAAC,GAAG,CAAC,2BAA2B,CAAC,CAAC;AAEzC,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,uCAAuC;IACvC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,qCAAqC,EAAE,CAAC,CAAC,EAAE;IAC/C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEnC,CAAC,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;AAChB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oCAAoC,EAAE,CAAC,CAAC,EAAE;IAC9C,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GACT,wFAAwF,CAAC;IAC1F,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,wCAAwC;IACxC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;IACnB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,0DAA0D,EAAE,CAAC,CAAC,EAAE;IACpE,MAAM,SAAS,GAAG,IAAI,cAAc,EAAE,CAAC;IAEvC,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,aAAa,CAAC,CAAC;AAC1C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,aAAa,CAAC,CAAC;IAEpD,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,mBAAmB,CAAC,CAAC;AAChD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,6CAA6C,EAAE,CAAC,CAAC,EAAE;IACvD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,qBAAqB;KAC9B,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,yDAAyD;IACzD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,WAAW;QACjB,OAAO,EAAE,6BAA6B;KACtC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,+CAA+C,EAAE,CAAC,CAAC,EAAE;IACzD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,8BAA8B;KACvC,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,kDAAkD,EAAE,CAAC,CAAC,EAAE;IAC5D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;KACX,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,oDAAoD,EAAE,CAAC,CAAC,EAAE;IAC9D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;KACD,CAAC;IAEb,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2BAA2B;IAC3B,CAAC,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sDAAsD,EAAE,CAAC,CAAC,EAAE;IAChE,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,YAAY,GAAY;QAC7B,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,IAAI;KACb,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;IAClD,MAAM,WAAW,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAE1C,gEAAgE;IAChE,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,WAAW,GAAG,QAAQ,CAAC,CAAC;AACxC,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG,oBAAoB,CAAC;IAClC,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,IAAI,GAAG;;;;EAIZ,CAAC;IACF,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAErC,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;AACpB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,yCAAyC,EAAE,CAAC,CAAC,EAAE;IACnD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,kBAAkB,CAAC,CAAC;AAC/C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,gBAAgB,CAAC,CAAC;AAC7C,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,2CAA2C,EAAE,CAAC,CAAC,EAAE;IACrD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,QAAQ;KACjB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,2CAA2C;IAC3C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAC;AACtB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,gDAAgD,EAAE,CAAC,CAAC,EAAE;IAC1D,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IAEnD,iDAAiD;IACjD,6CAA6C;IAC7C,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;IAE9C,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,8CAA8C,EAAE,CAAC,CAAC,EAAE;IACxD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,YAAY,CAAC,CAAC;IACnD,MAAM,OAAO,GAAY;QACxB,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,kBAAkB;QAC3B,YAAY,EAAE,KAAK;KACnB,CAAC;IAEF,MAAM,KAAK,GAAG,SAAS,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;IAE7C,8BAA8B;IAC9B,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;AACnB,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,uCAAuC,EAAE,CAAC,CAAC,EAAE;IACjD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,oBAAoB,CAAC,CAAC;IAC3D,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,0BAA0B,CAAC,CAAC;AACvD,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,sCAAsC,EAAE,CAAC,CAAC,EAAE;IAChD,MAAM,SAAS,GAAG,IAAI,cAAc,CAAC,cAAc,CAAC,CAAC;IACrD,MAAM,KAAK,GAAG,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC;IAEhD,CAAC,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,EAAE,oBAAoB,CAAC,CAAC;AACjD,CAAC,CAAC,CAAC"}

package/dist/tokenization/tokenizers/openai-tokenizer.d.ts

@@ -2,8 +2,8 @@
  * OpenAI tokenizer using tiktoken
  * Supports GPT-3.5, GPT-4, and other OpenAI models
  */
-import type { Tokenizer } from '../../types/tokenization.js';
 import type { Message } from '../../types/core.js';
+import type { Tokenizer } from '../../types/tokenization.js';
 /**
  * OpenAI tokenizer using tiktoken for accurate token counting
  */

package/dist/tokenization/tokenizers/openai-tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"openai-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACxD,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C;;GAEG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAChD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAU5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAS5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;IAIjB;;OAEG;IACH,IAAI,IAAI,IAAI;CAGZ"}
+{"version":3,"file":"openai-tokenizer.d.ts","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,KAAK,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAE1C,OAAO,KAAK,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AAExD;;GAEG;AACH,qBAAa,eAAgB,YAAW,SAAS;IAChD,OAAO,CAAC,QAAQ,CAAkC;IAClD,OAAO,CAAC,SAAS,CAAS;gBAEd,OAAO,CAAC,EAAE,MAAM;IAU5B,MAAM,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM;IAS5B,WAAW,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM;IAWrC,OAAO,IAAI,MAAM;IAIjB;;OAEG;IACH,IAAI,IAAI,IAAI;CAGZ"}

package/dist/tokenization/tokenizers/openai-tokenizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"openai-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAC,kBAAkB,EAAE,YAAY,EAAqB,MAAM,UAAU,CAAC;AAI9E;;GAEG;AACH,MAAM,OAAO,eAAe;IACnB,QAAQ,CAAkC;IAC1C,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,OAAO,CAAC;QAEpC,IAAI,CAAC;YACJ,IAAI,CAAC,QAAQ,GAAG,kBAAkB,CAAC,OAAwB,CAAC,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;QAC7C,CAAC;IACF,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,OAAO,MAAM,CAAC,MAAM,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,4DAA4D;QAC5D,sCAAsC;QACtC,MAAM,eAAe,GAAG,CAAC,CAAC,CAAC,mCAAmC;QAE9D,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,IAAI;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;CACD"}
+{"version":3,"file":"openai-tokenizer.js","sourceRoot":"","sources":["../../../source/tokenization/tokenizers/openai-tokenizer.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAqB,kBAAkB,EAAE,YAAY,EAAC,MAAM,UAAU,CAAC;AAG9E;;GAEG;AACH,MAAM,OAAO,eAAe;IACnB,QAAQ,CAAkC;IAC1C,SAAS,CAAS;IAE1B,YAAY,OAAgB;QAC3B,IAAI,CAAC,SAAS,GAAG,OAAO,IAAI,OAAO,CAAC;QAEpC,IAAI,CAAC;YACJ,IAAI,CAAC,QAAQ,GAAG,kBAAkB,CAAC,OAAwB,CAAC,CAAC;QAC9D,CAAC;QAAC,MAAM,CAAC;YACR,IAAI,CAAC,QAAQ,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;QAC7C,CAAC;IACF,CAAC;IAED,MAAM,CAAC,IAAY;QAClB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;YAC1C,OAAO,MAAM,CAAC,MAAM,CAAC;QACtB,CAAC;QAAC,MAAM,CAAC;YACR,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACnC,CAAC;IACF,CAAC;IAED,WAAW,CAAC,OAAgB;QAC3B,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAEhC,4DAA4D;QAC5D,sCAAsC;QACtC,MAAM,eAAe,GAAG,CAAC,CAAC,CAAC,mCAAmC;QAE9D,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,eAAe,CAAC;IACnE,CAAC;IAED,OAAO;QACN,OAAO,UAAU,IAAI,CAAC,SAAS,EAAE,CAAC;IACnC,CAAC;IAED;;OAEG;IACH,IAAI;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;IACtB,CAAC;CACD"}