@matperez/coderag 0.1.24

package/dist/ast-chunking.d.ts

@@ -0,0 +1,40 @@
+/**
+ * AST-Based Code Chunking using code-chunk (tree-sitter)
+ *
+ * Splits code at semantic boundaries (functions, classes, etc.)
+ * Supports TypeScript, JavaScript, Python, Rust, Go, Java.
+ * Other file types use character-based fallback.
+ */
+/**
+ * AST-based chunking options
+ */
+export interface ASTChunkOptions {
+    readonly maxChunkSize?: number;
+    readonly minChunkSize?: number;
+}
+/**
+ * Chunk result with metadata
+ */
+export interface ChunkResult {
+    readonly content: string;
+    readonly type: string;
+    readonly startLine: number;
+    readonly endLine: number;
+    readonly metadata: Record<string, unknown>;
+}
+/**
+ * Chunk code using AST analysis (via code-chunk / tree-sitter)
+ *
+ * Supported extensions: .ts, .tsx, .js, .jsx, .mjs, .cjs, .py, .pyi, .rs, .go, .java.
+ * Other files are chunked with character-based fallback.
+ */
+export declare function chunkCodeByAST(code: string, filePath: string, options?: ASTChunkOptions): Promise<readonly ChunkResult[]>;
+/**
+ * Simple wrapper for backward compatibility
+ */
+export declare function chunkCodeByASTSimple(code: string, filePath: string, options?: ASTChunkOptions): Promise<readonly string[]>;
+/**
+ * Get list of languages supported for AST chunking (code-chunk)
+ */
+export declare function getSupportedLanguages(): string[];
+//# sourceMappingURL=ast-chunking.d.ts.map

package/dist/ast-chunking.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast-chunking.d.ts","sourceRoot":"","sources":["../src/ast-chunking.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAUH;;GAEG;AACH,MAAM,WAAW,eAAe;IAC/B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;IAC9B,QAAQ,CAAC,YAAY,CAAC,EAAE,MAAM,CAAA;CAC9B;AAED;;GAEG;AACH,MAAM,WAAW,WAAW;IAC3B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAA;IAC1B,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAA;IACxB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC1C;AA8CD;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC3B,OAAO,CAAC,SAAS,WAAW,EAAE,CAAC,CAiBjC;AAED;;GAEG;AACH,wBAAsB,oBAAoB,CACzC,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,OAAO,GAAE,eAAoB,GAC3B,OAAO,CAAC,SAAS,MAAM,EAAE,CAAC,CAG5B;AAED;;GAEG;AACH,wBAAgB,qBAAqB,IAAI,MAAM,EAAE,CAEhD"}

package/dist/ast-chunking.js

@@ -0,0 +1,88 @@
+/**
+ * AST-Based Code Chunking using code-chunk (tree-sitter)
+ *
+ * Splits code at semantic boundaries (functions, classes, etc.)
+ * Supports TypeScript, JavaScript, Python, Rust, Go, Java.
+ * Other file types use character-based fallback.
+ */
+import { chunk as codeChunk } from 'code-chunk';
+import { chunkText } from './embeddings.js';
+/** Languages supported by code-chunk for AST chunking */
+const CODE_CHUNK_SUPPORTED_LANGUAGES = [
+    'typescript',
+    'javascript',
+    'python',
+    'rust',
+    'go',
+    'java',
+];
+/**
+ * Create fallback chunks using character-based splitting
+ */
+function createFallbackChunks(code, maxChunkSize) {
+    const chunks = chunkText(code, { maxChunkSize });
+    return chunks.map((content, i) => ({
+        content,
+        type: 'text',
+        startLine: 0,
+        endLine: 0,
+        metadata: { fallback: true, index: i },
+    }));
+}
+function mapCodeChunkToResult(c) {
+    const firstEntity = c.context.entities[0];
+    const type = firstEntity?.type ?? 'chunk';
+    return {
+        content: c.text,
+        type,
+        startLine: c.lineRange.start + 1,
+        endLine: c.lineRange.end + 1,
+        metadata: {
+            fallback: false,
+            scope: c.context.scope.map((s) => ({ name: s.name, type: s.type })),
+            entities: c.context.entities.map((e) => ({
+                name: e.name,
+                type: e.type,
+                signature: e.signature,
+            })),
+        },
+    };
+}
+/**
+ * Chunk code using AST analysis (via code-chunk / tree-sitter)
+ *
+ * Supported extensions: .ts, .tsx, .js, .jsx, .mjs, .cjs, .py, .pyi, .rs, .go, .java.
+ * Other files are chunked with character-based fallback.
+ */
+export async function chunkCodeByAST(code, filePath, options = {}) {
+    const maxChunkSize = options.maxChunkSize ?? 1000;
+    try {
+        const chunks = await codeChunk(filePath, code, {
+            maxChunkSize,
+            contextMode: 'full',
+        });
+        if (chunks.length === 0 && code.trim().length > 0) {
+            return createFallbackChunks(code, maxChunkSize);
+        }
+        return chunks.map(mapCodeChunkToResult);
+    }
+    catch (error) {
+        // UnsupportedLanguageError, ChunkingError, or any other: fall back to character chunking
+        console.error('[WARN] AST chunking failed, falling back to character chunking:', error instanceof Error ? error.message : String(error));
+        return createFallbackChunks(code, maxChunkSize);
+    }
+}
+/**
+ * Simple wrapper for backward compatibility
+ */
+export async function chunkCodeByASTSimple(code, filePath, options = {}) {
+    const chunks = await chunkCodeByAST(code, filePath, options);
+    return chunks.map((chunk) => chunk.content);
+}
+/**
+ * Get list of languages supported for AST chunking (code-chunk)
+ */
+export function getSupportedLanguages() {
+    return [...CODE_CHUNK_SUPPORTED_LANGUAGES];
+}
+//# sourceMappingURL=ast-chunking.js.map

package/dist/ast-chunking.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast-chunking.js","sourceRoot":"","sources":["../src/ast-chunking.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,YAAY,CAAA;AAE/C,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAA;AAyB3C,yDAAyD;AACzD,MAAM,8BAA8B,GAAG;IACtC,YAAY;IACZ,YAAY;IACZ,QAAQ;IACR,MAAM;IACN,IAAI;IACJ,MAAM;CACG,CAAA;AAEV;;GAEG;AACH,SAAS,oBAAoB,CAAC,IAAY,EAAE,YAAoB;IAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,EAAE,YAAY,EAAE,CAAC,CAAA;IAChD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;QACP,IAAI,EAAE,MAAM;QACZ,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;QACV,QAAQ,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE;KACtC,CAAC,CAAC,CAAA;AACJ,CAAC;AAED,SAAS,oBAAoB,CAAC,CAAiB;IAC9C,MAAM,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;IACzC,MAAM,IAAI,GAAG,WAAW,EAAE,IAAI,IAAI,OAAO,CAAA;IACzC,OAAO;QACN,OAAO,EAAE,CAAC,CAAC,IAAI;QACf,IAAI;QACJ,SAAS,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,GAAG,CAAC;QAChC,OAAO,EAAE,CAAC,CAAC,SAAS,CAAC,GAAG,GAAG,CAAC;QAC5B,QAAQ,EAAE;YACT,QAAQ,EAAE,KAAK;YACf,KAAK,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACnE,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACxC,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,SAAS,EAAE,CAAC,CAAC,SAAS;aACtB,CAAC,CAAC;SACH;KACD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,IAAY,EACZ,QAAgB,EAChB,UAA2B,EAAE;IAE7B,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAA;IAEjD,IAAI,CAAC;QACJ,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE;YAC9C,YAAY;YACZ,WAAW,EAAE,MAAM;SACnB,CAAC,CAAA;QACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnD,OAAO,oBAAoB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;QAChD,CAAC;QACD,OAAO,MAAM,CAAC,GAAG,CAAC,oBAAoB,CAAC,CAAA;IACxC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,yFAAyF;QACzF,OAAO,CAAC,KAAK,CAAC,iEAAiE,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAA;QACxI,OAAO,oBAAoB,CAAC,IAAI,EAAE,YAAY,CAAC,CAAA;IAChD,CAAC;AACF,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,IAAY,EACZ,QAAgB,EAChB,UAA2B,EAAE;IAE7B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAA;IAC5D,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;AAC5C,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB;IACpC,OAAO,CAAC,GAAG,8BAA8B,CAAC,CAAA;AAC3C,CAAC"}

package/dist/ast-chunking.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Tests for AST-based code chunking (code-chunk for supported languages, fallback for others)
+ */
+export {};
+//# sourceMappingURL=ast-chunking.test.d.ts.map

package/dist/ast-chunking.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast-chunking.test.d.ts","sourceRoot":"","sources":["../src/ast-chunking.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/ast-chunking.test.js

@@ -0,0 +1,173 @@
+/**
+ * Tests for AST-based code chunking (code-chunk for supported languages, fallback for others)
+ */
+import { describe, expect, it } from 'vitest';
+import { chunkCodeByAST, chunkCodeByASTSimple, getSupportedLanguages, } from './ast-chunking.js';
+describe('AST-based chunking', () => {
+    describe('getSupportedLanguages', () => {
+        it('returns code-chunk supported languages', () => {
+            const langs = getSupportedLanguages();
+            expect(langs).toContain('typescript');
+            expect(langs).toContain('javascript');
+            expect(langs).toContain('python');
+            expect(langs).toContain('rust');
+            expect(langs).toContain('go');
+            expect(langs).toContain('java');
+            expect(langs.length).toBe(6);
+        });
+    });
+    describe('Markdown (fallback)', () => {
+        it('uses character fallback for .md (no AST)', async () => {
+            const markdown = `# Introduction
+
+This is the introduction paragraph.
+
+## Section 1
+
+Some content here.
+`;
+            const chunks = await chunkCodeByAST(markdown, 'test.md');
+            expect(chunks.length).toBeGreaterThan(0);
+            // Fallback: type 'text', metadata.fallback true
+            chunks.forEach((c) => {
+                expect(c.type).toBe('text');
+                expect(c.metadata.fallback).toBe(true);
+            });
+        });
+        it('fallback preserves content and gives line 0 for fallback chunks', async () => {
+            const markdown = `# Title
+
+Paragraph 1
+
+Paragraph 2
+`;
+            const chunks = await chunkCodeByAST(markdown, 'test.md');
+            chunks.forEach((chunk) => {
+                expect(chunk.content).toBeTruthy();
+                expect(chunk.startLine).toBeGreaterThanOrEqual(0);
+                expect(chunk.endLine).toBeGreaterThanOrEqual(0);
+            });
+            expect(chunks.some((c) => c.content.includes('Title'))).toBe(true);
+        });
+    });
+    describe('JavaScript (code-chunk)', () => {
+        it('should split JavaScript by functions', async () => {
+            const code = `function foo() {
+  return 1;
+}
+
+function bar() {
+  return 2;
+}
+
+function baz() {
+  return 3;
+}
+`;
+            const chunks = await chunkCodeByASTSimple(code, 'test.js');
+            expect(chunks.length).toBeGreaterThanOrEqual(1);
+            expect(chunks.some((c) => c.includes('function foo'))).toBe(true);
+            expect(chunks.some((c) => c.includes('function bar'))).toBe(true);
+            expect(chunks.some((c) => c.includes('function baz'))).toBe(true);
+        });
+        it('should handle classes', async () => {
+            const code = `class MyClass {
+  constructor() {
+    this.value = 0;
+  }
+
+  increment() {
+    this.value++;
+  }
+}
+`;
+            const chunks = await chunkCodeByASTSimple(code, 'test.js');
+            expect(chunks.length).toBeGreaterThan(0);
+            expect(chunks.some((c) => c.includes('class MyClass'))).toBe(true);
+        });
+        it('should have meaningful line ranges for JS chunks', async () => {
+            const code = `function first() {
+  return 1;
+}
+
+function second() {
+  return 2;
+}
+`;
+            const chunks = await chunkCodeByAST(code, 'test.js');
+            expect(chunks.length).toBeGreaterThan(0);
+            chunks.forEach((chunk) => {
+                expect(chunk.startLine).toBeGreaterThanOrEqual(1);
+                expect(chunk.endLine).toBeGreaterThanOrEqual(chunk.startLine);
+                expect(chunk.content).toBeTruthy();
+                expect(chunk.metadata.fallback).toBe(false);
+            });
+        });
+    });
+    describe('Size constraints', () => {
+        it('should respect maxChunkSize', async () => {
+            const largeCode = `function veryLargeFunction() {
+  ${'return 1;\n'.repeat(100)}
+}
+`;
+            const chunks = await chunkCodeByASTSimple(largeCode, 'test.js', {
+                maxChunkSize: 500,
+            });
+            // code-chunk may exceed slightly; ensure we got multiple chunks and none is huge
+            expect(chunks.length).toBeGreaterThan(1);
+            chunks.forEach((chunk) => {
+                expect(chunk.length).toBeLessThanOrEqual(800);
+            });
+        });
+    });
+    describe('Fallback behavior', () => {
+        it('should fallback to character chunking for unknown languages', async () => {
+            const code = 'a'.repeat(2000);
+            const chunks = await chunkCodeByAST(code, 'test.unknown');
+            expect(chunks.length).toBeGreaterThan(1);
+            expect(chunks[0].metadata.fallback).toBe(true);
+        });
+        it('should return chunks when AST parsing fails (fallback or resilient parse)', async () => {
+            const invalidCode = 'function { syntax error }';
+            const chunks = await chunkCodeByAST(invalidCode, 'test.js');
+            // code-chunk may either throw (then we fallback) or return partial chunks
+            expect(chunks.length).toBeGreaterThan(0);
+            expect(chunks[0].content).toBeTruthy();
+        });
+        it('should handle empty input', async () => {
+            const chunks = await chunkCodeByAST('', 'test.js');
+            expect(chunks.length).toBe(0);
+        });
+    });
+    describe('Edge cases', () => {
+        it('should use fallback for HTML (unsupported by code-chunk)', async () => {
+            const html = `<div>
+  <p>Paragraph 1</p>
+</div>
+`;
+            const chunks = await chunkCodeByAST(html, 'test.html');
+            expect(chunks.length).toBeGreaterThan(0);
+            chunks.forEach((c) => expect(c.metadata.fallback).toBe(true));
+        });
+        it('should handle single-line content', async () => {
+            const code = 'const x = 42;';
+            const chunks = await chunkCodeByAST(code, 'test.js');
+            expect(chunks.length).toBeGreaterThan(0);
+            expect(chunks[0].content).toBeTruthy();
+        });
+    });
+    describe('Performance', () => {
+        it('should handle large files efficiently', async () => {
+            const sections = Array.from({ length: 100 }, (_, i) => {
+                return `## Section ${i + 1}\n\nContent for section ${i + 1}.\n`;
+            }).join('\n');
+            const markdown = `# Large Document\n\n${sections}`;
+            const start = Date.now();
+            const chunks = await chunkCodeByAST(markdown, 'test.md');
+            const duration = Date.now() - start;
+            expect(chunks.length).toBeGreaterThan(1);
+            expect(duration).toBeLessThan(5000);
+        });
+    });
+});
+//# sourceMappingURL=ast-chunking.test.js.map

package/dist/ast-chunking.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ast-chunking.test.js","sourceRoot":"","sources":["../src/ast-chunking.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AAC7C,OAAO,EACN,cAAc,EACd,oBAAoB,EACpB,qBAAqB,GACrB,MAAM,mBAAmB,CAAA;AAE1B,QAAQ,CAAC,oBAAoB,EAAE,GAAG,EAAE;IACnC,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;YACjD,MAAM,KAAK,GAAG,qBAAqB,EAAE,CAAA;YACrC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAA;YACrC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,YAAY,CAAC,CAAA;YACrC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAA;YACjC,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;YAC/B,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;YAC7B,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;YAC/B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC7B,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACpC,EAAE,CAAC,0CAA0C,EAAE,KAAK,IAAI,EAAE;YACzD,MAAM,QAAQ,GAAG;;;;;;;CAOnB,CAAA;YAEE,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;YAExD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,gDAAgD;YAChD,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE;gBACpB,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;gBAC3B,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACvC,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,iEAAiE,EAAE,KAAK,IAAI,EAAE;YAChF,MAAM,QAAQ,GAAG;;;;;CAKnB,CAAA;YAEE,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;YAExD,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBACxB,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,CAAA;gBAClC,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;gBACjD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;YAChD,CAAC,CAAC,CAAA;YACF,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnE,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACxC,EAAE,CAAC,sCAAsC,EAAE,KAAK,IAAI,EAAE;YACrD,MAAM,IAAI,GAAG;;;;;;;;;;;CAWf,CAAA;YAEE,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;YAE1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;YAC/C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACjE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACjE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAClE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,uBAAuB,EAAE,KAAK,IAAI,EAAE;YACtC,MAAM,IAAI,GAAG;;;;;;;;;CASf,CAAA;YAEE,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;YAE1D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACnE,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,kDAAkD,EAAE,KAAK,IAAI,EAAE;YACjE,MAAM,IAAI,GAAG;;;;;;;CAOf,CAAA;YAEE,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;YAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBACxB,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;gBACjD,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,sBAAsB,CAAC,KAAK,CAAC,SAAS,CAAC,CAAA;gBAC7D,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,CAAA;gBAClC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;YAC5C,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,kBAAkB,EAAE,GAAG,EAAE;QACjC,EAAE,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC5C,MAAM,SAAS,GAAG;IACjB,aAAa,CAAC,MAAM,CAAC,GAAG,CAAC;;CAE5B,CAAA;YAEE,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,SAAS,EAAE,SAAS,EAAE;gBAC/D,YAAY,EAAE,GAAG;aACjB,CAAC,CAAA;YAEF,iFAAiF;YACjF,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;gBACxB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,mBAAmB,CAAC,GAAG,CAAC,CAAA;YAC9C,CAAC,CAAC,CAAA;QACH,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,6DAA6D,EAAE,KAAK,IAAI,EAAE;YAC5E,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YAE7B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,cAAc,CAAC,CAAA;YAEzD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2EAA2E,EAAE,KAAK,IAAI,EAAE;YAC1F,MAAM,WAAW,GAAG,2BAA2B,CAAA;YAE/C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,WAAW,EAAE,SAAS,CAAC,CAAA;YAE3D,0EAA0E;YAC1E,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,CAAA;QACvC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YAC1C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;YAElD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC9B,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;QAC3B,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,IAAI,GAAG;;;CAGf,CAAA;YAEE,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,WAAW,CAAC,CAAA;YAEtD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAA;QAC9D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,mCAAmC,EAAE,KAAK,IAAI,EAAE;YAClD,MAAM,IAAI,GAAG,eAAe,CAAA;YAE5B,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,IAAI,EAAE,SAAS,CAAC,CAAA;YAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,CAAA;QACvC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;QAC5B,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACtD,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;gBACrD,OAAO,cAAc,CAAC,GAAG,CAAC,2BAA2B,CAAC,GAAG,CAAC,KAAK,CAAA;YAChE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAEb,MAAM,QAAQ,GAAG,uBAAuB,QAAQ,EAAE,CAAA;YAElD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YACxB,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAA;YACxD,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;YAEnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,QAAQ,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;QACpC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}

package/dist/code-tokenizer.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Code-Aware Tokenizer using StarCoder2
+ *
+ * StarCoder2 tokenizer is lightweight (only 4.7MB) and provides
+ * world-class code tokenization quality without requiring the full model.
+ */
+export interface CodeToken {
+    readonly text: string;
+    readonly id: number;
+}
+export interface TokenizerOptions {
+    readonly modelPath?: string;
+    readonly cacheDir?: string;
+}
+/**
+ * StarCoder2 Code Tokenizer
+ *
+ * Uses StarCoder2's tokenizer (4.7MB) for accurate code tokenization.
+ * Does NOT require downloading the full 15B parameter model.
+ */
+export declare class CodeTokenizer {
+    private tokenizer;
+    private initialized;
+    private initPromise;
+    private modelPath;
+    constructor(options?: TokenizerOptions);
+    /**
+     * Initialize tokenizer (downloads ~4.7MB on first use)
+     */
+    initialize(): Promise<void>;
+    private doInitialize;
+    /**
+     * Tokenize code into terms for TF-IDF indexing
+     */
+    tokenize(code: string): Promise<string[]>;
+    /**
+     * Extract unique terms with frequency counts
+     */
+    extractTerms(code: string): Promise<Map<string, number>>;
+    /**
+     * Check if tokenizer is ready
+     */
+    isReady(): boolean;
+}
+/**
+ * Get or create the global tokenizer instance
+ */
+export declare function getTokenizer(): CodeTokenizer;
+/**
+ * Tokenize code using StarCoder2 (async)
+ * This is the main entry point for tokenization
+ */
+export declare function tokenize(code: string): Promise<string[]>;
+/**
+ * Extract terms with frequency counts using StarCoder2 (async)
+ */
+export declare function extractTerms(code: string): Promise<Map<string, number>>;
+/**
+ * Initialize the global tokenizer (call early to avoid delay on first tokenize)
+ */
+export declare function initializeTokenizer(): Promise<void>;
+//# sourceMappingURL=code-tokenizer.d.ts.map

package/dist/code-tokenizer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-tokenizer.d.ts","sourceRoot":"","sources":["../src/code-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAIH,MAAM,WAAW,SAAS;IACzB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAA;IACrB,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAA;CACnB;AAED,MAAM,WAAW,gBAAgB;IAChC,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAA;IAC3B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;CAC1B;AAED;;;;;GAKG;AACH,qBAAa,aAAa;IACzB,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,WAAW,CAAQ;IAC3B,OAAO,CAAC,WAAW,CAA6B;IAChD,OAAO,CAAC,SAAS,CAAQ;gBAEb,OAAO,GAAE,gBAAqB;IAK1C;;OAEG;IACG,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAcnB,YAAY;IAiB1B;;OAEG;IACG,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IA8B/C;;OAEG;IACG,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAW9D;;OAEG;IACH,OAAO,IAAI,OAAO;CAGlB;AAKD;;GAEG;AACH,wBAAgB,YAAY,IAAI,aAAa,CAK5C;AAED;;;GAGG;AACH,wBAAsB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAG9D;AAED;;GAEG;AACH,wBAAsB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAG7E;AAED;;GAEG;AACH,wBAAsB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzD"}

package/dist/code-tokenizer.js

@@ -0,0 +1,129 @@
+/**
+ * Code-Aware Tokenizer using StarCoder2
+ *
+ * StarCoder2 tokenizer is lightweight (only 4.7MB) and provides
+ * world-class code tokenization quality without requiring the full model.
+ */
+import { AutoTokenizer } from '@huggingface/transformers';
+/**
+ * StarCoder2 Code Tokenizer
+ *
+ * Uses StarCoder2's tokenizer (4.7MB) for accurate code tokenization.
+ * Does NOT require downloading the full 15B parameter model.
+ */
+export class CodeTokenizer {
+    tokenizer;
+    initialized = false;
+    initPromise = null;
+    modelPath;
+    constructor(options = {}) {
+        // Default to StarCoder2 tokenizer (only downloads tokenizer files, not model)
+        this.modelPath = options.modelPath || 'bigcode/starcoder2-15b';
+    }
+    /**
+     * Initialize tokenizer (downloads ~4.7MB on first use)
+     */
+    async initialize() {
+        if (this.initialized) {
+            return;
+        }
+        // Prevent multiple concurrent initializations
+        if (this.initPromise) {
+            return this.initPromise;
+        }
+        this.initPromise = this.doInitialize();
+        return this.initPromise;
+    }
+    async doInitialize() {
+        try {
+            console.error('[INFO] Loading StarCoder2 tokenizer (4.7MB, one-time download)...');
+            const startTime = Date.now();
+            this.tokenizer = await AutoTokenizer.from_pretrained(this.modelPath);
+            const loadTime = Date.now() - startTime;
+            console.error(`[SUCCESS] Tokenizer loaded in ${loadTime}ms`);
+            this.initialized = true;
+        }
+        catch (error) {
+            this.initPromise = null;
+            throw new Error(`Failed to load tokenizer: ${error.message}`);
+        }
+    }
+    /**
+     * Tokenize code into terms for TF-IDF indexing
+     */
+    async tokenize(code) {
+        if (!this.initialized) {
+            await this.initialize();
+        }
+        if (!code || code.trim().length === 0) {
+            return [];
+        }
+        // Encode with StarCoder2
+        const encoded = await this.tokenizer(code);
+        const inputIds = encoded.input_ids.tolist()[0];
+        // Decode each token ID to get the actual tokens
+        const tokens = [];
+        for (const id of inputIds) {
+            const token = await this.tokenizer.decode([id], {
+                skip_special_tokens: true,
+            });
+            const cleaned = token.trim().toLowerCase();
+            // Filter: keep tokens with length > 1 (skip single chars and empty)
+            if (cleaned.length > 1) {
+                tokens.push(cleaned);
+            }
+        }
+        return tokens;
+    }
+    /**
+     * Extract unique terms with frequency counts
+     */
+    async extractTerms(code) {
+        const tokens = await this.tokenize(code);
+        const termFreq = new Map();
+        for (const token of tokens) {
+            termFreq.set(token, (termFreq.get(token) || 0) + 1);
+        }
+        return termFreq;
+    }
+    /**
+     * Check if tokenizer is ready
+     */
+    isReady() {
+        return this.initialized;
+    }
+}
+// Singleton instance for global use
+let globalTokenizer = null;
+/**
+ * Get or create the global tokenizer instance
+ */
+export function getTokenizer() {
+    if (!globalTokenizer) {
+        globalTokenizer = new CodeTokenizer();
+    }
+    return globalTokenizer;
+}
+/**
+ * Tokenize code using StarCoder2 (async)
+ * This is the main entry point for tokenization
+ */
+export async function tokenize(code) {
+    const tokenizer = getTokenizer();
+    return tokenizer.tokenize(code);
+}
+/**
+ * Extract terms with frequency counts using StarCoder2 (async)
+ */
+export async function extractTerms(code) {
+    const tokenizer = getTokenizer();
+    return tokenizer.extractTerms(code);
+}
+/**
+ * Initialize the global tokenizer (call early to avoid delay on first tokenize)
+ */
+export async function initializeTokenizer() {
+    const tokenizer = getTokenizer();
+    await tokenizer.initialize();
+}
+//# sourceMappingURL=code-tokenizer.js.map

package/dist/code-tokenizer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-tokenizer.js","sourceRoot":"","sources":["../src/code-tokenizer.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAA;AAYzD;;;;;GAKG;AACH,MAAM,OAAO,aAAa;IACjB,SAAS,CAAK;IACd,WAAW,GAAG,KAAK,CAAA;IACnB,WAAW,GAAyB,IAAI,CAAA;IACxC,SAAS,CAAQ;IAEzB,YAAY,UAA4B,EAAE;QACzC,8EAA8E;QAC9E,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,wBAAwB,CAAA;IAC/D,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,UAAU;QACf,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAM;QACP,CAAC;QAED,8CAA8C;QAC9C,IAAI,IAAI,CAAC,WAAW,EAAE,CAAC;YACtB,OAAO,IAAI,CAAC,WAAW,CAAA;QACxB,CAAC;QAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,YAAY,EAAE,CAAA;QACtC,OAAO,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;IAEO,KAAK,CAAC,YAAY;QACzB,IAAI,CAAC;YACJ,OAAO,CAAC,KAAK,CAAC,mEAAmE,CAAC,CAAA;YAClF,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;YAE5B,IAAI,CAAC,SAAS,GAAG,MAAM,aAAa,CAAC,eAAe,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;YAEpE,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAA;YACvC,OAAO,CAAC,KAAK,CAAC,iCAAiC,QAAQ,IAAI,CAAC,CAAA;YAE5D,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,IAAI,CAAC,WAAW,GAAG,IAAI,CAAA;YACvB,MAAM,IAAI,KAAK,CAAC,6BAA6B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAA;QAC9D,CAAC;IACF,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,QAAQ,CAAC,IAAY;QAC1B,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,CAAC;YACvB,MAAM,IAAI,CAAC,UAAU,EAAE,CAAA;QACxB,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvC,OAAO,EAAE,CAAA;QACV,CAAC;QAED,yBAAyB;QACzB,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAA;QAE9C,gDAAgD;QAChD,MAAM,MAAM,GAAa,EAAE,CAAA;QAC3B,KAAK,MAAM,EAAE,IAAI,QAAQ,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE;gBAC/C,mBAAmB,EAAE,IAAI;aACzB,CAAC,CAAA;YAEF,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;YAC1C,oEAAoE;YACpE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;YACrB,CAAC;QACF,CAAC;QAED,OAAO,MAAM,CAAA;IACd,CAAC;IAED;;OAEG;IACH,KAAK,CAAC,YAAY,CAAC,IAAY;QAC9B,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;QACxC,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAA;QAE1C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC5B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;QACpD,CAAC;QAED,OAAO,QAAQ,CAAA;IAChB,CAAC;IAED;;OAEG;IACH,OAAO;QACN,OAAO,IAAI,CAAC,WAAW,CAAA;IACxB,CAAC;CACD;AAED,oCAAoC;AACpC,IAAI,eAAe,GAAyB,IAAI,CAAA;AAEhD;;GAEG;AACH,MAAM,UAAU,YAAY;IAC3B,IAAI,CAAC,eAAe,EAAE,CAAC;QACtB,eAAe,GAAG,IAAI,aAAa,EAAE,CAAA;IACtC,CAAC;IACD,OAAO,eAAe,CAAA;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,IAAY;IAC1C,MAAM,SAAS,GAAG,YAAY,EAAE,CAAA;IAChC,OAAO,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAA;AAChC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,IAAY;IAC9C,MAAM,SAAS,GAAG,YAAY,EAAE,CAAA;IAChC,OAAO,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,CAAA;AACpC,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB;IACxC,MAAM,SAAS,GAAG,YAAY,EAAE,CAAA;IAChC,MAAM,SAAS,CAAC,UAAU,EAAE,CAAA;AAC7B,CAAC"}

package/dist/code-tokenizer.test.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Code Tokenizer Tests (StarCoder2)
+ */
+export {};
+//# sourceMappingURL=code-tokenizer.test.d.ts.map

package/dist/code-tokenizer.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-tokenizer.test.d.ts","sourceRoot":"","sources":["../src/code-tokenizer.test.ts"],"names":[],"mappings":"AAAA;;GAEG"}

package/dist/code-tokenizer.test.js

@@ -0,0 +1,96 @@
+/**
+ * Code Tokenizer Tests (StarCoder2)
+ */
+import { beforeAll, describe, expect, it } from 'vitest';
+import { CodeTokenizer, extractTerms, initializeTokenizer, tokenize } from './code-tokenizer.js';
+// Skip if running in CI without model cache
+const shouldSkip = process.env.CI === 'true' && !process.env.HF_HOME;
+describe('StarCoder2 Tokenizer', () => {
+    beforeAll(async () => {
+        if (shouldSkip)
+            return;
+        // Initialize tokenizer once for all tests
+        await initializeTokenizer();
+    }, 60000); // 60s timeout for download
+    describe('tokenize function', () => {
+        it.skipIf(shouldSkip)('should tokenize simple code', async () => {
+            const tokens = await tokenize('getUserData(userId)');
+            expect(tokens.length).toBeGreaterThan(0);
+            // StarCoder2 should understand code structure
+            expect(tokens.some((t) => t.includes('get') || t.includes('user'))).toBe(true);
+        });
+        it.skipIf(shouldSkip)('should tokenize complex code', async () => {
+            const code = `
+				async function authenticateUser(username: string, password: string) {
+					const user = await findUserByUsername(username);
+					if (!user) throw new Error('User not found');
+					return await verifyPassword(user, password);
+				}
+			`;
+            const tokens = await tokenize(code);
+            expect(tokens.length).toBeGreaterThan(10);
+            // Should extract meaningful tokens
+            expect(tokens.some((t) => t.includes('authenticate'))).toBe(true);
+            expect(tokens.some((t) => t.includes('user'))).toBe(true);
+            expect(tokens.some((t) => t.includes('password'))).toBe(true);
+        });
+        it.skipIf(shouldSkip)('should handle empty input', async () => {
+            const tokens = await tokenize('');
+            expect(tokens).toHaveLength(0);
+        });
+        it.skipIf(shouldSkip)('should handle whitespace', async () => {
+            const tokens = await tokenize('   \n\t  ');
+            expect(tokens).toHaveLength(0);
+        });
+        it.skipIf(shouldSkip)('should handle camelCase', async () => {
+            const tokens = await tokenize('getUserData');
+            expect(tokens.length).toBeGreaterThan(0);
+        });
+        it.skipIf(shouldSkip)('should handle snake_case', async () => {
+            const tokens = await tokenize('is_authenticated');
+            expect(tokens.length).toBeGreaterThan(0);
+        });
+    });
+    describe('extractTerms function', () => {
+        it.skipIf(shouldSkip)('should extract terms with frequencies', async () => {
+            const code = 'user user authenticate user';
+            const terms = await extractTerms(code);
+            expect(terms.size).toBeGreaterThan(0);
+            // Should count frequencies
+            const userFreq = terms.get('user');
+            expect(userFreq).toBeGreaterThanOrEqual(1);
+        });
+        it.skipIf(shouldSkip)('should handle code with duplicates', async () => {
+            const code = `
+				function getUserData(userId) {
+					const user = findUser(userId);
+					return user;
+				}
+			`;
+            const terms = await extractTerms(code);
+            expect(terms.size).toBeGreaterThan(0);
+        });
+    });
+    describe('CodeTokenizer class', () => {
+        let tokenizer;
+        beforeAll(async () => {
+            if (shouldSkip)
+                return;
+            tokenizer = new CodeTokenizer();
+            await tokenizer.initialize();
+        }, 30000);
+        it.skipIf(shouldSkip)('should initialize successfully', () => {
+            expect(tokenizer.isReady()).toBe(true);
+        });
+        it.skipIf(shouldSkip)('should tokenize code', async () => {
+            const tokens = await tokenizer.tokenize('function test() {}');
+            expect(tokens.length).toBeGreaterThan(0);
+            expect(tokens.some((t) => t.includes('function') || t.includes('test'))).toBe(true);
+        });
+        it.skipIf(shouldSkip)('should extract terms', async () => {
+            const terms = await tokenizer.extractTerms('const x = 1; const y = 2;');
+            expect(terms.size).toBeGreaterThan(0);
+        });
+    });
+});
+//# sourceMappingURL=code-tokenizer.test.js.map

package/dist/code-tokenizer.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"code-tokenizer.test.js","sourceRoot":"","sources":["../src/code-tokenizer.test.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAA;AACxD,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,mBAAmB,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAA;AAEhG,4CAA4C;AAC5C,MAAM,UAAU,GAAG,OAAO,CAAC,GAAG,CAAC,EAAE,KAAK,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,OAAO,CAAA;AAEpE,QAAQ,CAAC,sBAAsB,EAAE,GAAG,EAAE;IACrC,SAAS,CAAC,KAAK,IAAI,EAAE;QACpB,IAAI,UAAU;YAAE,OAAM;QACtB,0CAA0C;QAC1C,MAAM,mBAAmB,EAAE,CAAA;IAC5B,CAAC,EAAE,KAAK,CAAC,CAAA,CAAC,2BAA2B;IAErC,QAAQ,CAAC,mBAAmB,EAAE,GAAG,EAAE;QAClC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,6BAA6B,EAAE,KAAK,IAAI,EAAE;YAC/D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,qBAAqB,CAAC,CAAA;YAEpD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,8CAA8C;YAC9C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC/E,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,8BAA8B,EAAE,KAAK,IAAI,EAAE;YAChE,MAAM,IAAI,GAAG;;;;;;IAMZ,CAAA;YAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,CAAA;YAEnC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAA;YACzC,mCAAmC;YACnC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACjE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YACzD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC9D,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,2BAA2B,EAAE,KAAK,IAAI,EAAE;YAC7D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,EAAE,CAAC,CAAA;YACjC,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAC/B,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,CAAA;YAC1C,MAAM,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAC/B,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,yBAAyB,EAAE,KAAK,IAAI,EAAE;YAC3D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,aAAa,CAAC,CAAA;YAE5C,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QACzC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,0BAA0B,EAAE,KAAK,IAAI,EAAE;YAC5D,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,kBAAkB,CAAC,CAAA;YAEjD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QACzC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;QACtC,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;YACzE,MAAM,IAAI,GAAG,6BAA6B,CAAA;YAC1C,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;YAEtC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACrC,2BAA2B;YAC3B,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAA;YAClC,MAAM,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;QAC3C,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,oCAAoC,EAAE,KAAK,IAAI,EAAE;YACtE,MAAM,IAAI,GAAG;;;;;IAKZ,CAAA;YAED,MAAM,KAAK,GAAG,MAAM,YAAY,CAAC,IAAI,CAAC,CAAA;YAEtC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;IAEF,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;QACpC,IAAI,SAAwB,CAAA;QAE5B,SAAS,CAAC,KAAK,IAAI,EAAE;YACpB,IAAI,UAAU;gBAAE,OAAM;YACtB,SAAS,GAAG,IAAI,aAAa,EAAE,CAAA;YAC/B,MAAM,SAAS,CAAC,UAAU,EAAE,CAAA;QAC7B,CAAC,EAAE,KAAK,CAAC,CAAA;QAET,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,gCAAgC,EAAE,GAAG,EAAE;YAC5D,MAAM,CAAC,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACvC,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,QAAQ,CAAC,oBAAoB,CAAC,CAAA;YAE7D,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;YACxC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACpF,CAAC,CAAC,CAAA;QAEF,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC,sBAAsB,EAAE,KAAK,IAAI,EAAE;YACxD,MAAM,KAAK,GAAG,MAAM,SAAS,CAAC,YAAY,CAAC,2BAA2B,CAAC,CAAA;YAEvE,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QACtC,CAAC,CAAC,CAAA;IACH,CAAC,CAAC,CAAA;AACH,CAAC,CAAC,CAAA"}