@unerr-ai/unerr 0.0.1 → 0.1.1

package/dist/__tests__/search-index.test.js

@@ -0,0 +1,422 @@
+/**
+ * Tests for search-index.ts — tokenization, index building, and IDF-weighted search ranking.
+ *
+ * Tests validate:
+ *   - tokenize(): camelCase/PascalCase/snake_case splitting algorithm
+ *   - buildSearchIndex(): entity name tokenization + IDF weight computation
+ *   - searchLocal(): IDF-weighted scoring (rare tokens rank higher than common ones)
+ */
+import { describe, expect, it } from "vitest";
+import { buildSearchIndex, searchLocal, tokenize, } from "../intelligence/search-index.js";
+// ── In-memory CozoDB mock ─────────────────────────────────────────
+// Implements enough Datalog semantics for search_tokens + token_doc_frequency queries.
+function createMockDb() {
+    const entities = [];
+    const searchTokens = [];
+    const tokenDocFreq = []; // [token, doc_count, idf]
+    return {
+        async run(query, params) {
+            // Handle :create (schema init)
+            if (query.includes(":create"))
+                return { rows: [] };
+            // Handle :put search_tokens
+            if (query.includes(":put search_tokens")) {
+                const token = params?.token;
+                const key = params?.key;
+                const exists = searchTokens.some(([t, k]) => t === token && k === key);
+                if (!exists)
+                    searchTokens.push([token, key]);
+                return { rows: [] };
+            }
+            // Handle :put token_doc_frequency
+            if (query.includes(":put token_doc_frequency")) {
+                const token = params?.token;
+                const dc = params?.dc;
+                const idf = params?.idf;
+                // Upsert
+                const idx = tokenDocFreq.findIndex(([t]) => t === token);
+                if (idx >= 0) {
+                    tokenDocFreq[idx] = [token, dc, idf];
+                }
+                else {
+                    tokenDocFreq.push([token, dc, idf]);
+                }
+                return { rows: [] };
+            }
+            // Handle :put entities
+            if (query.includes(":put entities")) {
+                const key = params?.key;
+                const kind = params?.kind;
+                const name = params?.name;
+                const fp = params?.fp;
+                entities.push([key, kind, name, fp, 0, "", ""]);
+                return { rows: [] };
+            }
+            // Handle entity read for buildSearchIndex
+            if (query.includes("*entities") && query.includes("?[key, name]")) {
+                return { rows: entities.map(([key, _k, name]) => [key, name]) };
+            }
+            // Handle searchLocal Datalog query (IDF-weighted token intersection)
+            if (query.includes("search_tokens") && query.includes("matched")) {
+                // Parse token list from query — format: [["token1"], ["token2"], ...]
+                const tokenMatches = [...query.matchAll(/\["([^"]+)"\]/g)];
+                if (tokenMatches.length === 0)
+                    return { rows: [] };
+                const queryTokens = tokenMatches.map((m) => m[1]);
+                // Build IDF lookup
+                const idfMap = new Map();
+                for (const [token, _dc, idf] of tokenDocFreq) {
+                    idfMap.set(token, idf);
+                }
+                // Sum IDF weights per entity key
+                const entityScores = new Map();
+                for (const [token, entityKey] of searchTokens) {
+                    if (queryTokens.includes(token)) {
+                        const idf = idfMap.get(token) ?? 0;
+                        entityScores.set(entityKey, (entityScores.get(entityKey) ?? 0) + idf);
+                    }
+                }
+                // Join with entities, sort by score desc
+                const results = [];
+                for (const [entityKey, score] of entityScores) {
+                    const entity = entities.find(([k]) => k === entityKey);
+                    if (entity) {
+                        results.push([entityKey, score, entity[2], entity[1], entity[3]]);
+                    }
+                }
+                results.sort((a, b) => b[1] - a[1]);
+                // Parse limit
+                const limitMatch = query.match(/:limit (\d+)/);
+                const limit = limitMatch?.[1] ? Number.parseInt(limitMatch[1], 10) : 20;
+                return { rows: results.slice(0, limit) };
+            }
+            return { rows: [] };
+        },
+    };
+}
+async function seedEntities(db, items) {
+    for (const item of items) {
+        await db.run("?[key, kind, name, fp] <- [[$key, $kind, $name, $fp]] :put entities { key => kind, name, file_path: fp }", { key: item.key, kind: item.kind, name: item.name, fp: item.file_path });
+    }
+}
+// ── tokenize ──────────────────────────────────────────────────────
+describe("search-index", () => {
+    describe("tokenize", () => {
+        it("splits camelCase", () => {
+            expect(tokenize("doSomething")).toEqual(["do", "something"]);
+        });
+        it("splits PascalCase", () => {
+            expect(tokenize("MyClassName")).toEqual(["my", "class", "name"]);
+        });
+        it("splits snake_case", () => {
+            expect(tokenize("get_user_name")).toEqual(["get", "user", "name"]);
+        });
+        it("splits kebab-case", () => {
+            expect(tokenize("my-component")).toEqual(["my", "component"]);
+        });
+        it("handles mixed patterns", () => {
+            const tokens = tokenize("getUserName_v2");
+            expect(tokens).toContain("get");
+            expect(tokens).toContain("user");
+            expect(tokens).toContain("name");
+            expect(tokens).toContain("v2");
+        });
+        it("deduplicates tokens", () => {
+            const tokens = tokenize("test_test");
+            expect(tokens).toEqual(["test"]);
+        });
+        it("lowercases all tokens", () => {
+            const tokens = tokenize("HTTPRequest");
+            for (const t of tokens) {
+                expect(t).toBe(t.toLowerCase());
+            }
+        });
+        it("handles single word", () => {
+            expect(tokenize("hello")).toEqual(["hello"]);
+        });
+        it("handles empty string", () => {
+            expect(tokenize("")).toEqual([]);
+        });
+        it("splits consecutive uppercase (acronyms)", () => {
+            const tokens = tokenize("XMLHTTPRequest");
+            expect(tokens).toContain("request");
+            // Acronym splitting: XML, HTTP split from Request
+            expect(tokens.length).toBeGreaterThanOrEqual(2);
+        });
+        it("handles numeric suffixes", () => {
+            const tokens = tokenize("processV2");
+            expect(tokens).toContain("process");
+            expect(tokens).toContain("v2");
+        });
+        it("handles dots and special chars", () => {
+            const tokens = tokenize("file.name.ext");
+            expect(tokens).toContain("file");
+            expect(tokens).toContain("name");
+            expect(tokens).toContain("ext");
+        });
+    });
+    // ── buildSearchIndex ────────────────────────────────────────────
+    describe("buildSearchIndex", () => {
+        it("tokenizes entity names and stores all tokens", async () => {
+            const db = createMockDb();
+            await seedEntities(db, [
+                {
+                    key: "fn1",
+                    kind: "function",
+                    name: "processPayment",
+                    file_path: "src/billing.ts",
+                },
+                {
+                    key: "cls1",
+                    kind: "class",
+                    name: "UserService",
+                    file_path: "src/user.ts",
+                },
+            ]);
+            await buildSearchIndex(db);
+            // processPayment → ["process", "payment"]
+            // UserService → ["user", "service"]
+            // Verify by searching — tokens should be indexed
+            const results = await searchLocal(db, "process");
+            expect(results).toHaveLength(1);
+            expect(results[0]?.key).toBe("fn1");
+            expect(results[0]?.name).toBe("processPayment");
+        });
+        it("handles empty entity set without error", async () => {
+            const db = createMockDb();
+            // No entities seeded
+            await expect(buildSearchIndex(db)).resolves.not.toThrow();
+            const results = await searchLocal(db, "anything");
+            expect(results).toHaveLength(0);
+        });
+    });
+    // ── searchLocal — IDF-weighted ranking ─────────────────────────
+    describe("searchLocal", () => {
+        it("returns empty array for empty query", async () => {
+            const db = createMockDb();
+            await seedEntities(db, [
+                { key: "fn1", kind: "function", name: "doStuff", file_path: "a.ts" },
+            ]);
+            await buildSearchIndex(db);
+            const results = await searchLocal(db, "");
+            expect(results).toHaveLength(0);
+        });
+        it("ranks rare tokens higher than common tokens (IDF)", async () => {
+            const db = createMockDb();
+            await seedEntities(db, [
+                // "get" appears in 3 entities (very common → low IDF)
+                { key: "fn1", kind: "function", name: "getUser", file_path: "a.ts" },
+                { key: "fn2", kind: "function", name: "getOrder", file_path: "b.ts" },
+                {
+                    key: "fn3",
+                    kind: "function",
+                    name: "getPayment",
+                    file_path: "c.ts",
+                },
+                // "process" appears in 1 entity (rare → high IDF)
+                {
+                    key: "fn4",
+                    kind: "function",
+                    name: "processPayment",
+                    file_path: "d.ts",
+                },
+            ]);
+            await buildSearchIndex(db);
+            // Search for "payment" — appears in fn3 (getPayment) and fn4 (processPayment)
+            // fn4 has "process" (rare, IDF=ln(4/1)) + "payment" (doc_count=2, IDF=ln(4/2))
+            // fn3 has "get" (common, IDF=ln(4/3)) + "payment" (IDF=ln(4/2))
+            // fn4 should rank higher because "process" is rarer than "get"
+            const results = await searchLocal(db, "processPayment");
+            expect(results.length).toBeGreaterThanOrEqual(1);
+            expect(results[0]?.key).toBe("fn4");
+        });
+        it("IDF scores: single rare token outranks single common token", async () => {
+            const db = createMockDb();
+            await seedEntities(db, [
+                // "handle" in 5 entities
+                {
+                    key: "fn1",
+                    kind: "function",
+                    name: "handleRequest",
+                    file_path: "a.ts",
+                },
+                {
+                    key: "fn2",
+                    kind: "function",
+                    name: "handleResponse",
+                    file_path: "b.ts",
+                },
+                {
+                    key: "fn3",
+                    kind: "function",
+                    name: "handleError",
+                    file_path: "c.ts",
+                },
+                {
+                    key: "fn4",
+                    kind: "function",
+                    name: "handleTimeout",
+                    file_path: "d.ts",
+                },
+                {
+                    key: "fn5",
+                    kind: "function",
+                    name: "handleRetry",
+                    file_path: "e.ts",
+                },
+                // "orchestrate" in 1 entity (unique)
+                {
+                    key: "fn6",
+                    kind: "function",
+                    name: "orchestrateWorkflow",
+                    file_path: "f.ts",
+                },
+            ]);
+            await buildSearchIndex(db);
+            // Searching "orchestrate" should return fn6 with higher IDF than searching "handle"
+            const rareResults = await searchLocal(db, "orchestrate");
+            const commonResults = await searchLocal(db, "handle");
+            expect(rareResults).toHaveLength(1);
+            expect(commonResults).toHaveLength(5);
+            // The rare token's IDF weight should be higher
+            expect(rareResults[0]?.score).toBeGreaterThan(commonResults[0]?.score ?? 0);
+        });
+        it("multi-token query sums IDF weights correctly", async () => {
+            const db = createMockDb();
+            await seedEntities(db, [
+                // Matches 1 token: "user"
+                { key: "fn1", kind: "function", name: "getUser", file_path: "a.ts" },
+                // Matches 2 tokens: "user" + "name"
+                {
+                    key: "fn2",
+                    kind: "function",
+                    name: "getUserName",
+                    file_path: "b.ts",
+                },
+                // Matches 0 tokens
+                {
+                    key: "fn3",
+                    kind: "function",
+                    name: "processPayment",
+                    file_path: "c.ts",
+                },
+            ]);
+            await buildSearchIndex(db);
+            const results = await searchLocal(db, "userName");
+            // fn2 matches both tokens (sum of 2 IDF weights), fn1 matches one
+            expect(results.length).toBeGreaterThanOrEqual(2);
+            expect(results[0]?.key).toBe("fn2");
+            expect(results[0]?.score).toBeGreaterThan(results[1]?.score ?? 0);
+            expect(results[1]?.key).toBe("fn1");
+        });
+        it("respects limit parameter", async () => {
+            const db = createMockDb();
+            // Use names that all share the same token "action"
+            const entities = Array.from({ length: 30 }, (_, i) => ({
+                key: `fn${i}`,
+                kind: "function",
+                name: "handleAction",
+                file_path: `src/f${i}.ts`,
+            }));
+            await seedEntities(db, entities);
+            await buildSearchIndex(db);
+            const results = await searchLocal(db, "action", 5);
+            expect(results).toHaveLength(5);
+        });
+        it("uses default limit of 20", async () => {
+            const db = createMockDb();
+            const entities = Array.from({ length: 30 }, (_, i) => ({
+                key: `fn${i}`,
+                kind: "function",
+                name: "handleRequest",
+                file_path: `src/f${i}.ts`,
+            }));
+            await seedEntities(db, entities);
+            await buildSearchIndex(db);
+            const results = await searchLocal(db, "handle");
+            expect(results.length).toBeLessThanOrEqual(20);
+        });
+        it("returns correct entity metadata in results", async () => {
+            const db = createMockDb();
+            await seedEntities(db, [
+                {
+                    key: "cls1",
+                    kind: "class",
+                    name: "PaymentService",
+                    file_path: "src/billing/payment.ts",
+                },
+                {
+                    key: "fn1",
+                    kind: "function",
+                    name: "handleRequest",
+                    file_path: "src/server.ts",
+                },
+            ]);
+            await buildSearchIndex(db);
+            const results = await searchLocal(db, "payment");
+            expect(results).toHaveLength(1);
+            expect(results[0]?.key).toBe("cls1");
+            expect(results[0]?.name).toBe("PaymentService");
+            expect(results[0]?.kind).toBe("class");
+            expect(results[0]?.file_path).toBe("src/billing/payment.ts");
+            // IDF > 0 because "payment" appears in 1 of 2 entities
+            expect(results[0]?.score).toBeGreaterThan(0);
+        });
+        it("matches across different naming conventions", async () => {
+            const db = createMockDb();
+            await seedEntities(db, [
+                {
+                    key: "fn1",
+                    kind: "function",
+                    name: "getUserById",
+                    file_path: "a.ts",
+                },
+                {
+                    key: "fn2",
+                    kind: "function",
+                    name: "get_user_by_id",
+                    file_path: "b.py",
+                },
+                {
+                    key: "fn3",
+                    kind: "function",
+                    name: "GetUserById",
+                    file_path: "c.go",
+                },
+            ]);
+            await buildSearchIndex(db);
+            // All three should match "user" query regardless of casing convention
+            const results = await searchLocal(db, "user");
+            expect(results).toHaveLength(3);
+        });
+        it("single-token query matches all entities with that token", async () => {
+            const db = createMockDb();
+            await seedEntities(db, [
+                {
+                    key: "fn1",
+                    kind: "function",
+                    name: "processOrder",
+                    file_path: "a.ts",
+                },
+                {
+                    key: "fn2",
+                    kind: "function",
+                    name: "processPayment",
+                    file_path: "b.ts",
+                },
+                {
+                    key: "fn3",
+                    kind: "function",
+                    name: "handleOrder",
+                    file_path: "c.ts",
+                },
+            ]);
+            await buildSearchIndex(db);
+            const results = await searchLocal(db, "process");
+            expect(results).toHaveLength(2);
+            const keys = results.map((r) => r.key);
+            expect(keys).toContain("fn1");
+            expect(keys).toContain("fn2");
+        });
+    });
+});