akm-cli 0.7.0-rc1 → 0.7.1

package/dist/tests/ranking-regression.test.js

@@ -1,548 +0,0 @@
-/**
- * Ranking regression tests for akm search system.
- *
- * Uses the shared `ranking-baseline` fixture under
- * tests/fixtures/stashes/ranking-baseline/ to validate search ranking
- * invariants: score differentiation, exact name matching, type ranking,
- * fuzzy/prefix matching, score preservation, and provider merge behavior.
- *
- * The fixture stash is materialised once in beforeAll via the shared
- * `loadFixtureStash` helper, then re-indexed in place through the internal
- * indexer DB API so all tests share the same index.
- */
-import { afterAll, beforeAll, describe, expect, test } from "bun:test";
-import fs from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { akmSearch } from "../src/commands/search";
-import { saveConfig } from "../src/core/config";
-import { getDbPath } from "../src/core/paths";
-import { closeDatabase, openDatabase, rebuildFts, setMeta, upsertEntry } from "../src/indexer/db";
-import { buildSearchText } from "../src/indexer/search-fields";
-import { loadFixtureStash } from "./fixtures/stashes/load";
-// Local test helper — mirrors the pre-v1 mergeStashHits logic that was removed
-// from production code when the OpenViking provider was dropped (Phase 1).
-function mergeStashHits(localHits, additionalHits, limit) {
-    if (additionalHits.length === 0)
-        return localHits.slice(0, limit);
-    const localKeys = new Set();
-    for (const h of localHits)
-        localKeys.add(h.path ?? h.ref ?? h.name);
-    const providerOnly = additionalHits.filter((h) => !localKeys.has(h.path ?? h.ref ?? h.name));
-    return [...localHits, ...providerOnly].sort((a, b) => (b.score ?? 0) - (a.score ?? 0)).slice(0, limit);
-}
-// ── Fixture path ────────────────────────────────────────────────────────────
-let FIXTURE_STASH;
-let fixtureCleanup;
-// ── Temp directory tracking ─────────────────────────────────────────────────
-const createdTmpDirs = [];
-function createTmpDir(prefix = "akm-ranking-") {
-    const dir = fs.mkdtempSync(path.join(os.tmpdir(), prefix));
-    createdTmpDirs.push(dir);
-    return dir;
-}
-// ── Environment isolation ───────────────────────────────────────────────────
-let originalXdgCacheHome;
-let originalXdgConfigHome;
-let originalAkmStashDir;
-let testCacheDir;
-let testConfigDir;
-beforeAll(async () => {
-    originalXdgCacheHome = process.env.XDG_CACHE_HOME;
-    originalXdgConfigHome = process.env.XDG_CONFIG_HOME;
-    originalAkmStashDir = process.env.AKM_STASH_DIR;
-    testCacheDir = createTmpDir("akm-ranking-cache-");
-    testConfigDir = createTmpDir("akm-ranking-config-");
-    // Materialise the shared ranking-baseline fixture into a tmp dir. This
-    // test rebuilds the index from scratch via `buildFixtureIndex()` below
-    // against its own XDG_CACHE_HOME, so we skip the helper's `akm index`
-    // spawn (~200-300ms saved per run).
-    const loaded = loadFixtureStash("ranking-baseline", { skipIndex: true });
-    FIXTURE_STASH = loaded.stashDir;
-    fixtureCleanup = loaded.cleanup;
-    process.env.XDG_CACHE_HOME = testCacheDir;
-    process.env.XDG_CONFIG_HOME = testConfigDir;
-    process.env.AKM_STASH_DIR = FIXTURE_STASH;
-    saveConfig({
-        semanticSearchMode: "off",
-        sources: [{ type: "filesystem", path: FIXTURE_STASH }],
-        registries: [],
-    });
-    buildFixtureIndex();
-});
-afterAll(() => {
-    if (originalXdgCacheHome === undefined)
-        delete process.env.XDG_CACHE_HOME;
-    else
-        process.env.XDG_CACHE_HOME = originalXdgCacheHome;
-    if (originalXdgConfigHome === undefined)
-        delete process.env.XDG_CONFIG_HOME;
-    else
-        process.env.XDG_CONFIG_HOME = originalXdgConfigHome;
-    if (originalAkmStashDir === undefined)
-        delete process.env.AKM_STASH_DIR;
-    else
-        process.env.AKM_STASH_DIR = originalAkmStashDir;
-    fixtureCleanup?.();
-    for (const dir of createdTmpDirs) {
-        fs.rmSync(dir, { recursive: true, force: true });
-    }
-});
-// ── Index builder ───────────────────────────────────────────────────────────
-/**
- * Walk the fixture stash, read .stash.json files, and index all entries
- * directly into the SQLite database.
- */
-function buildFixtureIndex() {
-    const dbPath = getDbPath();
-    const db = openDatabase(dbPath);
-    try {
-        const stashJsonPaths = findStashJsonFiles(FIXTURE_STASH);
-        for (const stashJsonPath of stashJsonPaths) {
-            const dirPath = path.dirname(stashJsonPath);
-            const raw = JSON.parse(fs.readFileSync(stashJsonPath, "utf8"));
-            if (!raw || !Array.isArray(raw.entries))
-                continue;
-            const stash = { entries: raw.entries };
-            for (const entry of stash.entries) {
-                const entryPath = entry.filename ? path.join(dirPath, entry.filename) : dirPath;
-                const entryKey = `${FIXTURE_STASH}:${entry.type}:${entry.name}`;
-                const searchText = buildSearchText(entry);
-                let entryWithSize = entry;
-                try {
-                    const size = fs.statSync(entryPath).size;
-                    entryWithSize = { ...entry, fileSize: size };
-                }
-                catch {
-                    // File might not exist for some entries
-                }
-                upsertEntry(db, entryKey, dirPath, entryPath, FIXTURE_STASH, entryWithSize, searchText);
-            }
-        }
-        rebuildFts(db);
-        setMeta(db, "stashDir", FIXTURE_STASH);
-        setMeta(db, "builtAt", new Date().toISOString());
-        setMeta(db, "stashDirs", JSON.stringify([FIXTURE_STASH]));
-        setMeta(db, "hasEmbeddings", "0");
-    }
-    finally {
-        closeDatabase(db);
-    }
-}
-function findStashJsonFiles(dir) {
-    const results = [];
-    const entries = fs.readdirSync(dir, { withFileTypes: true });
-    for (const entry of entries) {
-        const fullPath = path.join(dir, entry.name);
-        if (entry.isDirectory()) {
-            results.push(...findStashJsonFiles(fullPath));
-        }
-        else if (entry.name === ".stash.json") {
-            results.push(fullPath);
-        }
-    }
-    return results;
-}
-// ── Helpers ─────────────────────────────────────────────────────────────────
-async function search(query, limit = 20) {
-    const result = await akmSearch({ query, source: "stash", limit });
-    return result.hits.filter((h) => h.type !== "registry");
-}
-function findHit(hits, name) {
-    return hits.find((h) => h.name === name);
-}
-/** Assert that a hit exists and return it (avoids non-null assertions). */
-function expectHit(hits, name) {
-    const hit = findHit(hits, name);
-    expect(hit).toBeDefined();
-    // biome-ignore lint/style/noNonNullAssertion: guarded by expect above
-    return hit;
-}
-/** Get the score of a hit, asserting it is defined. */
-function scoreOf(hit) {
-    expect(hit.score).toBeDefined();
-    return hit.score ?? 0;
-}
-function rankOf(hits, name) {
-    const idx = hits.findIndex((h) => h.name === name);
-    return idx === -1 ? Infinity : idx + 1; // 1-based rank
-}
-// ── Tests ───────────────────────────────────────────────────────────────────
-describe("Score differentiation", () => {
-    test('"docker homelab" returns skill:docker-homelab in top 3', async () => {
-        const hits = await search("docker homelab");
-        expect(hits.length).toBeGreaterThanOrEqual(2);
-        // docker-homelab should appear in the top results (within top 3)
-        // Sub-references also contain "docker-homelab" in their name, so they
-        // may rank highly on FTS name-field matching.
-        const skillRank = rankOf(hits, "docker-homelab");
-        expect(skillRank).toBeLessThanOrEqual(3);
-        // The skill should have a meaningful score (not RRF-compressed)
-        const skillHit = expectHit(hits, "docker-homelab");
-        expect(scoreOf(skillHit)).toBeGreaterThan(0.5);
-    });
-    test('"docker" returns docker-homelab and docker-clean', async () => {
-        const hits = await search("docker");
-        expect(hits.length).toBeGreaterThanOrEqual(2);
-        // Both docker-related assets should appear in results
-        expectHit(hits, "docker-homelab");
-        expectHit(hits, "docker-clean");
-    });
-    test('"svelte component" -> skill:svelte-components ranks #1, above sub-references', async () => {
-        const hits = await search("svelte component");
-        expect(hits.length).toBeGreaterThanOrEqual(1);
-        const skillRank = rankOf(hits, "svelte-components");
-        expect(skillRank).toBe(1);
-        // Sub-reference should rank below the skill
-        const refHit = findHit(hits, "svelte-components/references/web-components");
-        if (refHit) {
-            const refRank = rankOf(hits, "svelte-components/references/web-components");
-            expect(refRank).toBeGreaterThan(skillRank);
-        }
-    });
-    test('"code review" -> command or agent ranks above knowledge docs', async () => {
-        const hits = await search("code review");
-        expect(hits.length).toBeGreaterThanOrEqual(2);
-        // Find the top-ranked actionable asset (skill, command, or agent)
-        const actionableTypes = new Set(["skill", "command", "agent"]);
-        const topActionable = hits.find((h) => actionableTypes.has(h.type));
-        expect(topActionable).toBeDefined();
-        // Find the top-ranked knowledge doc
-        const topKnowledge = hits.find((h) => h.type === "knowledge");
-        if (topKnowledge && topActionable) {
-            const actionableRank = rankOf(hits, topActionable.name);
-            const knowledgeRank = rankOf(hits, topKnowledge.name);
-            expect(actionableRank).toBeLessThan(knowledgeRank);
-        }
-    });
-    test('"mem0 search" -> script:mem0-search ranks #1', async () => {
-        const hits = await search("mem0 search");
-        expect(hits.length).toBeGreaterThanOrEqual(1);
-        expect(rankOf(hits, "mem0-search")).toBe(1);
-    });
-});
-describe("Exact/near-exact name matching", () => {
-    test('"docker-homelab" (exact) -> skill:docker-homelab appears in top 3', async () => {
-        const hits = await search("docker-homelab");
-        expect(hits.length).toBeGreaterThanOrEqual(2);
-        // The skill entry and its sub-references all contain "docker-homelab"
-        // in their names. The skill gets a name-match boost but sub-references
-        // also match on FTS name field. Verify the skill is in the top 3.
-        const skillRank = rankOf(hits, "docker-homelab");
-        expect(skillRank).toBeLessThanOrEqual(3);
-        // The skill should have a strong score
-        const skillHit = expectHit(hits, "docker-homelab");
-        expect(scoreOf(skillHit)).toBeGreaterThan(0.5);
-    });
-    test('"mem0-search" (exact) -> script:mem0-search is #1', async () => {
-        const hits = await search("mem0-search");
-        expect(hits.length).toBeGreaterThanOrEqual(1);
-        expect(hits[0].name).toBe("mem0-search");
-    });
-    test('"security-review" (exact) -> command:security-review is #1', async () => {
-        const hits = await search("security-review");
-        expect(hits.length).toBeGreaterThanOrEqual(1);
-        expect(hits[0].name).toBe("security-review");
-        expect(hits[0].type).toBe("command");
-    });
-    test('"k8s-deploy" (exact) -> skill:k8s-deploy is #1', async () => {
-        const hits = await search("k8s-deploy");
-        expect(hits.length).toBeGreaterThanOrEqual(1);
-        expect(hits[0].name).toBe("k8s-deploy");
-        expect(hits[0].type).toBe("skill");
-    });
-    test('"code-reviewer" (exact) -> agent:code-reviewer is #1', async () => {
-        const hits = await search("code-reviewer");
-        expect(hits.length).toBeGreaterThanOrEqual(1);
-        expect(hits[0].name).toBe("code-reviewer");
-        expect(hits[0].type).toBe("agent");
-    });
-});
-describe("Type ranking", () => {
-    test('for "deploy", skills/commands/scripts rank above knowledge docs', async () => {
-        const hits = await search("deploy");
-        expect(hits.length).toBeGreaterThanOrEqual(2);
-        const actionableTypes = new Set(["skill", "command", "agent", "script"]);
-        const topActionable = hits.find((h) => actionableTypes.has(h.type));
-        const topKnowledge = hits.find((h) => h.type === "knowledge");
-        expect(topActionable).toBeDefined();
-        if (topKnowledge && topActionable) {
-            expect(rankOf(hits, topActionable.name)).toBeLessThan(rankOf(hits, topKnowledge.name));
-        }
-    });
-    test('for "review", agents/commands/skills rank above knowledge docs', async () => {
-        const hits = await search("review");
-        expect(hits.length).toBeGreaterThanOrEqual(2);
-        const actionableTypes = new Set(["skill", "command", "agent"]);
-        const topActionable = hits.find((h) => actionableTypes.has(h.type));
-        const topKnowledge = hits.find((h) => h.type === "knowledge");
-        expect(topActionable).toBeDefined();
-        if (topKnowledge && topActionable) {
-            expect(rankOf(hits, topActionable.name)).toBeLessThan(rankOf(hits, topKnowledge.name));
-        }
-    });
-});
-describe("Fuzzy/prefix matching", () => {
-    test('"kube" finds k8s-deploy via alias', async () => {
-        const hits = await search("kube");
-        expectHit(hits, "k8s-deploy");
-    });
-    test('"dock" finds docker-homelab via prefix', async () => {
-        const hits = await search("dock");
-        expectHit(hits, "docker-homelab");
-    });
-    test('"incident" finds the runbook', async () => {
-        const hits = await search("incident");
-        expectHit(hits, "incident-response-runbook");
-    });
-});
-describe("Score preservation (not RRF-flattened)", () => {
-    test("top result score > 0.5 (not capped at 0.0164)", async () => {
-        const hits = await search("docker homelab");
-        expect(hits.length).toBeGreaterThanOrEqual(1);
-        expect(scoreOf(hits[0])).toBeGreaterThan(0.5);
-    });
-    test("top result for exact name query has strong differentiation", async () => {
-        // Use a query that uniquely targets one asset.
-        // Per the locked v1 contract (CLAUDE.md / spec §9), SearchHit.score is
-        // bounded in [0,1]. An exact-name match accumulates large additive
-        // boosts that are clamped at the final emit step, so the top score for
-        // a uniquely-matching exact-name query must reach the ceiling (1.0).
-        const hits = await search("mem0 search");
-        expect(hits.length).toBeGreaterThanOrEqual(1);
-        const topScore = scoreOf(hits[0]);
-        expect(topScore).toBe(1);
-        // If there are additional results, the top should be at least as high.
-        // Below-ceiling differentiation is asserted by the broader
-        // "scores are monotonically decreasing" case below; here we just
-        // confirm the top hit reaches the maximum.
-        if (hits.length >= 2) {
-            expect(topScore).toBeGreaterThanOrEqual(scoreOf(hits[1]));
-        }
-    });
-    test("scores are monotonically decreasing", async () => {
-        const hits = await search("docker");
-        for (let i = 1; i < hits.length; i++) {
-            const prev = hits[i - 1].score ?? 0;
-            const curr = hits[i].score ?? 0;
-            expect(prev).toBeGreaterThanOrEqual(curr);
-        }
-    });
-    test("scores are not compressed to a narrow range", async () => {
-        const hits = await search("docker");
-        expect(hits.length).toBeGreaterThanOrEqual(3);
-        const topScore = scoreOf(hits[0]);
-        const lastScore = scoreOf(hits[hits.length - 1]);
-        const range = topScore - lastScore;
-        // Score range should be meaningful, not compressed to ~0.001 like RRF.
-        // Per the locked v1 contract (CLAUDE.md / spec §9), scores are bounded
-        // in [0,1] — multiple top hits on a popular query may all clamp to 1.0,
-        // which collapses the visible top-end differential. The bottom of the
-        // range still shows clear separation from the top, well above what RRF
-        // would compress to (~0.001).
-        expect(range).toBeGreaterThan(0.01);
-    });
-});
-describe("Provider merge (score not destroyed)", () => {
-    test("when additional provider hits exist, local scores are preserved", () => {
-        const localHits = [
-            {
-                type: "skill",
-                name: "local-skill-1",
-                path: "/test/skills/local-1/SKILL.md",
-                ref: "skill:local-skill-1",
-                origin: null,
-                score: 2.5,
-            },
-            {
-                type: "command",
-                name: "local-cmd-1",
-                path: "/test/commands/local-1.md",
-                ref: "command:local-cmd-1",
-                origin: null,
-                score: 1.8,
-            },
-            {
-                type: "knowledge",
-                name: "local-doc-1",
-                path: "/test/knowledge/local-1.md",
-                ref: "knowledge:local-doc-1",
-                origin: null,
-                score: 0.9,
-            },
-        ];
-        const additionalHits = [
-            {
-                type: "skill",
-                name: "remote-skill-1",
-                path: "/remote/skills/remote-1/SKILL.md",
-                ref: "skill:remote-skill-1",
-                origin: "remote",
-                score: 0.85,
-            },
-        ];
-        const merged = mergeStashHits(localHits, additionalHits, 20);
-        // Local hits should retain their original scores
-        const mergedLocal1 = expectHit(merged, "local-skill-1");
-        expect(mergedLocal1.score).toBe(2.5);
-        const mergedLocal2 = expectHit(merged, "local-cmd-1");
-        expect(mergedLocal2.score).toBe(1.8);
-    });
-    test("provider hits sort fairly by score alongside local hits", () => {
-        const localHits = [
-            {
-                type: "skill",
-                name: "local-high",
-                path: "/test/skills/high/SKILL.md",
-                ref: "skill:local-high",
-                origin: null,
-                score: 2.0,
-            },
-            {
-                type: "skill",
-                name: "local-low",
-                path: "/test/skills/low/SKILL.md",
-                ref: "skill:local-low",
-                origin: null,
-                score: 0.5,
-            },
-        ];
-        const additionalHits = [
-            {
-                type: "skill",
-                name: "remote-1",
-                path: "/remote/skills/1/SKILL.md",
-                ref: "skill:remote-1",
-                origin: "remote",
-                score: 1.0, // Normalized provider score between local-high and local-low
-            },
-        ];
-        const merged = mergeStashHits(localHits, additionalHits, 20);
-        // Provider hit keeps its original score and sorts by score
-        const remoteRank = merged.findIndex((h) => h.name === "remote-1") + 1;
-        const localHighRank = merged.findIndex((h) => h.name === "local-high") + 1;
-        const localLowRank = merged.findIndex((h) => h.name === "local-low") + 1;
-        // remote-1 (1.0) should rank between local-high (2.0) and local-low (0.5)
-        expect(remoteRank).toBeGreaterThan(localHighRank);
-        expect(remoteRank).toBeLessThan(localLowRank);
-    });
-    test("duplicate provider hits are deduplicated (local version wins)", () => {
-        const localHits = [
-            {
-                type: "skill",
-                name: "shared-skill",
-                path: "/test/skills/shared/SKILL.md",
-                ref: "skill:shared-skill",
-                origin: null,
-                score: 2.0,
-            },
-        ];
-        const additionalHits = [
-            {
-                type: "skill",
-                name: "shared-skill",
-                path: "/test/skills/shared/SKILL.md", // Same path = duplicate
-                ref: "skill:shared-skill",
-                origin: "remote",
-                score: 0.5,
-            },
-        ];
-        const merged = mergeStashHits(localHits, additionalHits, 20);
-        // Only one instance of the shared skill should appear
-        const sharedHits = merged.filter((h) => h.name === "shared-skill");
-        expect(sharedHits.length).toBe(1);
-        // And it should have the local score
-        expect(sharedHits[0].score).toBe(2.0);
-    });
-    test("merge preserves sort order by score descending", () => {
-        const localHits = [
-            { type: "skill", name: "a", path: "/a", ref: "skill:a", origin: null, score: 3.0 },
-            { type: "skill", name: "b", path: "/b", ref: "skill:b", origin: null, score: 1.0 },
-        ];
-        const additionalHits = [
-            { type: "skill", name: "c", path: "/c", ref: "skill:c", origin: "remote", score: 2.0 },
-        ];
-        const merged = mergeStashHits(localHits, additionalHits, 20);
-        for (let i = 1; i < merged.length; i++) {
-            const prev = merged[i - 1].score ?? 0;
-            const curr = merged[i].score ?? 0;
-            expect(prev).toBeGreaterThanOrEqual(curr);
-        }
-    });
-});
-describe("Cross-type search consistency", () => {
-    test("searching for 'docker' returns docker-homelab and docker-clean", async () => {
-        const hits = await search("docker");
-        expect(hits.length).toBeGreaterThanOrEqual(2);
-        const dockerNames = hits.map((h) => h.name);
-        expect(dockerNames).toContain("docker-homelab");
-        expect(dockerNames).toContain("docker-clean");
-    });
-    test("multi-word queries narrow results appropriately", async () => {
-        const narrowHits = await search("deploy check");
-        // The narrow query should return deploy-check at a high rank
-        const deployCheckRank = rankOf(narrowHits, "deploy-check");
-        expect(deployCheckRank).toBeLessThanOrEqual(3);
-    });
-    test("searching for svelte returns both skill and agent", async () => {
-        const hits = await search("svelte");
-        const svelteNames = hits.map((h) => h.name);
-        expect(svelteNames).toContain("svelte-components");
-        expect(svelteNames).toContain("svelte-expert");
-    });
-    test("searching for 'release' finds the release-manager command", async () => {
-        const hits = await search("release");
-        const releaseHit = expectHit(hits, "release-manager");
-        expect(releaseHit.type).toBe("command");
-    });
-});
-describe("Metadata signal strength", () => {
-    test("skill with rich metadata appears in results for broad queries", async () => {
-        // docker-homelab has rich tags, aliases, searchHints, and curated quality
-        const hits = await search("container management");
-        const skillHit = expectHit(hits, "docker-homelab");
-        expect(scoreOf(skillHit)).toBeGreaterThan(0);
-    });
-    test("curated quality assets include the curated metadata boost reason", async () => {
-        const hits = await search("kubernetes deploy");
-        expect(hits.length).toBeGreaterThanOrEqual(1);
-        const k8sHit = expectHit(hits, "k8s-deploy");
-        expect(k8sHit.whyMatched).toContain("curated metadata boost");
-    });
-    test("searchHints contribute to matching", async () => {
-        // "troubleshoot docker" is a search hint on docker-homelab
-        const hits = await search("troubleshoot docker");
-        const skillHit = expectHit(hits, "docker-homelab");
-        expect(skillHit.whyMatched).toContain("matched searchHints");
-    });
-    test("aliases contribute to matching", async () => {
-        // "docker-compose" is an alias for docker-homelab
-        const hits = await search("docker compose");
-        const skillHit = expectHit(hits, "docker-homelab");
-        expect(skillHit.whyMatched).toContain("matched aliases");
-    });
-    test("tags contribute to matching", async () => {
-        const hits = await search("homelab");
-        const skillHit = expectHit(hits, "docker-homelab");
-        expect(skillHit.whyMatched).toContain("matched tags");
-    });
-});
-describe("Empty and edge case queries", () => {
-    test("empty query returns all entries", async () => {
-        const result = await akmSearch({ query: "", source: "stash" });
-        expect(result.hits.length).toBeGreaterThan(0);
-    });
-    test("query with no matches returns empty results with tip", async () => {
-        const result = await akmSearch({ query: "xyznonexistent123", source: "stash" });
-        const hits = result.hits.filter((h) => h.type !== "registry");
-        expect(hits.length).toBe(0);
-    });
-    test("single character query returns results when prefix matches", async () => {
-        // Single char queries are too short for prefix expansion (< 3 chars)
-        // but may still match on exact tokens
-        const result = await akmSearch({ query: "k", source: "stash" });
-        // This may or may not return results depending on FTS tokenizer behavior
-        // The important thing is it does not crash
-        expect(result).toBeDefined();
-    });
-});