agentikit 0.0.8 → 0.0.12

package/dist/src/stash-search.js

@@ -1,192 +1,329 @@
 import fs from "node:fs";
 import path from "node:path";
-import { hasErrnoCode, resolveStashDir } from "./common";
 import { ASSET_TYPES, TYPE_DIRS, deriveCanonicalAssetName } from "./asset-spec";
-import { loadSearchIndex, buildSearchText } from "./indexer";
-import { TfIdfAdapter } from "./similarity";
-import { buildToolInfo } from "./tool-runner";
 import { walkStash } from "./walker";
-import { makeOpenRef } from "./stash-ref";
+import { makeAssetRef } from "./stash-ref";
 import { loadConfig } from "./config";
+import { searchRegistry } from "./registry-search";
+import { openDatabase, closeDatabase, getDbPath, getMeta, searchFts, searchVec, getAllEntries, getEntryCount, getEntryById, isVecAvailable, } from "./db";
+import { tryGetHandler } from "./asset-type-handler";
+import { resolveStashSources, findSourceForPath } from "./stash-source";
 const DEFAULT_LIMIT = 20;
 export async function agentikitSearch(input) {
     const t0 = Date.now();
-    const query = input.query.trim().toLowerCase();
+    const query = input.query.trim();
+    const normalizedQuery = query.toLowerCase();
     const searchType = input.type ?? "any";
     const limit = normalizeLimit(input.limit);
-    const stashDir = resolveStashDir();
-    const config = loadConfig(stashDir);
-    const allStashDirs = [
-        stashDir,
-        ...config.additionalStashDirs.filter((d) => {
-            try {
-                return fs.statSync(d).isDirectory();
-            }
-            catch {
-                return false;
-            }
-        }),
-    ];
-    // Try indexed search (single unified pipeline: embedding + TF-IDF as weighted features)
-    const index = loadSearchIndex();
-    if (index && index.entries && index.entries.length > 0 && index.stashDir === stashDir) {
-        const { hits, embedMs, rankMs } = await searchIndex(index, query, searchType, limit, stashDir, allStashDirs, config);
+    const usageMode = parseSearchUsageMode(input.usage);
+    const source = parseSearchSource(input.source);
+    const sources = resolveStashSources();
+    const stashDir = sources[0].path;
+    const localResult = source === "registry"
+        ? undefined
+        : await searchLocal({
+            query: normalizedQuery,
+            searchType,
+            limit,
+            usageMode,
+            stashDir,
+            sources,
+        });
+    const config = loadConfig();
+    const registryResult = source === "local"
+        ? undefined
+        : await searchRegistry(query, { limit, registryUrls: config.registryUrls });
+    if (source === "local") {
+        return {
+            stashDir,
+            source,
+            hits: localResult?.hits ?? [],
+            usageGuide: localResult?.usageGuide,
+            tip: localResult?.tip,
+            timing: { totalMs: Date.now() - t0, rankMs: localResult?.rankMs, embedMs: localResult?.embedMs },
+        };
+    }
+    const registryHits = (registryResult?.hits ?? []).map((hit) => {
+        const installRef = hit.source === "npm" ? `npm:${hit.ref}` : `github:${hit.ref}`;
+        return {
+            hitSource: "registry",
+            type: "registry",
+            name: hit.title,
+            id: hit.id,
+            registrySource: hit.source,
+            ref: hit.ref,
+            description: hit.description,
+            homepage: hit.homepage,
+            score: hit.score,
+            metadata: hit.metadata,
+            curated: hit.curated,
+            installRef,
+            installCmd: `akm add ${installRef}`,
+        };
+    });
+    if (source === "registry") {
+        const hits = registryHits.slice(0, limit);
         return {
             stashDir,
+            source,
             hits,
-            tip: hits.length === 0 ? "No matching stash assets were found. Try running 'akm index' to rebuild." : undefined,
-            timing: { totalMs: Date.now() - t0, rankMs, embedMs },
+            tip: hits.length === 0 ? "No matching registry entries were found." : undefined,
+            warnings: registryResult?.warnings.length ? registryResult.warnings : undefined,
+            timing: { totalMs: Date.now() - t0 },
         };
     }
-    // No index: fall back to filesystem walk + substring match across all stash dirs
+    const mergedHits = mergeSearchHits(localResult?.hits ?? [], registryHits, limit);
+    return {
+        stashDir,
+        source,
+        hits: mergedHits,
+        usageGuide: localResult?.usageGuide,
+        tip: mergedHits.length === 0 ? "No matching stash assets or registry entries were found." : undefined,
+        warnings: registryResult?.warnings.length ? registryResult.warnings : undefined,
+        timing: { totalMs: Date.now() - t0 },
+    };
+}
+async function searchLocal(input) {
+    const { query, searchType, limit, usageMode, stashDir, sources } = input;
+    const config = loadConfig();
+    const allStashDirs = sources.map((s) => s.path);
+    // Try to open the database
+    const dbPath = getDbPath();
+    try {
+        if (fs.existsSync(dbPath)) {
+            const embeddingDim = config.embedding?.dimension;
+            const db = openDatabase(dbPath, embeddingDim ? { embeddingDim } : undefined);
+            try {
+                const entryCount = getEntryCount(db);
+                const storedStashDir = getMeta(db, "stashDir");
+                if (entryCount > 0 && storedStashDir === stashDir) {
+                    const { hits, usageGuide, embedMs, rankMs } = await searchDatabase(db, query, searchType, limit, stashDir, allStashDirs, config, usageMode, sources);
+                    return {
+                        hits,
+                        usageGuide,
+                        tip: hits.length === 0 ? "No matching stash assets were found. Try running 'akm index' to rebuild." : undefined,
+                        embedMs,
+                        rankMs,
+                    };
+                }
+            }
+            finally {
+                closeDatabase(db);
+            }
+        }
+    }
+    catch (error) {
+        console.warn("Search index unavailable, falling back to substring search:", error instanceof Error ? error.message : String(error));
+    }
     const hits = allStashDirs
-        .flatMap((dir) => substringSearch(query, searchType, limit, dir))
+        .flatMap((dir) => substringSearch(query, searchType, limit, dir, sources))
         .slice(0, limit);
+    const usageGuide = shouldIncludeUsageGuide(usageMode) ? buildUsageGuide(hits.map((hit) => hit.type), searchType) : undefined;
     return {
-        stashDir,
         hits,
+        usageGuide,
         tip: hits.length === 0 ? "No matching stash assets were found. Try running 'akm index' to rebuild." : undefined,
-        timing: { totalMs: Date.now() - t0 },
     };
 }
-// ── Unified indexed search ──────────────────────────────────────────────────
-async function searchIndex(index, query, searchType, limit, stashDir, allStashDirs, config) {
-    // Filter candidates by type
-    let candidates = index.entries;
-    if (searchType !== "any") {
-        candidates = candidates.filter((ie) => ie.entry.type === searchType);
-    }
-    if (candidates.length === 0)
-        return { hits: [] };
-    // Empty query: return all entries (no scoring needed)
+// ── Database search ─────────────────────────────────────────────────────────
+async function searchDatabase(db, query, searchType, limit, stashDir, allStashDirs, config, usageMode, sources) {
+    // Empty query: return all entries
     if (!query) {
-        return { hits: candidates.slice(0, limit).map((ie) => buildIndexedHit({ entry: ie.entry, path: ie.path, score: 1, query, rankingMode: "tfidf", defaultStashDir: stashDir, allStashDirs })) };
+        const typeFilter = searchType === "any" ? undefined : searchType;
+        const allEntries = getAllEntries(db, typeFilter);
+        const selected = allEntries.slice(0, limit);
+        const hits = selected.map((ie) => buildDbHit({
+            entry: ie.entry,
+            path: ie.filePath,
+            score: 1,
+            query,
+            rankingMode: "fts",
+            defaultStashDir: stashDir,
+            allStashDirs,
+            sources,
+            includeItemUsage: shouldIncludeItemUsage(usageMode),
+        }));
+        return {
+            hits,
+            usageGuide: shouldIncludeUsageGuide(usageMode)
+                ? buildUsageGuideFromEntries(selected.map((e) => e.entry), searchType)
+                : undefined,
+        };
     }
-    // Score each candidate using available signals
+    // Score using FTS5 (BM25) and optionally sqlite-vec
     const tEmbed0 = Date.now();
-    const embeddingScores = await tryEmbeddingScores(candidates, query, config);
+    const embeddingScores = await tryVecScores(db, query, limit * 3, config);
     const embedMs = Date.now() - tEmbed0;
     const tRank0 = Date.now();
-    const tfidfScores = computeTfidfScores(index, candidates, query, searchType);
+    const typeFilter = searchType === "any" ? undefined : searchType;
+    const ftsResults = searchFts(db, query, limit * 3, typeFilter);
+    // Build score map from FTS results (normalize BM25 scores)
+    const ftsScoreMap = new Map();
+    for (const r of ftsResults) {
+        // BM25 returns negative scores (more negative = better match), normalize to 0-1
+        const absScore = Math.abs(r.bm25Score);
+        const normalized = absScore / (1 + absScore);
+        ftsScoreMap.set(r.id, { score: normalized, result: r });
+    }
+    // Blend scores
     const scored = [];
-    for (const ie of candidates) {
-        const key = ie.path;
-        const embScore = embeddingScores?.get(key);
-        const tfidfScore = tfidfScores.get(key) ?? 0;
+    const seenIds = new Set();
+    // Process FTS results
+    for (const [id, { score: ftsScore, result }] of ftsScoreMap) {
+        seenIds.add(id);
+        const embScore = embeddingScores?.get(id);
         if (embScore !== undefined) {
-            // Weighted blend: embedding dominates when available, TF-IDF boosts lexical matches
-            const blended = embScore * 0.7 + tfidfScore * 0.3;
+            const blended = embScore * 0.7 + ftsScore * 0.3;
             if (blended > 0)
-                scored.push({ ie, score: blended, rankingMode: "semantic" });
+                scored.push({ id, entry: result.entry, filePath: result.filePath, score: blended, rankingMode: "semantic" });
         }
-        else if (tfidfScore > 0) {
-            scored.push({ ie, score: tfidfScore, rankingMode: "tfidf" });
+        else if (ftsScore > 0) {
+            scored.push({ id, entry: result.entry, filePath: result.filePath, score: ftsScore, rankingMode: "fts" });
         }
     }
+    // Add vec-only results not already in FTS results
+    if (embeddingScores) {
+        for (const [id, embScore] of embeddingScores) {
+            if (seenIds.has(id))
+                continue;
+            const found = getEntryById(db, id);
+            if (found) {
+                if (typeFilter && found.entry.type !== typeFilter)
+                    continue;
+                scored.push({
+                    id,
+                    entry: found.entry,
+                    filePath: found.filePath,
+                    score: embScore,
+                    rankingMode: "semantic",
+                });
+            }
+        }
+    }
+    // Apply boosts (tag, intent, name matches)
+    const queryTokens = query.toLowerCase().split(/\s+/).filter(Boolean);
+    for (const item of scored) {
+        const entry = item.entry;
+        // Tag boost
+        if (entry.tags) {
+            for (const tag of entry.tags) {
+                if (queryTokens.some((t) => tag.toLowerCase() === t)) {
+                    item.score += 0.15;
+                }
+            }
+        }
+        // Intent boost
+        if (entry.intents) {
+            for (const intent of entry.intents) {
+                const intentLower = intent.toLowerCase();
+                for (const token of queryTokens) {
+                    if (intentLower.includes(token)) {
+                        item.score += 0.12;
+                        break;
+                    }
+                }
+            }
+        }
+        // Name boost
+        const nameLower = entry.name.toLowerCase().replace(/[-_]/g, " ");
+        if (queryTokens.some((t) => nameLower.includes(t))) {
+            item.score += 0.1;
+        }
+    }
+    for (const item of scored) {
+        item.score = Math.min(item.score, 1.0);
+    }
     scored.sort((a, b) => b.score - a.score);
     const rankMs = Date.now() - tRank0;
-    return { embedMs, rankMs, hits: scored.slice(0, limit).map(({ ie, score, rankingMode }) => buildIndexedHit({
-            entry: ie.entry,
-            path: ie.path,
-            score: Math.round(score * 1000) / 1000,
-            query,
-            rankingMode,
-            defaultStashDir: stashDir,
-            allStashDirs,
-        })) };
+    const selected = scored.slice(0, limit);
+    const hits = selected.map(({ entry, filePath, score, rankingMode }) => buildDbHit({
+        entry,
+        path: filePath,
+        score: Math.round(score * 1000) / 1000,
+        query,
+        rankingMode,
+        defaultStashDir: stashDir,
+        allStashDirs,
+        sources,
+        includeItemUsage: shouldIncludeItemUsage(usageMode),
+    }));
+    return {
+        embedMs,
+        rankMs,
+        hits,
+        usageGuide: shouldIncludeUsageGuide(usageMode)
+            ? buildUsageGuideFromEntries(selected.map((item) => item.entry), searchType)
+            : undefined,
+    };
 }
-// ── Embedding scorer ────────────────────────────────────────────────────────
-async function tryEmbeddingScores(candidates, query, config) {
-    if (!config.semanticSearch)
+// ── Vector scorer ───────────────────────────────────────────────────────────
+async function tryVecScores(db, query, k, config) {
+    if (!config.semanticSearch || !isVecAvailable())
         return null;
-    const withEmbeddings = candidates.filter((ie) => ie.embedding && ie.embedding.length > 0);
-    if (withEmbeddings.length === 0)
+    const hasEmbeddings = getMeta(db, "hasEmbeddings");
+    if (hasEmbeddings !== "1")
         return null;
     try {
-        const { embed, cosineSimilarity } = await import("./embedder.js");
+        const { embed } = await import("./embedder.js");
         const queryEmbedding = await embed(query, config.embedding);
+        const vecResults = searchVec(db, queryEmbedding, k);
         const scores = new Map();
-        for (const ie of withEmbeddings) {
-            scores.set(ie.path, cosineSimilarity(queryEmbedding, ie.embedding));
+        for (const { id, distance } of vecResults) {
+            // Convert L2 distance to cosine similarity (vectors are normalized)
+            const cosineSim = 1 - (distance * distance) / 2;
+            scores.set(id, Math.max(0, cosineSim));
         }
         return scores;
     }
-    catch {
+    catch (error) {
+        console.warn("Vector search failed, skipping:", error instanceof Error ? error.message : String(error));
         return null;
     }
 }
-// ── TF-IDF scorer ───────────────────────────────────────────────────────────
-function computeTfidfScores(index, candidates, query, searchType) {
-    const candidateScoredEntries = toScoredEntries(candidates);
-    let adapter;
-    if (index.tfidf) {
-        const allScored = toScoredEntries(index.entries);
-        adapter = TfIdfAdapter.deserialize(index.tfidf, allScored);
-    }
-    else {
-        adapter = new TfIdfAdapter();
-        adapter.buildIndex(candidateScoredEntries);
-    }
-    const typeFilter = searchType === "any" ? undefined : searchType;
-    const results = adapter.search(query, candidates.length, typeFilter);
-    const scores = new Map();
-    for (const r of results) {
-        scores.set(r.path, r.score);
-    }
-    return scores;
-}
 // ── Substring fallback (no index) ───────────────────────────────────────────
-function substringSearch(query, searchType, limit, stashDir) {
+function substringSearch(query, searchType, limit, stashDir, sources) {
     const assets = indexAssets(stashDir, searchType);
     return assets
         .filter((asset) => asset.name.toLowerCase().includes(query))
         .sort(compareAssets)
         .slice(0, limit)
-        .map((asset) => assetToSearchHit(asset, stashDir));
+        .map((asset) => assetToSearchHit(asset, stashDir, sources));
 }
 // ── Hit building ────────────────────────────────────────────────────────────
-function findStashDirForPath(filePath, stashDirs) {
-    const resolved = path.resolve(filePath);
-    for (const dir of stashDirs) {
-        if (resolved.startsWith(path.resolve(dir) + path.sep))
-            return dir;
-    }
-    return undefined;
-}
-function buildIndexedHit(input) {
-    const entryStashDir = findStashDirForPath(input.path, input.allStashDirs) ?? input.defaultStashDir;
+function buildDbHit(input) {
+    const entryStashDir = findSourceForPath(input.path, input.sources)?.path ?? input.defaultStashDir;
     const typeRoot = path.join(entryStashDir, TYPE_DIRS[input.entry.type]);
     const openRefName = deriveCanonicalAssetName(input.entry.type, typeRoot, input.path)
         ?? input.entry.name;
     const qualityBoost = input.entry.generated === true ? 0 : 0.05;
     const confidenceBoost = typeof input.entry.confidence === "number" ? Math.min(0.05, Math.max(0, input.entry.confidence) * 0.05) : 0;
-    const score = Math.round((input.score + qualityBoost + confidenceBoost) * 1000) / 1000;
+    const score = Math.min(Math.round((input.score + qualityBoost + confidenceBoost) * 1000) / 1000, 1.0);
     const whyMatched = buildWhyMatched(input.entry, input.query, input.rankingMode, qualityBoost, confidenceBoost);
+    const source = findSourceForPath(input.path, input.sources);
     const hit = {
+        hitSource: "local",
         type: input.entry.type,
         name: input.entry.name,
         path: input.path,
-        openRef: makeOpenRef(input.entry.type, openRefName),
+        openRef: makeAssetRef(input.entry.type, openRefName, source?.registryId),
+        registryId: source?.registryId,
+        editable: source?.writable ?? false,
         description: input.entry.description,
         tags: input.entry.tags,
         score,
         whyMatched,
     };
-    if (input.entry.type === "tool") {
-        try {
-            const toolInfo = buildToolInfo(entryStashDir, input.path);
-            hit.runCmd = toolInfo.runCmd;
-            hit.kind = toolInfo.kind;
-        }
-        catch (error) {
-            if (!hasErrnoCode(error, "ENOENT"))
-                throw error;
-        }
+    if (input.includeItemUsage && input.entry.usage && input.entry.usage.length > 0) {
+        hit.usage = input.entry.usage;
+    }
+    const handler = tryGetHandler(input.entry.type);
+    if (handler?.enrichSearchHit) {
+        handler.enrichSearchHit(hit, entryStashDir);
     }
     return hit;
 }
 function buildWhyMatched(entry, query, rankingMode, qualityBoost, confidenceBoost) {
-    const reasons = [rankingMode === "semantic" ? "semantic similarity" : "tf-idf lexical relevance"];
+    const reasons = [rankingMode === "semantic" ? "semantic similarity" : "fts bm25 relevance"];
     const tokens = query.toLowerCase().split(/\s+/).filter(Boolean);
     const name = entry.name.toLowerCase();
     const tags = entry.tags?.join(" ").toLowerCase() ?? "";
@@ -207,32 +344,22 @@ function buildWhyMatched(entry, query, rankingMode, qualityBoost, confidenceBoos
     return reasons;
 }
 // ── Helpers ─────────────────────────────────────────────────────────────────
-function toScoredEntries(entries) {
-    return entries.map((ie) => ({
-        id: `${ie.entry.type}:${ie.entry.name}`,
-        text: buildSearchText(ie.entry),
-        entry: ie.entry,
-        path: ie.path,
-    }));
-}
-function assetToSearchHit(asset, stashDir) {
-    if (asset.type !== "tool") {
-        return {
-            type: asset.type,
-            name: asset.name,
-            path: asset.path,
-            openRef: makeOpenRef(asset.type, asset.name),
-        };
-    }
-    const toolInfo = buildToolInfo(stashDir, asset.path);
-    return {
-        type: "tool",
+function assetToSearchHit(asset, stashDir, sources) {
+    const source = findSourceForPath(asset.path, sources);
+    const hit = {
+        hitSource: "local",
+        type: asset.type,
         name: asset.name,
         path: asset.path,
-        openRef: makeOpenRef("tool", asset.name),
-        runCmd: toolInfo.runCmd,
-        kind: toolInfo.kind,
+        openRef: makeAssetRef(asset.type, asset.name, source?.registryId),
+        registryId: source?.registryId,
+        editable: source?.writable ?? false,
     };
+    const handler = tryGetHandler(asset.type);
+    if (handler?.enrichSearchHit) {
+        handler.enrichSearchHit(hit, stashDir);
+    }
+    return hit;
 }
 function normalizeLimit(limit) {
     if (typeof limit !== "number" || Number.isNaN(limit) || limit <= 0) {
@@ -240,6 +367,98 @@ function normalizeLimit(limit) {
     }
     return Math.min(Math.floor(limit), 200);
 }
+function parseSearchUsageMode(mode) {
+    if (mode === "none" || mode === "both" || mode === "item" || mode === "guide") {
+        return mode;
+    }
+    if (typeof mode === "undefined")
+        return "both";
+    throw new Error(`Invalid usage mode: ${String(mode)}. Expected one of: none|both|item|guide`);
+}
+function parseSearchSource(source) {
+    if (source === "local" || source === "registry" || source === "both")
+        return source;
+    if (typeof source === "undefined")
+        return "local";
+    throw new Error(`Invalid search source: ${String(source)}. Expected one of: local|registry|both`);
+}
+function mergeSearchHits(localHits, registryHits, limit) {
+    const merged = [];
+    let localIndex = 0;
+    let registryIndex = 0;
+    while (merged.length < limit && (localIndex < localHits.length || registryIndex < registryHits.length)) {
+        if (localIndex < localHits.length) {
+            merged.push(localHits[localIndex]);
+            localIndex += 1;
+            if (merged.length >= limit)
+                break;
+        }
+        if (registryIndex < registryHits.length) {
+            merged.push(registryHits[registryIndex]);
+            registryIndex += 1;
+        }
+    }
+    return merged;
+}
+function shouldIncludeUsageGuide(mode) {
+    return mode === "both" || mode === "guide";
+}
+function shouldIncludeItemUsage(mode) {
+    return mode === "both" || mode === "item";
+}
+function buildUsageGuideFromEntries(entries, searchType) {
+    const types = entries.map((entry) => entry.type);
+    const fallbackGuide = buildUsageGuide(types, searchType);
+    const metadataByType = new Map();
+    for (const entry of entries) {
+        if (!entry.usage || entry.usage.length === 0)
+            continue;
+        const current = metadataByType.get(entry.type) ?? [];
+        for (const item of entry.usage) {
+            const trimmed = item.trim();
+            if (trimmed && !current.includes(trimmed))
+                current.push(trimmed);
+        }
+        if (current.length > 0)
+            metadataByType.set(entry.type, current);
+    }
+    if (!fallbackGuide && metadataByType.size === 0)
+        return undefined;
+    const result = {};
+    for (const assetType of resolveGuideTypes(types, searchType)) {
+        const lines = [];
+        const metadataLines = metadataByType.get(assetType);
+        if (metadataLines && metadataLines.length > 0) {
+            lines.push(...metadataLines);
+        }
+        const fallbackLines = fallbackGuide?.[assetType];
+        if (fallbackLines && fallbackLines.length > 0) {
+            for (const line of fallbackLines) {
+                if (!lines.includes(line))
+                    lines.push(line);
+            }
+        }
+        if (lines.length > 0)
+            result[assetType] = lines;
+    }
+    return Object.keys(result).length > 0 ? result : undefined;
+}
+function buildUsageGuide(hitTypes, searchType) {
+    const result = {};
+    for (const assetType of resolveGuideTypes(hitTypes, searchType)) {
+        result[assetType] = usageGuideByType(assetType);
+    }
+    return Object.keys(result).length > 0 ? result : undefined;
+}
+function resolveGuideTypes(hitTypes, searchType) {
+    if (searchType !== "any")
+        return [searchType];
+    return Array.from(new Set(hitTypes));
+}
+function usageGuideByType(type) {
+    const handler = tryGetHandler(type);
+    return handler?.defaultUsageGuide ?? [];
+}
 function fileToAsset(assetType, root, file) {
     const name = deriveCanonicalAssetName(assetType, root, file);
     if (!name)

package/dist/src/stash-show.d.ts

@@ -2,4 +2,4 @@ import type { KnowledgeView, ShowResponse } from "./stash-types";
 export declare function agentikitShow(input: {
     ref: string;
     view?: KnowledgeView;
-}): ShowResponse;
+}): Promise<ShowResponse>;