@appchy/jarvis 0.1.37 → 0.1.39

package/dist/data/chunk-7REP35VA.mjs

@@ -0,0 +1,462 @@
+// ../../packages/data/src/embeddings.ts
+async function buildEmbeddings(config, graph) {
+  const cfg = config.search;
+  if (!cfg) return void 0;
+  const embedder = cfg.embedder.resolve ? await cfg.embedder.resolve() : cfg.embedder;
+  const storeImpl = cfg.store;
+  const nodes = cfg.types ? cfg.types.flatMap((t) => graph.nodes(t)) : graph.nodes();
+  const ctx = { repoRoot: config.repoRoot };
+  const cache = await readCache(config, ctx, embedder.name);
+  const items = [];
+  const toEmbed = [];
+  for (const node of nodes) {
+    const text = searchText(node);
+    const hash = hashText(text);
+    const cached = cache.entries[node.id];
+    if (cached && cached.hash === hash) items.push({ id: node.id, vector: cached.vector });
+    else toEmbed.push({ id: node.id, text, hash });
+  }
+  const batchSize = Math.max(1, embedder.batchSize ?? DEFAULT_BATCH_SIZE);
+  const batches = [];
+  for (let i = 0; i < toEmbed.length; i += batchSize) batches.push(toEmbed.slice(i, i + batchSize));
+  let done = 0;
+  await pool(batches, Math.max(1, embedder.concurrency ?? 1), async (batch) => {
+    const out = await embedder.embed(batch.map((b) => b.text));
+    batch.forEach((b, j) => {
+      const vector = out[j] ?? [];
+      items.push({ id: b.id, vector });
+      cache.entries[b.id] = { hash: b.hash, vector };
+    });
+    done += 1;
+    process.stderr.write(
+      `[data] embedding ${done}/${batches.length} batch(es)${process.stderr.isTTY ? "\r" : "\n"}`
+    );
+  });
+  if (batches.length > 0) process.stderr.write(`[data] embedded ${toEmbed.length} node(s)
+`);
+  const live = new Set(nodes.map((n) => n.id));
+  let pruned = 0;
+  for (const id of Object.keys(cache.entries)) {
+    if (!live.has(id)) {
+      delete cache.entries[id];
+      pruned += 1;
+    }
+  }
+  if (toEmbed.length > 0 || pruned > 0) await writeCache(config, ctx, cache);
+  await storeImpl.add(items);
+  return toRuntime(cfg, embedder, storeImpl, items);
+}
+var DEFAULT_BATCH_SIZE = 64;
+async function pool(items, limit, work) {
+  let next = 0;
+  const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
+    for (let i = next++; i < items.length; i = next++) {
+      const item = items[i];
+      if (item !== void 0) await work(item);
+    }
+  });
+  await Promise.all(runners);
+}
+async function loadEmbeddings(config, graph) {
+  const cfg = config.search;
+  if (!cfg) return void 0;
+  const embedder = cfg.embedder.resolve ? await cfg.embedder.resolve() : cfg.embedder;
+  await probeEmbedder(embedder);
+  const storeImpl = cfg.store;
+  const nodes = cfg.types ? cfg.types.flatMap((t) => graph.nodes(t)) : graph.nodes();
+  const cache = await readCache(config, { repoRoot: config.repoRoot }, embedder.name);
+  const items = [];
+  for (const node of nodes) {
+    const cached = cache.entries[node.id];
+    if (cached) items.push({ id: node.id, vector: cached.vector });
+  }
+  if (nodes.length > 0 && items.length === 0) {
+    process.stderr.write(
+      `[data:warn] no cached vectors for ${nodes.length} node(s) under embedder ${embedder.name} \u2014 dense search is empty (stale cache or a different build-time embedder). Run \`jarvis build graph\`; lexical (BM25) still applies.
+`
+    );
+  }
+  await storeImpl.add(items);
+  return toRuntime(cfg, embedder, storeImpl, items);
+}
+function toRuntime(cfg, embedder, storeImpl, items) {
+  return {
+    embedder,
+    store: storeImpl,
+    vectorById: new Map(items.map((it) => [it.id, it.vector])),
+    hybrid: cfg.hybrid !== false,
+    ...cfg.reranker ? { reranker: cfg.reranker } : {},
+    ...cfg.reranker?.limit !== void 0 ? { rerankLimit: cfg.reranker.limit } : {},
+    ...cfg.expand ? { expand: true } : {},
+    ...cfg.types ? { types: cfg.types } : {},
+    ...cfg.confidence ? { confidence: cfg.confidence } : {}
+  };
+}
+async function probeEmbedder(embedder) {
+  try {
+    await embedder.embed(["probe"]);
+  } catch (err) {
+    throw new Error(
+      `embedder ${embedder.name} unavailable \u2014 ${err instanceof Error ? err.message : String(err)}`
+    );
+  }
+}
+function searchText(node) {
+  const label = node.metadata.label ?? node.metadata.title;
+  const parts = [node.id, typeof label === "string" ? label : "", node.type];
+  const body = node.metadata.embedText ?? node.metadata.excerpt;
+  if (typeof body === "string") parts.push(body);
+  return parts.filter(Boolean).join(" \xB7 ");
+}
+var parsedCache = /* @__PURE__ */ new Map();
+var cacheKey = (ctx, model) => `${ctx.repoRoot}::${model}`;
+async function readCache(config, ctx, model) {
+  const held = parsedCache.get(cacheKey(ctx, model));
+  if (held) return held;
+  const fresh = await loadCache(config, ctx, model);
+  parsedCache.set(cacheKey(ctx, model), fresh);
+  return fresh;
+}
+async function loadCache(config, ctx, model) {
+  try {
+    const raw = await config.persistence.load(ctx, "embeddings");
+    if (raw) {
+      const parsed = JSON.parse(raw);
+      if (parsed.model === model && parsed.entries) return parsed;
+    }
+  } catch {
+  }
+  return { model, entries: {} };
+}
+async function writeCache(config, ctx, cache) {
+  parsedCache.set(cacheKey(ctx, cache.model), cache);
+  await config.persistence.save(ctx, "embeddings", `${JSON.stringify(cache)}
+`);
+}
+function hashText(text) {
+  let h = 5381;
+  for (let i = 0; i < text.length; i++) h = (h << 5) + h + text.charCodeAt(i) | 0;
+  return (h >>> 0).toString(36);
+}
+function buildSearchRuntime(config, graph) {
+  return runSearchRuntime(config, graph, "building", buildEmbeddings);
+}
+function loadSearchRuntime(config, graph) {
+  return runSearchRuntime(config, graph, "loading", loadEmbeddings);
+}
+async function runSearchRuntime(config, graph, verb, make) {
+  const cfg = config.search;
+  if (!cfg) return void 0;
+  try {
+    process.stderr.write(`[data] ${verb} embeddings (${cfg.embedder.name})\u2026
+`);
+    const runtime = await make(config, graph);
+    process.stderr.write(
+      `[data] semantic search ready (hybrid, ${verb === "loading" ? "loaded" : "built"})
+`
+    );
+    return runtime;
+  } catch (err) {
+    process.stderr.write(
+      `[data:warn] semantic search unavailable \u2014 falling back to lexical (BM25) search. ${err instanceof Error ? err.message : String(err)}
+`
+    );
+    return void 0;
+  }
+}
+
+// ../../packages/data/src/tokenize.ts
+var STOPWORDS = /* @__PURE__ */ new Set([
+  "a",
+  "an",
+  "and",
+  "are",
+  "as",
+  "at",
+  "be",
+  "but",
+  "by",
+  "can",
+  "do",
+  "does",
+  "for",
+  "from",
+  "how",
+  "i",
+  "if",
+  "in",
+  "into",
+  "is",
+  "it",
+  "its",
+  "of",
+  "on",
+  "or",
+  "that",
+  "the",
+  "their",
+  "then",
+  "there",
+  "this",
+  "to",
+  "was",
+  "we",
+  "what",
+  "when",
+  "where",
+  "which",
+  "who",
+  "why",
+  "will",
+  "with",
+  "you",
+  "your"
+]);
+function tokenize(text) {
+  const out = [];
+  for (const raw of text.split(/[\s_\-/.:]+/)) {
+    if (!raw) continue;
+    const parts = splitIdentifier(raw);
+    for (const p of parts) out.push(p.toLowerCase());
+    if (parts.length > 1) out.push(raw.toLowerCase());
+  }
+  return out;
+}
+function splitIdentifier(s) {
+  return s.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2").replace(/([a-zA-Z])([0-9])/g, "$1 $2").replace(/([0-9])([a-zA-Z])/g, "$1 $2").split(/\s+/).filter(Boolean);
+}
+
+// ../../packages/data/src/search.ts
+var SEED_COUNT = 10;
+var FRONTIER_CAP = 50;
+var RERANK_LIMIT = 50;
+async function rankNodes(req) {
+  const { candidates, query, graph, runtime } = req;
+  const lexical = lexicalRank(candidates, query);
+  if (!runtime) return lexical;
+  const dense = await denseRank(runtime, candidates, query, graph);
+  let ranked = fuseRankings(lexical, dense, runtime.hybrid, candidates);
+  if ((req.expand ?? runtime.expand) && dense.length > 0) {
+    ranked = expandFrontier(ranked, graph);
+  }
+  if (req.deep && !isSingleTerm(query) && runtime.reranker && ranked.length > 0) {
+    ranked = await rerankHead(runtime.reranker, query, ranked, runtime.rerankLimit ?? RERANK_LIMIT);
+  }
+  return ranked;
+}
+function fuseRankings(lexical, dense, hybrid, candidates) {
+  if (dense.length === 0) return lexical;
+  if (!hybrid) return dense;
+  const fused = rrf({
+    lists: [lexical.map((r) => r.node.id), dense.map((r) => r.node.id)],
+    weights: [1, DENSE_FUSION_WEIGHT]
+  });
+  const byId = new Map(candidates.map((n) => [n.id, n]));
+  const lexicalById = new Map(lexical.map((r) => [r.node.id, r.score]));
+  const denseById = new Map(dense.map((r) => [r.node.id, r.score]));
+  const ranked = [];
+  for (const [id, score] of fused) {
+    const node = byId.get(id);
+    if (!node) continue;
+    const lex = lexicalById.get(id);
+    const den = denseById.get(id);
+    ranked.push({
+      node,
+      score,
+      ...lex !== void 0 ? { lexical: lex } : {},
+      ...den !== void 0 ? { dense: den } : {}
+    });
+  }
+  const pinned = ranked.filter((r) => (r.lexical ?? 0) >= IDENTIFIER_PIN_FLOOR).sort((a, b) => (b.lexical ?? 0) - (a.lexical ?? 0) || tieByNodeId(a, b));
+  const rest = ranked.filter((r) => (r.lexical ?? 0) < IDENTIFIER_PIN_FLOOR).sort((a, b) => b.score - a.score || tieByNodeId(a, b));
+  return [...pinned, ...rest];
+}
+function expandFrontier(ranked, graph) {
+  const present = new Set(ranked.map((r) => r.node.id));
+  const seeds = ranked.slice(0, SEED_COUNT);
+  const floor = seeds.length > 0 ? seeds[seeds.length - 1]?.score ?? 0 : 0;
+  const added = /* @__PURE__ */ new Map();
+  const communitiesDone = /* @__PURE__ */ new Set();
+  for (const seed of seeds) {
+    for (const nb of graph.neighbors(seed.node.id)) {
+      if (!present.has(nb.id) && !added.has(nb.id)) added.set(nb.id, nb);
+    }
+    const community = seed.node.metadata.community;
+    if (typeof community === "string" && community && !communitiesDone.has(community)) {
+      communitiesDone.add(community);
+      for (const m of graph.nodes()) {
+        if (m.metadata.community === community && !present.has(m.id) && !added.has(m.id))
+          added.set(m.id, m);
+      }
+    }
+    if (added.size >= FRONTIER_CAP) break;
+  }
+  const frontier = [...added.values()].slice(0, FRONTIER_CAP).map((node) => ({ node, score: floor * 0.5 }));
+  return [...ranked, ...frontier];
+}
+async function rerankHead(reranker, query, ranked, limit) {
+  const head = ranked.slice(0, limit);
+  const tail = ranked.slice(limit);
+  const hits = head.map((r) => ({
+    id: r.node.id,
+    text: searchText(r.node),
+    score: r.score
+  }));
+  let reordered;
+  try {
+    reordered = await reranker.rerank(query, hits, { topN: head.length });
+  } catch {
+    return ranked;
+  }
+  const byId = new Map(head.map((r) => [r.node.id, r]));
+  const out = [];
+  const seen = /* @__PURE__ */ new Set();
+  for (const hit of reordered) {
+    const prior = byId.get(hit.id);
+    if (prior) {
+      out.push({ ...prior, score: hit.score ?? 0 });
+      seen.add(hit.id);
+    }
+  }
+  for (const r of head) if (!seen.has(r.node.id)) out.push(r);
+  return [...out, ...tail];
+}
+var BM25_K1 = 1.2;
+var BM25_B = 0.25;
+var EXACT_ID_BOOST = 1e3;
+var SYMBOL_HIT_BOOST = 100;
+var IDENTIFIER_PIN_FLOOR = SYMBOL_HIT_BOOST;
+var DENSE_FUSION_WEIGHT = 3;
+function lexicalRank(candidates, query) {
+  const raw = query.trim().toLowerCase();
+  if (!raw) return [];
+  const rawTerms = raw.split(/\s+/).filter(Boolean);
+  const single = rawTerms.length === 1;
+  let queryTerms = [...new Set(tokenize(query))];
+  if (!single) {
+    const stripped = queryTerms.filter((t) => !STOPWORDS.has(t));
+    if (stripped.length > 0) queryTerms = stripped;
+  }
+  const docs = candidates.map((node) => ({ node, tokens: tokenize(haystack(node)) }));
+  const total = docs.length;
+  const df = /* @__PURE__ */ new Map();
+  let lengthSum = 0;
+  for (const d of docs) {
+    lengthSum += d.tokens.length;
+    for (const t of new Set(d.tokens)) df.set(t, (df.get(t) ?? 0) + 1);
+  }
+  const avgdl = total > 0 ? lengthSum / total : 0;
+  const ranked = [];
+  for (const d of docs) {
+    const tf = /* @__PURE__ */ new Map();
+    for (const t of d.tokens) tf.set(t, (tf.get(t) ?? 0) + 1);
+    let score = 0;
+    for (const term of queryTerms) {
+      const f = tf.get(term);
+      if (!f) continue;
+      const n = df.get(term) ?? 0;
+      const idf = Math.log(1 + (total - n + 0.5) / (n + 0.5));
+      const norm = avgdl > 0 ? 1 - BM25_B + BM25_B * (d.tokens.length / avgdl) : 1;
+      score += idf * (f * (BM25_K1 + 1) / (f + BM25_K1 * norm));
+    }
+    score += identifierBoost(d.node, raw, rawTerms[0] ?? "", single);
+    if (score > 0) ranked.push({ node: d.node, score, lexical: score });
+  }
+  ranked.sort((a, b) => b.score - a.score || tieByNodeId(a, b));
+  return ranked;
+}
+function tieByNodeId(a, b) {
+  return a.node.id < b.node.id ? -1 : a.node.id > b.node.id ? 1 : 0;
+}
+function isSingleTerm(query) {
+  return query.trim().split(/\s+/).filter(Boolean).length === 1;
+}
+function diversify(ranked, key) {
+  const byKey = /* @__PURE__ */ new Map();
+  const order = [];
+  for (const r of ranked) {
+    const k = key(r.node);
+    const kept = byKey.get(k);
+    if (kept === void 0) {
+      byKey.set(k, { ...r });
+      order.push(k);
+    } else {
+      kept.collapsed = (kept.collapsed ?? 0) + 1;
+    }
+  }
+  const out = [];
+  for (const k of order) {
+    const r = byKey.get(k);
+    if (r) out.push(r);
+  }
+  return out;
+}
+var CONFIDENT_LEXICAL = 1;
+var CONFIDENT_DENSE = 0.6;
+function isConfident(r, confidence) {
+  const lexicalFloor = confidence?.lexical ?? CONFIDENT_LEXICAL;
+  const denseFloor = confidence?.dense ?? CONFIDENT_DENSE;
+  return (r.lexical ?? 0) >= lexicalFloor || (r.dense ?? 0) >= denseFloor;
+}
+function haystack(node) {
+  const label = node.metadata.label ?? node.metadata.title;
+  const parts = [node.id, typeof label === "string" ? label : "", node.type];
+  const community = node.metadata.community;
+  if (typeof community === "string") parts.push(community);
+  const excerpt = node.metadata.excerpt;
+  if (typeof excerpt === "string") parts.push(excerpt);
+  return parts.filter(Boolean).join(" ");
+}
+function identifierBoost(node, raw, term, single) {
+  const id = node.id.toLowerCase();
+  let boost = 0;
+  if (id === raw) boost += EXACT_ID_BOOST;
+  if (single && id !== raw) {
+    const label = node.metadata.label ?? node.metadata.title;
+    const labelText = typeof label === "string" ? label.toLowerCase() : "";
+    if (id.includes(term) || labelText.includes(term)) boost += SYMBOL_HIT_BOOST;
+  }
+  return boost;
+}
+async function denseRank(runtime, candidates, query, graph) {
+  const queryVector = await embedQuery(runtime.embedder, query);
+  const filterIds = new Set(candidates.map((n) => n.id));
+  const hits = await runtime.store.query(queryVector, { topK: 50, filterIds });
+  const ranked = [];
+  for (const hit of hits) {
+    const node = graph.node(hit.id);
+    if (node) ranked.push({ node, score: hit.score, dense: hit.score });
+  }
+  return ranked;
+}
+async function embedQuery(embedder, text) {
+  if (embedder.embedQuery) return embedder.embedQuery(text);
+  const [vector] = await embedder.embed([text]);
+  return vector ?? [];
+}
+function rrf(req) {
+  const { lists, k = 60, weights } = req;
+  const scores = /* @__PURE__ */ new Map();
+  lists.forEach((list, i) => {
+    const w = weights?.[i] ?? 1;
+    list.forEach((id, rank) => {
+      scores.set(id, (scores.get(id) ?? 0) + w / (k + rank + 1));
+    });
+  });
+  return scores;
+}
+
+export {
+  buildEmbeddings,
+  loadEmbeddings,
+  searchText,
+  buildSearchRuntime,
+  loadSearchRuntime,
+  rankNodes,
+  lexicalRank,
+  isSingleTerm,
+  diversify,
+  isConfident,
+  embedQuery,
+  rrf
+};