@appchy/jarvis 0.1.37 → 0.1.39

package/dist/data/chunk-AYOJSS2F.mjs

@@ -0,0 +1,72 @@
+import {
+  files
+} from "./chunk-YWSWQEJN.mjs";
+
+// ../../packages/data/src/define.ts
+function config(entries, settings) {
+  return { ...settings, plugins: entries };
+}
+function map(where, how, options = {}) {
+  const from = typeof where === "string" || Array.isArray(where) ? files(where) : where;
+  return {
+    entry: "map",
+    name: options.name ?? from.name,
+    from,
+    how,
+    ...options.dedupe !== void 0 ? { dedupe: options.dedupe } : {},
+    ...options.lens !== void 0 ? { lens: options.lens } : {}
+  };
+}
+function connect(run, options = {}) {
+  return { entry: "connect", name: options.name ?? "connect", run };
+}
+function source(spec) {
+  return { ...spec, seam: "source" };
+}
+function linker(spec) {
+  return { ...spec, seam: "linker" };
+}
+function enforcer(spec) {
+  return { ...spec, seam: "enforcer" };
+}
+function enricher(spec) {
+  return { ...spec, seam: "enricher" };
+}
+function backend(spec) {
+  return { ...spec, seam: "backend" };
+}
+function persistence(spec) {
+  return { ...spec, seam: "persistence" };
+}
+function labeler(spec) {
+  return { ...spec, seam: "labeler" };
+}
+function embedder(spec) {
+  return { ...spec, seam: "embedder" };
+}
+function store(spec) {
+  return { ...spec, seam: "store" };
+}
+function reranker(spec) {
+  return { ...spec, seam: "reranker" };
+}
+function extractor(spec) {
+  return { ...spec, seam: "extractor" };
+}
+
+export {
+  config,
+  map,
+  connect,
+  source,
+  linker,
+  enforcer,
+  enricher,
+  backend,
+  persistence,
+  labeler,
+  embedder,
+  store,
+  reranker,
+  extractor
+};

package/dist/data/chunk-RRJ6KKYL.mjs

@@ -0,0 +1,42 @@
+import {
+  persistence
+} from "./chunk-AYOJSS2F.mjs";
+
+// ../../packages/data/src/persistences/local.ts
+import { mkdir, readFile, writeFile } from "fs/promises";
+import { dirname, join, resolve } from "path";
+function localFiles(options = {}) {
+  const root = options.root ?? ".data";
+  const rel = {
+    snapshot: options.snapshot ?? "snapshot.json",
+    overlay: options.overlay ?? "trace.json",
+    embeddings: options.embeddings ?? "embeddings.json",
+    state: options.state ?? "state.json",
+    memory: options.memory ?? "memory.json"
+  };
+  const pathOf = (ctx, key) => resolve(
+    ctx.repoRoot,
+    root,
+    key.startsWith("plugin:") ? join("plugins", `${key.slice(7)}.json`) : rel[key]
+  );
+  return persistence({
+    name: "local-files",
+    load: async (ctx, key) => {
+      try {
+        return await readFile(pathOf(ctx, key), "utf8");
+      } catch {
+        return void 0;
+      }
+    },
+    save: async (ctx, key, bytes) => {
+      const path = pathOf(ctx, key);
+      await mkdir(dirname(path), { recursive: true });
+      await writeFile(path, bytes, "utf8");
+    },
+    locate: (ctx, key) => pathOf(ctx, key)
+  });
+}
+
+export {
+  localFiles
+};

package/dist/data/chunk-YWSWQEJN.mjs

@@ -0,0 +1,138 @@
+// ../../packages/data/src/frontmatter.ts
+var FENCE = /^---\s*$/;
+function parseFrontmatter(source) {
+  const lines = source.split(/\r?\n/);
+  if (lines[0] === void 0 || !FENCE.test(lines[0])) {
+    return { data: {}, body: source };
+  }
+  const data = {};
+  let i = 1;
+  let pendingKey = null;
+  let pendingList = [];
+  const flushList = () => {
+    if (pendingKey !== null) {
+      data[pendingKey] = pendingList;
+      pendingKey = null;
+      pendingList = [];
+    }
+  };
+  for (; i < lines.length; i++) {
+    const line = lines[i] ?? "";
+    if (FENCE.test(line)) {
+      flushList();
+      i++;
+      break;
+    }
+    const listItem = /^\s*-\s+(.*)$/.exec(line);
+    if (listItem && pendingKey !== null) {
+      pendingList.push(stripQuotes((listItem[1] ?? "").trim()));
+      continue;
+    }
+    const kv = /^([A-Za-z0-9_.-]+):\s*(.*)$/.exec(line);
+    if (kv) {
+      flushList();
+      const key = kv[1] ?? "";
+      const rawValue = (kv[2] ?? "").trim();
+      if (rawValue === "") {
+        pendingKey = key;
+        pendingList = [];
+      } else if (rawValue.startsWith("[") && rawValue.endsWith("]")) {
+        data[key] = rawValue.slice(1, -1).split(",").map((s) => stripQuotes(s.trim())).filter((s) => s.length > 0);
+      } else {
+        data[key] = stripQuotes(rawValue);
+      }
+    }
+  }
+  flushList();
+  return { data, body: lines.slice(i).join("\n") };
+}
+function stripQuotes(value) {
+  if (value.length >= 2) {
+    const first = value[0];
+    const last = value[value.length - 1];
+    if (first === '"' && last === '"' || first === "'" && last === "'") {
+      return value.slice(1, -1);
+    }
+  }
+  return value;
+}
+
+// ../../packages/data/src/paths.ts
+import { relative } from "path";
+import fg from "fast-glob";
+import picomatch from "picomatch";
+var DEFAULT_IGNORE = ["**/node_modules/**", "**/dist/**", "**/.git/**"];
+async function glob(patterns, request = {}) {
+  const cwd = request.cwd ?? process.cwd();
+  const found = await fg(patterns, {
+    cwd,
+    absolute: true,
+    onlyFiles: true,
+    ignore: request.ignore ?? DEFAULT_IGNORE
+  });
+  const paths = request.absolute === true ? found : found.map((path) => relative(cwd, path));
+  return paths.sort();
+}
+function matcher(patterns) {
+  const list = Array.isArray(patterns) ? patterns : [patterns];
+  const include = list.filter((pattern) => !pattern.startsWith("!"));
+  const exclude = list.filter((pattern) => pattern.startsWith("!")).map((pattern) => pattern.slice(1));
+  if (include.length === 0) return () => false;
+  return picomatch(include, exclude.length > 0 ? { ignore: exclude } : {});
+}
+function beneath(path) {
+  const root = path.trim().replace(/\/+$/, "");
+  return (file) => file === root || file.startsWith(`${root}/`);
+}
+function dirOf(path) {
+  const cut = path.lastIndexOf("/");
+  return cut === -1 ? "" : path.slice(0, cut);
+}
+
+// ../../packages/data/src/finders/files.ts
+import { readFile } from "fs/promises";
+import { relative as relative2, resolve } from "path";
+import fg2 from "fast-glob";
+function files(patterns, options = {}) {
+  const globs = Array.isArray(patterns) ? patterns : [patterns];
+  return {
+    name: `files(${globs.join(", ")})`,
+    ...options.schema !== void 0 ? { schema: options.schema } : {},
+    ...options.lens !== void 0 ? { lens: options.lens } : {},
+    list: async (ctx) => {
+      const root = ctx.repoRoot;
+      const matched = await fg2(globs, {
+        cwd: options.cwd ? resolve(root, options.cwd) : root,
+        absolute: true,
+        onlyFiles: true,
+        ignore: options.ignore ?? DEFAULT_IGNORE
+      });
+      const whole = options.text === "file";
+      const read = await Promise.all(matched.map((file) => toItem(root, file, whole)));
+      return read.filter((item) => item !== void 0);
+    }
+  };
+}
+async function toItem(root, file, whole) {
+  const raw = await readFile(file, "utf8").catch(() => "");
+  if (!raw) return void 0;
+  const ref = relative2(root, file);
+  const { data, body } = parseFrontmatter(raw);
+  const text = whole ? raw : body;
+  return {
+    ref,
+    file: ref,
+    data,
+    text: async () => text
+  };
+}
+
+export {
+  parseFrontmatter,
+  DEFAULT_IGNORE,
+  glob,
+  matcher,
+  beneath,
+  dirOf,
+  files
+};

package/dist/data/embedders.mjs

@@ -0,0 +1,80 @@
+import "./chunk-AKQQC5IT.mjs";
+import "./chunk-7REP35VA.mjs";
+import "./chunk-RRJ6KKYL.mjs";
+import {
+  embedder
+} from "./chunk-AYOJSS2F.mjs";
+import "./chunk-YWSWQEJN.mjs";
+
+// ../../packages/data/src/embedders/local.ts
+var BGE_QUERY_INSTRUCTION = "Represent this sentence for searching relevant passages: ";
+function resolveQueryInstruction(model) {
+  return /bge/i.test(model) ? BGE_QUERY_INSTRUCTION : "";
+}
+function localEmbeddings(options = {}) {
+  const model = options.model ?? "Xenova/bge-small-en-v1.5";
+  const pkg = "@huggingface/transformers";
+  let pipe;
+  const extractor = async () => {
+    if (!pipe) {
+      const mod = await import(pkg).catch(() => {
+        throw new Error(
+          "localEmbeddings needs '@huggingface/transformers' \u2014 run `pnpm add @huggingface/transformers` (optional dep)."
+        );
+      });
+      pipe = mod.pipeline(
+        "feature-extraction",
+        model
+      );
+    }
+    return pipe;
+  };
+  const queryInstruction = options.queryInstruction ?? resolveQueryInstruction(model);
+  const embed = async (texts) => {
+    if (texts.length === 0) return [];
+    const run = await extractor();
+    const out = await run(texts, { pooling: "mean", normalize: true });
+    return out.tolist();
+  };
+  return embedder({
+    name: `local:${model}`,
+    family: "localEmbeddings",
+    dimensions: options.dimensions ?? 384,
+    // On-device inference is CPU-bound: overlapping batches only fights itself for cores (the whole
+    // point of "no heavy local lifting"), so this embedder declares itself serial.
+    concurrency: 1,
+    embed,
+    // Query path prepends the model's instruction (bge); passages are embedded raw via `embed`.
+    ...queryInstruction ? { embedQuery: async (text) => (await embed([queryInstruction + text]))[0] ?? [] } : {}
+  });
+}
+
+// ../../packages/data/src/embedders/openai.ts
+function openaiEmbeddings(options = {}) {
+  const model = options.model ?? "text-embedding-3-small";
+  const baseUrl = options.baseUrl ?? "https://api.openai.com/v1";
+  return embedder({
+    name: `openai:${model}`,
+    family: "openaiEmbeddings",
+    // A hosted endpoint is network-bound, so a cold start overlaps batches — kept modest so a large
+    // corpus doesn't trip per-minute rate limits; `concurrency` lets a self-hosted endpoint raise it.
+    concurrency: options.concurrency ?? 4,
+    embed: async (texts) => {
+      if (texts.length === 0) return [];
+      const apiKey = options.apiKey ?? process.env.OPENAI_API_KEY;
+      if (!apiKey) throw new Error("openaiEmbeddings needs an apiKey or OPENAI_API_KEY.");
+      const res = await fetch(`${baseUrl}/embeddings`, {
+        method: "POST",
+        headers: { "content-type": "application/json", authorization: `Bearer ${apiKey}` },
+        body: JSON.stringify({ model, input: texts })
+      });
+      if (!res.ok) throw new Error(`openai embeddings ${res.status}: ${await res.text()}`);
+      const json = await res.json();
+      return json.data.map((d) => d.embedding);
+    }
+  });
+}
+export {
+  localEmbeddings,
+  openaiEmbeddings
+};

package/dist/data/finders.mjs

@@ -0,0 +1,35 @@
+import {
+  DEFAULT_IGNORE,
+  files
+} from "./chunk-YWSWQEJN.mjs";
+
+// ../../packages/data/src/finders/declared.ts
+function declared(items) {
+  return {
+    name: `declared(${items.map((one) => one.ref).join(", ")})`,
+    list: async () => items.map((one) => ({
+      ref: one.ref,
+      data: one.data,
+      text: async () => one.ref
+    }))
+  };
+}
+
+// ../../packages/data/src/finders/expand.ts
+function expand(from, into, name) {
+  return {
+    name: name === void 0 ? from.name : `${name}(${from.name})`,
+    ...from.lens !== void 0 ? { lens: from.lens } : {},
+    list: async (ctx) => {
+      const out = [];
+      for (const item of await from.list(ctx)) out.push(...await into(item, ctx));
+      return out;
+    }
+  };
+}
+export {
+  DEFAULT_IGNORE,
+  declared,
+  expand,
+  files
+};

package/dist/data/index.mjs

@@ -0,0 +1,137 @@
+import {
+  COVERAGE_VIA,
+  DERIVED_FROM,
+  FUZZY_EDGE_META,
+  FUZZY_RELATIONS,
+  GraphStore,
+  IMPORT_RELATIONS,
+  RESERVED,
+  TIER,
+  applyMemoryOverlay,
+  asString,
+  buildGraph,
+  buildIdRegex,
+  categoryOf,
+  countOccurrences,
+  dot,
+  edgeMatchesConfidence,
+  everywhere,
+  filesOf,
+  formatLedger,
+  isPrecise,
+  lineOf,
+  loadConfig,
+  matches,
+  mintId,
+  normalize,
+  printDiagnostics,
+  readJson,
+  runEnforcers,
+  sections,
+  slug,
+  stripExampleSpans,
+  summarize,
+  validateConfig,
+  values
+} from "./chunk-AKQQC5IT.mjs";
+import {
+  buildEmbeddings,
+  buildSearchRuntime,
+  embedQuery,
+  lexicalRank,
+  loadEmbeddings,
+  loadSearchRuntime,
+  rankNodes,
+  rrf,
+  searchText
+} from "./chunk-7REP35VA.mjs";
+import "./chunk-RRJ6KKYL.mjs";
+import {
+  backend,
+  config,
+  connect,
+  embedder,
+  enforcer,
+  enricher,
+  extractor,
+  labeler,
+  linker,
+  map,
+  persistence,
+  reranker,
+  source,
+  store
+} from "./chunk-AYOJSS2F.mjs";
+import {
+  DEFAULT_IGNORE,
+  beneath,
+  dirOf,
+  glob,
+  matcher,
+  parseFrontmatter
+} from "./chunk-YWSWQEJN.mjs";
+export {
+  COVERAGE_VIA,
+  DEFAULT_IGNORE,
+  DERIVED_FROM,
+  FUZZY_EDGE_META,
+  FUZZY_RELATIONS,
+  GraphStore,
+  IMPORT_RELATIONS,
+  RESERVED,
+  TIER,
+  applyMemoryOverlay,
+  asString,
+  backend,
+  beneath,
+  buildEmbeddings,
+  buildGraph,
+  buildIdRegex,
+  buildSearchRuntime,
+  categoryOf,
+  config,
+  connect,
+  countOccurrences,
+  dirOf,
+  dot,
+  edgeMatchesConfidence,
+  embedQuery,
+  embedder,
+  enforcer,
+  enricher,
+  everywhere,
+  extractor,
+  filesOf,
+  formatLedger,
+  glob,
+  isPrecise,
+  labeler,
+  lexicalRank,
+  lineOf,
+  linker,
+  loadConfig,
+  loadEmbeddings,
+  loadSearchRuntime,
+  map,
+  matcher,
+  matches,
+  mintId,
+  normalize,
+  parseFrontmatter,
+  persistence,
+  printDiagnostics,
+  rankNodes,
+  readJson,
+  reranker,
+  rrf,
+  runEnforcers,
+  searchText,
+  sections,
+  slug,
+  source,
+  store,
+  stripExampleSpans,
+  summarize,
+  validateConfig,
+  values
+};

package/dist/data/linkers.mjs

@@ -0,0 +1,83 @@
+import {
+  embedQuery,
+  loadEmbeddings
+} from "./chunk-7REP35VA.mjs";
+import {
+  linker
+} from "./chunk-AYOJSS2F.mjs";
+import "./chunk-YWSWQEJN.mjs";
+
+// ../../packages/data/src/linkers/semantic.ts
+var RELATION_RELATED = "relates_to";
+var DEFAULT_TOP_K = 6;
+var DEFAULT_SCORE_FLOOR = 0.2;
+function relatesTo(options = {}) {
+  const requested = Number.isFinite(options.topK) ? options.topK ?? DEFAULT_TOP_K : DEFAULT_TOP_K;
+  const topK = Math.max(1, Math.floor(requested));
+  const scoreFloor = clampScore(options.scoreFloor ?? DEFAULT_SCORE_FLOOR);
+  const relation = options.relation?.trim() || RELATION_RELATED;
+  const allowed = options.artifactTypes ? new Set(options.artifactTypes) : void 0;
+  return linker({
+    name: "relates-to",
+    apply: async (ctx) => {
+      if (!ctx.config.search) return;
+      const runtime = await loadEmbeddings(ctx.config, ctx.graph).catch((err) => {
+        ctx.logger.warn(
+          `[data] semantic linker disabled: ${err instanceof Error ? err.message : String(err)}`
+        );
+        return void 0;
+      });
+      if (!runtime) return;
+      const filterIds = collectCodeNodes(ctx);
+      if (filterIds.size === 0) return;
+      const backendName = ctx.config.backend.name;
+      for (const artifact of ctx.graph.nodes()) {
+        if (artifact.owner === backendName) continue;
+        if (allowed && !allowed.has(artifact.type)) continue;
+        const query = queryText(artifact);
+        if (!query) continue;
+        const candidates = await semanticCandidates(runtime, query, filterIds, topK);
+        if (candidates.length === 0) continue;
+        const seen = /* @__PURE__ */ new Set();
+        for (const hit of candidates) {
+          if (hit.score < scoreFloor) break;
+          const code = ctx.graph.node(hit.id);
+          if (!code || code.owner !== backendName) continue;
+          const file = typeof code.metadata.sourceFile === "string" ? code.metadata.sourceFile : code.id;
+          if (seen.has(file)) continue;
+          seen.add(file);
+          ctx.addEdge({
+            from: code.id,
+            to: artifact.id,
+            relation,
+            metadata: { score: hit.score, source: "semantic" }
+          });
+          if (seen.size >= topK) break;
+        }
+      }
+    }
+  });
+}
+function collectCodeNodes(ctx) {
+  const ids = /* @__PURE__ */ new Set();
+  for (const n of ctx.graph.nodes()) {
+    if (n.owner === ctx.config.backend.name) ids.add(n.id);
+  }
+  return ids;
+}
+async function semanticCandidates(runtime, query, filterIds, topK) {
+  const q = await embedQuery(runtime.embedder, query);
+  return runtime.store.query(q, { topK, filterIds });
+}
+function queryText(node) {
+  const title = typeof node.metadata.title === "string" ? node.metadata.title : void 0;
+  const excerpt = typeof node.metadata.excerpt === "string" ? node.metadata.excerpt : void 0;
+  return [node.id, title, excerpt].filter((value) => typeof value === "string" && value.length > 0).join(" ");
+}
+function clampScore(value) {
+  if (!Number.isFinite(value)) return DEFAULT_SCORE_FLOOR;
+  return Math.max(0, Math.min(1, value));
+}
+export {
+  relatesTo
+};