@functional-systems/srcx 0.1.0-dev.3 → 0.1.0-dev.30

package/src/remote.ts

@@ -7,7 +7,7 @@ import {
   tagTarget,
   type IndexConfigsUnion,
 } from "@functional-systems/lambdadb";
-import { invariant, type Doc } from "./common.js";
+import { hash, invariant, type Doc } from "./common.js";
 import type { Settings } from "./settings.js";
 export type Ref = { kind: "branch" | "tag"; name: string };
 export type Snapshot = { name: string; snapshotId: string };
@@ -96,6 +96,43 @@ export class LambdaRemote {
     const c = this.client.collection(name);
     const ref = (r: Ref) =>
       r.kind === "tag" ? tagRef(r.name) : branchRef(r.name);
+    // Some deployments omit managed vectors from list/query responses despite
+    // includeVectors. Keep the same immutable Tag and exact non-vector payload.
+    const hydrate = async (r: Ref, docs: Doc[]): Promise<Doc[]> => {
+      const missing = docs.filter(
+        (d) => d.embeddingStatus === "managed" && d.embedding === undefined,
+      );
+      const hydrated = new Map<string, Doc>();
+      if (missing.length) {
+        invariant(
+          r.kind === "tag",
+          "Managed vector hydration requires an immutable Tag.",
+        );
+        const fetched = await request("fetch omitted vectors", () =>
+          c.docs.fetch({
+            ref: tagRef(r.name),
+            ids: missing.map((d) => d.id),
+            consistentRead: false,
+            includeVectors: true,
+          }),
+        );
+        const expected = new Map(missing.map((d) => [d.id, hash(d)]));
+        for (const item of fetched.docs) {
+          const doc = item.doc as Doc;
+          const { embedding: _, ...source } = doc;
+          invariant(
+            !hydrated.has(doc.id) && expected.get(doc.id) === hash(source),
+            "Returned/fetched managed payloads disagree.",
+          );
+          hydrated.set(doc.id, doc);
+        }
+        invariant(
+          hydrated.size === expected.size,
+          "Returned managed records are missing from fetch.",
+        );
+      }
+      return docs.map((doc) => hydrated.get(doc.id) ?? doc);
+    };
     return {
       fetch: (r, ids, consistent = false) =>
         request("fetch", async () => {
@@ -132,7 +169,11 @@ export class LambdaRemote {
               pageToken: cursor,
             }),
           );
-          for (const d of page.docs) yield d.doc as Doc;
+          for (const doc of await hydrate(
+            r,
+            page.docs.map((d) => d.doc as Doc),
+          ))
+            yield doc;
           cursor = page.nextPageToken || undefined;
           if (cursor) {
             invariant(
@@ -199,17 +240,22 @@ export class LambdaRemote {
             : c.aliases.create({ aliasName: name, target: tagTarget(target) }),
         );
       },
-      query: (tag, query, size) =>
-        request("query", async () =>
-          (
-            await c.query({
-              ref: tagRef(tag),
-              consistentRead: false,
-              query,
-              size,
-            })
-          ).docs.map((h) => ({ doc: h.doc as Doc, score: h.score })),
-        ),
+      query: async (name, query, size) => {
+        const result = await request("query", () =>
+          c.query({
+            ref: tagRef(name),
+            consistentRead: false,
+            query,
+            size,
+            includeVectors: true,
+          }),
+        );
+        const docs = await hydrate(
+          { kind: "tag", name },
+          result.docs.map((h) => h.doc as Doc),
+        );
+        return result.docs.map((h, i) => ({ doc: docs[i]!, score: h.score }));
+      },
     };
   }
 }

package/src/repository.ts

@@ -8,13 +8,14 @@ import {
   PURPOSE,
   type Doc,
 } from "./common.js";
-import { PRESET, INDEX_CONFIGS } from "./build.js";
+import { PRESET, indexConfigs, supportedPreset, type Preset } from "./build.js";
 import { identity, type Identity } from "./git.js";
 import { LambdaRemote, one, branch } from "./remote.js";
 import { exclusive, type Binding } from "./publish.js";
 import { stateRoot, type Settings } from "./settings.js";
 import type { IndexConfigsUnion } from "@functional-systems/lambdadb";
 export type Repository = Binding & {
+  preset?: Preset;
   collection: string;
   name: string;
   description: string;
@@ -42,12 +43,15 @@ export function collectionName(
   return `code-${slug}-${hash(["collection-v1", key, configHash]).slice(0, 16)}`;
 }
 /** LambdaDB adds its built-in keyword id index to collection metadata. */
-export function matchesIndexSchema(actual: unknown): boolean {
+export function matchesIndexSchema(
+  actual: unknown,
+  preset: Preset = PRESET,
+): boolean {
   if (!actual || typeof actual !== "object" || Array.isArray(actual))
     return false;
   const { id, ...fields } = actual as Record<string, unknown>;
   if (id !== undefined && hash(id) !== hash({ type: "keyword" })) return false;
-  return hash(fields) === hash(INDEX_CONFIGS);
+  return hash(fields) === hash(indexConfigs(preset));
 }
 function descriptor(r: Repository): Doc {
   return {
@@ -60,7 +64,7 @@ function descriptor(r: Repository): Doc {
     repoId: r.repoId,
     indexId: r.indexId,
     name: r.name,
-    preset: PRESET,
+    preset: r.preset ?? PRESET,
     initialization: "ready",
   };
 }
@@ -76,8 +80,8 @@ function decode(
   invariant(
     doc?.role === "repository" &&
       doc.initialization === "ready" &&
-      doc.configHash === hash(PRESET) &&
-      hash(doc.preset) === hash(PRESET),
+      supportedPreset(doc.preset) &&
+      doc.configHash === hash(doc.preset),
     "Repository is partially initialized or uses an unsupported preset.",
   );
   invariant(
@@ -87,7 +91,7 @@ function decode(
     "Repository descriptor and Collection labels disagree.",
   );
   invariant(
-    matchesIndexSchema(c.indexConfigs),
+    matchesIndexSchema(c.indexConfigs, doc.preset as Preset),
     "Collection schema differs from the pinned preset.",
   );
   invariant(
@@ -98,6 +102,7 @@ function decode(
     "Malformed repository descriptor.",
   );
   return {
+    preset: doc.preset as Preset,
     collection: c.collectionName,
     repoKey: doc.repoKey,
     repoId: doc.repoId,
@@ -148,7 +153,7 @@ export async function selectRepository(
   invariant(
     selected.length === 1,
     selected.length
-      ? "Ambiguous repository; use its full source identity or Collection name."
+      ? "Ambiguous repository; use its exact Collection name (presets have separate Collections)."
       : `Repository not found${partial.length ? "; partially initialized collections exist (see repo list)" : ""}.`,
   );
   return selected[0]!;
@@ -160,10 +165,13 @@ export async function register(
     remote?: string;
     description?: string;
     labels?: Record<string, string>;
+    preset?: Preset;
   },
 ): Promise<Repository> {
   const source = await identity(args.path, args.remote);
-  const configHash = hash(PRESET),
+  const preset = args.preset ?? PRESET;
+  invariant(supportedPreset(preset), "Unsupported repository preset.");
+  const configHash = hash(preset),
     collection = collectionName(source.key, source.name, configHash),
     state = destination(remote.settings, collection);
   return exclusive(state, async () => {
@@ -218,6 +226,7 @@ export async function register(
       );
       const indexId = randomUUID();
       intent = {
+        preset,
         repoKey: source.key,
         name: source.name,
         repoId: randomUUID(),
@@ -244,13 +253,13 @@ export async function register(
     if (existing)
       invariant(
         hash(existing.tags) === hash(intent.tags) &&
-          matchesIndexSchema(existing.indexConfigs),
+          matchesIndexSchema(existing.indexConfigs, preset),
         "Existing Collection differs from provisioning journal.",
       );
     else
       await remote.create(
         collection,
-        JSON.parse(JSON.stringify(INDEX_CONFIGS)) as Record<
+        JSON.parse(JSON.stringify(indexConfigs(preset))) as Record<
           string,
           IndexConfigsUnion
         >,

package/src/rerank.ts

@@ -0,0 +1,131 @@
+import { execFile } from "node:child_process";
+import { fileURLToPath } from "node:url";
+import { invariant } from "./common.js";
+
+export const QWEN_MODEL = "Qwen/Qwen3-Reranker-0.6B";
+export const QWEN_REVISION = "e61197ed45024b0ed8a2d74b80b4d909f1255473";
+// Reserved worker exit codes; never interpret or forward library stderr.
+const WORKER_FAILURES: Record<number, string> = {
+  20: "Qwen Python dependencies are missing or incompatible. Install runtime/requirements.txt with the Python selected by SRCX_RERANK_PYTHON.",
+  21: "Qwen model cache is missing or incomplete. Download the pinned model revision from README and check HF_HUB_CACHE.",
+  22: "Qwen device is invalid or unavailable. Set SRCX_RERANK_DEVICE to auto or cpu, or a supported mps/cuda device.",
+  23: "Qwen query/chunk pair exceeds 8,192 tokens. Shorten the query or use search without --rerank; source is not truncated.",
+  24: "Qwen request exceeds 4 MiB. Reduce --candidates or shorten the query.",
+};
+export type Candidate = { id: string; text: string };
+export type RerankOptions = {
+  rerank?: "qwen";
+  candidates?: number;
+  onTiming?: (timing: Record<string, unknown>) => void;
+};
+
+export function candidateLimit(limit: number, options: RerankOptions): number {
+  invariant(
+    Number.isInteger(limit) && limit >= 1 && limit <= 100,
+    "Limit must be an integer from 1 to 100.",
+  );
+  invariant(
+    options.rerank === undefined || options.rerank === "qwen",
+    "Unknown reranker.",
+  );
+  invariant(
+    options.rerank || options.candidates === undefined,
+    "--candidates requires --rerank qwen.",
+  );
+  const count = options.rerank
+    ? (options.candidates ?? Math.max(50, limit))
+    : limit;
+  invariant(
+    Number.isInteger(count) && count >= limit && count <= 100,
+    "Candidates must be an integer between --limit and 100.",
+  );
+  if (options.rerank)
+    invariant(
+      process.env.SRCX_RERANK_PYTHON,
+      "Set SRCX_RERANK_PYTHON to the prepared Python executable; see README reranker setup.",
+    );
+  return count;
+}
+
+export async function qwenScores(query: string, candidates: Candidate[]) {
+  invariant(
+    candidates.length > 0 &&
+      candidates.length <= 100 &&
+      new Set(candidates.map((c) => c.id)).size === candidates.length,
+    "Invalid reranker candidates.",
+  );
+  const input = JSON.stringify({ query, candidates });
+  invariant(
+    Buffer.byteLength(input) <= 4 * 1024 * 1024,
+    "Reranker input exceeds 4 MiB. Reduce --candidates or shorten the query; no fallback was applied.",
+  );
+  const python = process.env.SRCX_RERANK_PYTHON;
+  invariant(
+    python,
+    "Set SRCX_RERANK_PYTHON to the prepared Python executable.",
+  );
+  const start = performance.now();
+  const stdout = await new Promise<string>((resolve, reject) => {
+    const child = execFile(
+      python,
+      [fileURLToPath(new URL("../runtime/qwen.py", import.meta.url))],
+      {
+        encoding: "utf8",
+        timeout: 120_000,
+        killSignal: "SIGKILL",
+        maxBuffer: 1024 * 1024,
+        env: {
+          ...process.env,
+          HF_HUB_OFFLINE: "1",
+          HF_HUB_DISABLE_TELEMETRY: "1",
+          TOKENIZERS_PARALLELISM: "false",
+        },
+      },
+      (error, output) => {
+        if (!error) return resolve(output);
+        let message =
+          "Qwen reranking failed. Check Python dependencies, cached model and SRCX_RERANK_DEVICE.";
+        if (error.code === "ENOENT")
+          message =
+            "Qwen Python executable was not found. Set SRCX_RERANK_PYTHON to an existing Python executable.";
+        else if (error.code === "EACCES")
+          message =
+            "Qwen Python executable cannot be started. Check execute permissions for SRCX_RERANK_PYTHON.";
+        else if (error.killed && error.signal === "SIGKILL" && !error.code)
+          message =
+            "Qwen reranking exceeded 120 seconds. Reduce --candidates or use an available faster device via SRCX_RERANK_DEVICE.";
+        else if (typeof error.code === "number")
+          message = WORKER_FAILURES[error.code] ?? message;
+        reject(new Error(`${message} No fallback was applied.`));
+      },
+    );
+    // An early worker exit can close stdin before Node finishes writing.
+    child.stdin?.on("error", () => {});
+    child.stdin?.end(input);
+  });
+  let response;
+  try {
+    response = JSON.parse(stdout);
+  } catch {
+    throw new Error("Invalid Qwen response.");
+  }
+  invariant(
+    response?.model === QWEN_MODEL &&
+      response?.revision === QWEN_REVISION &&
+      Array.isArray(response.scores) &&
+      response.scores.length === candidates.length,
+    "Qwen response model or candidate count mismatch.",
+  );
+  const scores: number[] = response.scores.map(
+    (entry: { id?: unknown; score?: unknown }, index: number) => {
+      invariant(
+        entry?.id === candidates[index]!.id &&
+          typeof entry.score === "number" &&
+          Number.isFinite(entry.score),
+        "Qwen response candidate identity or score mismatch.",
+      );
+      return entry.score;
+    },
+  );
+  return { scores, elapsedMs: performance.now() - start };
+}

package/src/search.ts

@@ -5,7 +5,15 @@ import { one, tag, type CollectionStore } from "./remote.js";
 import { stateRoot, type Settings } from "./settings.js";
 import type { Repository } from "./repository.js";
 import type { Published } from "./publish.js";
-import type { InventoryItem } from "./build.js";
+import { PRESET, recordHash, type InventoryItem } from "./build.js";
+import { implementationSpan, type Implementation } from "./implementation.js";
+import {
+  candidateLimit,
+  qwenScores,
+  QWEN_MODEL,
+  QWEN_REVISION,
+  type RerankOptions,
+} from "./rerank.js";
 export type Handle = {
   id: string;
   endpoint: string;
@@ -84,6 +92,33 @@ export function lexicalQuery(
     ],
   };
 }
+export type SearchMode = "lexical" | "semantic" | "hybrid";
+export function retrievalQuery(
+  query: string,
+  size: number,
+  filters: { path?: string; language?: string },
+  mode: SearchMode,
+): Record<string, unknown> {
+  invariant(
+    ["lexical", "semantic", "hybrid"].includes(mode),
+    "Unknown search mode.",
+  );
+  const lexical = lexicalQuery(query, filters);
+  if (mode === "lexical") return lexical;
+  const semantic = {
+    knn: {
+      field: "embedding",
+      queryText: query,
+      k: size,
+      filter: {
+        bool: (lexical.bool as { occur: string }[]).filter(
+          (c) => c.occur === "filter",
+        ),
+      },
+    },
+  };
+  return mode === "semantic" ? semantic : { rrf: [lexical, semantic] };
+}
 export async function search(
   store: CollectionStore,
   s: Settings,
@@ -92,21 +127,28 @@ export async function search(
   query: string,
   size = 10,
   filters: { path?: string; language?: string } = {},
+  mode: SearchMode = "lexical",
+  options: RerankOptions = {},
 ): Promise<unknown[]> {
+  const started = performance.now();
+  const count = candidateLimit(size, options);
   invariant(
-    Number.isInteger(size) && size > 0 && size <= 100,
-    "Limit must be an integer from 1 to 100.",
+    mode === "lexical" || r.preset?.embedding?.managed,
+    "Semantic/hybrid search requires a managed embedding Collection; register with --embedding text-embedding-3-small or text-embedding-3-large.",
   );
+  const request = retrievalQuery(query, count, filters, mode);
   const entries = await inventoryAt(store, r, v);
   const files = new Map(
     entries.filter((e) => e.status === "included").map((e) => [e.fileId, e]),
   );
-  const result = [];
-  for (const hit of await store.query(
-    v.tagName,
-    lexicalQuery(query, filters),
-    size,
-  )) {
+  const candidates = [];
+  const hits = await store.query(v.tagName, request, count);
+  invariant(
+    hits.length <= count &&
+      new Set(hits.map((h) => h.doc.id)).size === hits.length,
+    "Invalid search candidate count or duplicate IDs.",
+  );
+  for (const hit of hits) {
     const d = hit.doc,
       e = files.get(d.fileId as string);
     invariant(
@@ -128,28 +170,78 @@ export async function search(
       path: String(d.path),
       contentHash: String(d.contentHash),
       chunkId: d.id,
-      chunkHash: hash(d),
+      chunkHash: recordHash(d, r.preset ?? PRESET),
       startLine: Number(d.startLine),
       endLine: Number(d.endLine),
     };
     // Check the exact source/ranges before persisting an evidence handle.
-    const evidence = await readHandle(store, s, handle);
-    await atomic(join(stateRoot(), "results", id + ".json"), handle);
+    const evidence = await readHandle(store, s, handle, {
+      exactChunk: !!options.rerank,
+    });
+    candidates.push({
+      handle,
+      text: evidence.sourceText,
+      result: {
+        resultId: id,
+        repository: r.repoKey,
+        commitOid: v.commitOid,
+        version: v.tagName,
+        snapshotId: v.snapshotId,
+        path: d.path,
+        startLine: d.startLine,
+        endLine: d.endLine,
+        symbol: d.symbol,
+        score: hit.score,
+        excerpt: evidence.sourceText.slice(0, 600),
+        citation: evidence.citation,
+      },
+    });
+  }
+  const retrievalMs = performance.now() - started;
+  let rerankMs = 0;
+  let ranked = candidates.map((candidate, index) => ({
+    ...candidate,
+    index,
+    rerankScore: 0,
+  }));
+  if (options.rerank && candidates.length) {
+    const response = await qwenScores(
+      query,
+      candidates.map((c) => ({ id: c.handle.chunkId!, text: c.text })),
+    );
+    rerankMs = response.elapsedMs;
+    ranked = ranked
+      .map((c, index) => ({ ...c, rerankScore: response.scores[index]! }))
+      .sort((a, b) => b.rerankScore - a.rerankScore || a.index - b.index);
+  }
+  if (options.rerank)
+    invariant(
+      (await store.tags()).find((t) => t.name === v.tagName)?.snapshotId ===
+        v.snapshotId,
+      "Pinned Tag was deleted or recreated during reranking.",
+    );
+  const result = [];
+  for (const c of ranked.slice(0, size)) {
+    await atomic(join(stateRoot(), "results", c.handle.id + ".json"), c.handle);
     result.push({
-      resultId: id,
-      repository: r.repoKey,
-      commitOid: v.commitOid,
-      version: v.tagName,
-      snapshotId: v.snapshotId,
-      path: d.path,
-      startLine: d.startLine,
-      endLine: d.endLine,
-      symbol: d.symbol,
-      score: hit.score,
-      excerpt: evidence.sourceText.slice(0, 600),
-      citation: evidence.citation,
+      ...c.result,
+      ...(options.rerank
+        ? {
+            rerankScore: c.rerankScore,
+            retrievalRank: c.index + 1,
+            reranker: { model: QWEN_MODEL, revision: QWEN_REVISION },
+          }
+        : {}),
     });
   }
+  if (options.rerank)
+    options.onTiming?.({
+      candidates: candidates.length,
+      returned: result.length,
+      retrievalMs,
+      rerankMs,
+      searchMs: performance.now() - started,
+    });
   return result;
 }
 export async function loadHandle(id: string): Promise<Handle> {
@@ -191,8 +283,19 @@ export async function readHandle(
     context?: number;
     fullFile?: boolean;
     lines?: [number, number];
+    exactChunk?: boolean;
+    implementation?: boolean;
   } = {},
 ) {
+  invariant(
+    !options.implementation ||
+      (h.chunkId &&
+        !options.fullFile &&
+        !options.lines &&
+        !options.context &&
+        !options.exactChunk),
+    "Implementation reads require a result handle without range overrides.",
+  );
   invariant(
     h.endpoint === s.endpoint && h.project === s.project,
     "Result belongs to another endpoint/project; restore that connection before reading.",
@@ -218,11 +321,13 @@ export async function readHandle(
     raw.length === 0
       ? 0
       : lineAt(raw, raw.length - (raw.at(-1) === 10 ? 1 : 0));
+  let chunkSource: string | undefined;
+  let implementation: Implementation | undefined;
   if (h.chunkId) {
     const d = await one(store, tag(v.tagName), h.chunkId);
     invariant(
       d &&
-        hash(d) === h.chunkHash &&
+        recordHash(d, h.repository.preset ?? PRESET) === h.chunkHash &&
         d.fileId === h.fileId &&
         d.contentHash === h.contentHash,
       "Pinned chunk has changed or is missing.",
@@ -239,6 +344,12 @@ export async function readHandle(
         lineAt(raw, z - 1) === h.endLine,
       "Invalid source range.",
     );
+    chunkSource = raw.subarray(a, z).toString("utf8");
+    if (options.implementation)
+      implementation = await implementationSpan(file.sourceText, h.path, {
+        startByte: a,
+        endByte: z,
+      });
   }
   const context = options.context ?? 0;
   invariant(
@@ -247,6 +358,10 @@ export async function readHandle(
   );
   let start = options.lines?.[0] ?? h.startLine,
     end = options.lines?.[1] ?? h.endLine;
+  if (implementation?.status === "resolved") {
+    start = implementation.startLine!;
+    end = implementation.endLine!;
+  }
   if (options.fullFile) {
     start = 1;
     end = total;
@@ -264,9 +379,19 @@ export async function readHandle(
     end = Math.min(total, end + context);
   }
   const lines = file.sourceText.match(/[^\n]*\n|[^\n]+$/g) ?? [];
-  const sourceText = options.fullFile
-    ? file.sourceText
-    : lines.slice(start - 1, end).join("");
+  invariant(
+    !options.exactChunk ||
+      (chunkSource !== undefined &&
+        !options.fullFile &&
+        !options.lines &&
+        !options.context),
+    "Exact chunk reads require a chunk handle without range overrides.",
+  );
+  const sourceText = options.exactChunk
+    ? chunkSource!
+    : options.fullFile
+      ? file.sourceText
+      : lines.slice(start - 1, end).join("");
   return {
     repository: h.repository.repoKey,
     commitOid: v.commitOid,
@@ -276,6 +401,7 @@ export async function readHandle(
     startLine: start,
     endLine: end,
     contentHash: h.contentHash,
+    ...(implementation ? { implementation } : {}),
     sourceText,
     citation: `${h.repository.repoKey}@${v.commitOid}:${h.path}:${start}-${end}`,
   };