@functional-systems/srcx 0.1.0-dev.7 → 0.1.0-dev.9

package/src/cli.ts

@@ -1,12 +1,20 @@
 #!/usr/bin/env node
 import { readFileSync } from "node:fs";
-import { Command, InvalidArgumentError } from "commander";
+import { Command, InvalidArgumentError, Option } from "commander";
 import { mkdir } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
 import { configure, loadSettings, stateRoot } from "./settings.js";
 import { identity, resolveCommit } from "./git.js";
-import { loadBuild, materialize, validateBuild, type Build } from "./build.js";
+import {
+  loadBuild,
+  materialize,
+  validateBuild,
+  type Build,
+  type Preset,
+  PRESET,
+  presetFor,
+} from "./build.js";
 import { invariant, optionalJson } from "./common.js";
 import { LambdaRemote } from "./remote.js";
 import {
@@ -87,6 +95,14 @@ repo
   .option("--remote <name>")
   .option("--description <text>")
   .option("--tag <key=value>", "One optional Collection metadata tag")
+  .addOption(
+    new Option(
+      "--embedding <model>",
+      "Managed model (source text is sent to the provider on import)",
+    )
+      .choices(["none", "text-embedding-3-small", "text-embedding-3-large"])
+      .default("none"),
+  )
   .action(async (o) => {
     const { remote } = await connected();
     let labels: Record<string, string> | undefined;
@@ -101,6 +117,7 @@ repo
         remote: o.remote,
         description: o.description,
         labels,
+        preset: presetFor(o.embedding),
       }),
     );
   });
@@ -128,6 +145,12 @@ cli
   .option("--repo <name>", "Registered remote repository")
   .option("--ref <ref>", "Local branch, Git tag, or commit OID")
   .option("--dry-run", "Only materialize and validate a local build")
+  .addOption(
+    new Option(
+      "--embedding <model>",
+      "Preset for --dry-run --path; connected imports use the repository preset",
+    ).choices(["none", "text-embedding-3-small", "text-embedding-3-large"]),
+  )
   .option("--output <directory>", "New artifact directory")
   .option(
     "--previous <directory>",
@@ -140,6 +163,10 @@ cli
   )
   .option("--timeout <seconds>", "Validation wait deadline", integer, 120)
   .action(async (o) => {
+    invariant(
+      !o.embedding || (o.dryRun && o.path),
+      "--embedding is only accepted with --dry-run --path; connected imports use the repository preset.",
+    );
     invariant(!(o.path && o.repo), "Choose --path or --repo.");
     invariant(
       !o.previous || o.dryRun,
@@ -154,6 +181,7 @@ cli
       source: Awaited<ReturnType<typeof identity>>,
       previous?: Build,
       state?: string,
+      preset: Preset = PRESET,
     ) => {
       invariant(o.ref, "--ref is required to build a commit.");
       const commit = await resolveCommit(source.path, o.ref);
@@ -178,6 +206,7 @@ cli
         resolvedCommit: commit,
         output: path,
         previous,
+        preset,
       });
     };
     if (o.dryRun) {
@@ -187,18 +216,20 @@ cli
         await validateBuild(b);
       } else {
         let source;
+        let preset = presetFor(o.embedding);
         if (o.path) source = await identity(o.path, o.remote);
         else {
           invariant(o.repo, "Use --path for a credential-free dry run.");
           const { remote, settings } = await connected();
-          source = await attachment(
-            settings,
-            await selectRepository(remote, o.repo),
-          );
+          const selected = await selectRepository(remote, o.repo);
+          preset = selected.preset ?? PRESET;
+          source = await attachment(settings, selected);
         }
         b = await buildLocal(
           source,
           o.previous ? await loadBuild(o.previous) : undefined,
+          undefined,
+          preset,
         );
       }
       output({
@@ -239,6 +270,7 @@ cli
               await attachment(settings, r),
               undefined,
               pending ? undefined : state,
+              r.preset ?? PRESET,
             );
         let baseline: { version: Published; build: Build } | undefined;
         // Tracked branches discover their own immutable baseline remotely.
@@ -295,6 +327,14 @@ cli
   .requiredOption("--version <commit-branch-or-release>")
   .requiredOption("--query <text>")
   .option("--limit <count>", "Maximum hits", integer, 10)
+  .addOption(
+    new Option(
+      "--mode <mode>",
+      "Retrieval method; semantic/hybrid incur managed query embedding usage",
+    )
+      .choices(["lexical", "semantic", "hybrid"])
+      .default("lexical"),
+  )
   .option("--path <path>", "Exact path filter")
   .option("--language <name>")
   .action(async (o) => {
@@ -303,10 +343,19 @@ cli
       store = remote.store(r.collection),
       v = await resolveVersion(store, r, o.version);
     output(
-      await search(store, settings, r, v, o.query, o.limit, {
-        path: o.path,
-        language: o.language,
-      }),
+      await search(
+        store,
+        settings,
+        r,
+        v,
+        o.query,
+        o.limit,
+        {
+          path: o.path,
+          language: o.language,
+        },
+        o.mode,
+      ),
     );
   });
 cli

package/src/publish.ts

@@ -10,7 +10,14 @@ import {
   sleep,
   type Doc,
 } from "./common.js";
-import { corpusMetadata, records, validateBuild, type Build } from "./build.js";
+import {
+  corpusMetadata,
+  records,
+  recordHash,
+  supportedPreset,
+  validateBuild,
+  type Build,
+} from "./build.js";
 import { branch, tag, one, type CollectionStore } from "./remote.js";
 export type Binding = {
   repoId: string;
@@ -154,7 +161,11 @@ export async function validateCandidate(
   for (const d of corpusMetadata(b, binding.repoId, binding.indexId, attemptId))
     expected.set(d.id, hash(d));
   for await (const d of store.list(tag(name))) {
-    if (expected.get(d.id) !== hash(d)) return false;
+    try {
+      if (expected.get(d.id) !== recordHash(d, b.preset)) return false;
+    } catch {
+      return false;
+    }
     expected.delete(d.id);
   }
   if (expected.size) return false;
@@ -203,7 +214,8 @@ export async function validateCandidate(
       !found.length ||
       found.some(
         (h) =>
-          h.doc.kind !== "chunk" || b.recordHashes[h.doc.id] !== hash(h.doc),
+          h.doc.kind !== "chunk" ||
+          b.recordHashes[h.doc.id] !== recordHash(h.doc, b.preset),
       )
     )
       return false;
@@ -284,8 +296,8 @@ export async function publish(args: {
     "Build does not belong to this index.",
   );
   invariant(
-    !b.preset.embedding,
-    "CLI publication currently supports only the embedding=none preset.",
+    !b.preset.embedding || supportedPreset(b.preset),
+    "Publication requires a supported lexical or managed embedding preset.",
   );
   await validateBuild(b);
   const journalPath = join(state, "pending.json");
@@ -522,7 +534,7 @@ export async function publish(args: {
     if (j.baseline)
       for await (const d of store.list(tag(j.baseline.tagName))) {
         oldIds.add(d.id);
-        oldHashes.set(d.id, hash(d));
+        oldHashes.set(d.id, recordHash(d, b.preset));
       }
     const newIds = new Set([
       ...Object.keys(b.recordHashes),

package/src/remote.ts

@@ -7,7 +7,7 @@ import {
   tagTarget,
   type IndexConfigsUnion,
 } from "@functional-systems/lambdadb";
-import { invariant, type Doc } from "./common.js";
+import { hash, invariant, type Doc } from "./common.js";
 import type { Settings } from "./settings.js";
 export type Ref = { kind: "branch" | "tag"; name: string };
 export type Snapshot = { name: string; snapshotId: string };
@@ -96,6 +96,43 @@ export class LambdaRemote {
     const c = this.client.collection(name);
     const ref = (r: Ref) =>
       r.kind === "tag" ? tagRef(r.name) : branchRef(r.name);
+    // Some deployments omit managed vectors from list/query responses despite
+    // includeVectors. Keep the same immutable Tag and exact non-vector payload.
+    const hydrate = async (r: Ref, docs: Doc[]): Promise<Doc[]> => {
+      const missing = docs.filter(
+        (d) => d.embeddingStatus === "managed" && d.embedding === undefined,
+      );
+      const hydrated = new Map<string, Doc>();
+      if (missing.length) {
+        invariant(
+          r.kind === "tag",
+          "Managed vector hydration requires an immutable Tag.",
+        );
+        const fetched = await request("fetch omitted vectors", () =>
+          c.docs.fetch({
+            ref: tagRef(r.name),
+            ids: missing.map((d) => d.id),
+            consistentRead: false,
+            includeVectors: true,
+          }),
+        );
+        const expected = new Map(missing.map((d) => [d.id, hash(d)]));
+        for (const item of fetched.docs) {
+          const doc = item.doc as Doc;
+          const { embedding: _, ...source } = doc;
+          invariant(
+            !hydrated.has(doc.id) && expected.get(doc.id) === hash(source),
+            "Returned/fetched managed payloads disagree.",
+          );
+          hydrated.set(doc.id, doc);
+        }
+        invariant(
+          hydrated.size === expected.size,
+          "Returned managed records are missing from fetch.",
+        );
+      }
+      return docs.map((doc) => hydrated.get(doc.id) ?? doc);
+    };
     return {
       fetch: (r, ids, consistent = false) =>
         request("fetch", async () => {
@@ -132,7 +169,11 @@ export class LambdaRemote {
               pageToken: cursor,
             }),
           );
-          for (const d of page.docs) yield d.doc as Doc;
+          for (const doc of await hydrate(
+            r,
+            page.docs.map((d) => d.doc as Doc),
+          ))
+            yield doc;
           cursor = page.nextPageToken || undefined;
           if (cursor) {
             invariant(
@@ -199,17 +240,22 @@ export class LambdaRemote {
             : c.aliases.create({ aliasName: name, target: tagTarget(target) }),
         );
       },
-      query: (tag, query, size) =>
-        request("query", async () =>
-          (
-            await c.query({
-              ref: tagRef(tag),
-              consistentRead: false,
-              query,
-              size,
-            })
-          ).docs.map((h) => ({ doc: h.doc as Doc, score: h.score })),
-        ),
+      query: async (name, query, size) => {
+        const result = await request("query", () =>
+          c.query({
+            ref: tagRef(name),
+            consistentRead: false,
+            query,
+            size,
+            includeVectors: true,
+          }),
+        );
+        const docs = await hydrate(
+          { kind: "tag", name },
+          result.docs.map((h) => h.doc as Doc),
+        );
+        return result.docs.map((h, i) => ({ doc: docs[i]!, score: h.score }));
+      },
     };
   }
 }

package/src/repository.ts

@@ -8,13 +8,14 @@ import {
   PURPOSE,
   type Doc,
 } from "./common.js";
-import { PRESET, INDEX_CONFIGS } from "./build.js";
+import { PRESET, indexConfigs, supportedPreset, type Preset } from "./build.js";
 import { identity, type Identity } from "./git.js";
 import { LambdaRemote, one, branch } from "./remote.js";
 import { exclusive, type Binding } from "./publish.js";
 import { stateRoot, type Settings } from "./settings.js";
 import type { IndexConfigsUnion } from "@functional-systems/lambdadb";
 export type Repository = Binding & {
+  preset?: Preset;
   collection: string;
   name: string;
   description: string;
@@ -42,12 +43,15 @@ export function collectionName(
   return `code-${slug}-${hash(["collection-v1", key, configHash]).slice(0, 16)}`;
 }
 /** LambdaDB adds its built-in keyword id index to collection metadata. */
-export function matchesIndexSchema(actual: unknown): boolean {
+export function matchesIndexSchema(
+  actual: unknown,
+  preset: Preset = PRESET,
+): boolean {
   if (!actual || typeof actual !== "object" || Array.isArray(actual))
     return false;
   const { id, ...fields } = actual as Record<string, unknown>;
   if (id !== undefined && hash(id) !== hash({ type: "keyword" })) return false;
-  return hash(fields) === hash(INDEX_CONFIGS);
+  return hash(fields) === hash(indexConfigs(preset));
 }
 function descriptor(r: Repository): Doc {
   return {
@@ -60,7 +64,7 @@ function descriptor(r: Repository): Doc {
     repoId: r.repoId,
     indexId: r.indexId,
     name: r.name,
-    preset: PRESET,
+    preset: r.preset ?? PRESET,
     initialization: "ready",
   };
 }
@@ -76,8 +80,8 @@ function decode(
   invariant(
     doc?.role === "repository" &&
       doc.initialization === "ready" &&
-      doc.configHash === hash(PRESET) &&
-      hash(doc.preset) === hash(PRESET),
+      supportedPreset(doc.preset) &&
+      doc.configHash === hash(doc.preset),
     "Repository is partially initialized or uses an unsupported preset.",
   );
   invariant(
@@ -87,7 +91,7 @@ function decode(
     "Repository descriptor and Collection labels disagree.",
   );
   invariant(
-    matchesIndexSchema(c.indexConfigs),
+    matchesIndexSchema(c.indexConfigs, doc.preset as Preset),
     "Collection schema differs from the pinned preset.",
   );
   invariant(
@@ -98,6 +102,7 @@ function decode(
     "Malformed repository descriptor.",
   );
   return {
+    preset: doc.preset as Preset,
     collection: c.collectionName,
     repoKey: doc.repoKey,
     repoId: doc.repoId,
@@ -148,7 +153,7 @@ export async function selectRepository(
   invariant(
     selected.length === 1,
     selected.length
-      ? "Ambiguous repository; use its full source identity or Collection name."
+      ? "Ambiguous repository; use its exact Collection name (presets have separate Collections)."
       : `Repository not found${partial.length ? "; partially initialized collections exist (see repo list)" : ""}.`,
   );
   return selected[0]!;
@@ -160,10 +165,13 @@ export async function register(
     remote?: string;
     description?: string;
     labels?: Record<string, string>;
+    preset?: Preset;
   },
 ): Promise<Repository> {
   const source = await identity(args.path, args.remote);
-  const configHash = hash(PRESET),
+  const preset = args.preset ?? PRESET;
+  invariant(supportedPreset(preset), "Unsupported repository preset.");
+  const configHash = hash(preset),
     collection = collectionName(source.key, source.name, configHash),
     state = destination(remote.settings, collection);
   return exclusive(state, async () => {
@@ -218,6 +226,7 @@ export async function register(
       );
       const indexId = randomUUID();
       intent = {
+        preset,
         repoKey: source.key,
         name: source.name,
         repoId: randomUUID(),
@@ -244,13 +253,13 @@ export async function register(
     if (existing)
       invariant(
         hash(existing.tags) === hash(intent.tags) &&
-          matchesIndexSchema(existing.indexConfigs),
+          matchesIndexSchema(existing.indexConfigs, preset),
         "Existing Collection differs from provisioning journal.",
       );
     else
       await remote.create(
         collection,
-        JSON.parse(JSON.stringify(INDEX_CONFIGS)) as Record<
+        JSON.parse(JSON.stringify(indexConfigs(preset))) as Record<
           string,
           IndexConfigsUnion
         >,

package/src/search.ts

@@ -5,7 +5,7 @@ import { one, tag, type CollectionStore } from "./remote.js";
 import { stateRoot, type Settings } from "./settings.js";
 import type { Repository } from "./repository.js";
 import type { Published } from "./publish.js";
-import type { InventoryItem } from "./build.js";
+import { PRESET, recordHash, type InventoryItem } from "./build.js";
 export type Handle = {
   id: string;
   endpoint: string;
@@ -84,6 +84,33 @@ export function lexicalQuery(
     ],
   };
 }
+export type SearchMode = "lexical" | "semantic" | "hybrid";
+export function retrievalQuery(
+  query: string,
+  size: number,
+  filters: { path?: string; language?: string },
+  mode: SearchMode,
+): Record<string, unknown> {
+  invariant(
+    ["lexical", "semantic", "hybrid"].includes(mode),
+    "Unknown search mode.",
+  );
+  const lexical = lexicalQuery(query, filters);
+  if (mode === "lexical") return lexical;
+  const semantic = {
+    knn: {
+      field: "embedding",
+      queryText: query,
+      k: size,
+      filter: {
+        bool: (lexical.bool as { occur: string }[]).filter(
+          (c) => c.occur === "filter",
+        ),
+      },
+    },
+  };
+  return mode === "semantic" ? semantic : { rrf: [lexical, semantic] };
+}
 export async function search(
   store: CollectionStore,
   s: Settings,
@@ -92,21 +119,23 @@ export async function search(
   query: string,
   size = 10,
   filters: { path?: string; language?: string } = {},
+  mode: SearchMode = "lexical",
 ): Promise<unknown[]> {
   invariant(
     Number.isInteger(size) && size > 0 && size <= 100,
     "Limit must be an integer from 1 to 100.",
   );
+  invariant(
+    mode === "lexical" || r.preset?.embedding?.managed,
+    "Semantic/hybrid search requires a managed embedding Collection; register with --embedding text-embedding-3-small or text-embedding-3-large.",
+  );
+  const request = retrievalQuery(query, size, filters, mode);
   const entries = await inventoryAt(store, r, v);
   const files = new Map(
     entries.filter((e) => e.status === "included").map((e) => [e.fileId, e]),
   );
   const result = [];
-  for (const hit of await store.query(
-    v.tagName,
-    lexicalQuery(query, filters),
-    size,
-  )) {
+  for (const hit of await store.query(v.tagName, request, size)) {
     const d = hit.doc,
       e = files.get(d.fileId as string);
     invariant(
@@ -128,7 +157,7 @@ export async function search(
       path: String(d.path),
       contentHash: String(d.contentHash),
       chunkId: d.id,
-      chunkHash: hash(d),
+      chunkHash: recordHash(d, r.preset ?? PRESET),
       startLine: Number(d.startLine),
       endLine: Number(d.endLine),
     };
@@ -222,7 +251,7 @@ export async function readHandle(
     const d = await one(store, tag(v.tagName), h.chunkId);
     invariant(
       d &&
-        hash(d) === h.chunkHash &&
+        recordHash(d, h.repository.preset ?? PRESET) === h.chunkHash &&
         d.fileId === h.fileId &&
         d.contentHash === h.contentHash,
       "Pinned chunk has changed or is missing.",