@functional-systems/srcx 0.1.0-dev.6 → 0.1.0-dev.8

package/src/remote.ts

@@ -7,7 +7,7 @@ import {
   tagTarget,
   type IndexConfigsUnion,
 } from "@functional-systems/lambdadb";
-import { invariant, type Doc } from "./common.js";
+import { hash, invariant, type Doc } from "./common.js";
 import type { Settings } from "./settings.js";
 export type Ref = { kind: "branch" | "tag"; name: string };
 export type Snapshot = { name: string; snapshotId: string };
@@ -132,7 +132,48 @@ export class LambdaRemote {
               pageToken: cursor,
             }),
           );
-          for (const d of page.docs) yield d.doc as Doc;
+          // Some deployments omit managed vectors from list responses even with
+          // includeVectors. Fetch only those records from the same immutable Tag;
+          // require identical non-vector payloads rather than weakening validation.
+          const missing = page.docs
+            .map((d) => d.doc as Doc)
+            .filter(
+              (d) =>
+                d.embeddingStatus === "managed" && d.embedding === undefined,
+            );
+          const hydrated = new Map<string, Doc>();
+          if (missing.length) {
+            invariant(
+              r.kind === "tag",
+              "Managed vector hydration requires an immutable Tag.",
+            );
+            const fetched = await request("fetch listed vectors", () =>
+              c.docs.fetch({
+                ref: tagRef(r.name),
+                ids: missing.map((d) => d.id),
+                consistentRead: false,
+                includeVectors: true,
+              }),
+            );
+            const expected = new Map(missing.map((d) => [d.id, hash(d)]));
+            for (const item of fetched.docs) {
+              const doc = item.doc as Doc;
+              const { embedding: _, ...source } = doc;
+              invariant(
+                !hydrated.has(doc.id) && expected.get(doc.id) === hash(source),
+                "Listed/fetched managed payloads disagree.",
+              );
+              hydrated.set(doc.id, doc);
+            }
+            invariant(
+              hydrated.size === expected.size,
+              "Listed managed records are missing from fetch.",
+            );
+          }
+          for (const d of page.docs) {
+            const doc = d.doc as Doc;
+            yield hydrated.get(doc.id) ?? doc;
+          }
           cursor = page.nextPageToken || undefined;
           if (cursor) {
             invariant(
@@ -207,6 +248,7 @@ export class LambdaRemote {
               consistentRead: false,
               query,
               size,
+              includeVectors: true,
             })
           ).docs.map((h) => ({ doc: h.doc as Doc, score: h.score })),
         ),

package/src/repository.ts

@@ -8,13 +8,14 @@ import {
   PURPOSE,
   type Doc,
 } from "./common.js";
-import { PRESET, INDEX_CONFIGS } from "./build.js";
+import { PRESET, indexConfigs, supportedPreset, type Preset } from "./build.js";
 import { identity, type Identity } from "./git.js";
 import { LambdaRemote, one, branch } from "./remote.js";
 import { exclusive, type Binding } from "./publish.js";
 import { stateRoot, type Settings } from "./settings.js";
 import type { IndexConfigsUnion } from "@functional-systems/lambdadb";
 export type Repository = Binding & {
+  preset?: Preset;
   collection: string;
   name: string;
   description: string;
@@ -42,12 +43,15 @@ export function collectionName(
   return `code-${slug}-${hash(["collection-v1", key, configHash]).slice(0, 16)}`;
 }
 /** LambdaDB adds its built-in keyword id index to collection metadata. */
-export function matchesIndexSchema(actual: unknown): boolean {
+export function matchesIndexSchema(
+  actual: unknown,
+  preset: Preset = PRESET,
+): boolean {
   if (!actual || typeof actual !== "object" || Array.isArray(actual))
     return false;
   const { id, ...fields } = actual as Record<string, unknown>;
   if (id !== undefined && hash(id) !== hash({ type: "keyword" })) return false;
-  return hash(fields) === hash(INDEX_CONFIGS);
+  return hash(fields) === hash(indexConfigs(preset));
 }
 function descriptor(r: Repository): Doc {
   return {
@@ -60,7 +64,7 @@ function descriptor(r: Repository): Doc {
     repoId: r.repoId,
     indexId: r.indexId,
     name: r.name,
-    preset: PRESET,
+    preset: r.preset ?? PRESET,
     initialization: "ready",
   };
 }
@@ -76,8 +80,8 @@ function decode(
   invariant(
     doc?.role === "repository" &&
       doc.initialization === "ready" &&
-      doc.configHash === hash(PRESET) &&
-      hash(doc.preset) === hash(PRESET),
+      supportedPreset(doc.preset) &&
+      doc.configHash === hash(doc.preset),
     "Repository is partially initialized or uses an unsupported preset.",
   );
   invariant(
@@ -87,7 +91,7 @@ function decode(
     "Repository descriptor and Collection labels disagree.",
   );
   invariant(
-    matchesIndexSchema(c.indexConfigs),
+    matchesIndexSchema(c.indexConfigs, doc.preset as Preset),
     "Collection schema differs from the pinned preset.",
   );
   invariant(
@@ -98,6 +102,7 @@ function decode(
     "Malformed repository descriptor.",
   );
   return {
+    preset: doc.preset as Preset,
     collection: c.collectionName,
     repoKey: doc.repoKey,
     repoId: doc.repoId,
@@ -148,7 +153,7 @@ export async function selectRepository(
   invariant(
     selected.length === 1,
     selected.length
-      ? "Ambiguous repository; use its full source identity or Collection name."
+      ? "Ambiguous repository; use its exact Collection name (presets have separate Collections)."
       : `Repository not found${partial.length ? "; partially initialized collections exist (see repo list)" : ""}.`,
   );
   return selected[0]!;
@@ -160,10 +165,13 @@ export async function register(
     remote?: string;
     description?: string;
     labels?: Record<string, string>;
+    preset?: Preset;
   },
 ): Promise<Repository> {
   const source = await identity(args.path, args.remote);
-  const configHash = hash(PRESET),
+  const preset = args.preset ?? PRESET;
+  invariant(supportedPreset(preset), "Unsupported repository preset.");
+  const configHash = hash(preset),
     collection = collectionName(source.key, source.name, configHash),
     state = destination(remote.settings, collection);
   return exclusive(state, async () => {
@@ -218,6 +226,7 @@ export async function register(
       );
       const indexId = randomUUID();
       intent = {
+        preset,
         repoKey: source.key,
         name: source.name,
         repoId: randomUUID(),
@@ -244,13 +253,13 @@ export async function register(
     if (existing)
       invariant(
         hash(existing.tags) === hash(intent.tags) &&
-          matchesIndexSchema(existing.indexConfigs),
+          matchesIndexSchema(existing.indexConfigs, preset),
         "Existing Collection differs from provisioning journal.",
       );
     else
       await remote.create(
         collection,
-        JSON.parse(JSON.stringify(INDEX_CONFIGS)) as Record<
+        JSON.parse(JSON.stringify(indexConfigs(preset))) as Record<
           string,
           IndexConfigsUnion
         >,

package/src/search.ts

@@ -5,7 +5,7 @@ import { one, tag, type CollectionStore } from "./remote.js";
 import { stateRoot, type Settings } from "./settings.js";
 import type { Repository } from "./repository.js";
 import type { Published } from "./publish.js";
-import type { InventoryItem } from "./build.js";
+import { PRESET, recordHash, type InventoryItem } from "./build.js";
 export type Handle = {
   id: string;
   endpoint: string;
@@ -84,6 +84,33 @@ export function lexicalQuery(
     ],
   };
 }
+export type SearchMode = "lexical" | "semantic" | "hybrid";
+export function retrievalQuery(
+  query: string,
+  size: number,
+  filters: { path?: string; language?: string },
+  mode: SearchMode,
+): Record<string, unknown> {
+  invariant(
+    ["lexical", "semantic", "hybrid"].includes(mode),
+    "Unknown search mode.",
+  );
+  const lexical = lexicalQuery(query, filters);
+  if (mode === "lexical") return lexical;
+  const semantic = {
+    knn: {
+      field: "embedding",
+      queryText: query,
+      k: size,
+      filter: {
+        bool: (lexical.bool as { occur: string }[]).filter(
+          (c) => c.occur === "filter",
+        ),
+      },
+    },
+  };
+  return mode === "semantic" ? semantic : { rrf: [lexical, semantic] };
+}
 export async function search(
   store: CollectionStore,
   s: Settings,
@@ -92,21 +119,23 @@ export async function search(
   query: string,
   size = 10,
   filters: { path?: string; language?: string } = {},
+  mode: SearchMode = "lexical",
 ): Promise<unknown[]> {
   invariant(
     Number.isInteger(size) && size > 0 && size <= 100,
     "Limit must be an integer from 1 to 100.",
   );
+  invariant(
+    mode === "lexical" || r.preset?.embedding?.managed,
+    "Semantic/hybrid search requires a managed embedding Collection; register with --embedding text-embedding-3-small.",
+  );
+  const request = retrievalQuery(query, size, filters, mode);
   const entries = await inventoryAt(store, r, v);
   const files = new Map(
     entries.filter((e) => e.status === "included").map((e) => [e.fileId, e]),
   );
   const result = [];
-  for (const hit of await store.query(
-    v.tagName,
-    lexicalQuery(query, filters),
-    size,
-  )) {
+  for (const hit of await store.query(v.tagName, request, size)) {
     const d = hit.doc,
       e = files.get(d.fileId as string);
     invariant(
@@ -128,7 +157,7 @@ export async function search(
       path: String(d.path),
       contentHash: String(d.contentHash),
       chunkId: d.id,
-      chunkHash: hash(d),
+      chunkHash: recordHash(d, r.preset ?? PRESET),
       startLine: Number(d.startLine),
       endLine: Number(d.endLine),
     };
@@ -222,7 +251,7 @@ export async function readHandle(
     const d = await one(store, tag(v.tagName), h.chunkId);
     invariant(
       d &&
-        hash(d) === h.chunkHash &&
+        recordHash(d, h.repository.preset ?? PRESET) === h.chunkHash &&
         d.fileId === h.fileId &&
         d.contentHash === h.contentHash,
       "Pinned chunk has changed or is missing.",