@functional-systems/srcx 0.1.0-dev.4 → 0.1.0-dev.40

package/src/publish.ts

@@ -10,7 +10,15 @@ import {
   sleep,
   type Doc,
 } from "./common.js";
-import { corpusMetadata, records, validateBuild, type Build } from "./build.js";
+import {
+  corpusMetadata,
+  loadBuild,
+  records,
+  recordHash,
+  supportedPreset,
+  validateBuild,
+  type Build,
+} from "./build.js";
 import { branch, tag, one, type CollectionStore } from "./remote.js";
 export type Binding = {
   repoId: string;
@@ -36,6 +44,8 @@ export type Journal = {
   attemptId: string;
   writer: string;
   source: string;
+  gitRef?: string;
+  reuseWriter?: boolean;
   baseline?: Published;
   phase: "branch" | "writing" | "waiting" | "publishing";
   batches: {
@@ -45,6 +55,43 @@ export type Journal = {
   }[];
   candidate?: { name: string; snapshotId: string };
 };
+/** The applied snapshot may differ from the shared canonical commit Tag. */
+export type TrackedBranch = Doc & {
+  gitRef: string;
+  writer: string;
+  applied?: Published;
+  publishedTagName?: string;
+  pending?: { attemptId: string; buildId: string; commitOid: string };
+};
+export function gitBranchName(ref: string): string {
+  invariant(
+    ref.startsWith("refs/heads/") && ref.length > 11,
+    "Expected a full Git branch ref.",
+  );
+  return `git-${hash(ref).slice(0, 40)}`;
+}
+export const gitBranchId = (ref: string): string => `branch-${hash(ref)}`;
+export const lastBuildPath = (state: string, ref?: string): string =>
+  join(state, ref ? `last-build-${hash(ref)}.json` : "last-build.json");
+export async function trackedBranch(
+  store: CollectionStore,
+  binding: Binding,
+  ref: string,
+): Promise<TrackedBranch | undefined> {
+  const doc = await one(store, branch("main"), gitBranchId(ref), true);
+  if (!doc) return undefined;
+  invariant(
+    doc.kind === "manifest" &&
+      doc.role === "git-branch" &&
+      doc.gitRef === ref &&
+      doc.writer === gitBranchName(ref) &&
+      doc.repoId === binding.repoId &&
+      doc.indexId === binding.indexId &&
+      doc.configHash === binding.configHash,
+    "Tracked branch identity mismatch.",
+  );
+  return doc as TrackedBranch;
+}
 export function versionTag(oid: string): string {
   return `ver-${hash(oid).slice(0, 40)}`;
 }
@@ -115,7 +162,11 @@ export async function validateCandidate(
   for (const d of corpusMetadata(b, binding.repoId, binding.indexId, attemptId))
     expected.set(d.id, hash(d));
   for await (const d of store.list(tag(name))) {
-    if (expected.get(d.id) !== hash(d)) return false;
+    try {
+      if (expected.get(d.id) !== recordHash(d, b.preset)) return false;
+    } catch {
+      return false;
+    }
     expected.delete(d.id);
   }
   if (expected.size) return false;
@@ -123,12 +174,10 @@ export async function validateCandidate(
     let query: Record<string, unknown> = {
       queryString: { query: "kind:chunk" },
     };
-    for await (const d of records(b.directory)) {
+    for await (const d of records(b)) {
       if (d.kind !== "chunk") continue;
-      // Keep a complete surface token: the standard analyzer may retain dots
-      // (e.g. code.ts), so extracting an alphanumeric substring is not a valid probe.
-      const term = String(d.symbol ?? d.searchText).match(/\S+/u)?.[0];
-      if (!term) continue;
+      // Probe an indexed chunk ID: a valid source may contain only stop words
+      // or punctuation, so text-query matches cannot establish readiness.
       query = {
         bool: [
           {
@@ -141,16 +190,8 @@ export async function validateCandidate(
           },
           {
             queryString: {
-              query: d.fileId,
-              defaultField: "fileId",
-              skipSyntax: true,
-            },
-            occur: "filter",
-          },
-          {
-            queryString: {
-              query: term,
-              defaultField: "searchText",
+              query: d.id,
+              defaultField: "id",
               skipSyntax: true,
             },
             occur: "must",
@@ -164,7 +205,8 @@ export async function validateCandidate(
       !found.length ||
       found.some(
         (h) =>
-          h.doc.kind !== "chunk" || b.recordHashes[h.doc.id] !== hash(h.doc),
+          h.doc.kind !== "chunk" ||
+          b.recordHashes[h.doc.id] !== recordHash(h.doc, b.preset),
       )
     )
       return false;
@@ -228,6 +270,72 @@ export async function published(
       a.commitOid.localeCompare(b.commitOid),
   );
 }
+/** A local artifact is only a cache for this exact, previously published baseline. */
+async function baselineHashes(
+  store: CollectionStore,
+  binding: Binding,
+  preset: Build["preset"],
+  baseline: Published,
+  state: string,
+  gitRef?: string,
+  local?: Build,
+): Promise<Map<string, string>> {
+  const pin = (await store.tags()).find((t) => t.name === baseline.tagName);
+  const root = await one(store, tag(baseline.tagName), "__manifest__");
+  invariant(
+    pin?.snapshotId === baseline.snapshotId &&
+      root?.repoId === binding.repoId &&
+      root.indexId === binding.indexId &&
+      root.configHash === binding.configHash &&
+      root.buildId === baseline.buildId &&
+      root.commitOid === baseline.commitOid &&
+      root.attemptId === baseline.attemptId &&
+      root.recordsHash === baseline.recordsHash &&
+      root.inventoryHash === baseline.inventoryHash,
+    "Baseline Tag is missing or changed.",
+  );
+  if (!local) {
+    const last = await optionalJson<{ artifact: string }>(
+      lastBuildPath(state, gitRef),
+    );
+    if (last) {
+      try {
+        local = await loadBuild(last.artifact);
+      } catch (error) {
+        if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+      }
+    }
+  }
+  if (
+    local &&
+    local.repoKey === binding.repoKey &&
+    local.configHash === binding.configHash &&
+    local.buildId === baseline.buildId &&
+    local.commitOid === baseline.commitOid &&
+    local.recordsHash === baseline.recordsHash &&
+    local.inventoryHash === baseline.inventoryHash
+  ) {
+    try {
+      await validateBuild(local);
+      const hashes = new Map(Object.entries(local.recordHashes));
+      for (const doc of corpusMetadata(
+        local,
+        binding.repoId,
+        binding.indexId,
+        baseline.attemptId,
+      ))
+        hashes.set(doc.id, hash(doc));
+      return hashes;
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+    }
+  }
+  // Retain recovery without local files, including explicit same-artifact retries.
+  const hashes = new Map<string, string>();
+  for await (const doc of store.list(tag(baseline.tagName)))
+    hashes.set(doc.id, recordHash(doc, preset));
+  return hashes;
+}
 /** Caller holds the per-collection lock for the complete import. */
 export async function publish(args: {
   store: CollectionStore;
@@ -245,8 +353,8 @@ export async function publish(args: {
     "Build does not belong to this index.",
   );
   invariant(
-    !b.preset.embedding,
-    "CLI publication currently supports only the embedding=none preset.",
+    !b.preset.embedding || supportedPreset(b.preset),
+    "Publication requires a supported lexical or managed embedding preset.",
   );
   await validateBuild(b);
   const journalPath = join(state, "pending.json");
@@ -255,7 +363,9 @@ export async function publish(args: {
     invariant(
       j.buildId === b.buildId &&
         j.recordsHash === b.recordsHash &&
-        j.inventoryHash === b.inventoryHash,
+        j.inventoryHash === b.inventoryHash &&
+        (j.gitRef === b.branchRef ||
+          (j.reuseWriter === undefined && j.writer.startsWith("work-"))),
       "Another build is pending. Resume the artifact recorded in pending.json before importing a new commit.",
     );
     invariant(
@@ -266,6 +376,22 @@ export async function publish(args: {
   const existing = (await published(store, binding)).find(
     (v) => v.commitOid === b.commitOid,
   );
+  const tracked = b.branchRef
+    ? await trackedBranch(store, binding, b.branchRef)
+    : undefined;
+  if (tracked?.pending)
+    invariant(
+      j?.attemptId === tracked.pending.attemptId,
+      "Git branch has a pending import; resume with its original retry journal.",
+    );
+  if (j?.gitRef && j.phase !== "branch")
+    invariant(
+      tracked?.pending?.attemptId === j.attemptId ||
+        (j.phase === "publishing" &&
+          !tracked?.pending &&
+          tracked?.applied?.attemptId === j.attemptId),
+      "Tracked branch import ownership changed; refusing to replay this journal.",
+    );
   if (existing) {
     invariant(
       await validateCandidate(
@@ -277,17 +403,79 @@ export async function publish(args: {
       ),
       "Existing published version differs from this build.",
     );
-    await atomic(join(state, "last-build.json"), {
-      artifact: b.directory,
-      version: existing,
-    });
-    if (j) await rm(journalPath);
-    return existing;
+    // A shared commit Tag does not mean this Git branch has applied that commit.
+    if (!j && (!b.branchRef || tracked?.applied?.commitOid === b.commitOid)) {
+      if (tracked?.applied) {
+        const writer = (await store.branches()).find(
+          (x) => x.name === tracked.writer,
+        );
+        invariant(
+          writer?.snapshotId === tracked.applied.snapshotId,
+          "Writer baseline has changed; refusing to update tracked branch.",
+        );
+        invariant(
+          await validateCandidate(
+            store,
+            tracked.applied.tagName,
+            b,
+            binding,
+            tracked.applied.attemptId,
+          ),
+          "Tracked branch baseline validation failed.",
+        );
+      }
+      await atomic(lastBuildPath(state, b.branchRef), {
+        artifact: b.directory,
+        version: existing,
+      });
+      return existing;
+    }
+    // Legacy manual imports can finish bookkeeping after publication on retry.
+    if (j && !j.gitRef) {
+      await atomic(lastBuildPath(state), {
+        artifact: b.directory,
+        version: existing,
+      });
+      await rm(journalPath);
+      return existing;
+    }
   }
   if (!j) {
     let source = "checkpoint-empty";
     let baseline: Published | undefined;
-    if (args.baseline) {
+    if (b.branchRef) {
+      const writer = (await store.branches()).find(
+        (x) => x.name === gitBranchName(b.branchRef!),
+      );
+      if (tracked?.applied) {
+        baseline = tracked.applied;
+        invariant(
+          writer?.snapshotId === baseline.snapshotId,
+          "Writer baseline has changed; refusing to update tracked branch.",
+        );
+        const pin = (await store.tags()).find(
+          (t) => t.name === baseline!.tagName,
+        );
+        const root = await one(store, tag(baseline.tagName), "__manifest__");
+        invariant(
+          pin?.snapshotId === baseline.snapshotId &&
+            root?.repoId === binding.repoId &&
+            root.indexId === binding.indexId &&
+            root.configHash === binding.configHash &&
+            root.buildId === baseline.buildId &&
+            root.attemptId === baseline.attemptId &&
+            root.recordsHash === baseline.recordsHash &&
+            root.inventoryHash === baseline.inventoryHash &&
+            root.commitOid === baseline.commitOid,
+          "Tracked branch baseline Tag is missing or changed.",
+        );
+      } else {
+        invariant(
+          !writer && !tracked,
+          "Unowned or unfinished tracked branch; retain its retry journal.",
+        );
+      }
+    } else if (args.baseline?.version.writer.startsWith("work-")) {
       const prev = args.baseline;
       await validateBuild(prev.build);
       invariant(
@@ -322,7 +510,9 @@ export async function publish(args: {
       inventoryHash: b.inventoryHash,
       artifact: b.directory,
       attemptId,
-      writer: `work-${attemptId}`,
+      writer: b.branchRef ? gitBranchName(b.branchRef) : `work-${attemptId}`,
+      gitRef: b.branchRef,
+      reuseWriter: !!tracked?.applied,
       source,
       baseline,
       phase: "branch",
@@ -332,15 +522,48 @@ export async function publish(args: {
   }
   const save = () => atomic(journalPath, j);
   if (j.phase === "branch") {
+    if (j.gitRef) {
+      // Record intent before any corpus mutation, including before branch creation.
+      // A fresh local state must not treat a lagging, partially written head as clean.
+      const current = await trackedBranch(store, binding, j.gitRef);
+      invariant(
+        (!current?.pending || current.pending.attemptId === j.attemptId) &&
+          current?.applied?.snapshotId === j.baseline?.snapshotId,
+        "Git branch ownership or baseline changed before import.",
+      );
+      await store.upsert("main", [
+        {
+          ...current,
+          id: gitBranchId(j.gitRef),
+          kind: "manifest",
+          role: "git-branch",
+          schemaVersion: 1,
+          repoId: binding.repoId,
+          indexId: binding.indexId,
+          configHash: binding.configHash,
+          gitRef: j.gitRef,
+          writer: j.writer,
+          pending: {
+            attemptId: j.attemptId,
+            buildId: j.buildId,
+            commitOid: b.commitOid,
+          },
+        },
+      ]);
+    }
     const branches = await store.branches();
     const found = branches.find((x) => x.name === j!.writer);
     if (found)
       invariant(
-        found.parent === j.source &&
+        (j.reuseWriter || found.parent === j.source) &&
           found.snapshotId === (j.baseline?.snapshotId ?? null),
         "Partially created writer has an unexpected baseline.",
       );
     else {
+      invariant(
+        !j.reuseWriter,
+        "Tracked writer is missing; refusing to recreate a nonempty baseline.",
+      );
       const source = branches.find((x) => x.name === j!.source);
       invariant(
         source &&
@@ -363,18 +586,22 @@ export async function publish(args: {
   const marker = metadata.at(-1)!;
   if (j.phase === "writing") {
     // Reconcile from the immutable old Tag, never from a half-written writer.
-    const oldIds = new Set<string>();
-    const oldHashes = new Map<string, string>();
-    if (j.baseline)
-      for await (const d of store.list(tag(j.baseline.tagName))) {
-        oldIds.add(d.id);
-        oldHashes.set(d.id, hash(d));
-      }
+    const oldHashes = j.baseline
+      ? await baselineHashes(
+          store,
+          binding,
+          b.preset,
+          j.baseline,
+          state,
+          j.gitRef,
+          args.baseline?.build,
+        )
+      : new Map<string, string>();
     const newIds = new Set([
       ...Object.keys(b.recordHashes),
       ...metadata.map((d) => d.id),
     ]);
-    const obsolete = [...oldIds].filter((id) => !newIds.has(id));
+    const obsolete = [...oldHashes.keys()].filter((id) => !newIds.has(id));
     const mutate = async (
       operation: string,
       payload: unknown,
@@ -396,9 +623,10 @@ export async function publish(args: {
       await mutate("delete", ids, () => store.delete(j!.writer, ids));
     }
     async function* changed() {
-      for await (const d of records(b.directory))
+      for await (const d of records(b))
+        if (oldHashes.get(d.id) !== hash(d)) yield d;
+      for (const d of metadata.slice(0, -1))
         if (oldHashes.get(d.id) !== hash(d)) yield d;
-      for (const d of metadata.slice(0, -1)) yield d;
     }
     for await (const docs of batches(changed()))
       await mutate("upsert", docs, () => store.upsert(j!.writer, docs));
@@ -414,6 +642,7 @@ export async function publish(args: {
   const deadline = Date.now() + (args.timeoutMs ?? 120_000);
   const pause = args.pollMs ?? 1000;
   let rejectedSnapshot: string | undefined;
+  let validatedHere = false;
   while (j.phase === "waiting") {
     invariant(
       Date.now() < deadline,
@@ -442,22 +671,33 @@ export async function publish(args: {
       continue;
     }
     j.candidate = candidate;
+    validatedHere = true;
     j.phase = "publishing";
     await save();
   }
   invariant(j.candidate, "Missing validated candidate.");
+  // Never persist this shortcut: a resumed journal must revalidate the full corpus.
+  if (!validatedHere)
+    invariant(
+      await validateCandidate(store, j.candidate.name, b, binding, j.attemptId),
+      "Validated candidate is missing or changed.",
+    );
+  const pins = await store.tags();
   invariant(
-    await validateCandidate(store, j.candidate.name, b, binding, j.attemptId),
-    "Validated candidate is missing or changed.",
+    pins.find((t) => t.name === j!.candidate!.name)?.snapshotId ===
+      j.candidate.snapshotId,
+    "Validated candidate Tag is missing or changed.",
   );
   const name = versionTag(b.commitOid);
-  const found = (await store.tags()).find((t) => t.name === name);
+  const found = pins.find((t) => t.name === name);
   const final = found ?? (await store.tag(name, tag(j.candidate.name)));
   invariant(
-    final.snapshotId === j.candidate.snapshotId,
+    existing
+      ? final.snapshotId === existing.snapshotId
+      : final.snapshotId === j.candidate.snapshotId,
     "Published Tag name already points to a different snapshot.",
   );
-  const result: Published = {
+  const result: Published = existing ?? {
     id: versionId(b.commitOid),
     kind: "manifest",
     role: "published",
@@ -477,8 +717,42 @@ export async function publish(args: {
     counts: b.counts,
     validation: { inventory: true, content: true, query: true },
   };
-  await store.upsert("main", [result]);
-  await atomic(join(state, "last-build.json"), {
+  if (!existing) await store.upsert("main", [result]);
+  if (j.gitRef) {
+    const current = await trackedBranch(store, binding, j.gitRef);
+    invariant(
+      current?.pending?.attemptId === j.attemptId ||
+        (!current?.pending && current?.applied?.attemptId === j.attemptId),
+      "Tracked branch publication ownership changed.",
+    );
+    // Pin the exact writer baseline even when another branch published this commit first.
+    const applied: Published = {
+      ...result,
+      writer: j.writer,
+      snapshotId: j.candidate.snapshotId,
+      tagName:
+        final.snapshotId === j.candidate.snapshotId
+          ? final.name
+          : j.candidate.name,
+      attemptId: j.attemptId,
+    };
+    await store.upsert("main", [
+      {
+        id: gitBranchId(j.gitRef),
+        kind: "manifest",
+        role: "git-branch",
+        schemaVersion: 1,
+        repoId: binding.repoId,
+        indexId: binding.indexId,
+        configHash: binding.configHash,
+        gitRef: j.gitRef,
+        writer: j.writer,
+        applied,
+        publishedTagName: result.tagName,
+      },
+    ]);
+  }
+  await atomic(lastBuildPath(state, j.gitRef), {
     artifact: b.directory,
     version: result,
   });

package/src/releases.ts

@@ -1,7 +1,12 @@
 import { hash, invariant, type Doc } from "./common.js";
 import { gitTags } from "./git.js";
 import { one, branch, type CollectionStore } from "./remote.js";
-import { published, type Binding, type Published } from "./publish.js";
+import {
+  published,
+  trackedBranch,
+  type Binding,
+  type Published,
+} from "./publish.js";
 export const releaseAlias = (ref: string) => `rel-${hash(ref).slice(0, 40)}`;
 export const releaseId = (ref: string) => `ref-${hash(ref)}`;
 /** Only observed local tags. Absence is not deletion authority; this command never prunes. */
@@ -58,11 +63,39 @@ export async function resolveVersion(
   binding: Binding,
   selector: string,
 ): Promise<Published> {
-  const versions = await published(store, binding);
   const ref = selector.startsWith("refs/tags/")
     ? selector
     : `refs/tags/${selector}`;
-  const control = await one(store, branch("main"), releaseId(ref), true);
+  const control = selector.startsWith("refs/heads/")
+    ? undefined
+    : await one(store, branch("main"), releaseId(ref), true);
+  const headRef = selector.startsWith("refs/heads/")
+    ? selector
+    : `refs/heads/${selector}`;
+  const head =
+    !selector.startsWith("refs/") || selector.startsWith("refs/heads/")
+      ? await trackedBranch(store, binding, headRef)
+      : undefined;
+  invariant(
+    !(head && control),
+    "Ambiguous branch/tag name: use refs/heads/... or refs/tags/...",
+  );
+  // Pin moving ref control before listing versions. Otherwise publication can
+  // advance the ref to a version absent from the earlier list.
+  const versions = await published(store, binding);
+  if (head) {
+    invariant(
+      head.applied,
+      "Git branch has no published version yet; finish its import.",
+    );
+    const result = versions.find(
+      (v) =>
+        v.commitOid === head.applied!.commitOid &&
+        v.tagName === head.publishedTagName,
+    );
+    invariant(result, "Git branch target is not a published version.");
+    return result;
+  }
   // Like Git import, a known tag name takes precedence over an OID spelling.
   // Its pending or invalid state must not fall back to an unrelated commit.
   if (!control && !selector.startsWith("refs/")) {

package/src/remote.ts

@@ -7,7 +7,7 @@ import {
   tagTarget,
   type IndexConfigsUnion,
 } from "@functional-systems/lambdadb";
-import { invariant, type Doc } from "./common.js";
+import { hash, invariant, type Doc } from "./common.js";
 import type { Settings } from "./settings.js";
 export type Ref = { kind: "branch" | "tag"; name: string };
 export type Snapshot = { name: string; snapshotId: string };
@@ -77,6 +77,17 @@ export class LambdaRemote {
         (await this.client.listAllCollections({ size: 100 })).collections,
     );
   }
+  async collection(name: string) {
+    return request("get collection", async () => {
+      try {
+        return (await this.client.collection(name).get()).collection;
+      } catch (error) {
+        if ((error as { statusCode?: number }).statusCode === 404)
+          return undefined;
+        throw error;
+      }
+    });
+  }
   async create(
     name: string,
     indexConfigs: Record<string, IndexConfigsUnion>,
@@ -96,6 +107,43 @@ export class LambdaRemote {
     const c = this.client.collection(name);
     const ref = (r: Ref) =>
       r.kind === "tag" ? tagRef(r.name) : branchRef(r.name);
+    // Some deployments omit managed vectors from list/query responses despite
+    // includeVectors. Keep the same immutable Tag and exact non-vector payload.
+    const hydrate = async (r: Ref, docs: Doc[]): Promise<Doc[]> => {
+      const missing = docs.filter(
+        (d) => d.embeddingStatus === "managed" && d.embedding === undefined,
+      );
+      const hydrated = new Map<string, Doc>();
+      if (missing.length) {
+        invariant(
+          r.kind === "tag",
+          "Managed vector hydration requires an immutable Tag.",
+        );
+        const fetched = await request("fetch omitted vectors", () =>
+          c.docs.fetch({
+            ref: tagRef(r.name),
+            ids: missing.map((d) => d.id),
+            consistentRead: false,
+            includeVectors: true,
+          }),
+        );
+        const expected = new Map(missing.map((d) => [d.id, hash(d)]));
+        for (const item of fetched.docs) {
+          const doc = item.doc as Doc;
+          const { embedding: _, ...source } = doc;
+          invariant(
+            !hydrated.has(doc.id) && expected.get(doc.id) === hash(source),
+            "Returned/fetched managed payloads disagree.",
+          );
+          hydrated.set(doc.id, doc);
+        }
+        invariant(
+          hydrated.size === expected.size,
+          "Returned managed records are missing from fetch.",
+        );
+      }
+      return docs.map((doc) => hydrated.get(doc.id) ?? doc);
+    };
     return {
       fetch: (r, ids, consistent = false) =>
         request("fetch", async () => {
@@ -132,7 +180,11 @@ export class LambdaRemote {
               pageToken: cursor,
             }),
           );
-          for (const d of page.docs) yield d.doc as Doc;
+          for (const doc of await hydrate(
+            r,
+            page.docs.map((d) => d.doc as Doc),
+          ))
+            yield doc;
           cursor = page.nextPageToken || undefined;
           if (cursor) {
             invariant(
@@ -199,17 +251,22 @@ export class LambdaRemote {
             : c.aliases.create({ aliasName: name, target: tagTarget(target) }),
         );
       },
-      query: (tag, query, size) =>
-        request("query", async () =>
-          (
-            await c.query({
-              ref: tagRef(tag),
-              consistentRead: false,
-              query,
-              size,
-            })
-          ).docs.map((h) => ({ doc: h.doc as Doc, score: h.score })),
-        ),
+      query: async (name, query, size) => {
+        const result = await request("query", () =>
+          c.query({
+            ref: tagRef(name),
+            consistentRead: false,
+            query,
+            size,
+            includeVectors: true,
+          }),
+        );
+        const docs = await hydrate(
+          { kind: "tag", name },
+          result.docs.map((h) => h.doc as Doc),
+        );
+        return result.docs.map((h, i) => ({ doc: docs[i]!, score: h.score }));
+      },
     };
   }
 }