@functional-systems/srcx 0.1.0-dev.1 → 0.1.0-dev.10

package/src/cli.ts

@@ -1,12 +1,20 @@
 #!/usr/bin/env node
 import { readFileSync } from "node:fs";
-import { Command, InvalidArgumentError } from "commander";
+import { Command, InvalidArgumentError, Option } from "commander";
 import { mkdir } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
 import { configure, loadSettings, stateRoot } from "./settings.js";
-import { identity } from "./git.js";
-import { loadBuild, materialize, validateBuild, type Build } from "./build.js";
+import { identity, resolveCommit } from "./git.js";
+import {
+  loadBuild,
+  materialize,
+  validateBuild,
+  type Build,
+  type Preset,
+  PRESET,
+  presetFor,
+} from "./build.js";
 import { invariant, optionalJson } from "./common.js";
 import { LambdaRemote } from "./remote.js";
 import {
@@ -16,7 +24,13 @@ import {
   register,
   selectRepository,
 } from "./repository.js";
-import { exclusive, publish, published, type Published } from "./publish.js";
+import {
+  exclusive,
+  lastBuildPath,
+  publish,
+  published,
+  type Published,
+} from "./publish.js";
 import { resolveVersion, syncTags } from "./releases.js";
 import { directHandle, loadHandle, readHandle, search } from "./search.js";
 const cli = new Command()
@@ -81,6 +95,14 @@ repo
   .option("--remote <name>")
   .option("--description <text>")
   .option("--tag <key=value>", "One optional Collection metadata tag")
+  .addOption(
+    new Option(
+      "--embedding <model>",
+      "Managed model (source text is sent to the provider on import)",
+    )
+      .choices(["none", "text-embedding-3-small", "text-embedding-3-large"])
+      .default("none"),
+  )
   .action(async (o) => {
     const { remote } = await connected();
     let labels: Record<string, string> | undefined;
@@ -95,6 +117,7 @@ repo
         remote: o.remote,
         description: o.description,
         labels,
+        preset: presetFor(o.embedding),
       }),
     );
   });
@@ -122,6 +145,12 @@ cli
   .option("--repo <name>", "Registered remote repository")
   .option("--ref <ref>", "Local branch, Git tag, or commit OID")
   .option("--dry-run", "Only materialize and validate a local build")
+  .addOption(
+    new Option(
+      "--embedding <model>",
+      "Preset for --dry-run --path; connected imports use the repository preset",
+    ).choices(["none", "text-embedding-3-small", "text-embedding-3-large"]),
+  )
   .option("--output <directory>", "New artifact directory")
   .option(
     "--previous <directory>",
@@ -134,6 +163,10 @@ cli
   )
   .option("--timeout <seconds>", "Validation wait deadline", integer, 120)
   .action(async (o) => {
+    invariant(
+      !o.embedding || (o.dryRun && o.path),
+      "--embedding is only accepted with --dry-run --path; connected imports use the repository preset.",
+    );
     invariant(!(o.path && o.repo), "Choose --path or --repo.");
     invariant(
       !o.previous || o.dryRun,
@@ -147,16 +180,33 @@ cli
     const buildLocal = async (
       source: Awaited<ReturnType<typeof identity>>,
       previous?: Build,
+      state?: string,
+      preset: Preset = PRESET,
     ) => {
       invariant(o.ref, "--ref is required to build a commit.");
+      const commit = await resolveCommit(source.path, o.ref);
+      if (state) {
+        const last = await optionalJson<{ artifact: string }>(
+          lastBuildPath(state, commit.branch),
+        );
+        if (last) {
+          try {
+            previous = await loadBuild(last.artifact);
+          } catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+          }
+        }
+      }
       const parent = join(stateRoot(), "builds");
       await mkdir(parent, { recursive: true, mode: 0o700 });
       const path = o.output ? resolve(o.output) : join(parent, randomUUID());
       return materialize({
         identity: source,
         ref: o.ref,
+        resolvedCommit: commit,
         output: path,
         previous,
+        preset,
       });
     };
     if (o.dryRun) {
@@ -166,18 +216,20 @@ cli
         await validateBuild(b);
       } else {
         let source;
+        let preset = presetFor(o.embedding);
         if (o.path) source = await identity(o.path, o.remote);
         else {
           invariant(o.repo, "Use --path for a credential-free dry run.");
           const { remote, settings } = await connected();
-          source = await attachment(
-            settings,
-            await selectRepository(remote, o.repo),
-          );
+          const selected = await selectRepository(remote, o.repo);
+          preset = selected.preset ?? PRESET;
+          source = await attachment(settings, selected);
         }
         b = await buildLocal(
           source,
           o.previous ? await loadBuild(o.previous) : undefined,
+          undefined,
+          preset,
         );
       }
       output({
@@ -212,20 +264,34 @@ cli
         // Pending attempts reconcile against the journal's immutable remote
         // baseline; they do not need the previous build's local files.
         const pending = await optionalJson(join(state, "pending.json"));
+        const b = o.artifact
+          ? await loadBuild(o.artifact)
+          : await buildLocal(
+              await attachment(settings, r),
+              undefined,
+              pending ? undefined : state,
+              r.preset ?? PRESET,
+            );
         let baseline: { version: Published; build: Build } | undefined;
-        if (!pending) {
+        // Tracked branches discover their own immutable baseline remotely.
+        // Manual SHA/tag imports retain the previous frozen workspace optimization.
+        if (!pending && !b.branchRef) {
           const last = await optionalJson<{
             artifact: string;
             version: Published;
-          }>(join(state, "last-build.json"));
+          }>(lastBuildPath(state));
           if (last) {
-            const b = await loadBuild(last.artifact);
-            baseline = { version: last.version, build: b };
+            try {
+              baseline = {
+                version: last.version,
+                build: await loadBuild(last.artifact),
+              };
+            } catch (error) {
+              if ((error as NodeJS.ErrnoException).code !== "ENOENT")
+                throw error;
+            }
           }
         }
-        const b = o.artifact
-          ? await loadBuild(o.artifact)
-          : await buildLocal(await attachment(settings, r), baseline?.build);
         return publish({
           store,
           binding: r,
@@ -249,7 +315,7 @@ cli
 cli
   .command("resolve")
   .requiredOption("--repo <name>")
-  .requiredOption("--ref <commit-or-release>")
+  .requiredOption("--ref <commit-branch-or-release>")
   .action(async (o) => {
     const { remote } = await connected();
     const r = await selectRepository(remote, o.repo);
@@ -258,9 +324,17 @@ cli
 cli
   .command("search")
   .requiredOption("--repo <name>")
-  .requiredOption("--version <commit-or-release>")
+  .requiredOption("--version <commit-branch-or-release>")
   .requiredOption("--query <text>")
   .option("--limit <count>", "Maximum hits", integer, 10)
+  .addOption(
+    new Option(
+      "--mode <mode>",
+      "Retrieval method; semantic/hybrid incur managed query embedding usage",
+    )
+      .choices(["lexical", "semantic", "hybrid"])
+      .default("lexical"),
+  )
   .option("--path <path>", "Exact path filter")
   .option("--language <name>")
   .action(async (o) => {
@@ -269,17 +343,26 @@ cli
       store = remote.store(r.collection),
       v = await resolveVersion(store, r, o.version);
     output(
-      await search(store, settings, r, v, o.query, o.limit, {
-        path: o.path,
-        language: o.language,
-      }),
+      await search(
+        store,
+        settings,
+        r,
+        v,
+        o.query,
+        o.limit,
+        {
+          path: o.path,
+          language: o.language,
+        },
+        o.mode,
+      ),
     );
   });
 cli
   .command("read")
   .option("--result <id>")
   .option("--repo <name>")
-  .option("--version <commit-or-release>")
+  .option("--version <commit-branch-or-release>")
   .option("--path <path>")
   .option("--lines <start:end>")
   .option("--context <lines>", "Expand an evidence span", integer, 0)

package/src/publish.ts

@@ -10,7 +10,14 @@ import {
   sleep,
   type Doc,
 } from "./common.js";
-import { corpusMetadata, records, validateBuild, type Build } from "./build.js";
+import {
+  corpusMetadata,
+  records,
+  recordHash,
+  supportedPreset,
+  validateBuild,
+  type Build,
+} from "./build.js";
 import { branch, tag, one, type CollectionStore } from "./remote.js";
 export type Binding = {
   repoId: string;
@@ -36,6 +43,8 @@ export type Journal = {
   attemptId: string;
   writer: string;
   source: string;
+  gitRef?: string;
+  reuseWriter?: boolean;
   baseline?: Published;
   phase: "branch" | "writing" | "waiting" | "publishing";
   batches: {
@@ -45,6 +54,43 @@ export type Journal = {
   }[];
   candidate?: { name: string; snapshotId: string };
 };
+/** The applied snapshot may differ from the shared canonical commit Tag. */
+export type TrackedBranch = Doc & {
+  gitRef: string;
+  writer: string;
+  applied?: Published;
+  publishedTagName?: string;
+  pending?: { attemptId: string; buildId: string; commitOid: string };
+};
+export function gitBranchName(ref: string): string {
+  invariant(
+    ref.startsWith("refs/heads/") && ref.length > 11,
+    "Expected a full Git branch ref.",
+  );
+  return `git-${hash(ref).slice(0, 40)}`;
+}
+export const gitBranchId = (ref: string): string => `branch-${hash(ref)}`;
+export const lastBuildPath = (state: string, ref?: string): string =>
+  join(state, ref ? `last-build-${hash(ref)}.json` : "last-build.json");
+export async function trackedBranch(
+  store: CollectionStore,
+  binding: Binding,
+  ref: string,
+): Promise<TrackedBranch | undefined> {
+  const doc = await one(store, branch("main"), gitBranchId(ref), true);
+  if (!doc) return undefined;
+  invariant(
+    doc.kind === "manifest" &&
+      doc.role === "git-branch" &&
+      doc.gitRef === ref &&
+      doc.writer === gitBranchName(ref) &&
+      doc.repoId === binding.repoId &&
+      doc.indexId === binding.indexId &&
+      doc.configHash === binding.configHash,
+    "Tracked branch identity mismatch.",
+  );
+  return doc as TrackedBranch;
+}
 export function versionTag(oid: string): string {
   return `ver-${hash(oid).slice(0, 40)}`;
 }
@@ -115,7 +161,11 @@ export async function validateCandidate(
   for (const d of corpusMetadata(b, binding.repoId, binding.indexId, attemptId))
     expected.set(d.id, hash(d));
   for await (const d of store.list(tag(name))) {
-    if (expected.get(d.id) !== hash(d)) return false;
+    try {
+      if (expected.get(d.id) !== recordHash(d, b.preset)) return false;
+    } catch {
+      return false;
+    }
     expected.delete(d.id);
   }
   if (expected.size) return false;
@@ -164,7 +214,8 @@ export async function validateCandidate(
       !found.length ||
       found.some(
         (h) =>
-          h.doc.kind !== "chunk" || b.recordHashes[h.doc.id] !== hash(h.doc),
+          h.doc.kind !== "chunk" ||
+          b.recordHashes[h.doc.id] !== recordHash(h.doc, b.preset),
       )
     )
       return false;
@@ -245,8 +296,8 @@ export async function publish(args: {
     "Build does not belong to this index.",
   );
   invariant(
-    !b.preset.embedding,
-    "CLI publication currently supports only the embedding=none preset.",
+    !b.preset.embedding || supportedPreset(b.preset),
+    "Publication requires a supported lexical or managed embedding preset.",
   );
   await validateBuild(b);
   const journalPath = join(state, "pending.json");
@@ -255,7 +306,9 @@ export async function publish(args: {
     invariant(
       j.buildId === b.buildId &&
         j.recordsHash === b.recordsHash &&
-        j.inventoryHash === b.inventoryHash,
+        j.inventoryHash === b.inventoryHash &&
+        (j.gitRef === b.branchRef ||
+          (j.reuseWriter === undefined && j.writer.startsWith("work-"))),
       "Another build is pending. Resume the artifact recorded in pending.json before importing a new commit.",
     );
     invariant(
@@ -266,6 +319,22 @@ export async function publish(args: {
   const existing = (await published(store, binding)).find(
     (v) => v.commitOid === b.commitOid,
   );
+  const tracked = b.branchRef
+    ? await trackedBranch(store, binding, b.branchRef)
+    : undefined;
+  if (tracked?.pending)
+    invariant(
+      j?.attemptId === tracked.pending.attemptId,
+      "Git branch has a pending import; resume with its original retry journal.",
+    );
+  if (j?.gitRef && j.phase !== "branch")
+    invariant(
+      tracked?.pending?.attemptId === j.attemptId ||
+        (j.phase === "publishing" &&
+          !tracked?.pending &&
+          tracked?.applied?.attemptId === j.attemptId),
+      "Tracked branch import ownership changed; refusing to replay this journal.",
+    );
   if (existing) {
     invariant(
       await validateCandidate(
@@ -277,17 +346,79 @@ export async function publish(args: {
       ),
       "Existing published version differs from this build.",
     );
-    await atomic(join(state, "last-build.json"), {
-      artifact: b.directory,
-      version: existing,
-    });
-    if (j) await rm(journalPath);
-    return existing;
+    // A shared commit Tag does not mean this Git branch has applied that commit.
+    if (!j && (!b.branchRef || tracked?.applied?.commitOid === b.commitOid)) {
+      if (tracked?.applied) {
+        const writer = (await store.branches()).find(
+          (x) => x.name === tracked.writer,
+        );
+        invariant(
+          writer?.snapshotId === tracked.applied.snapshotId,
+          "Writer baseline has changed; refusing to update tracked branch.",
+        );
+        invariant(
+          await validateCandidate(
+            store,
+            tracked.applied.tagName,
+            b,
+            binding,
+            tracked.applied.attemptId,
+          ),
+          "Tracked branch baseline validation failed.",
+        );
+      }
+      await atomic(lastBuildPath(state, b.branchRef), {
+        artifact: b.directory,
+        version: existing,
+      });
+      return existing;
+    }
+    // Legacy manual imports can finish bookkeeping after publication on retry.
+    if (j && !j.gitRef) {
+      await atomic(lastBuildPath(state), {
+        artifact: b.directory,
+        version: existing,
+      });
+      await rm(journalPath);
+      return existing;
+    }
   }
   if (!j) {
     let source = "checkpoint-empty";
     let baseline: Published | undefined;
-    if (args.baseline) {
+    if (b.branchRef) {
+      const writer = (await store.branches()).find(
+        (x) => x.name === gitBranchName(b.branchRef!),
+      );
+      if (tracked?.applied) {
+        baseline = tracked.applied;
+        invariant(
+          writer?.snapshotId === baseline.snapshotId,
+          "Writer baseline has changed; refusing to update tracked branch.",
+        );
+        const pin = (await store.tags()).find(
+          (t) => t.name === baseline!.tagName,
+        );
+        const root = await one(store, tag(baseline.tagName), "__manifest__");
+        invariant(
+          pin?.snapshotId === baseline.snapshotId &&
+            root?.repoId === binding.repoId &&
+            root.indexId === binding.indexId &&
+            root.configHash === binding.configHash &&
+            root.buildId === baseline.buildId &&
+            root.attemptId === baseline.attemptId &&
+            root.recordsHash === baseline.recordsHash &&
+            root.inventoryHash === baseline.inventoryHash &&
+            root.commitOid === baseline.commitOid,
+          "Tracked branch baseline Tag is missing or changed.",
+        );
+      } else {
+        invariant(
+          !writer && !tracked,
+          "Unowned or unfinished tracked branch; retain its retry journal.",
+        );
+      }
+    } else if (args.baseline?.version.writer.startsWith("work-")) {
       const prev = args.baseline;
       await validateBuild(prev.build);
       invariant(
@@ -322,7 +453,9 @@ export async function publish(args: {
       inventoryHash: b.inventoryHash,
       artifact: b.directory,
       attemptId,
-      writer: `work-${attemptId}`,
+      writer: b.branchRef ? gitBranchName(b.branchRef) : `work-${attemptId}`,
+      gitRef: b.branchRef,
+      reuseWriter: !!tracked?.applied,
       source,
       baseline,
       phase: "branch",
@@ -332,15 +465,48 @@ export async function publish(args: {
   }
   const save = () => atomic(journalPath, j);
   if (j.phase === "branch") {
+    if (j.gitRef) {
+      // Record intent before any corpus mutation, including before branch creation.
+      // A fresh local state must not treat a lagging, partially written head as clean.
+      const current = await trackedBranch(store, binding, j.gitRef);
+      invariant(
+        (!current?.pending || current.pending.attemptId === j.attemptId) &&
+          current?.applied?.snapshotId === j.baseline?.snapshotId,
+        "Git branch ownership or baseline changed before import.",
+      );
+      await store.upsert("main", [
+        {
+          ...current,
+          id: gitBranchId(j.gitRef),
+          kind: "manifest",
+          role: "git-branch",
+          schemaVersion: 1,
+          repoId: binding.repoId,
+          indexId: binding.indexId,
+          configHash: binding.configHash,
+          gitRef: j.gitRef,
+          writer: j.writer,
+          pending: {
+            attemptId: j.attemptId,
+            buildId: j.buildId,
+            commitOid: b.commitOid,
+          },
+        },
+      ]);
+    }
     const branches = await store.branches();
     const found = branches.find((x) => x.name === j!.writer);
     if (found)
       invariant(
-        found.parent === j.source &&
+        (j.reuseWriter || found.parent === j.source) &&
           found.snapshotId === (j.baseline?.snapshotId ?? null),
         "Partially created writer has an unexpected baseline.",
       );
     else {
+      invariant(
+        !j.reuseWriter,
+        "Tracked writer is missing; refusing to recreate a nonempty baseline.",
+      );
       const source = branches.find((x) => x.name === j!.source);
       invariant(
         source &&
@@ -368,7 +534,7 @@ export async function publish(args: {
     if (j.baseline)
       for await (const d of store.list(tag(j.baseline.tagName))) {
         oldIds.add(d.id);
-        oldHashes.set(d.id, hash(d));
+        oldHashes.set(d.id, recordHash(d, b.preset));
       }
     const newIds = new Set([
       ...Object.keys(b.recordHashes),
@@ -454,10 +620,12 @@ export async function publish(args: {
   const found = (await store.tags()).find((t) => t.name === name);
   const final = found ?? (await store.tag(name, tag(j.candidate.name)));
   invariant(
-    final.snapshotId === j.candidate.snapshotId,
+    existing
+      ? final.snapshotId === existing.snapshotId
+      : final.snapshotId === j.candidate.snapshotId,
     "Published Tag name already points to a different snapshot.",
   );
-  const result: Published = {
+  const result: Published = existing ?? {
     id: versionId(b.commitOid),
     kind: "manifest",
     role: "published",
@@ -477,8 +645,42 @@ export async function publish(args: {
     counts: b.counts,
     validation: { inventory: true, content: true, query: true },
   };
-  await store.upsert("main", [result]);
-  await atomic(join(state, "last-build.json"), {
+  if (!existing) await store.upsert("main", [result]);
+  if (j.gitRef) {
+    const current = await trackedBranch(store, binding, j.gitRef);
+    invariant(
+      current?.pending?.attemptId === j.attemptId ||
+        (!current?.pending && current?.applied?.attemptId === j.attemptId),
+      "Tracked branch publication ownership changed.",
+    );
+    // Pin the exact writer baseline even when another branch published this commit first.
+    const applied: Published = {
+      ...result,
+      writer: j.writer,
+      snapshotId: j.candidate.snapshotId,
+      tagName:
+        final.snapshotId === j.candidate.snapshotId
+          ? final.name
+          : j.candidate.name,
+      attemptId: j.attemptId,
+    };
+    await store.upsert("main", [
+      {
+        id: gitBranchId(j.gitRef),
+        kind: "manifest",
+        role: "git-branch",
+        schemaVersion: 1,
+        repoId: binding.repoId,
+        indexId: binding.indexId,
+        configHash: binding.configHash,
+        gitRef: j.gitRef,
+        writer: j.writer,
+        applied,
+        publishedTagName: result.tagName,
+      },
+    ]);
+  }
+  await atomic(lastBuildPath(state, j.gitRef), {
     artifact: b.directory,
     version: result,
   });

package/src/releases.ts

@@ -1,7 +1,12 @@
 import { hash, invariant, type Doc } from "./common.js";
 import { gitTags } from "./git.js";
 import { one, branch, type CollectionStore } from "./remote.js";
-import { published, type Binding, type Published } from "./publish.js";
+import {
+  published,
+  trackedBranch,
+  type Binding,
+  type Published,
+} from "./publish.js";
 export const releaseAlias = (ref: string) => `rel-${hash(ref).slice(0, 40)}`;
 export const releaseId = (ref: string) => `ref-${hash(ref)}`;
 /** Only observed local tags. Absence is not deletion authority; this command never prunes. */
@@ -58,11 +63,39 @@ export async function resolveVersion(
   binding: Binding,
   selector: string,
 ): Promise<Published> {
-  const versions = await published(store, binding);
   const ref = selector.startsWith("refs/tags/")
     ? selector
     : `refs/tags/${selector}`;
-  const control = await one(store, branch("main"), releaseId(ref), true);
+  const control = selector.startsWith("refs/heads/")
+    ? undefined
+    : await one(store, branch("main"), releaseId(ref), true);
+  const headRef = selector.startsWith("refs/heads/")
+    ? selector
+    : `refs/heads/${selector}`;
+  const head =
+    !selector.startsWith("refs/") || selector.startsWith("refs/heads/")
+      ? await trackedBranch(store, binding, headRef)
+      : undefined;
+  invariant(
+    !(head && control),
+    "Ambiguous branch/tag name: use refs/heads/... or refs/tags/...",
+  );
+  // Pin moving ref control before listing versions. Otherwise publication can
+  // advance the ref to a version absent from the earlier list.
+  const versions = await published(store, binding);
+  if (head) {
+    invariant(
+      head.applied,
+      "Git branch has no published version yet; finish its import.",
+    );
+    const result = versions.find(
+      (v) =>
+        v.commitOid === head.applied!.commitOid &&
+        v.tagName === head.publishedTagName,
+    );
+    invariant(result, "Git branch target is not a published version.");
+    return result;
+  }
   // Like Git import, a known tag name takes precedence over an OID spelling.
   // Its pending or invalid state must not fall back to an unrelated commit.
   if (!control && !selector.startsWith("refs/")) {