@functional-systems/srcx 0.1.0-dev.4 → 0.1.0-dev.6

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@functional-systems/srcx",
-  "version": "0.1.0-dev.4",
+  "version": "0.1.0-dev.6",
   "private": false,
   "description": "Version-aware code search on LambdaDB",
   "type": "module",
@@ -28,11 +28,14 @@
     "start": "node dist/cli.js",
     "demo": "npm run build && node scripts/demo.mjs",
     "prepack": "npm run build",
-    "format": "prettier --write src test scripts README.md VALIDATION.md CONTRIBUTING.md RELEASING.md CHANGELOG.md .github package.json tsconfig.json",
-    "format:check": "prettier --check src test scripts README.md VALIDATION.md CONTRIBUTING.md RELEASING.md CHANGELOG.md .github package.json tsconfig.json",
+    "format": "prettier --write src test scripts eval README.md VALIDATION.md CONTRIBUTING.md RELEASING.md CHANGELOG.md .github package.json tsconfig.json",
+    "format:check": "prettier --check src test scripts eval README.md VALIDATION.md CONTRIBUTING.md RELEASING.md CHANGELOG.md .github package.json tsconfig.json",
     "test:live": "npm run build && node --env-file=.env.local scripts/live.mjs",
     "check:version": "node scripts/check-release.mjs",
-    "test:package": "node scripts/test-package.mjs"
+    "test:package": "node scripts/test-package.mjs",
+    "test:live:branches": "npm run build && node --env-file=.env.local scripts/live-branches.mjs",
+    "eval:prepare": "npm run build && node scripts/retrieval-eval.mjs prepare",
+    "eval:run": "node scripts/retrieval-eval.mjs run"
   },
   "dependencies": {
     "@functional-systems/lambdadb": "0.5.1",
@@ -55,5 +58,5 @@
     "url": "https://github.com/lambdadb/srcx/issues"
   },
   "license": "Apache-2.0",
-  "gitHead": "9612cf3c342dbcf898b0ed8ca1526c0c3a04648f"
+  "gitHead": "8c0d1656d4d66a6461a32e9072b2e5f7fda9cec1"
 }

package/src/build.ts

@@ -27,6 +27,8 @@ export type Preset = {
   schemaVersion: 1;
   chunker: typeof CHUNKER;
   mode: "syntax" | "window";
+  /** Internal evaluation override; the CLI keeps its original enrichment. */
+  enrichment?: "path-only-v1";
   maxFileBytes: number;
   embedding: { model: string; dimensions: number } | null;
 };
@@ -123,6 +125,7 @@ function exclude(path: string): string | undefined {
 export async function materialize(args: {
   identity: Identity;
   ref: string;
+  resolvedCommit?: Awaited<ReturnType<typeof resolveCommit>>;
   output: string;
   previous?: Build;
   preset?: Preset;
@@ -135,6 +138,8 @@ export async function materialize(args: {
   invariant(
     hash(preset.chunker) === hash(CHUNKER) &&
       ["syntax", "window"].includes(preset.mode) &&
+      (preset.enrichment === undefined ||
+        preset.enrichment === "path-only-v1") &&
       preset.maxFileBytes === PRESET.maxFileBytes,
     "Unsupported build preset.",
   );
@@ -151,7 +156,8 @@ export async function materialize(args: {
         args.embedder.dimensions === preset.embedding.dimensions),
     "Embedding provider must match the pinned preset.",
   );
-  const commit = await resolveCommit(args.identity.path, args.ref);
+  const commit =
+    args.resolvedCommit ?? (await resolveCommit(args.identity.path, args.ref));
   const directory = resolve(args.output);
   await mkdir(directory, { recursive: false, mode: 0o700 });
   const output = await open(join(directory, "records.jsonl"), "wx", 0o600);
@@ -233,7 +239,12 @@ export async function materialize(args: {
         item.reason = "lfs-pointer";
         continue;
       }
-      const parsed = await chunk(source, e.path, preset.mode);
+      const parsed = await chunk(
+        source,
+        e.path,
+        preset.mode,
+        preset.enrichment,
+      );
       const contentHash = hash(data);
       const fileId = `f-${hash([e.pathBase64, contentHash])}`;
       const base = { schemaVersion: 1, configHash };

package/src/chunk.ts

@@ -75,7 +75,9 @@ export function detectLanguage(path: string): string {
     )[ext ?? ""] ?? "text"
   );
 }
-function enrichment(path: string, u: Unit): string {
+export type Enrichment = "path-scope-symbol-v1" | "path-only-v1";
+function enrichment(path: string, u: Unit, policy: Enrichment): string {
+  if (policy === "path-only-v1") return path.slice(0, 240) + "\n";
   return (
     [path.slice(0, 240), u.scope?.slice(0, 160), u.symbol?.slice(0, 160)]
       .filter(Boolean)
@@ -106,10 +108,11 @@ function splitUnit(
   path: string,
   u: Unit,
   overlap: number,
+  policy: Enrichment,
 ): Unit[] {
   const result: Unit[] = [];
   let start = u.start;
-  const prefix = enrichment(path, u);
+  const prefix = enrichment(path, u, policy);
   invariant(
     tokens(prefix) < CHUNKER.maxTokens,
     "Path/scope enrichment exceeds token limit.",
@@ -185,6 +188,7 @@ export async function chunk(
   source: string,
   path: string,
   mode: "syntax" | "window" = "syntax",
+  policy: Enrichment = CHUNKER.enrichment,
 ): Promise<{ spans: Span[]; parseStatus: string; language: string }> {
   const lang = detectLanguage(path);
   if (!source.length)
@@ -224,7 +228,7 @@ export async function chunk(
           if (
             n.namedChildCount &&
             (container ||
-              tokens(enrichment(path, u) + n.text) > CHUNKER.maxTokens)
+              tokens(enrichment(path, u, policy) + n.text) > CHUNKER.maxTokens)
           ) {
             for (const child of n.namedChildren) if (child) visit(child);
           } else if (n.endIndex > n.startIndex) units.push(u);
@@ -271,8 +275,9 @@ export async function chunk(
       prev.kind !== "function" &&
       u.kind === prev.kind &&
       u.scope === prev.scope &&
-      tokens(enrichment(path, prev) + source.slice(prev.start, u.end)) <=
-        CHUNKER.targetTokens
+      tokens(
+        enrichment(path, prev, policy) + source.slice(prev.start, u.end),
+      ) <= CHUNKER.targetTokens
     ) {
       prev.end = u.end;
       delete prev.symbol;
@@ -280,7 +285,7 @@ export async function chunk(
     } else merged.push({ ...u });
   }
   const bounded = merged.flatMap((u) =>
-    splitUnit(source, path, u, fallback ? CHUNKER.fallbackOverlap : 0),
+    splitUnit(source, path, u, fallback ? CHUNKER.fallbackOverlap : 0, policy),
   );
   const offsets = new Uint32Array(source.length + 1);
   let byte = 0;
@@ -298,7 +303,7 @@ export async function chunk(
     const startByte = offsets[u.start]!,
       endByte = offsets[u.end]!;
     const text = source.slice(u.start, u.end);
-    const searchText = enrichment(path, u) + text;
+    const searchText = enrichment(path, u, policy) + text;
     const kind = text.trim() === "" ? "structural" : u.kind;
     return {
       startByte,

package/src/cli.ts

@@ -5,7 +5,7 @@ import { mkdir } from "node:fs/promises";
 import { join, resolve } from "node:path";
 import { randomUUID } from "node:crypto";
 import { configure, loadSettings, stateRoot } from "./settings.js";
-import { identity } from "./git.js";
+import { identity, resolveCommit } from "./git.js";
 import { loadBuild, materialize, validateBuild, type Build } from "./build.js";
 import { invariant, optionalJson } from "./common.js";
 import { LambdaRemote } from "./remote.js";
@@ -16,7 +16,13 @@ import {
   register,
   selectRepository,
 } from "./repository.js";
-import { exclusive, publish, published, type Published } from "./publish.js";
+import {
+  exclusive,
+  lastBuildPath,
+  publish,
+  published,
+  type Published,
+} from "./publish.js";
 import { resolveVersion, syncTags } from "./releases.js";
 import { directHandle, loadHandle, readHandle, search } from "./search.js";
 const cli = new Command()
@@ -147,14 +153,29 @@ cli
     const buildLocal = async (
       source: Awaited<ReturnType<typeof identity>>,
       previous?: Build,
+      state?: string,
     ) => {
       invariant(o.ref, "--ref is required to build a commit.");
+      const commit = await resolveCommit(source.path, o.ref);
+      if (state) {
+        const last = await optionalJson<{ artifact: string }>(
+          lastBuildPath(state, commit.branch),
+        );
+        if (last) {
+          try {
+            previous = await loadBuild(last.artifact);
+          } catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+          }
+        }
+      }
       const parent = join(stateRoot(), "builds");
       await mkdir(parent, { recursive: true, mode: 0o700 });
       const path = o.output ? resolve(o.output) : join(parent, randomUUID());
       return materialize({
         identity: source,
         ref: o.ref,
+        resolvedCommit: commit,
         output: path,
         previous,
       });
@@ -212,20 +233,33 @@ cli
         // Pending attempts reconcile against the journal's immutable remote
         // baseline; they do not need the previous build's local files.
         const pending = await optionalJson(join(state, "pending.json"));
+        const b = o.artifact
+          ? await loadBuild(o.artifact)
+          : await buildLocal(
+              await attachment(settings, r),
+              undefined,
+              pending ? undefined : state,
+            );
         let baseline: { version: Published; build: Build } | undefined;
-        if (!pending) {
+        // Tracked branches discover their own immutable baseline remotely.
+        // Manual SHA/tag imports retain the previous frozen workspace optimization.
+        if (!pending && !b.branchRef) {
           const last = await optionalJson<{
             artifact: string;
             version: Published;
-          }>(join(state, "last-build.json"));
+          }>(lastBuildPath(state));
           if (last) {
-            const b = await loadBuild(last.artifact);
-            baseline = { version: last.version, build: b };
+            try {
+              baseline = {
+                version: last.version,
+                build: await loadBuild(last.artifact),
+              };
+            } catch (error) {
+              if ((error as NodeJS.ErrnoException).code !== "ENOENT")
+                throw error;
+            }
           }
         }
-        const b = o.artifact
-          ? await loadBuild(o.artifact)
-          : await buildLocal(await attachment(settings, r), baseline?.build);
         return publish({
           store,
           binding: r,
@@ -249,7 +283,7 @@ cli
 cli
   .command("resolve")
   .requiredOption("--repo <name>")
-  .requiredOption("--ref <commit-or-release>")
+  .requiredOption("--ref <commit-branch-or-release>")
   .action(async (o) => {
     const { remote } = await connected();
     const r = await selectRepository(remote, o.repo);
@@ -258,7 +292,7 @@ cli
 cli
   .command("search")
   .requiredOption("--repo <name>")
-  .requiredOption("--version <commit-or-release>")
+  .requiredOption("--version <commit-branch-or-release>")
   .requiredOption("--query <text>")
   .option("--limit <count>", "Maximum hits", integer, 10)
   .option("--path <path>", "Exact path filter")
@@ -279,7 +313,7 @@ cli
   .command("read")
   .option("--result <id>")
   .option("--repo <name>")
-  .option("--version <commit-or-release>")
+  .option("--version <commit-branch-or-release>")
   .option("--path <path>")
   .option("--lines <start:end>")
   .option("--context <lines>", "Expand an evidence span", integer, 0)

package/src/publish.ts

@@ -36,6 +36,8 @@ export type Journal = {
   attemptId: string;
   writer: string;
   source: string;
+  gitRef?: string;
+  reuseWriter?: boolean;
   baseline?: Published;
   phase: "branch" | "writing" | "waiting" | "publishing";
   batches: {
@@ -45,6 +47,43 @@ export type Journal = {
   }[];
   candidate?: { name: string; snapshotId: string };
 };
+/** The applied snapshot may differ from the shared canonical commit Tag. */
+export type TrackedBranch = Doc & {
+  gitRef: string;
+  writer: string;
+  applied?: Published;
+  publishedTagName?: string;
+  pending?: { attemptId: string; buildId: string; commitOid: string };
+};
+export function gitBranchName(ref: string): string {
+  invariant(
+    ref.startsWith("refs/heads/") && ref.length > 11,
+    "Expected a full Git branch ref.",
+  );
+  return `git-${hash(ref).slice(0, 40)}`;
+}
+export const gitBranchId = (ref: string): string => `branch-${hash(ref)}`;
+export const lastBuildPath = (state: string, ref?: string): string =>
+  join(state, ref ? `last-build-${hash(ref)}.json` : "last-build.json");
+export async function trackedBranch(
+  store: CollectionStore,
+  binding: Binding,
+  ref: string,
+): Promise<TrackedBranch | undefined> {
+  const doc = await one(store, branch("main"), gitBranchId(ref), true);
+  if (!doc) return undefined;
+  invariant(
+    doc.kind === "manifest" &&
+      doc.role === "git-branch" &&
+      doc.gitRef === ref &&
+      doc.writer === gitBranchName(ref) &&
+      doc.repoId === binding.repoId &&
+      doc.indexId === binding.indexId &&
+      doc.configHash === binding.configHash,
+    "Tracked branch identity mismatch.",
+  );
+  return doc as TrackedBranch;
+}
 export function versionTag(oid: string): string {
   return `ver-${hash(oid).slice(0, 40)}`;
 }
@@ -255,7 +294,9 @@ export async function publish(args: {
     invariant(
       j.buildId === b.buildId &&
         j.recordsHash === b.recordsHash &&
-        j.inventoryHash === b.inventoryHash,
+        j.inventoryHash === b.inventoryHash &&
+        (j.gitRef === b.branchRef ||
+          (j.reuseWriter === undefined && j.writer.startsWith("work-"))),
       "Another build is pending. Resume the artifact recorded in pending.json before importing a new commit.",
     );
     invariant(
@@ -266,6 +307,22 @@ export async function publish(args: {
   const existing = (await published(store, binding)).find(
     (v) => v.commitOid === b.commitOid,
   );
+  const tracked = b.branchRef
+    ? await trackedBranch(store, binding, b.branchRef)
+    : undefined;
+  if (tracked?.pending)
+    invariant(
+      j?.attemptId === tracked.pending.attemptId,
+      "Git branch has a pending import; resume with its original retry journal.",
+    );
+  if (j?.gitRef && j.phase !== "branch")
+    invariant(
+      tracked?.pending?.attemptId === j.attemptId ||
+        (j.phase === "publishing" &&
+          !tracked?.pending &&
+          tracked?.applied?.attemptId === j.attemptId),
+      "Tracked branch import ownership changed; refusing to replay this journal.",
+    );
   if (existing) {
     invariant(
       await validateCandidate(
@@ -277,17 +334,79 @@ export async function publish(args: {
       ),
       "Existing published version differs from this build.",
     );
-    await atomic(join(state, "last-build.json"), {
-      artifact: b.directory,
-      version: existing,
-    });
-    if (j) await rm(journalPath);
-    return existing;
+    // A shared commit Tag does not mean this Git branch has applied that commit.
+    if (!j && (!b.branchRef || tracked?.applied?.commitOid === b.commitOid)) {
+      if (tracked?.applied) {
+        const writer = (await store.branches()).find(
+          (x) => x.name === tracked.writer,
+        );
+        invariant(
+          writer?.snapshotId === tracked.applied.snapshotId,
+          "Writer baseline has changed; refusing to update tracked branch.",
+        );
+        invariant(
+          await validateCandidate(
+            store,
+            tracked.applied.tagName,
+            b,
+            binding,
+            tracked.applied.attemptId,
+          ),
+          "Tracked branch baseline validation failed.",
+        );
+      }
+      await atomic(lastBuildPath(state, b.branchRef), {
+        artifact: b.directory,
+        version: existing,
+      });
+      return existing;
+    }
+    // Legacy manual imports can finish bookkeeping after publication on retry.
+    if (j && !j.gitRef) {
+      await atomic(lastBuildPath(state), {
+        artifact: b.directory,
+        version: existing,
+      });
+      await rm(journalPath);
+      return existing;
+    }
   }
   if (!j) {
     let source = "checkpoint-empty";
     let baseline: Published | undefined;
-    if (args.baseline) {
+    if (b.branchRef) {
+      const writer = (await store.branches()).find(
+        (x) => x.name === gitBranchName(b.branchRef!),
+      );
+      if (tracked?.applied) {
+        baseline = tracked.applied;
+        invariant(
+          writer?.snapshotId === baseline.snapshotId,
+          "Writer baseline has changed; refusing to update tracked branch.",
+        );
+        const pin = (await store.tags()).find(
+          (t) => t.name === baseline!.tagName,
+        );
+        const root = await one(store, tag(baseline.tagName), "__manifest__");
+        invariant(
+          pin?.snapshotId === baseline.snapshotId &&
+            root?.repoId === binding.repoId &&
+            root.indexId === binding.indexId &&
+            root.configHash === binding.configHash &&
+            root.buildId === baseline.buildId &&
+            root.attemptId === baseline.attemptId &&
+            root.recordsHash === baseline.recordsHash &&
+            root.inventoryHash === baseline.inventoryHash &&
+            root.commitOid === baseline.commitOid,
+          "Tracked branch baseline Tag is missing or changed.",
+        );
+      } else {
+        invariant(
+          !writer && !tracked,
+          "Unowned or unfinished tracked branch; retain its retry journal.",
+        );
+      }
+    } else if (args.baseline?.version.writer.startsWith("work-")) {
       const prev = args.baseline;
       await validateBuild(prev.build);
       invariant(
@@ -322,7 +441,9 @@ export async function publish(args: {
       inventoryHash: b.inventoryHash,
       artifact: b.directory,
       attemptId,
-      writer: `work-${attemptId}`,
+      writer: b.branchRef ? gitBranchName(b.branchRef) : `work-${attemptId}`,
+      gitRef: b.branchRef,
+      reuseWriter: !!tracked?.applied,
       source,
       baseline,
       phase: "branch",
@@ -332,15 +453,48 @@ export async function publish(args: {
   }
   const save = () => atomic(journalPath, j);
   if (j.phase === "branch") {
+    if (j.gitRef) {
+      // Record intent before any corpus mutation, including before branch creation.
+      // A fresh local state must not treat a lagging, partially written head as clean.
+      const current = await trackedBranch(store, binding, j.gitRef);
+      invariant(
+        (!current?.pending || current.pending.attemptId === j.attemptId) &&
+          current?.applied?.snapshotId === j.baseline?.snapshotId,
+        "Git branch ownership or baseline changed before import.",
+      );
+      await store.upsert("main", [
+        {
+          ...current,
+          id: gitBranchId(j.gitRef),
+          kind: "manifest",
+          role: "git-branch",
+          schemaVersion: 1,
+          repoId: binding.repoId,
+          indexId: binding.indexId,
+          configHash: binding.configHash,
+          gitRef: j.gitRef,
+          writer: j.writer,
+          pending: {
+            attemptId: j.attemptId,
+            buildId: j.buildId,
+            commitOid: b.commitOid,
+          },
+        },
+      ]);
+    }
     const branches = await store.branches();
     const found = branches.find((x) => x.name === j!.writer);
     if (found)
       invariant(
-        found.parent === j.source &&
+        (j.reuseWriter || found.parent === j.source) &&
           found.snapshotId === (j.baseline?.snapshotId ?? null),
         "Partially created writer has an unexpected baseline.",
       );
     else {
+      invariant(
+        !j.reuseWriter,
+        "Tracked writer is missing; refusing to recreate a nonempty baseline.",
+      );
       const source = branches.find((x) => x.name === j!.source);
       invariant(
         source &&
@@ -454,10 +608,12 @@ export async function publish(args: {
   const found = (await store.tags()).find((t) => t.name === name);
   const final = found ?? (await store.tag(name, tag(j.candidate.name)));
   invariant(
-    final.snapshotId === j.candidate.snapshotId,
+    existing
+      ? final.snapshotId === existing.snapshotId
+      : final.snapshotId === j.candidate.snapshotId,
     "Published Tag name already points to a different snapshot.",
   );
-  const result: Published = {
+  const result: Published = existing ?? {
     id: versionId(b.commitOid),
     kind: "manifest",
     role: "published",
@@ -477,8 +633,42 @@ export async function publish(args: {
     counts: b.counts,
     validation: { inventory: true, content: true, query: true },
   };
-  await store.upsert("main", [result]);
-  await atomic(join(state, "last-build.json"), {
+  if (!existing) await store.upsert("main", [result]);
+  if (j.gitRef) {
+    const current = await trackedBranch(store, binding, j.gitRef);
+    invariant(
+      current?.pending?.attemptId === j.attemptId ||
+        (!current?.pending && current?.applied?.attemptId === j.attemptId),
+      "Tracked branch publication ownership changed.",
+    );
+    // Pin the exact writer baseline even when another branch published this commit first.
+    const applied: Published = {
+      ...result,
+      writer: j.writer,
+      snapshotId: j.candidate.snapshotId,
+      tagName:
+        final.snapshotId === j.candidate.snapshotId
+          ? final.name
+          : j.candidate.name,
+      attemptId: j.attemptId,
+    };
+    await store.upsert("main", [
+      {
+        id: gitBranchId(j.gitRef),
+        kind: "manifest",
+        role: "git-branch",
+        schemaVersion: 1,
+        repoId: binding.repoId,
+        indexId: binding.indexId,
+        configHash: binding.configHash,
+        gitRef: j.gitRef,
+        writer: j.writer,
+        applied,
+        publishedTagName: result.tagName,
+      },
+    ]);
+  }
+  await atomic(lastBuildPath(state, j.gitRef), {
     artifact: b.directory,
     version: result,
   });

package/src/releases.ts

@@ -1,7 +1,12 @@
 import { hash, invariant, type Doc } from "./common.js";
 import { gitTags } from "./git.js";
 import { one, branch, type CollectionStore } from "./remote.js";
-import { published, type Binding, type Published } from "./publish.js";
+import {
+  published,
+  trackedBranch,
+  type Binding,
+  type Published,
+} from "./publish.js";
 export const releaseAlias = (ref: string) => `rel-${hash(ref).slice(0, 40)}`;
 export const releaseId = (ref: string) => `ref-${hash(ref)}`;
 /** Only observed local tags. Absence is not deletion authority; this command never prunes. */
@@ -58,11 +63,39 @@ export async function resolveVersion(
   binding: Binding,
   selector: string,
 ): Promise<Published> {
-  const versions = await published(store, binding);
   const ref = selector.startsWith("refs/tags/")
     ? selector
     : `refs/tags/${selector}`;
-  const control = await one(store, branch("main"), releaseId(ref), true);
+  const control = selector.startsWith("refs/heads/")
+    ? undefined
+    : await one(store, branch("main"), releaseId(ref), true);
+  const headRef = selector.startsWith("refs/heads/")
+    ? selector
+    : `refs/heads/${selector}`;
+  const head =
+    !selector.startsWith("refs/") || selector.startsWith("refs/heads/")
+      ? await trackedBranch(store, binding, headRef)
+      : undefined;
+  invariant(
+    !(head && control),
+    "Ambiguous branch/tag name: use refs/heads/... or refs/tags/...",
+  );
+  // Pin moving ref control before listing versions. Otherwise publication can
+  // advance the ref to a version absent from the earlier list.
+  const versions = await published(store, binding);
+  if (head) {
+    invariant(
+      head.applied,
+      "Git branch has no published version yet; finish its import.",
+    );
+    const result = versions.find(
+      (v) =>
+        v.commitOid === head.applied!.commitOid &&
+        v.tagName === head.publishedTagName,
+    );
+    invariant(result, "Git branch target is not a published version.");
+    return result;
+  }
   // Like Git import, a known tag name takes precedence over an OID spelling.
   // Its pending or invalid state must not fall back to an unrelated commit.
   if (!control && !selector.startsWith("refs/")) {