@functional-systems/srcx 0.1.0-dev.3 → 0.1.0-dev.31

package/src/implementation.ts

@@ -0,0 +1,253 @@
+import type { Node } from "web-tree-sitter";
+import { CHUNKER, detectLanguage, tokens, withSyntaxTree } from "./chunk.js";
+import { invariant, lineAt } from "./common.js";
+
+export type Implementation = {
+  status:
+    "resolved" | "not-linked" | "unsupported" | "parse-error" | "too-large";
+  startLine?: number;
+  endLine?: number;
+  symbol?: string;
+};
+type Definition = {
+  node: Node;
+  name: string;
+  key: string;
+  declaration: boolean;
+};
+const children = (n: Node): Node[] =>
+  n.namedChildren.filter((c): c is Node => !!c);
+
+/** Only imported typing overload aliases used exclusively as decorators are safe here. */
+function overloadAliases(root: Node): Set<string> {
+  const aliases = new Set<string>(),
+    imports = new Set<number>();
+  for (const n of children(root)) {
+    if (
+      n.type !== "import_from_statement" ||
+      !["typing", "typing_extensions"].includes(
+        n.childForFieldName("module_name")?.text ?? "",
+      )
+    )
+      continue;
+    for (const name of n.childrenForFieldName("name")) {
+      if (!name) continue;
+      if (name.type === "dotted_name" && name.text === "overload") {
+        aliases.add("overload");
+        imports.add(name.id);
+      } else if (
+        name.type === "aliased_import" &&
+        name.childForFieldName("name")?.text === "overload"
+      ) {
+        const alias = name.childForFieldName("alias");
+        if (alias) {
+          aliases.add(alias.text);
+          imports.add(name.id);
+        }
+      }
+    }
+  }
+  function visit(n: Node): void {
+    if (imports.has(n.id)) return;
+    if (n.type === "wildcard_import") aliases.clear();
+    if (
+      n.type === "identifier" &&
+      aliases.has(n.text) &&
+      !(n.parent?.type === "decorator" && n.parent.text === `@${n.text}`)
+    )
+      aliases.delete(n.text);
+    for (const child of children(n)) visit(child);
+  }
+  visit(root);
+  return aliases;
+}
+function definition(
+  n: Node,
+  python: boolean,
+  aliases: Set<string>,
+): Definition | undefined {
+  const d = n.childForFieldName(python ? "definition" : "declaration") ?? n;
+  const name = d.childForFieldName("name");
+  if (
+    !name ||
+    ![
+      "identifier",
+      "property_identifier",
+      "private_property_identifier",
+    ].includes(name.type)
+  )
+    return;
+  if (python) {
+    if (d.type !== "function_definition") return;
+    const decorators = children(n).filter((c) => c.type === "decorator");
+    const declaration = decorators.some((c) => aliases.has(c.text.slice(1)));
+    const body = d.childForFieldName("body");
+    // Stubs without an overload marker do not supply an implementation.
+    const executable =
+      body &&
+      children(body).some(
+        (c) =>
+          c.type !== "comment" &&
+          c.type !== "pass_statement" &&
+          !(
+            c.type === "expression_statement" &&
+            ["ellipsis", "string"].includes(c.firstNamedChild?.type ?? "")
+          ),
+      );
+    if (!declaration && !executable) return;
+    // Other decorators may change dispatch (property, singledispatch, etc.).
+    if (
+      decorators.some(
+        (c) =>
+          !aliases.has(c.text.slice(1)) &&
+          !["@staticmethod", "@classmethod"].includes(c.text),
+      )
+    )
+      return;
+    const modifier = decorators
+      .filter((c) => ["@staticmethod", "@classmethod"].includes(c.text))
+      .map((c) => c.text)
+      .sort()
+      .join(":");
+    const async = d.children.some((c) => c?.type === "async");
+    return {
+      node: n,
+      name: name.text,
+      key: `${name.text}:${modifier}:${async}`,
+      declaration,
+    };
+  }
+  if (n !== d && n.type !== "export_statement") return;
+  if (
+    ![
+      "function_signature",
+      "function_declaration",
+      "method_signature",
+      "method_definition",
+    ].includes(d.type)
+  )
+    return;
+  const prefix = d.text.slice(0, name.startIndex - d.startIndex);
+  if (/\b(abstract|get|set|declare)\b/.test(prefix)) return;
+  const method = d.type.startsWith("method");
+  const declaration = d.type.endsWith("signature");
+  if (!declaration && !d.childForFieldName("body")) return;
+  return {
+    node: n,
+    name: name.text,
+    key: `${method}:${/\bstatic\b/.test(prefix)}:${name.text}`,
+    declaration,
+  };
+}
+
+/** Follow only a contiguous, unambiguous overload group in the same AST container. */
+export async function implementationSpan(
+  source: string,
+  path: string,
+  span: { startByte: number; endByte: number },
+): Promise<Implementation> {
+  const raw = Buffer.from(source),
+    lang = detectLanguage(path);
+  invariant(
+    Number.isInteger(span.startByte) &&
+      Number.isInteger(span.endByte) &&
+      span.startByte >= 0 &&
+      span.endByte > span.startByte &&
+      span.endByte <= raw.length,
+    "Invalid implementation source range.",
+  );
+  if (
+    !["python", "typescript", "tsx"].includes(lang) ||
+    /\.(pyi|d\.ts|d\.tsx)$/i.test(path)
+  )
+    return { status: "unsupported" };
+  const start = raw.subarray(0, span.startByte).toString("utf8").length;
+  const end = raw.subarray(0, span.endByte).toString("utf8").length;
+  return withSyntaxTree(source, lang, (root) => {
+    if (root.hasError) return { status: "parse-error" };
+    const python = lang === "python",
+      aliases = python ? overloadAliases(root) : new Set<string>();
+    const targets: Definition[] = [];
+    function visit(container: Node): void {
+      const siblings = children(container).filter((n) => n.type !== "comment");
+      const definitions = siblings.map((n) => definition(n, python, aliases));
+      for (let i = 0; i < definitions.length; i++) {
+        const d = definitions[i];
+        // A class/container hit is not a declaration hit. Require the selected
+        // span to end before the implementation, and contain a whole declaration.
+        if (
+          !d?.declaration ||
+          d.node.startIndex < start ||
+          d.node.endIndex > end
+        )
+          continue;
+        let first = i;
+        while (
+          first > 0 &&
+          definitions[first - 1]?.declaration &&
+          definitions[first - 1]?.key === d.key
+        )
+          first--;
+        let next = first;
+        while (
+          definitions[next]?.declaration &&
+          definitions[next]?.key === d.key
+        )
+          next++;
+        const target = definitions[next];
+        if (
+          !target ||
+          target.declaration ||
+          target.key !== d.key ||
+          target.node.startIndex < end
+        )
+          continue;
+        const matches = siblings.filter((n) => {
+          const node =
+            n.childForFieldName(python ? "definition" : "declaration") ?? n;
+          // TypeScript namespaces augment a function without adding another body.
+          // Keep other same-name declarations in the ambiguity check.
+          if (!python && node.type === "internal_module") return false;
+          return node.childForFieldName("name")?.text === d.name;
+        });
+        if (matches.length !== next - first + 1) continue;
+        // Reject unrelated code included in the selected chunk, but allow trivia
+        // between declarations (the chunker can attach comments/whitespace).
+        const selected = siblings.filter(
+          (n) => n.startIndex < end && n.endIndex > start,
+        );
+        if (
+          selected.some(
+            (n) =>
+              !definitions.slice(first, next).some((x) => x?.node.id === n.id),
+          )
+        )
+          continue;
+        if (!targets.some((t) => t.node.id === target.node.id))
+          targets.push(target);
+      }
+      for (const child of siblings) visit(child);
+    }
+    visit(root);
+    if (targets.length !== 1) return { status: "not-linked" };
+    const target = targets[0]!;
+    const startLine = lineAt(
+      raw,
+      Buffer.byteLength(source.slice(0, target.node.startIndex)),
+    );
+    const endLine = lineAt(
+      raw,
+      Buffer.byteLength(source.slice(0, target.node.endIndex)) - 1,
+    );
+    const text = (source.match(/[^\n]*\n|[^\n]+$/g) ?? [])
+      .slice(startLine - 1, endLine)
+      .join("");
+    if (tokens(text) > CHUNKER.maxTokens) return { status: "too-large" };
+    return {
+      status: "resolved",
+      startLine,
+      endLine,
+      symbol: target.name,
+    };
+  });
+}

package/src/publish.ts

@@ -10,7 +10,14 @@ import {
   sleep,
   type Doc,
 } from "./common.js";
-import { corpusMetadata, records, validateBuild, type Build } from "./build.js";
+import {
+  corpusMetadata,
+  records,
+  recordHash,
+  supportedPreset,
+  validateBuild,
+  type Build,
+} from "./build.js";
 import { branch, tag, one, type CollectionStore } from "./remote.js";
 export type Binding = {
   repoId: string;
@@ -36,6 +43,8 @@ export type Journal = {
   attemptId: string;
   writer: string;
   source: string;
+  gitRef?: string;
+  reuseWriter?: boolean;
   baseline?: Published;
   phase: "branch" | "writing" | "waiting" | "publishing";
   batches: {
@@ -45,6 +54,43 @@ export type Journal = {
   }[];
   candidate?: { name: string; snapshotId: string };
 };
+/** The applied snapshot may differ from the shared canonical commit Tag. */
+export type TrackedBranch = Doc & {
+  gitRef: string;
+  writer: string;
+  applied?: Published;
+  publishedTagName?: string;
+  pending?: { attemptId: string; buildId: string; commitOid: string };
+};
+export function gitBranchName(ref: string): string {
+  invariant(
+    ref.startsWith("refs/heads/") && ref.length > 11,
+    "Expected a full Git branch ref.",
+  );
+  return `git-${hash(ref).slice(0, 40)}`;
+}
+export const gitBranchId = (ref: string): string => `branch-${hash(ref)}`;
+export const lastBuildPath = (state: string, ref?: string): string =>
+  join(state, ref ? `last-build-${hash(ref)}.json` : "last-build.json");
+export async function trackedBranch(
+  store: CollectionStore,
+  binding: Binding,
+  ref: string,
+): Promise<TrackedBranch | undefined> {
+  const doc = await one(store, branch("main"), gitBranchId(ref), true);
+  if (!doc) return undefined;
+  invariant(
+    doc.kind === "manifest" &&
+      doc.role === "git-branch" &&
+      doc.gitRef === ref &&
+      doc.writer === gitBranchName(ref) &&
+      doc.repoId === binding.repoId &&
+      doc.indexId === binding.indexId &&
+      doc.configHash === binding.configHash,
+    "Tracked branch identity mismatch.",
+  );
+  return doc as TrackedBranch;
+}
 export function versionTag(oid: string): string {
   return `ver-${hash(oid).slice(0, 40)}`;
 }
@@ -115,7 +161,11 @@ export async function validateCandidate(
   for (const d of corpusMetadata(b, binding.repoId, binding.indexId, attemptId))
     expected.set(d.id, hash(d));
   for await (const d of store.list(tag(name))) {
-    if (expected.get(d.id) !== hash(d)) return false;
+    try {
+      if (expected.get(d.id) !== recordHash(d, b.preset)) return false;
+    } catch {
+      return false;
+    }
     expected.delete(d.id);
   }
   if (expected.size) return false;
@@ -125,10 +175,8 @@ export async function validateCandidate(
     };
     for await (const d of records(b.directory)) {
       if (d.kind !== "chunk") continue;
-      // Keep a complete surface token: the standard analyzer may retain dots
-      // (e.g. code.ts), so extracting an alphanumeric substring is not a valid probe.
-      const term = String(d.symbol ?? d.searchText).match(/\S+/u)?.[0];
-      if (!term) continue;
+      // Probe an indexed chunk ID: a valid source may contain only stop words
+      // or punctuation, so text-query matches cannot establish readiness.
       query = {
         bool: [
           {
@@ -141,16 +189,8 @@ export async function validateCandidate(
           },
           {
             queryString: {
-              query: d.fileId,
-              defaultField: "fileId",
-              skipSyntax: true,
-            },
-            occur: "filter",
-          },
-          {
-            queryString: {
-              query: term,
-              defaultField: "searchText",
+              query: d.id,
+              defaultField: "id",
               skipSyntax: true,
             },
             occur: "must",
@@ -164,7 +204,8 @@ export async function validateCandidate(
       !found.length ||
       found.some(
         (h) =>
-          h.doc.kind !== "chunk" || b.recordHashes[h.doc.id] !== hash(h.doc),
+          h.doc.kind !== "chunk" ||
+          b.recordHashes[h.doc.id] !== recordHash(h.doc, b.preset),
       )
     )
       return false;
@@ -245,8 +286,8 @@ export async function publish(args: {
     "Build does not belong to this index.",
   );
   invariant(
-    !b.preset.embedding,
-    "CLI publication currently supports only the embedding=none preset.",
+    !b.preset.embedding || supportedPreset(b.preset),
+    "Publication requires a supported lexical or managed embedding preset.",
   );
   await validateBuild(b);
   const journalPath = join(state, "pending.json");
@@ -255,7 +296,9 @@ export async function publish(args: {
     invariant(
       j.buildId === b.buildId &&
         j.recordsHash === b.recordsHash &&
-        j.inventoryHash === b.inventoryHash,
+        j.inventoryHash === b.inventoryHash &&
+        (j.gitRef === b.branchRef ||
+          (j.reuseWriter === undefined && j.writer.startsWith("work-"))),
       "Another build is pending. Resume the artifact recorded in pending.json before importing a new commit.",
     );
     invariant(
@@ -266,6 +309,22 @@ export async function publish(args: {
   const existing = (await published(store, binding)).find(
     (v) => v.commitOid === b.commitOid,
   );
+  const tracked = b.branchRef
+    ? await trackedBranch(store, binding, b.branchRef)
+    : undefined;
+  if (tracked?.pending)
+    invariant(
+      j?.attemptId === tracked.pending.attemptId,
+      "Git branch has a pending import; resume with its original retry journal.",
+    );
+  if (j?.gitRef && j.phase !== "branch")
+    invariant(
+      tracked?.pending?.attemptId === j.attemptId ||
+        (j.phase === "publishing" &&
+          !tracked?.pending &&
+          tracked?.applied?.attemptId === j.attemptId),
+      "Tracked branch import ownership changed; refusing to replay this journal.",
+    );
   if (existing) {
     invariant(
       await validateCandidate(
@@ -277,17 +336,79 @@ export async function publish(args: {
       ),
       "Existing published version differs from this build.",
     );
-    await atomic(join(state, "last-build.json"), {
-      artifact: b.directory,
-      version: existing,
-    });
-    if (j) await rm(journalPath);
-    return existing;
+    // A shared commit Tag does not mean this Git branch has applied that commit.
+    if (!j && (!b.branchRef || tracked?.applied?.commitOid === b.commitOid)) {
+      if (tracked?.applied) {
+        const writer = (await store.branches()).find(
+          (x) => x.name === tracked.writer,
+        );
+        invariant(
+          writer?.snapshotId === tracked.applied.snapshotId,
+          "Writer baseline has changed; refusing to update tracked branch.",
+        );
+        invariant(
+          await validateCandidate(
+            store,
+            tracked.applied.tagName,
+            b,
+            binding,
+            tracked.applied.attemptId,
+          ),
+          "Tracked branch baseline validation failed.",
+        );
+      }
+      await atomic(lastBuildPath(state, b.branchRef), {
+        artifact: b.directory,
+        version: existing,
+      });
+      return existing;
+    }
+    // Legacy manual imports can finish bookkeeping after publication on retry.
+    if (j && !j.gitRef) {
+      await atomic(lastBuildPath(state), {
+        artifact: b.directory,
+        version: existing,
+      });
+      await rm(journalPath);
+      return existing;
+    }
   }
   if (!j) {
     let source = "checkpoint-empty";
     let baseline: Published | undefined;
-    if (args.baseline) {
+    if (b.branchRef) {
+      const writer = (await store.branches()).find(
+        (x) => x.name === gitBranchName(b.branchRef!),
+      );
+      if (tracked?.applied) {
+        baseline = tracked.applied;
+        invariant(
+          writer?.snapshotId === baseline.snapshotId,
+          "Writer baseline has changed; refusing to update tracked branch.",
+        );
+        const pin = (await store.tags()).find(
+          (t) => t.name === baseline!.tagName,
+        );
+        const root = await one(store, tag(baseline.tagName), "__manifest__");
+        invariant(
+          pin?.snapshotId === baseline.snapshotId &&
+            root?.repoId === binding.repoId &&
+            root.indexId === binding.indexId &&
+            root.configHash === binding.configHash &&
+            root.buildId === baseline.buildId &&
+            root.attemptId === baseline.attemptId &&
+            root.recordsHash === baseline.recordsHash &&
+            root.inventoryHash === baseline.inventoryHash &&
+            root.commitOid === baseline.commitOid,
+          "Tracked branch baseline Tag is missing or changed.",
+        );
+      } else {
+        invariant(
+          !writer && !tracked,
+          "Unowned or unfinished tracked branch; retain its retry journal.",
+        );
+      }
+    } else if (args.baseline?.version.writer.startsWith("work-")) {
       const prev = args.baseline;
       await validateBuild(prev.build);
       invariant(
@@ -322,7 +443,9 @@ export async function publish(args: {
       inventoryHash: b.inventoryHash,
       artifact: b.directory,
       attemptId,
-      writer: `work-${attemptId}`,
+      writer: b.branchRef ? gitBranchName(b.branchRef) : `work-${attemptId}`,
+      gitRef: b.branchRef,
+      reuseWriter: !!tracked?.applied,
       source,
       baseline,
       phase: "branch",
@@ -332,15 +455,48 @@ export async function publish(args: {
   }
   const save = () => atomic(journalPath, j);
   if (j.phase === "branch") {
+    if (j.gitRef) {
+      // Record intent before any corpus mutation, including before branch creation.
+      // A fresh local state must not treat a lagging, partially written head as clean.
+      const current = await trackedBranch(store, binding, j.gitRef);
+      invariant(
+        (!current?.pending || current.pending.attemptId === j.attemptId) &&
+          current?.applied?.snapshotId === j.baseline?.snapshotId,
+        "Git branch ownership or baseline changed before import.",
+      );
+      await store.upsert("main", [
+        {
+          ...current,
+          id: gitBranchId(j.gitRef),
+          kind: "manifest",
+          role: "git-branch",
+          schemaVersion: 1,
+          repoId: binding.repoId,
+          indexId: binding.indexId,
+          configHash: binding.configHash,
+          gitRef: j.gitRef,
+          writer: j.writer,
+          pending: {
+            attemptId: j.attemptId,
+            buildId: j.buildId,
+            commitOid: b.commitOid,
+          },
+        },
+      ]);
+    }
     const branches = await store.branches();
     const found = branches.find((x) => x.name === j!.writer);
     if (found)
       invariant(
-        found.parent === j.source &&
+        (j.reuseWriter || found.parent === j.source) &&
           found.snapshotId === (j.baseline?.snapshotId ?? null),
         "Partially created writer has an unexpected baseline.",
       );
     else {
+      invariant(
+        !j.reuseWriter,
+        "Tracked writer is missing; refusing to recreate a nonempty baseline.",
+      );
       const source = branches.find((x) => x.name === j!.source);
       invariant(
         source &&
@@ -368,7 +524,7 @@ export async function publish(args: {
     if (j.baseline)
       for await (const d of store.list(tag(j.baseline.tagName))) {
         oldIds.add(d.id);
-        oldHashes.set(d.id, hash(d));
+        oldHashes.set(d.id, recordHash(d, b.preset));
       }
     const newIds = new Set([
       ...Object.keys(b.recordHashes),
@@ -454,10 +610,12 @@ export async function publish(args: {
   const found = (await store.tags()).find((t) => t.name === name);
   const final = found ?? (await store.tag(name, tag(j.candidate.name)));
   invariant(
-    final.snapshotId === j.candidate.snapshotId,
+    existing
+      ? final.snapshotId === existing.snapshotId
+      : final.snapshotId === j.candidate.snapshotId,
     "Published Tag name already points to a different snapshot.",
   );
-  const result: Published = {
+  const result: Published = existing ?? {
     id: versionId(b.commitOid),
     kind: "manifest",
     role: "published",
@@ -477,8 +635,42 @@ export async function publish(args: {
     counts: b.counts,
     validation: { inventory: true, content: true, query: true },
   };
-  await store.upsert("main", [result]);
-  await atomic(join(state, "last-build.json"), {
+  if (!existing) await store.upsert("main", [result]);
+  if (j.gitRef) {
+    const current = await trackedBranch(store, binding, j.gitRef);
+    invariant(
+      current?.pending?.attemptId === j.attemptId ||
+        (!current?.pending && current?.applied?.attemptId === j.attemptId),
+      "Tracked branch publication ownership changed.",
+    );
+    // Pin the exact writer baseline even when another branch published this commit first.
+    const applied: Published = {
+      ...result,
+      writer: j.writer,
+      snapshotId: j.candidate.snapshotId,
+      tagName:
+        final.snapshotId === j.candidate.snapshotId
+          ? final.name
+          : j.candidate.name,
+      attemptId: j.attemptId,
+    };
+    await store.upsert("main", [
+      {
+        id: gitBranchId(j.gitRef),
+        kind: "manifest",
+        role: "git-branch",
+        schemaVersion: 1,
+        repoId: binding.repoId,
+        indexId: binding.indexId,
+        configHash: binding.configHash,
+        gitRef: j.gitRef,
+        writer: j.writer,
+        applied,
+        publishedTagName: result.tagName,
+      },
+    ]);
+  }
+  await atomic(lastBuildPath(state, j.gitRef), {
     artifact: b.directory,
     version: result,
   });

package/src/releases.ts

@@ -1,7 +1,12 @@
 import { hash, invariant, type Doc } from "./common.js";
 import { gitTags } from "./git.js";
 import { one, branch, type CollectionStore } from "./remote.js";
-import { published, type Binding, type Published } from "./publish.js";
+import {
+  published,
+  trackedBranch,
+  type Binding,
+  type Published,
+} from "./publish.js";
 export const releaseAlias = (ref: string) => `rel-${hash(ref).slice(0, 40)}`;
 export const releaseId = (ref: string) => `ref-${hash(ref)}`;
 /** Only observed local tags. Absence is not deletion authority; this command never prunes. */
@@ -58,11 +63,39 @@ export async function resolveVersion(
   binding: Binding,
   selector: string,
 ): Promise<Published> {
-  const versions = await published(store, binding);
   const ref = selector.startsWith("refs/tags/")
     ? selector
     : `refs/tags/${selector}`;
-  const control = await one(store, branch("main"), releaseId(ref), true);
+  const control = selector.startsWith("refs/heads/")
+    ? undefined
+    : await one(store, branch("main"), releaseId(ref), true);
+  const headRef = selector.startsWith("refs/heads/")
+    ? selector
+    : `refs/heads/${selector}`;
+  const head =
+    !selector.startsWith("refs/") || selector.startsWith("refs/heads/")
+      ? await trackedBranch(store, binding, headRef)
+      : undefined;
+  invariant(
+    !(head && control),
+    "Ambiguous branch/tag name: use refs/heads/... or refs/tags/...",
+  );
+  // Pin moving ref control before listing versions. Otherwise publication can
+  // advance the ref to a version absent from the earlier list.
+  const versions = await published(store, binding);
+  if (head) {
+    invariant(
+      head.applied,
+      "Git branch has no published version yet; finish its import.",
+    );
+    const result = versions.find(
+      (v) =>
+        v.commitOid === head.applied!.commitOid &&
+        v.tagName === head.publishedTagName,
+    );
+    invariant(result, "Git branch target is not a published version.");
+    return result;
+  }
   // Like Git import, a known tag name takes precedence over an OID spelling.
   // Its pending or invalid state must not fall back to an unrelated commit.
   if (!control && !selector.startsWith("refs/")) {