@functional-systems/srcx 0.1.0-dev.1

package/src/remote.ts

@@ -0,0 +1,230 @@
+import {
+  LambdaDBClient,
+  branchRef,
+  tagRef,
+  branchSource,
+  tagSource,
+  tagTarget,
+  type IndexConfigsUnion,
+} from "@functional-systems/lambdadb";
+import { invariant, type Doc } from "./common.js";
+import type { Settings } from "./settings.js";
+export type Ref = { kind: "branch" | "tag"; name: string };
+export type Snapshot = { name: string; snapshotId: string };
+export type Branch = {
+  name: string;
+  snapshotId: string | null;
+  parent: string | null;
+};
+export type Alias = {
+  name: string;
+  target: string;
+  kind: string;
+  dangling: boolean;
+};
+export type Hit = { doc: Doc; score?: number };
+/** Narrow protocol boundary for fault-injection tests; LambdaDB is the only CLI backend. */
+export interface CollectionStore {
+  fetch(ref: Ref, ids: string[], consistent?: boolean): Promise<Doc[]>;
+  list(ref: Ref): AsyncIterable<Doc>;
+  upsert(branch: string, docs: Doc[]): Promise<void>;
+  delete(branch: string, ids: string[]): Promise<void>;
+  branches(): Promise<Branch[]>;
+  branch(name: string, source: string): Promise<void>;
+  tags(): Promise<Snapshot[]>;
+  tag(name: string, source: Ref): Promise<Snapshot>;
+  aliases(): Promise<Alias[]>;
+  alias(name: string, target: string, exists: boolean): Promise<void>;
+  query(
+    tag: string,
+    query: Record<string, unknown>,
+    size: number,
+  ): Promise<Hit[]>;
+}
+export class RemoteError extends Error {}
+async function request<T>(
+  operation: string,
+  call: () => Promise<T>,
+): Promise<T> {
+  try {
+    return await call();
+  } catch (e) {
+    // SDK errors can contain response bodies/source text. Never echo them or credentials.
+    const status = (e as { statusCode?: number }).statusCode;
+    throw new RemoteError(
+      `LambdaDB ${operation} failed${typeof status === "number" ? ` (HTTP ${status})` : ""}. Mutation outcome may be unknown; retain the retry journal.`,
+      { cause: e },
+    );
+  }
+}
+export class LambdaRemote {
+  readonly client: LambdaDBClient;
+  constructor(readonly settings: Settings) {
+    const key = process.env[settings.apiKeyEnv];
+    invariant(key, `Missing environment variable ${settings.apiKeyEnv}.`);
+    this.client = new LambdaDBClient({
+      baseUrl: settings.endpoint,
+      projectName: settings.project,
+      projectApiKey: key,
+      retryConfig: { strategy: "none" },
+      timeoutMs: 30_000,
+    });
+  }
+  async collections() {
+    return request(
+      "list collections",
+      async () =>
+        (await this.client.listAllCollections({ size: 100 })).collections,
+    );
+  }
+  async create(
+    name: string,
+    indexConfigs: Record<string, IndexConfigsUnion>,
+    description: string,
+    tags: Record<string, string>,
+  ) {
+    await request("create collection", () =>
+      this.client.createCollection({
+        collectionName: name,
+        indexConfigs,
+        description,
+        tags,
+      }),
+    );
+  }
+  store(name: string): CollectionStore {
+    const c = this.client.collection(name);
+    const ref = (r: Ref) =>
+      r.kind === "tag" ? tagRef(r.name) : branchRef(r.name);
+    return {
+      fetch: (r, ids, consistent = false) =>
+        request("fetch", async () => {
+          invariant(
+            !consistent || r.kind === "branch",
+            "Consistent reads require a Branch.",
+          );
+          const result = await c.docs.fetch(
+            r.kind === "branch"
+              ? {
+                  ref: branchRef(r.name),
+                  ids,
+                  consistentRead: consistent,
+                  includeVectors: true,
+                }
+              : {
+                  ref: tagRef(r.name),
+                  ids,
+                  consistentRead: false,
+                  includeVectors: true,
+                },
+          );
+          return result.docs.map((d) => d.doc as Doc);
+        }),
+      async *list(r) {
+        let cursor: string | undefined;
+        const seen = new Set<string>();
+        do {
+          const page = await request("list documents", () =>
+            c.docs.list({
+              ref: ref(r),
+              size: 100,
+              includeVectors: true,
+              pageToken: cursor,
+            }),
+          );
+          for (const d of page.docs) yield d.doc as Doc;
+          cursor = page.nextPageToken || undefined;
+          if (cursor) {
+            invariant(
+              !seen.has(cursor),
+              "LambdaDB repeated a pagination cursor.",
+            );
+            seen.add(cursor);
+          }
+        } while (cursor);
+      },
+      upsert: async (branch, docs) => {
+        await request("upsert", () => c.docs.upsert({ branch, docs }));
+      },
+      delete: async (branch, ids) => {
+        if (ids.length)
+          await request("delete", () => c.docs.delete({ branch, ids }));
+      },
+      branches: () =>
+        request("list branches", async () =>
+          (await c.branches.list()).branches.map((b) => ({
+            name: b.name,
+            snapshotId: b.headSnapshot?.snapshotId ?? null,
+            parent: b.parentBranch?.name ?? null,
+          })),
+        ),
+      branch: async (name, source) => {
+        await request("create branch", () =>
+          c.branches.create({ branchName: name, source: branchSource(source) }),
+        );
+      },
+      tags: () =>
+        request("list tags", async () =>
+          (await c.tags.list()).tags.map((t) => ({
+            name: t.name,
+            snapshotId: t.snapshotId,
+          })),
+        ),
+      tag: (name, source) =>
+        request("create tag", async () => {
+          const t = (
+            await c.tags.create({
+              tagName: name,
+              source:
+                source.kind === "tag"
+                  ? tagSource(source.name)
+                  : branchSource(source.name),
+            })
+          ).tag;
+          return { name: t.name, snapshotId: t.snapshotId };
+        }),
+      aliases: () =>
+        request("list aliases", async () =>
+          (await c.aliases.list()).aliases.map((a) => ({
+            name: a.aliasName,
+            target: a.targetName,
+            kind: a.targetKind,
+            dangling: a.dangling,
+          })),
+        ),
+      alias: async (name, target, exists) => {
+        await request("set alias", () =>
+          exists
+            ? c.aliases.retarget(name, { target: tagTarget(target) })
+            : c.aliases.create({ aliasName: name, target: tagTarget(target) }),
+        );
+      },
+      query: (tag, query, size) =>
+        request("query", async () =>
+          (
+            await c.query({
+              ref: tagRef(tag),
+              consistentRead: false,
+              query,
+              size,
+            })
+          ).docs.map((h) => ({ doc: h.doc as Doc, score: h.score })),
+        ),
+    };
+  }
+}
+export const branch = (name: string): Ref => ({ kind: "branch", name });
+export const tag = (name: string): Ref => ({ kind: "tag", name });
+export async function one(
+  store: CollectionStore,
+  ref: Ref,
+  id: string,
+  consistent = false,
+): Promise<Doc | undefined> {
+  const docs = await store.fetch(ref, [id], consistent);
+  invariant(
+    docs.length <= 1 && docs.every((d) => d.id === id),
+    "Unexpected fetch result.",
+  );
+  return docs[0];
+}

package/src/repository.ts

@@ -0,0 +1,306 @@
+import { randomUUID } from "node:crypto";
+import { join } from "node:path";
+import {
+  atomic,
+  hash,
+  invariant,
+  optionalJson,
+  PURPOSE,
+  type Doc,
+} from "./common.js";
+import { PRESET, INDEX_CONFIGS } from "./build.js";
+import { identity, type Identity } from "./git.js";
+import { LambdaRemote, one, branch } from "./remote.js";
+import { exclusive, type Binding } from "./publish.js";
+import { stateRoot, type Settings } from "./settings.js";
+import type { IndexConfigsUnion } from "@functional-systems/lambdadb";
+export type Repository = Binding & {
+  collection: string;
+  name: string;
+  description: string;
+  tags: Record<string, string>;
+};
+export function destination(s: Settings, collection: string): string {
+  return join(
+    stateRoot(),
+    "destinations",
+    hash([s.endpoint, s.project]),
+    collection,
+  );
+}
+export function collectionName(
+  key: string,
+  name: string,
+  configHash: string,
+): string {
+  const slug =
+    name
+      .toLowerCase()
+      .replace(/[^a-z0-9]+/g, "-")
+      .replace(/^-+|-+$/g, "")
+      .slice(0, 24) || "repo";
+  return `code-${slug}-${hash(["collection-v1", key, configHash]).slice(0, 16)}`;
+}
+/** LambdaDB adds its built-in keyword id index to collection metadata. */
+export function matchesIndexSchema(actual: unknown): boolean {
+  if (!actual || typeof actual !== "object" || Array.isArray(actual))
+    return false;
+  const { id, ...fields } = actual as Record<string, unknown>;
+  if (id !== undefined && hash(id) !== hash({ type: "keyword" })) return false;
+  return hash(fields) === hash(INDEX_CONFIGS);
+}
+function descriptor(r: Repository): Doc {
+  return {
+    id: "__repo__",
+    kind: "manifest",
+    role: "repository",
+    schemaVersion: 1,
+    configHash: r.configHash,
+    repoKey: r.repoKey,
+    repoId: r.repoId,
+    indexId: r.indexId,
+    name: r.name,
+    preset: PRESET,
+    initialization: "ready",
+  };
+}
+function decode(
+  doc: Doc | undefined,
+  c: {
+    collectionName: string;
+    description: string;
+    tags: Record<string, string>;
+    indexConfigs: unknown;
+  },
+): Repository {
+  invariant(
+    doc?.role === "repository" &&
+      doc.initialization === "ready" &&
+      doc.configHash === hash(PRESET) &&
+      hash(doc.preset) === hash(PRESET),
+    "Repository is partially initialized or uses an unsupported preset.",
+  );
+  invariant(
+    doc.indexId === c.tags["index-id"] &&
+      doc.configHash === c.tags["config-hash"] &&
+      c.tags.purpose === PURPOSE,
+    "Repository descriptor and Collection labels disagree.",
+  );
+  invariant(
+    matchesIndexSchema(c.indexConfigs),
+    "Collection schema differs from the pinned preset.",
+  );
+  invariant(
+    typeof doc.repoKey === "string" &&
+      typeof doc.repoId === "string" &&
+      typeof doc.indexId === "string" &&
+      typeof doc.name === "string",
+    "Malformed repository descriptor.",
+  );
+  return {
+    collection: c.collectionName,
+    repoKey: doc.repoKey,
+    repoId: doc.repoId,
+    indexId: doc.indexId,
+    name: doc.name,
+    configHash: String(doc.configHash),
+    description: c.description,
+    tags: c.tags,
+  };
+}
+export async function discover(remote: LambdaRemote): Promise<{
+  repositories: Repository[];
+  partial: { collection: string; reason: string }[];
+}> {
+  const repositories: Repository[] = [];
+  const partial: { collection: string; reason: string }[] = [];
+  for (const c of await remote.collections()) {
+    if (c.tags.purpose !== PURPOSE) continue;
+    const doc = await one(
+      remote.store(c.collectionName),
+      branch("main"),
+      "__repo__",
+      true,
+    );
+    try {
+      repositories.push(decode(doc, c));
+    } catch {
+      partial.push({
+        collection: c.collectionName,
+        reason: "Missing/incompatible repository descriptor or schema.",
+      });
+    }
+  }
+  return { repositories, partial };
+}
+export async function selectRepository(
+  remote: LambdaRemote,
+  selector: string,
+): Promise<Repository> {
+  const { repositories, partial } = await discover(remote);
+  const selected = repositories.filter(
+    (r) =>
+      r.name === selector ||
+      r.repoKey === selector ||
+      r.repoKey.split("/").slice(1).join("/") === selector ||
+      r.collection === selector,
+  );
+  invariant(
+    selected.length === 1,
+    selected.length
+      ? "Ambiguous repository; use its full source identity or Collection name."
+      : `Repository not found${partial.length ? "; partially initialized collections exist (see repo list)" : ""}.`,
+  );
+  return selected[0]!;
+}
+export async function register(
+  remote: LambdaRemote,
+  args: {
+    path: string;
+    remote?: string;
+    description?: string;
+    labels?: Record<string, string>;
+  },
+): Promise<Repository> {
+  const source = await identity(args.path, args.remote);
+  const configHash = hash(PRESET),
+    collection = collectionName(source.key, source.name, configHash),
+    state = destination(remote.settings, collection);
+  return exclusive(state, async () => {
+    const initPath = join(state, "initialization.json");
+    let intent = await optionalJson<Repository>(initPath);
+    const existing = (await remote.collections()).find(
+      (c) => c.collectionName === collection,
+    );
+    if (existing) {
+      const d = await one(
+        remote.store(collection),
+        branch("main"),
+        "__repo__",
+        true,
+      );
+      if (d) {
+        const result = decode(d, existing);
+        invariant(result.repoKey === source.key, "Collection name collision.");
+        await atomic(join(state, "attachment.json"), source);
+        return result;
+      }
+    }
+    if (!intent) {
+      invariant(
+        !existing,
+        "Collection exists without a descriptor or a local provisioning journal; refusing adoption.",
+      );
+      const labels = args.labels ?? {};
+      const reserved = ["purpose", "repository", "index-id", "config-hash"];
+      invariant(
+        Object.keys(labels).length <= 1 &&
+          !Object.keys(labels).some((k) => reserved.includes(k)),
+        "At most one nonreserved metadata tag is allowed.",
+      );
+      invariant(
+        Object.entries(labels).every(
+          ([k, v]) =>
+            /^[A-Za-z0-9_.-]{1,63}$/.test(k) &&
+            v.length >= 1 &&
+            v.length <= 127 &&
+            v.trim().length > 0 &&
+            !/[:#,]/.test(v),
+        ),
+        "Invalid metadata tag.",
+      );
+      const description =
+        args.description ??
+        `Source, tests, and documentation for ${source.key.slice(0, 170)}; version-aware code search.`;
+      invariant(
+        description.length <= 255,
+        "Description exceeds 255 characters.",
+      );
+      const indexId = randomUUID();
+      intent = {
+        repoKey: source.key,
+        name: source.name,
+        repoId: randomUUID(),
+        indexId,
+        collection,
+        configHash,
+        description,
+        tags: {
+          purpose: PURPOSE,
+          ...(source.key.length <= 127 && !/[:#,]/.test(source.key)
+            ? { repository: source.key }
+            : {}),
+          "index-id": indexId,
+          "config-hash": configHash,
+          ...labels,
+        },
+      };
+      await atomic(initPath, intent);
+    }
+    invariant(
+      intent.repoKey === source.key && intent.configHash === configHash,
+      "Provisioning journal identity mismatch.",
+    );
+    if (existing)
+      invariant(
+        hash(existing.tags) === hash(intent.tags) &&
+          matchesIndexSchema(existing.indexConfigs),
+        "Existing Collection differs from provisioning journal.",
+      );
+    else
+      await remote.create(
+        collection,
+        JSON.parse(JSON.stringify(INDEX_CONFIGS)) as Record<
+          string,
+          IndexConfigsUnion
+        >,
+        intent.description,
+        intent.tags,
+      );
+    const store = remote.store(collection);
+    const branches = await store.branches();
+    const empty = branches.find((b) => b.name === "checkpoint-empty");
+    if (empty)
+      invariant(
+        empty.snapshotId === null && empty.parent === "main",
+        "Empty checkpoint has been modified.",
+      );
+    else {
+      const main = branches.find((b) => b.name === "main");
+      invariant(
+        main && main.snapshotId === null,
+        "Main is not empty; cannot safely initialize checkpoint.",
+      );
+      await store.branch("checkpoint-empty", "main");
+    }
+    await store.upsert("main", [descriptor(intent)]);
+    const c = (await remote.collections()).find(
+      (c) => c.collectionName === collection,
+    );
+    invariant(c, "Created Collection not discoverable; retry repo add.");
+    const result = decode(
+      await one(store, branch("main"), "__repo__", true),
+      c,
+    );
+    await atomic(join(state, "attachment.json"), source);
+    return result;
+  });
+}
+export async function attachment(
+  s: Settings,
+  r: Repository,
+): Promise<Identity> {
+  const saved = await optionalJson<Identity>(
+    join(destination(s, r.collection), "attachment.json"),
+  );
+  invariant(
+    saved,
+    "No local checkout attached. Use repo add --path for this repository.",
+  );
+  const current = await identity(saved.path, saved.remote);
+  invariant(
+    current.key === r.repoKey,
+    "Attached checkout now has a different source identity.",
+  );
+  return current;
+}