@agentshouse/kit 0.0.1 → 0.1.0-alpha.1

package/src/mount.js

@@ -0,0 +1,582 @@
+import { createHash } from "node:crypto";
+import { join } from "node:path";
+import { readFile, writeFile } from "node:fs/promises";
+import { apply, links, parse, serialize } from "@agentshouse/mdmodel";
+import {
+  KIT_DOOR_BATCH_PATH,
+  KIT_DOOR_BOOTSTRAP_PATH,
+  KIT_DOOR_DISCOVER_PATH,
+  KIT_DOOR_UNMOUNT_PATH,
+  MIGRATOR_ATTACHMENTS_HOME,
+  MIGRATOR_CONTENT_HOME,
+  MOUNT_REPORT_NAME,
+} from "./contract.js";
+import { heldControl, houseGetJson, houseRequest, maintainControl, openSession } from "./house.js";
+import { operationId } from "./operation.js";
+import { placeAttachment } from "./original.js";
+import { KitRefusal } from "./refusal.js";
+import {
+  OPERATIONS,
+  assertAdmitted,
+  admitsOperation,
+  isProtected,
+  protectedPathRefusal,
+  operationDeniedRefusal,
+  housePath,
+  VERSION,
+} from "./sync-library.js";
+
+const COLLECTION_HOME = "library";
+const MODEL_HOME = "settings/model";
+
+function admitModel(authority, relative) {
+  if (isProtected(relative, authority)) throw protectedPathRefusal(authority, relative);
+  if (!admitsOperation(relative, authority, "write")) {
+    throw operationDeniedRefusal(authority, relative, "write");
+  }
+}
+
+const PENDING_LINK_SCHEME = "house-attachment";
+const ATTACHMENT_PREFIX = `${MIGRATOR_ATTACHMENTS_HOME}/`;
+
+function refuse(code, detail = null) {
+  throw new KitRefusal(code, detail);
+}
+
+function sessionDependencies(options, dependencies) {
+  return {
+    fetch: dependencies.fetch ?? options.fetch,
+    now: dependencies.now ?? options.now,
+  };
+}
+
+async function withSession(options, dependencies, operation) {
+  const session = await openSession(options, sessionDependencies(options, dependencies));
+  return maintainControl(session, (signal) => operation(session, signal), options.signal);
+}
+
+async function door(session, path, body, signal) {
+  return houseRequest(
+    session.house,
+    await heldControl(session, signal),
+    path,
+    body,
+    session.dependencies,
+    signal,
+  );
+}
+
+export function mountTarget(subtree) {
+  if (typeof subtree !== "string" || !subtree.startsWith("/") || subtree.includes("//")) {
+    refuse("kit_mount_subtree_invalid", subtree);
+  }
+  const segments = subtree.slice(1).split("/");
+  const own = segments[0] === "private";
+  if (!own && (segments[0] !== "rooms" || segments.length < 2)) {
+    refuse("kit_mount_subtree_invalid", subtree);
+  }
+  const relative = segments.slice(own ? 1 : 2);
+  if (
+    relative.length < 2 ||
+    relative[0] !== COLLECTION_HOME ||
+    segments.some((segment) => !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(segment))
+  ) {
+    refuse("kit_mount_subtree_invalid", subtree);
+  }
+  return {
+    kind: own ? "private" : "room",
+    handle: own ? null : segments[1],
+    root: own ? "/private" : `/rooms/${segments[1]}`,
+    subtree,
+    relative: relative.join("/"),
+  };
+}
+
+function discoveryOf(value) {
+  if (typeof value?.md_model_version !== "string" || !Array.isArray(value.authorities)) {
+    refuse("kit_mount_rejected", "discover");
+  }
+  if (value.md_model_version !== VERSION) {
+    refuse("kit_md_model_version_mismatch", value.md_model_version);
+  }
+  if (!Number.isSafeInteger(value.bounds?.max_batch_operations) || value.bounds.max_batch_operations <= 0) {
+    refuse("kit_mount_rejected", "discover.bounds.max_batch_operations");
+  }
+  return value;
+}
+
+function authorityFor(discovery, target) {
+  const found = discovery.authorities.find((entry) =>
+    target.kind === "private" ? entry.kind === "private" : entry.room_handle === target.handle,
+  );
+  if (found === undefined) refuse("kit_mount_authority_unknown", target.root);
+  if (
+    !Array.isArray(found.protected_paths) ||
+    found.operations === null ||
+    typeof found.operations !== "object" ||
+    OPERATIONS.some((operation) => !Array.isArray(found.operations[operation]))
+  ) {
+    refuse("kit_mount_rejected", "discover");
+  }
+  return {
+    kind: found.kind,
+    authority: found.kind === "private" ? "private" : found.room_ref,
+    path: target.root,
+    protected_paths: found.protected_paths,
+    operations: found.operations,
+    files: [],
+  };
+}
+
+function hrefSuffix(href) {
+  const match = /[?#]/.exec(href);
+  return match === null ? "" : href.slice(match.index);
+}
+
+function replaceLink(document, href, destination) {
+  const marker = `house-kit-link:${operationId()}`;
+  const located = apply(document, { kind: "link", href, to: marker });
+  if (located.outcome !== "applied") refuse("kit_mount_link_unwritable", href);
+  const cleared = parse(serialize(located.document).replaceAll(`${marker}${hrefSuffix(href)}`, marker));
+  if (!cleared.ok) refuse("kit_mount_link_unwritable", href);
+  const changed = apply(cleared.document, { kind: "link", href: marker, to: destination });
+  if (changed.outcome !== "applied") refuse("kit_mount_link_unwritable", href);
+  return changed.document;
+}
+
+function sourceHref(source, suffix) {
+  const sourceFragment = source.indexOf("#");
+  const base = sourceFragment === -1 ? source : source.slice(0, sourceFragment);
+  const priorFragment = sourceFragment === -1 ? "" : source.slice(sourceFragment);
+  const suffixFragment = suffix.indexOf("#");
+  const query = suffixFragment === -1 ? suffix : suffix.slice(0, suffixFragment);
+  const fragment = suffixFragment === -1 ? priorFragment : suffix.slice(suffixFragment);
+  if (query === "") return base + fragment;
+  const separator = base.includes("?") ? (/[?&]$/.test(base) ? "" : "&") : "?";
+  return base + separator + query.slice(1) + fragment;
+}
+
+export function rewriteAttachmentLinks(held, mintVersion) {
+  const content = {};
+  const saves = [];
+  for (const [relative, text] of Object.entries(held.content).sort()) {
+    const file = `${MIGRATOR_CONTENT_HOME}/${relative}`;
+    const parsed = parse(text, relative);
+    if (!parsed.ok) {
+      content[relative] = text;
+      continue;
+    }
+    let document = parsed.document;
+    for (const link of links(document, file)) {
+      if (!link.target.startsWith(ATTACHMENT_PREFIX)) continue;
+      const path = link.target.slice(ATTACHMENT_PREFIX.length);
+      const declared = held.manifest.attachments.find(
+        (entry) => entry.path === path && entry.download === "downloaded",
+      );
+      if (declared === undefined) refuse("kit_migrator_attachment_undeclared", link.target);
+      if (!declared.referrers.includes(relative)) {
+        refuse("kit_migrator_referrer_undeclared", `${relative} -> ${path}`);
+      }
+      const version = mintVersion(path);
+      const applied = apply(document, {
+        kind: "link",
+        href: link.href,
+        to: `${PENDING_LINK_SCHEME}:${version}`,
+      });
+      if (applied.outcome !== "applied") refuse("kit_mount_link_unwritable", link.href);
+      document = applied.document;
+      let save = saves.find((entry) => entry.path === path);
+      if (save === undefined) {
+        save = { path, version, documents: [], hrefs: [] };
+        saves.push(save);
+      }
+      if (!save.documents.includes(relative)) save.documents.push(relative);
+      const suffix = hrefSuffix(link.href);
+      const href = `${PENDING_LINK_SCHEME}:${version}${suffix}`;
+      if (!save.hrefs.some((entry) => entry.document === relative && entry.href === href)) {
+        save.hrefs.push({ document: relative, href, suffix });
+      }
+    }
+    content[relative] = serialize(document);
+  }
+  for (const entry of held.manifest.attachments) {
+    if (entry.download !== "downloaded" || saves.some((save) => save.path === entry.path)) continue;
+    refuse("kit_migrator_attachment_unlinked", entry.path);
+  }
+  return { content, saves };
+}
+
+export function withinBounds(bounds, target, content) {
+  if (bounds === null || typeof bounds !== "object") return;
+  const files = Object.keys(content);
+  if (typeof bounds.max_mount_files === "number" && files.length > bounds.max_mount_files) {
+    refuse("mount_files_exceeded", `${files.length}`);
+  }
+  let bytes = 0;
+  for (const relative of files) {
+    const size = Buffer.byteLength(content[relative], "utf8");
+    if (typeof bounds.max_authored_file_bytes === "number" && size > bounds.max_authored_file_bytes) {
+      refuse("content_too_large", `${target.subtree}/${relative}`);
+    }
+    bytes += size;
+    const path = `${target.subtree}/${relative}`;
+    if (typeof bounds.max_path_length === "number" && path.length > bounds.max_path_length) {
+      refuse("invalid_authored_path", path);
+    }
+    if (
+      typeof bounds.max_path_depth === "number" &&
+      `${target.relative}/${relative}`.split("/").length > bounds.max_path_depth
+    ) {
+      refuse("invalid_authored_path", path);
+    }
+  }
+  if (typeof bounds.max_mount_bytes === "number" && bytes > bounds.max_mount_bytes) {
+    refuse("batch_too_large", `${bytes}`);
+  }
+}
+
+async function currentFiles(session, prefixes, signal) {
+  const answer = await door(
+    session,
+    KIT_DOOR_BOOTSTRAP_PATH,
+    { prefixes },
+    signal,
+  );
+  const bundle =
+    typeof answer?.transfer?.url === "string"
+      ? await houseGetJson(answer.transfer.url, session.dependencies, signal)
+      : answer;
+  if (!Array.isArray(bundle?.scopes)) refuse("kit_mount_rejected", "bootstrap");
+  const held = new Map();
+  for (const scope of bundle.scopes) {
+    for (const entry of scope.contents ?? []) {
+      if (typeof entry?.path !== "string" || typeof entry.content !== "string") continue;
+      held.set(entry.path, entry.content);
+    }
+  }
+  return held;
+}
+
+async function prepareModel(session, target, authority, model, held, signal) {
+  const prepared = [];
+  const changes = [];
+  for (const [name, definition] of Object.entries(model).sort()) {
+    const relative = `${MODEL_HOME}/${name}`;
+    admitModel(authority, relative);
+    prepared.push(relative);
+    changes.push({ op: "create", path: housePath(target.root, relative), content: definition });
+  }
+  if (changes.length === 0) return { prepared, adopted: [], written: [] };
+  const adopted = [];
+  const pending = [];
+  for (const change of changes) {
+    const relative = change.path.slice(target.root.length + 1);
+    const current = held.get(relative);
+    if (current === undefined) {
+      pending.push(change);
+      continue;
+    }
+    if (current !== change.content) refuse("kit_mount_model_incompatible", change.path);
+    adopted.push(relative);
+  }
+  if (pending.length > 0) {
+    await door(
+      session,
+      KIT_DOOR_BATCH_PATH,
+      { operation_id: operationId(), changes: pending },
+      signal,
+    );
+  }
+  return { prepared, adopted, written: pending.map((change) => change.path) };
+}
+
+function mountedRevisions(outcome, target) {
+  if (
+    !Array.isArray(outcome?.contents) ||
+    typeof outcome.position !== "string" ||
+    typeof outcome.log_epoch !== "string"
+  ) {
+    refuse("kit_mount_rejected", "batch");
+  }
+  const revisions = new Map();
+  const prefix = `${target.subtree}/`;
+  for (const entry of outcome.contents) {
+    if (typeof entry?.path !== "string" || typeof entry.revision !== "string") continue;
+    if (!entry.path.startsWith(prefix)) continue;
+    revisions.set(entry.path.slice(prefix.length), entry.revision);
+  }
+  return revisions;
+}
+
+function transferFailure(error) {
+  return error instanceof KitRefusal ? error.code : "kit_attachment_transfer_failed";
+}
+
+function pendingTransfer(held, save) {
+  const declared = held.manifest.attachments.find((entry) => entry.path === save.path);
+  return {
+    path: save.path,
+    documents: save.documents,
+    source: declared.source,
+    name: declared.name,
+    version: save.version,
+    downloaded: true,
+    attachment: null,
+    save: { status: "pending", failure: null },
+  };
+}
+
+async function submitAttachments(session, held, target, authority, plan, signal) {
+  const transfers = [];
+  for (const save of plan.saves) {
+    const local = held.attachments.get(save.path);
+    const declared = held.manifest.attachments.find((entry) => entry.path === save.path);
+    const record = pendingTransfer(held, save);
+    try {
+      const body = await readFile(local.file);
+      if (body.length !== local.bytes || createHash("sha256").update(body).digest("hex") !== local.sha256) {
+        refuse("kit_migrator_attachment_changed", save.path);
+      }
+      const placed = await placeAttachment(session, {
+        version: save.version,
+        name: declared.name,
+        mediaType: declared.mediaType ?? undefined,
+        room: target.kind === "room" ? authority.authority : null,
+        body,
+        signal,
+      });
+      record.attachment = placed.attachment;
+      record.save = placed.save;
+    } catch (error) {
+      record.save = { status: "failed", failure: transferFailure(error) };
+    }
+    transfers.push(record);
+  }
+  return transfers;
+}
+
+function reportRow(values) {
+  return `| ${values.map((value) => String(value ?? "").replace(/\|/g, "&#124;").replace(/[\r\n]/g, " ")).join(" | ")} |`;
+}
+
+export function reportDocument(result) {
+  const lines = [
+    "# Migration report",
+    "",
+    `Migrator: ${result.migrator}`,
+    `Subtree: ${result.subtree}`,
+    `Documents mounted: ${result.documents}`,
+    `Model definitions prepared: ${result.model.prepared.length}`,
+    "",
+    "## Attachment transfers",
+    "",
+    reportRow(["Source", "Referring document", "Local download", "Save", "Attachment", "Version", "Detail"]),
+    reportRow(["---", "---", "---", "---", "---", "---", "---"]),
+  ];
+  for (const transfer of result.transfers) {
+    for (const document of transfer.documents) {
+      lines.push(
+        reportRow([
+          transfer.source,
+          document,
+          "downloaded",
+          transfer.save.status,
+          transfer.save.status === "saved" ? transfer.attachment : "",
+          transfer.version,
+          transfer.save.failure ?? "",
+        ]),
+      );
+    }
+  }
+  for (const missing of result.unavailable) {
+    for (const referrer of missing.referrers) {
+      lines.push(
+        reportRow([
+          missing.source,
+          referrer,
+          missing.download,
+          "skipped",
+          "",
+          "",
+          missing.failure ?? "",
+        ]),
+      );
+    }
+  }
+  lines.push("");
+  return `${lines.join("\n")}\n`;
+}
+
+export async function mountPackage(held, options, dependencies = {}) {
+  const target = mountTarget(options.subtree);
+  const minted = new Map();
+  const plan = rewriteAttachmentLinks(held, (path) => {
+    if (!minted.has(path)) minted.set(path, operationId());
+    return minted.get(path);
+  });
+  return withSession(options, dependencies, async (session, signal) => {
+    const discovery = discoveryOf(
+      await door(session, KIT_DOOR_DISCOVER_PATH, { prefixes: [target.root] }, signal),
+    );
+    const authority = authorityFor(discovery, target);
+    for (const relative of Object.keys(plan.content)) {
+      assertAdmitted(authority, `${target.relative}/${relative}`, "write");
+    }
+    const reportRelative = `${target.relative}/${MOUNT_REPORT_NAME}`;
+    assertAdmitted(authority, reportRelative, "write");
+    for (const name of Object.keys(held.model)) {
+      admitModel(authority, `${MODEL_HOME}/${name}`);
+    }
+    const existing = Object.keys(held.model).length > 0
+      ? await currentFiles(session, [`${target.root}/${MODEL_HOME}`], signal)
+      : new Map();
+    const unavailable = held.manifest.attachments
+      .filter((entry) => entry.download !== "downloaded")
+      .map((entry) => ({ source: entry.source, referrers: entry.referrers,
+        download: entry.download, failure: entry.failure }));
+    const pending = plan.saves.map((save) => pendingTransfer(held, save));
+    const initialReport = reportDocument({ migrator: held.manifest.migrator,
+      subtree: target.subtree, documents: Object.keys(plan.content).length,
+      model: { prepared: Object.keys(held.model) }, transfers: pending, unavailable });
+    const mounting = { ...plan.content, [MOUNT_REPORT_NAME]: initialReport };
+    withinBounds(discovery.bounds, target, mounting);
+    const model = await prepareModel(session, target, authority, held.model, existing, signal);
+    const outcome = await door(
+      session,
+      KIT_DOOR_BATCH_PATH,
+      {
+        operation_id: operationId(),
+        mount: target.subtree,
+        changes: Object.entries(mounting)
+          .sort()
+          .map(([relative, content]) => ({
+            op: "create",
+            path: `${target.subtree}/${relative}`,
+            content,
+          })),
+      },
+      signal,
+    );
+    const revisions = mountedRevisions(outcome, target);
+    const result = {
+      migrator: held.manifest.migrator,
+      subtree: target.subtree,
+      authority: outcome.authority,
+      position: outcome.position,
+      log_epoch: outcome.log_epoch,
+      documents: Object.keys(plan.content).length,
+      model,
+      transfers: pending,
+      unavailable,
+    };
+    const localReport = join(held.root, "MIGRATION.json");
+    await writeFile(localReport, `${JSON.stringify(result, null, 2)}\n`);
+    result.transfers = await submitAttachments(session, held, target, authority, plan, signal);
+
+    const report = `${target.subtree}/${MOUNT_REPORT_NAME}`;
+    await writeFile(localReport, `${JSON.stringify(result, null, 2)}\n`);
+    const changes = [];
+    for (const [relative, text] of Object.entries(plan.content)) {
+      let content = text;
+      for (const transfer of result.transfers.filter((entry) => entry.documents.includes(relative))) {
+        const save = plan.saves.find((entry) => entry.path === transfer.path);
+        for (const { href, suffix } of save.hrefs.filter((entry) => entry.document === relative)) {
+          const parsed = parse(content, relative);
+          const destination = transfer.save.status === "saved"
+            ? `${transfer.attachment}${suffix}`
+            : sourceHref(transfer.source, suffix);
+          content = serialize(replaceLink(parsed.document, href, destination));
+        }
+      }
+      if (content !== text) changes.push({ op: "replace", path: `${target.subtree}/${relative}`, base: revisions.get(relative), content });
+    }
+    changes.push({ op: "replace", path: report, base: revisions.get(MOUNT_REPORT_NAME), content: reportDocument(result) });
+    try {
+      let reported;
+      const limit = discovery.bounds.max_batch_operations;
+      for (let offset = 0; offset < changes.length; offset += limit) {
+        reported = await door(session, KIT_DOOR_BATCH_PATH, { operation_id: operationId(), changes: changes.slice(offset, offset + limit) }, signal);
+      }
+      return { ...result, report, reportPosition: reported.position, reportLogEpoch: reported.log_epoch, reportSaved: true };
+    } catch (error) {
+      return { ...result, report, reportSaved: false, reportFailure: transferFailure(error) };
+    }
+  });
+}
+
+export async function unmountSubtree(options, dependencies = {}) {
+  const target = mountTarget(options.subtree);
+  const confirm = options.confirm;
+  if (confirm !== undefined && confirm !== target.subtree) {
+    refuse("kit_unmount_confirmation_mismatch", confirm);
+  }
+  return withSession(options, dependencies, async (session, signal) => {
+    const discovery = discoveryOf(
+      await door(session, KIT_DOOR_DISCOVER_PATH, { prefixes: [target.root] }, signal),
+    );
+    const authority = authorityFor(discovery, target);
+    assertAdmitted(authority, target.relative, "delete");
+    assertAdmitted(authority, target.relative, "write");
+    const files = await currentFiles(session, [target.subtree], signal);
+    for (const path of files.keys()) {
+      if (!path.startsWith(`${target.relative}/`) || path.split("/").includes("_provenance")) continue;
+      assertAdmitted(authority, path, "delete");
+      assertAdmitted(authority, path, "write");
+    }
+    if (confirm === undefined) {
+      return {
+        status: "confirmation_required",
+        subtree: target.subtree,
+        summary: `Permanently remove every file and revision under ${target.subtree}.`,
+      };
+    }
+    const outcome = await door(
+      session,
+      KIT_DOOR_UNMOUNT_PATH,
+      { operation_id: operationId(), subtree: target.subtree },
+      signal,
+    );
+    if (outcome?.status !== "unmounted") refuse("kit_unmount_rejected", "unmount");
+    return {
+      status: "unmounted",
+      subtree: target.subtree,
+      authority: outcome.authority,
+      position: outcome.position,
+      log_epoch: outcome.log_epoch,
+      removed: Array.isArray(outcome.contents) ? outcome.contents.length : 0,
+    };
+  });
+}
+
+export function formatMount(result) {
+  const saved = result.transfers.filter((transfer) => transfer.save.status === "saved").length;
+  const lines = [
+    `Mounted ${result.documents} documents into ${result.subtree} at position ${result.position}.`,
+    `Prepared ${result.model.prepared.length} model definitions: ${result.model.written.length} written, ${result.model.adopted.length} already present.`,
+    `Downloaded ${result.transfers.length} attachments locally; ${saved} saved as House attachments.`,
+  ];
+  for (const transfer of result.transfers) {
+    if (transfer.save.status === "saved") continue;
+    lines.push(
+      `Attachment ${transfer.source} referenced by ${transfer.documents.join(", ")}: ${transfer.save.status}` +
+        `${transfer.save.failure === null ? "" : ` (${transfer.save.failure})`}.`,
+    );
+  }
+  for (const missing of result.unavailable) {
+    lines.push(
+      `Unavailable source ${missing.source} referenced by ${missing.referrers.join(", ")}: ${missing.download}` +
+        `${missing.failure === null ? "" : ` (${missing.failure})`}.`,
+    );
+  }
+  lines.push(result.reportSaved ? `Report written to ${result.report}.` : `Final report pending at ${result.report}: ${result.reportFailure}. Local outcome: MIGRATION.json in the package.`);
+  return lines;
+}
+
+export function formatUnmount(result) {
+  if (result.status === "confirmation_required") {
+    return [result.summary, `Confirm with --confirm ${result.subtree} to remove it.`];
+  }
+  return [
+    `Unmounted ${result.subtree}: removed ${result.removed} files at position ${result.position}.`,
+  ];
+}

package/src/operation.js

@@ -0,0 +1,20 @@
+import { createHash, randomBytes } from "node:crypto";
+
+function versioned(bytes) {
+  bytes[6] = (bytes[6] & 0x0f) | 0x70;
+  bytes[8] = (bytes[8] & 0x3f) | 0x80;
+  const hex = bytes.toString("hex");
+  return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
+}
+
+export function operationId() {
+  const bytes = randomBytes(16);
+  bytes.writeUIntBE(Date.now(), 0, 6);
+  return versioned(bytes);
+}
+
+export function derivedOperationId(operation, label) {
+  const bytes = createHash("sha256").update(`${operation}/${label}`).digest().subarray(0, 16);
+  bytes.write(operation.replaceAll("-", "").slice(0, 12), 0, "hex");
+  return versioned(bytes);
+}