@agentshouse/kit 0.0.1 → 0.1.0-alpha.2

package/src/repository.js

@@ -0,0 +1,521 @@
+import { createHash } from "node:crypto";
+import { spawn } from "node:child_process";
+import { constants } from "node:fs";
+import { open, readlink, realpath } from "node:fs/promises";
+import { join } from "node:path";
+import { deliverPublicationChunk, maintainControl, openPublicationManifest, openSession } from "./house.js";
+import { KitRefusal } from "./refusal.js";
+import { readRepositoryState, writeRepositoryState } from "./state.js";
+
+const BLOCK_BYTES = 512;
+const READ_BYTES = 64 * 1024;
+const MANIFEST_ROUNDS = 3;
+const USTAR_NAME_BYTES = 100;
+const PUBLICATION_REFERENCE = /^p_[0-9abcdefghjkmnpqrstvwxyz]+\.[0-9abcdefghjkmnpqrstvwxyz]$/;
+const SOURCE_HANDLE = /^[a-z0-9][a-z0-9-]{0,61}[a-z0-9]$/;
+const REFERENCE_ALPHABET = "0123456789abcdefghjkmnpqrstvwxyz";
+const MAX_REFERENCE_ORDINAL = (1n << 63n) - 1n;
+const ITEM_OUTCOMES = [
+  "accepted",
+  "duplicate",
+  "path_not_in_manifest",
+  "fingerprint_mismatch",
+  "refused",
+];
+const PUBLICATION_STATES = ["active", "completed"];
+const utf8 = new TextDecoder("utf-8", { fatal: true });
+
+function refuse(code, detail = null) {
+  throw new KitRefusal(code, detail);
+}
+
+function active(signal) {
+  if (signal?.aborted === true) refuse("kit_command_cancelled");
+}
+
+function validSourceReference(value) {
+  if (typeof value !== "string") return false;
+  const matched = /^s_([0-9a-z]+)\.([0-9a-z])$/.exec(value);
+  if (matched === null || (matched[1].length > 1 && matched[1][0] === "0")) return false;
+  let ordinal = 0n;
+  for (const character of matched[1]) {
+    const digit = REFERENCE_ALPHABET.indexOf(character);
+    if (digit === -1) return false;
+    ordinal = (ordinal << 5n) + BigInt(digit);
+    if (ordinal > MAX_REFERENCE_ORDINAL) return false;
+  }
+  return ordinal > 0n && REFERENCE_ALPHABET.includes(matched[2]);
+}
+
+function validRepositoryPath(value) {
+  return !(
+    value.length === 0 ||
+    value.startsWith("/") ||
+    value.endsWith("/") ||
+    value.includes("\\") ||
+    value.split("/").some((segment) => segment === "" || segment === "." || segment === "..")
+  );
+}
+
+function repositoryPath(value) {
+  if (!validRepositoryPath(value)) refuse("kit_repository_path_invalid", value);
+  return value;
+}
+
+function stateKey(identity) {
+  return createHash("sha256")
+    .update(JSON.stringify([identity.checkout, identity.room, identity.sourceHandle]))
+    .digest("hex");
+}
+
+function exactObject(value, keys) {
+  if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
+  const present = Object.keys(value).sort();
+  return present.length === keys.length && present.every((key, index) => key === keys[index]);
+}
+
+function stateOf(value, identity) {
+  if (value === null) {
+    return {
+      checkout: identity.checkout,
+      room: identity.room,
+      source_handle: identity.sourceHandle,
+      source_ref: null,
+    };
+  }
+  if (
+    !exactObject(value, ["checkout", "room", "source_handle", "source_ref"]) ||
+    value.checkout !== identity.checkout ||
+    value.room !== identity.room ||
+    value.source_handle !== identity.sourceHandle ||
+    (value.source_ref !== null && !validSourceReference(value.source_ref))
+  ) {
+    refuse("kit_state_unreadable");
+  }
+  return {
+    checkout: value.checkout,
+    room: value.room,
+    source_handle: value.source_handle,
+    source_ref: value.source_ref,
+  };
+}
+
+function octal(header, offset, length, value) {
+  const encoded = value.toString(8).padStart(length - 1, "0");
+  if (encoded.length >= length) refuse("kit_repository_archive_failed");
+  header.write(encoded, offset, length - 1, "ascii");
+  header[offset + length - 1] = 0;
+}
+
+function tarHeader(name, size, type) {
+  const header = Buffer.alloc(BLOCK_BYTES);
+  Buffer.from(name, "utf8").copy(header, 0, 0, USTAR_NAME_BYTES);
+  octal(header, 100, 8, 0o644);
+  octal(header, 108, 8, 0);
+  octal(header, 116, 8, 0);
+  octal(header, 124, 12, size);
+  octal(header, 136, 12, 0);
+  header.fill(32, 148, 156);
+  header[156] = type.charCodeAt(0);
+  header.write("ustar\0", 257, 6, "ascii");
+  header.write("00", 263, 2, "ascii");
+  const checksum = [...header]
+    .reduce((sum, byte) => sum + byte, 0)
+    .toString(8)
+    .padStart(6, "0");
+  header.write(checksum, 148, 6, "ascii");
+  header[154] = 0;
+  header[155] = 32;
+  return header;
+}
+
+function paxRecord(name, value) {
+  const content = `${name}=${value}\n`;
+  const contentBytes = Buffer.byteLength(content);
+  let length = contentBytes + 3;
+  for (;;) {
+    const next = String(length).length + 1 + contentBytes;
+    if (next === length) return Buffer.from(`${length} ${content}`, "utf8");
+    length = next;
+  }
+}
+
+function padding(size) {
+  const remainder = size % BLOCK_BYTES;
+  return remainder === 0 ? Buffer.alloc(0) : Buffer.alloc(BLOCK_BYTES - remainder);
+}
+
+async function material(path, signal) {
+  active(signal);
+  let handle;
+  try {
+    handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+  } catch (error) {
+    if (error.code === "ENOENT") return null;
+    if (error.code === "ELOOP") {
+      try {
+        const body = await readlink(path, { encoding: "buffer" });
+        active(signal);
+        return { kind: "buffer", body };
+      } catch {
+        active(signal);
+        refuse("kit_repository_checkout_unreadable", path);
+      }
+    }
+    refuse("kit_repository_checkout_unreadable", path);
+  }
+  try {
+    active(signal);
+    const status = await handle.stat();
+    active(signal);
+    if (!status.isFile()) {
+      await handle.close();
+      return null;
+    }
+    return { kind: "file", size: status.size, handle };
+  } catch (error) {
+    await handle.close().catch(() => undefined);
+    if (error instanceof KitRefusal) throw error;
+    if (signal?.aborted === true || error?.name === "AbortError") {
+      refuse("kit_command_cancelled");
+    } else {
+      refuse("kit_repository_checkout_unreadable", path);
+    }
+  }
+}
+
+async function* fileBody(selected, signal) {
+  if (selected.kind === "buffer") {
+    yield selected.body;
+    return;
+  }
+  const chunk = Buffer.allocUnsafe(READ_BYTES);
+  let read = 0;
+  while (read < selected.size) {
+    active(signal);
+    const { bytesRead } = await selected.handle.read(
+      chunk,
+      0,
+      Math.min(chunk.length, selected.size - read),
+      read,
+    );
+    if (bytesRead === 0) break;
+    read += bytesRead;
+    yield Buffer.from(chunk.subarray(0, bytesRead));
+  }
+  if (read < selected.size) yield Buffer.alloc(selected.size - read);
+}
+
+async function selectedFile(checkout, path, signal) {
+  const selected = await material(join(checkout, path), signal);
+  if (selected === null) return null;
+  try {
+    const digest = createHash("sha256");
+    const size = selected.kind === "buffer" ? selected.body.length : selected.size;
+    for await (const body of fileBody(selected, signal)) digest.update(body);
+    return { path, fingerprint: digest.digest("hex"), bytes: size };
+  } catch (error) {
+    if (error instanceof KitRefusal) throw error;
+    if (signal?.aborted === true || error?.name === "AbortError") {
+      refuse("kit_command_cancelled");
+    }
+    refuse("kit_repository_checkout_unreadable", path);
+  } finally {
+    if (selected.kind === "file") await selected.handle.close().catch(() => undefined);
+  }
+}
+
+async function manifestOf(checkout, paths, signal) {
+  const files = [];
+  for (const path of paths) {
+    const held = await selectedFile(checkout, path, signal);
+    if (held !== null) files.push(held);
+  }
+  return files;
+}
+
+async function* chunkArchive(checkout, entries, signal) {
+  for (const entry of entries) {
+    const selected = await material(join(checkout, entry.path), signal);
+    if (selected === null) continue;
+    try {
+      const size = selected.kind === "buffer" ? selected.body.length : selected.size;
+      if (Buffer.byteLength(entry.path, "utf8") > USTAR_NAME_BYTES) {
+        const record = paxRecord("path", entry.path);
+        yield tarHeader("PaxHeaders.X/entry", record.length, "x");
+        yield record;
+        yield padding(record.length);
+        yield tarHeader("entry", size, "0");
+      } else {
+        yield tarHeader(entry.path, size, "0");
+      }
+      for await (const body of fileBody(selected, signal)) yield body;
+      yield padding(size);
+    } finally {
+      if (selected.kind === "file") await selected.handle.close().catch(() => undefined);
+    }
+  }
+  yield Buffer.alloc(BLOCK_BYTES * 2);
+}
+
+function runGit(checkout, args, dependencies, capture, signal) {
+  active(signal);
+  const launcher = dependencies.spawn ?? spawn;
+  return new Promise((resolve, reject) => {
+    let child;
+    try {
+      child = launcher("git", args, {
+        cwd: checkout,
+        env: process.env,
+        stdio: capture ? ["ignore", "pipe", "inherit"] : "inherit",
+        signal,
+        killSignal: "SIGKILL",
+      });
+    } catch (error) {
+      reject(
+        signal?.aborted === true || error?.name === "AbortError"
+          ? new KitRefusal("kit_command_cancelled")
+          : new KitRefusal("kit_git_unavailable"),
+      );
+      return;
+    }
+    const output = [];
+    let settled = false;
+    const finish = (operation) => {
+      if (settled) return;
+      settled = true;
+      operation();
+    };
+    child.stdout?.on("data", (chunk) => output.push(Buffer.from(chunk)));
+    child.once("error", (error) =>
+      finish(() =>
+        reject(
+          signal?.aborted === true || error?.name === "AbortError"
+            ? new KitRefusal("kit_command_cancelled")
+            : new KitRefusal("kit_git_unavailable"),
+        ),
+      ),
+    );
+    child.once("close", (status) => {
+      finish(() => {
+        if (signal?.aborted === true) {
+          reject(new KitRefusal("kit_command_cancelled"));
+          return;
+        }
+        if (status !== 0) {
+          reject(
+            new KitRefusal(args[0] === "pull" ? "kit_git_pull_failed" : "kit_git_selection_failed"),
+          );
+          return;
+        }
+        resolve(Buffer.concat(output));
+      });
+    });
+  });
+}
+
+async function selectedPaths(checkout, dependencies, signal) {
+  const output = await runGit(
+    checkout,
+    ["ls-files", "--cached", "--others", "--exclude-standard", "-z"],
+    dependencies,
+    true,
+    signal,
+  );
+  const paths = [];
+  let offset = 0;
+  for (let end = output.indexOf(0); end !== -1; end = output.indexOf(0, offset)) {
+    let path;
+    try {
+      path = utf8.decode(output.subarray(offset, end));
+    } catch {
+      refuse("kit_repository_path_invalid");
+    }
+    paths.push(repositoryPath(path));
+    offset = end + 1;
+  }
+  if (offset !== output.length) refuse("kit_git_selection_failed");
+  return [...new Set(paths)].sort();
+}
+
+function openedPublication(answer, sourceRef) {
+  if (
+    answer === null ||
+    typeof answer !== "object" ||
+    typeof answer.publication !== "string" ||
+    !PUBLICATION_REFERENCE.test(answer.publication) ||
+    !validSourceReference(answer.source_ref) ||
+    !PUBLICATION_STATES.includes(answer.state) ||
+    !Array.isArray(answer.needed) ||
+    answer.needed.some((path) => typeof path !== "string" || !validRepositoryPath(path)) ||
+    answer.needed_count !== answer.needed.length ||
+    answer.chunk_bound === null ||
+    typeof answer.chunk_bound !== "object" ||
+    !Number.isSafeInteger(answer.chunk_bound.paths) ||
+    answer.chunk_bound.paths < 1 ||
+    !Number.isSafeInteger(answer.chunk_bound.bytes) ||
+    answer.chunk_bound.bytes < 1 ||
+    (sourceRef !== null && answer.source_ref !== sourceRef)
+  ) {
+    refuse("kit_publication_rejected");
+  }
+  return answer;
+}
+
+function deliveredChunk(answer, expected) {
+  if (
+    answer === null ||
+    typeof answer !== "object" ||
+    answer.publication !== expected ||
+    !PUBLICATION_STATES.includes(answer.state) ||
+    !Array.isArray(answer.outstanding) ||
+    answer.outstanding.some((path) => typeof path !== "string" || !validRepositoryPath(path)) ||
+    answer.outstanding_count !== answer.outstanding.length ||
+    !Array.isArray(answer.items) ||
+    answer.items.some(
+      (item) =>
+        item === null ||
+        typeof item !== "object" ||
+        typeof item.path !== "string" ||
+        !ITEM_OUTCOMES.includes(item.outcome),
+    )
+  ) {
+    refuse("kit_publication_rejected");
+  }
+  return answer;
+}
+
+function chunksOf(needed, files, bound) {
+  const bytes = new Map(files.map((file) => [file.path, file.bytes]));
+  const chunks = [];
+  let held = [];
+  let size = 0;
+  for (const path of needed) {
+    const entry = { path, bytes: bytes.get(path) ?? 0 };
+    if (held.length > 0 && (held.length >= bound.paths || size + entry.bytes > bound.bytes)) {
+      chunks.push(held);
+      held = [];
+      size = 0;
+    }
+    held.push(entry);
+    size += entry.bytes;
+  }
+  if (held.length > 0) chunks.push(held);
+  return chunks;
+}
+
+async function deliverRound(session, options, state, opened, files, signal) {
+  const delivered = [];
+  const refused = [];
+  let state_ = opened.state;
+  for (const chunk of chunksOf(opened.needed, files, opened.chunk_bound)) {
+    active(signal);
+    let answer;
+    try {
+      answer = deliveredChunk(
+        await deliverPublicationChunk(
+          session,
+          { room: state.room, publication: opened.publication },
+          chunkArchive(options.checkout, chunk, signal),
+          signal,
+        ),
+        opened.publication,
+      );
+    } catch (error) {
+      if (error instanceof KitRefusal && error.code === "publication_ended") {
+        return {
+          delivered,
+          refused,
+          state: "ended",
+          ending: error.payload?.error?.ending ?? "completed",
+        };
+      }
+      throw error;
+    }
+    for (const item of answer.items) {
+      if (item.outcome === "accepted" || item.outcome === "duplicate") delivered.push(item.path);
+      else refused.push(item);
+    }
+    state_ = answer.state;
+  }
+  return { delivered, refused, state: state_, ending: state_ === "completed" ? "completed" : null };
+}
+
+async function checkoutOf(value) {
+  try {
+    return await realpath(value);
+  } catch {
+    refuse("kit_repository_checkout_invalid", value);
+  }
+}
+
+export async function captureRepository(options, dependencies = {}) {
+  if (
+    typeof options.room !== "string" ||
+    options.room.length === 0 ||
+    typeof options.sourceHandle !== "string" ||
+    !SOURCE_HANDLE.test(options.sourceHandle)
+  ) {
+    refuse("kit_repository_declaration_invalid");
+  }
+  const identity = {
+    checkout: await checkoutOf(options.checkout),
+    room: options.room,
+    sourceHandle: options.sourceHandle,
+  };
+  const key = stateKey(identity);
+  const session = await openSession(options, dependencies);
+  return maintainControl(
+    session,
+    async (signal) => {
+      const state = stateOf(await readRepositoryState(options.directory, key), identity);
+      await runGit(identity.checkout, ["pull"], dependencies, false, signal);
+      const now = dependencies.now ?? (() => new Date());
+      let sourceRef = state.source_ref;
+      let publication = null;
+      let delivered = [];
+      let refused = [];
+      let ending = null;
+      for (let round = 0; round < MANIFEST_ROUNDS; round += 1) {
+        const paths = await selectedPaths(identity.checkout, dependencies, signal);
+        const files = await manifestOf(identity.checkout, paths, signal);
+        if (files.length === 0) refuse("kit_repository_empty");
+        const opened = openedPublication(
+          await openPublicationManifest(
+            session,
+            {
+              room_ref: state.room,
+              source_handle: state.source_handle,
+              observed_at: now().toISOString(),
+              paths: files,
+              ...(sourceRef === null ? {} : { source_ref: sourceRef }),
+            },
+            signal,
+          ),
+          sourceRef,
+        );
+        sourceRef = opened.source_ref;
+        publication = opened.publication;
+        if (sourceRef !== state.source_ref) {
+          state.source_ref = sourceRef;
+          await writeRepositoryState(options.directory, key, state, signal);
+        }
+        const round_ = await deliverRound(session, options, state, opened, files, signal);
+        delivered = round_.delivered;
+        refused = round_.refused;
+        ending = round_.ending;
+        if (round_.state === "completed") break;
+        if (round_.state === "ended" && round_.ending === "superseded") break;
+        if (round_.state === "active" && refused.length === 0) break;
+      }
+      return {
+        sourceRef,
+        publication,
+        delivered: delivered.length,
+        refused,
+        ending: ending ?? "active",
+      };
+    },
+    options.signal,
+  );
+}

package/src/session-bin/house

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import "../house-main.js";

package/src/settlement.js

@@ -0,0 +1,57 @@
+import { createHash } from "node:crypto";
+import { readdir, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { KitRefusal } from "./refusal.js";
+import { readNote, writeNote } from "./state.js";
+import { reportedNote, retainedReport } from "./terminal.js";
+
+function settlementDirectory(directory) {
+  return join(directory, "state", "settlements");
+}
+
+function settlementName(run) {
+  return `${createHash("sha256").update(run).digest("hex")}.json`;
+}
+
+export async function writeSettlement(directory, settlement) {
+  await writeNote(join(settlementDirectory(directory), settlementName(settlement.run)), {
+    installation: directory,
+    run: settlement.run,
+    session: settlement.session,
+    reported: reportedNote(settlement.reported),
+  });
+}
+
+export async function removeSettlement(directory, run) {
+  await rm(join(settlementDirectory(directory), settlementName(run)), { force: true });
+}
+
+function completeSettlement(directory, stored, name) {
+  if (typeof stored !== "object" || stored === null || Array.isArray(stored)) return null;
+  if (stored.installation !== directory) return null;
+  const named = [stored.run, stored.session];
+  if (named.some((value) => typeof value !== "string" || value.length === 0)) return null;
+  if (settlementName(stored.run) !== name) return null;
+  const reported = retainedReport(stored.reported);
+  if (reported === null || reported === undefined) return null;
+  return { run: stored.run, session: stored.session, reported };
+}
+
+export async function readSettlements(directory) {
+  let present;
+  try {
+    present = await readdir(settlementDirectory(directory));
+  } catch (error) {
+    if (error?.code === "ENOENT") return [];
+    throw new KitRefusal("kit_state_unreadable", settlementDirectory(directory));
+  }
+  const retained = [];
+  for (const name of present.filter((entry) => entry.endsWith(".json")).sort()) {
+    const stored = await readNote(join(settlementDirectory(directory), name), null);
+    if (stored === null) continue;
+    const settlement = completeSettlement(directory, stored, name);
+    if (settlement === null) continue;
+    retained.push(settlement);
+  }
+  return retained;
+}