@agentshouse/kit 0.0.1 → 0.1.0-alpha.1

package/src/sync.js

@@ -0,0 +1,477 @@
+import { readFile } from "node:fs/promises";
+import {
+  EXPLICIT_SYNC_VERSION,
+  KIT_DOOR_BATCH_PATH,
+  KIT_DOOR_BOOTSTRAP_PATH,
+  KIT_DOOR_DISCOVER_PATH,
+  KIT_DOOR_ENUMERATE_PATH,
+} from "./contract.js";
+import { heldControl, houseGetJson, houseRequest, maintainControl, openSession } from "./house.js";
+import { operationId } from "./operation.js";
+import { houseFailed, KitRefusal } from "./refusal.js";
+import {
+  assertWritableChanges,
+  authorityFor,
+  deriveChanges,
+  OPERATIONS,
+  relativeTo,
+  resultingFiles,
+  sidecarPath,
+  unknownPathRefusal,
+  VERSION,
+} from "./sync-library.js";
+import {
+  canonicalReplica,
+  clearPending,
+  locate,
+  listFiles,
+  readLocalMap,
+  readPending,
+  readReplicaState,
+  removeReplicaFile,
+  writePending,
+  writeReplicaFile,
+  writeReplicaState,
+} from "./sync-state.js";
+
+function sessionDependencies(options, dependencies) {
+  return {
+    fetch: dependencies.fetch ?? options.fetch,
+    now: dependencies.now ?? options.now,
+  };
+}
+
+async function withSession(options, dependencies, operation) {
+  const session = await openSession(options, sessionDependencies(options, dependencies));
+  return maintainControl(session, (signal) => operation(session, signal), options.signal);
+}
+
+async function door(session, path, body, signal) {
+  return houseRequest(
+    session.house,
+    await heldControl(session, signal),
+    path,
+    body,
+    session.dependencies,
+    signal,
+  );
+}
+
+async function replicaDoor(session, options, state, path, body, signal) {
+  try {
+    return await door(session, path, body, signal);
+  } catch (error) {
+    if (error instanceof KitRefusal && error.code === "position_expired") {
+      await writeReplicaState(options.directory, { ...state, expired: true }, signal);
+    }
+    throw error;
+  }
+}
+
+function positionsOf(state) {
+  return state.authorities.map((entry) => ({
+    authority: entry.authority,
+    position: entry.position,
+    log_epoch: entry.log_epoch,
+  }));
+}
+
+function changePosition(value, declaration) {
+  if (typeof value?.position !== "string" || typeof value.log_epoch !== "string") {
+    throw new KitRefusal("kit_sync_rejected", declaration);
+  }
+  return { position: value.position, log_epoch: value.log_epoch };
+}
+
+function discoveryOf(value) {
+  if (typeof value?.md_model_version !== "string" || !Array.isArray(value.authorities)) {
+    throw new KitRefusal("kit_sync_rejected", "discover");
+  }
+  if (value.md_model_version !== VERSION) {
+    throw new KitRefusal("kit_md_model_version_mismatch", value.md_model_version);
+  }
+  return value;
+}
+
+async function bootstrapBundle(session, prefixes, signal) {
+  const answer = await door(session, KIT_DOOR_BOOTSTRAP_PATH, { prefixes }, signal);
+  if (typeof answer?.transfer?.url === "string") {
+    const granted = await houseGetJson(answer.transfer.url, session.dependencies, signal);
+    if (!Array.isArray(granted?.scopes)) throw new KitRefusal("kit_bootstrap_rejected");
+    return granted;
+  }
+  if (!Array.isArray(answer?.scopes)) throw new KitRefusal("kit_bootstrap_rejected");
+  return answer;
+}
+
+function matchedDiscovery(discovery, scope) {
+  if (scope.manifest?.scope === "private") {
+    return discovery.authorities.find((entry) => entry.kind === "private");
+  }
+  return discovery.authorities.find(
+    (entry) => entry.kind === "room" && entry.room_ref === scope.manifest?.anchor,
+  );
+}
+
+async function writeScope(replica, scope, signal) {
+  if (
+    typeof scope?.manifest?.path !== "string" ||
+    typeof scope.manifest.position !== "string" ||
+    typeof scope.manifest.log_epoch !== "string"
+  ) {
+    throw new KitRefusal("kit_bootstrap_rejected");
+  }
+  if (!Array.isArray(scope.contents) || !Array.isArray(scope.manifest.files)) {
+    throw new KitRefusal("kit_bootstrap_rejected");
+  }
+  const files = [];
+  for (const entry of scope.contents) {
+    if (typeof entry?.path !== "string" || typeof entry.content !== "string") {
+      throw new KitRefusal("kit_bootstrap_rejected");
+    }
+    const meta = scope.manifest.files.find((file) => file.path === entry.path);
+    await writeReplicaFile(locate(replica, scope.manifest.path, entry.path), entry.content, signal);
+    files.push({
+      path: entry.path,
+      revision: typeof meta?.revision === "string" ? meta.revision : "",
+      sha256: typeof meta?.sha256 === "string" ? meta.sha256 : "",
+      content: entry.content,
+    });
+  }
+  return files;
+}
+
+async function replaceScope(replica, previous, scope, signal) {
+  const next = await writeScope(replica, scope, signal);
+  const kept = new Set(next.map((file) => file.path));
+  const root = locate(replica, scope.manifest.path);
+  for (const path of await listFiles(root)) {
+    if (!kept.has(path)) await removeReplicaFile(locate(replica, scope.manifest.path, path));
+  }
+  if (previous !== undefined) {
+    for (const file of previous.files) {
+      if (!kept.has(file.path)) await removeReplicaFile(locate(replica, previous.path, file.path));
+    }
+  }
+  return next;
+}
+
+function stringList(value) {
+  return Array.isArray(value) && value.every((entry) => typeof entry === "string");
+}
+
+function operationsOf(value) {
+  if (value === null || typeof value !== "object") return null;
+  const operations = {};
+  for (const operation of OPERATIONS) {
+    if (!stringList(value[operation])) return null;
+    operations[operation] = [...value[operation]];
+  }
+  return operations;
+}
+
+function authorityRecord(scope, discovery, files) {
+  const found = matchedDiscovery(discovery, scope);
+  const operations = found === undefined ? null : operationsOf(found.operations);
+  if (found === undefined || !stringList(found.protected_paths) || operations === null) {
+    throw new KitRefusal("kit_bootstrap_rejected");
+  }
+  return {
+    kind: scope.manifest.scope === "private" ? "private" : "room",
+    authority: scope.manifest.anchor,
+    room_handle: scope.manifest.room_handle,
+    path: scope.manifest.path,
+    position: scope.manifest.position,
+    log_epoch: scope.manifest.log_epoch,
+    protected_paths: found.protected_paths,
+    operations,
+    files,
+  };
+}
+
+async function materialize(options, discovery, bundle, signal, previous = null) {
+  const authorities = [];
+  for (const scope of bundle.scopes) {
+    const held = previous?.authorities.find((entry) => entry.authority === scope.manifest?.anchor);
+    const files = await replaceScope(options.replica, held, scope, signal);
+    authorities.push(authorityRecord(scope, discovery, files));
+  }
+  const state = {
+    version: EXPLICIT_SYNC_VERSION,
+    directory: options.canonical,
+    prefixes: options.prefixes,
+    md_model_version: discovery.md_model_version,
+    bounds: discovery.bounds ?? null,
+    authorities,
+  };
+  await writeReplicaState(options.directory, state, signal);
+  return state;
+}
+
+async function loadState(options) {
+  const canonical = await canonicalReplica(options.replica);
+  const state = await readReplicaState(options.directory, canonical);
+  if (state === null || state.version !== EXPLICIT_SYNC_VERSION) {
+    throw new KitRefusal("kit_sync_not_bootstrapped", canonical);
+  }
+  return { ...state, directory: canonical };
+}
+
+function enumeratedRevision(change) {
+  return typeof change.revision === "string" ? change.revision : "";
+}
+
+function enumeratedDigest(change) {
+  return typeof change.sha256 === "string" ? change.sha256 : "";
+}
+
+function enumeratedRecord(path, content, change) {
+  return {
+    path,
+    revision: enumeratedRevision(change),
+    sha256: enumeratedDigest(change),
+    content,
+  };
+}
+
+async function applyEnumeration(options, state, enumeration, signal) {
+  if (!Array.isArray(enumeration?.authorities)) throw new KitRefusal("kit_sync_rejected", "enumerate");
+  for (const view of enumeration.authorities) {
+    const authority = state.authorities.find((entry) => entry.authority === view.authority);
+    if (authority === undefined) throw new KitRefusal("kit_sync_rejected", view.authority);
+    const reached = changePosition(view, "enumerate");
+    const recorded = new Map(authority.files.map((file) => [file.path, file]));
+    for (const change of view.changes ?? []) {
+      if (typeof change?.path !== "string") throw new KitRefusal("kit_sync_rejected", "enumerate");
+      if (change.removed === true) {
+        await removeReplicaFile(locate(options.replica, authority.path, change.path));
+        recorded.delete(change.path);
+        continue;
+      }
+      if (typeof change.content === "string") {
+        await writeReplicaFile(
+          locate(options.replica, authority.path, change.path),
+          change.content,
+          signal,
+        );
+        recorded.set(change.path, enumeratedRecord(change.path, change.content, change));
+      }
+      if (typeof change.provenance === "string") {
+        const sidecar = sidecarPath(change.path);
+        await writeReplicaFile(
+          locate(options.replica, authority.path, sidecar),
+          change.provenance,
+          signal,
+        );
+        recorded.set(sidecar, enumeratedRecord(sidecar, change.provenance, change));
+      }
+    }
+    authority.files = [...recorded.values()];
+    Object.assign(authority, reached);
+  }
+  await writeReplicaState(options.directory, state, signal);
+  return state;
+}
+
+async function replicaText(path) {
+  try {
+    return await readFile(path, "utf8");
+  } catch (error) {
+    if (error.code === "ENOENT") return null;
+    throw error;
+  }
+}
+
+function predictedReplica(authority, pending) {
+  if (!Array.isArray(pending?.body?.changes)) return null;
+  return resultingFiles(authority, pending.body.changes);
+}
+
+function uncertainTransport(error) {
+  return (
+    !(error instanceof KitRefusal) ||
+    houseFailed(error) ||
+    error.code === "kit_command_cancelled"
+  );
+}
+
+async function acceptBatch(options, state, outcome, signal, pending = null) {
+  const authority = state.authorities.find((entry) => entry.authority === outcome.authority);
+  if (authority === undefined) throw new KitRefusal("kit_sync_rejected", outcome.authority);
+  const reached = changePosition(outcome, "batch");
+  const recorded = new Map(authority.files.map((file) => [file.path, file]));
+  const predicted = predictedReplica(authority, pending);
+  for (const file of outcome.contents ?? []) {
+    const relative = relativeTo(authority.path, file.path);
+    const destination = locate(options.replica, authority.path, relative);
+    const local = await replicaText(destination);
+    const held = recorded.get(relative)?.content ?? null;
+    const accepted = file.content;
+    const expected = predicted === null ? null : (predicted[relative] ?? null);
+    const preserve =
+      pending !== null &&
+      local !== null &&
+      local !== held &&
+      local !== accepted &&
+      local !== expected;
+    if (accepted === null) {
+      if (!preserve) await removeReplicaFile(destination);
+      recorded.delete(relative);
+      continue;
+    }
+    if (!preserve) await writeReplicaFile(destination, accepted, signal);
+    recorded.set(relative, {
+      path: relative,
+      revision: typeof file.revision === "string" ? file.revision : "",
+      sha256: "",
+      content: accepted,
+    });
+  }
+  authority.files = [...recorded.values()];
+  Object.assign(authority, reached);
+  await writeReplicaState(options.directory, state, signal);
+  return { authority: outcome.authority, ...reached, contents: outcome.contents };
+}
+
+function sameBody(left, right) {
+  return JSON.stringify(left) === JSON.stringify(right);
+}
+
+async function recoverPending(session, options, state, signal) {
+  const pending = await readPending(options.directory, state.directory);
+  if (pending === null) return null;
+  try {
+    const outcome = await replicaDoor(session, options, state, pending.path, pending.body, signal);
+    const accepted = await acceptBatch(options, state, outcome, signal, pending);
+    await clearPending(options.directory, state.directory);
+    return { pending, accepted };
+  } catch (error) {
+    if (uncertainTransport(error)) throw error;
+    await clearPending(options.directory, state.directory);
+    throw error;
+  }
+}
+
+function groupChanges(state, changes) {
+  const groups = [];
+  const index = new Map();
+  for (const change of changes) {
+    const authority = authorityFor(state, change.path);
+    if (authority === undefined) throw unknownPathRefusal(change.path);
+    const key = authority.authority;
+    if (!index.has(key)) {
+      index.set(key, groups.length);
+      groups.push({ authority, changes: [] });
+    }
+    groups[index.get(key)].changes.push(change);
+  }
+  return groups;
+}
+
+async function sendBatch(session, options, state, changes, signal) {
+  assertWritableChanges(state, changes);
+  const groups = groupChanges(state, changes);
+  let last = null;
+  for (const group of groups) {
+    const body = { operation_id: operationId(), changes: group.changes };
+    await writePending(
+      options.directory,
+      state.directory,
+      { operation_id: body.operation_id, path: KIT_DOOR_BATCH_PATH, body },
+      signal,
+    );
+    try {
+      const outcome = await replicaDoor(session, options, state, KIT_DOOR_BATCH_PATH, body, signal);
+      last = await acceptBatch(options, state, outcome, signal);
+      await clearPending(options.directory, state.directory);
+    } catch (error) {
+      if (uncertainTransport(error)) throw error;
+      await clearPending(options.directory, state.directory);
+      throw error;
+    }
+  }
+  return last;
+}
+
+async function derivedChanges(options, state) {
+  const changes = [];
+  for (const authority of state.authorities) {
+    const local = await readLocalMap(options.replica, authority.path);
+    changes.push(...deriveChanges(state, authority, local));
+  }
+  return changes;
+}
+
+export async function bootstrap(options, dependencies = {}) {
+  if (!Array.isArray(options.prefixes) || options.prefixes.length === 0) {
+    throw new KitRefusal("kit_sync_prefixes_required");
+  }
+  const canonical = await canonicalReplica(options.replica);
+  const existing = await readReplicaState(options.directory, canonical);
+  if (existing !== null) throw new KitRefusal("kit_sync_already_bootstrapped", canonical);
+  return withSession(options, dependencies, async (session, signal) => {
+    const discovery = discoveryOf(
+      await door(session, KIT_DOOR_DISCOVER_PATH, { prefixes: options.prefixes }, signal),
+    );
+    const bundle = await bootstrapBundle(session, options.prefixes, signal);
+    const state = await materialize(
+      { ...options, canonical, prefixes: options.prefixes },
+      discovery,
+      bundle,
+      signal,
+    );
+    return { positions: positionsOf(state) };
+  });
+}
+
+export async function pull(options, dependencies = {}) {
+  return withSession(options, dependencies, async (session, signal) => {
+    const state = await loadState(options);
+    const enumeration = await replicaDoor(
+      session,
+      options,
+      state,
+      KIT_DOOR_ENUMERATE_PATH,
+      { prefixes: state.prefixes, positions: positionsOf(state) },
+      signal,
+    );
+    const next = await applyEnumeration(options, state, enumeration, signal);
+    return { positions: positionsOf(next) };
+  });
+}
+
+export async function rebootstrap(options, dependencies = {}) {
+  return withSession(options, dependencies, async (session, signal) => {
+    const state = await loadState(options);
+    const discovery = discoveryOf(
+      await door(session, KIT_DOOR_DISCOVER_PATH, { prefixes: state.prefixes }, signal),
+    );
+    const bundle = await bootstrapBundle(session, state.prefixes, signal);
+    const next = await materialize(
+      { ...options, canonical: state.directory, prefixes: state.prefixes },
+      discovery,
+      bundle,
+      signal,
+      state,
+    );
+    await clearPending(options.directory, state.directory);
+    return { positions: positionsOf(next) };
+  });
+}
+
+export async function submit(options, dependencies = {}) {
+  return withSession(options, dependencies, async (session, signal) => {
+    const state = await loadState(options);
+    if (state.expired === true) throw new KitRefusal("position_expired", state.directory);
+    const recovered = await recoverPending(session, options, state, signal);
+    const requested = Array.isArray(options.changes) ? options.changes : null;
+    if (recovered !== null && requested !== null && sameBody(recovered.pending.body.changes, requested)) {
+      return recovered.accepted;
+    }
+    const changes = requested ?? (await derivedChanges(options, state));
+    if (changes.length === 0) {
+      return recovered !== null ? recovered.accepted : { submitted: false };
+    }
+    return sendBatch(session, options, state, changes, signal);
+  });
+}