@agentshouse/kit 0.1.0-alpha.1 → 0.1.0-alpha.11

package/src/house.js

@@ -1,6 +1,8 @@
-import { readFile } from "node:fs/promises";
-import { join } from "node:path";
+import { chmod, mkdir, readFile, rm } from "node:fs/promises";
+import { connect, createServer } from "node:net";
+import { dirname, join } from "node:path";
 import { assertAcquiredCommandAdapter } from "./adapter.js";
+import { readFrames, writeFrame } from "./channel.js";
 import {
   assistantDraftsPath,
   assistantTextPath,
@@ -35,6 +37,8 @@ import { declarationReader } from "./declaration.js";
 import { houseRefusal, KitRefusal, throwHouseAnswer } from "./refusal.js";
 
 const LEASE = /^ahl_[A-Za-z0-9_-]+$/;
+const RESIDENT = "Resident";
+const LEASE_ACTIVE = "environment_control_lease_active";
 const ASSISTANT_DRAFT_STATES = ["streamed", "unsupported"];
 const ITEM_OUTCOMES = ["accepted", "duplicate", "rejected"];
 
@@ -105,6 +109,172 @@ function heldLease(answer) {
   return { lease: answer.lease, expiresAt: Date.parse(answer.expires_at) };
 }
 
+function residentSocket(directory) {
+  return join(directory, "run", "resident.sock");
+}
+
+function residentAnswer(status, code) {
+  const body = Buffer.from(JSON.stringify({ error: { code } })).toString("base64");
+  return { status, type: "application/json", body };
+}
+
+async function residentRelay(session, request, signal) {
+  const path = typeof request.path === "string" ? request.path : "";
+  if (!path.startsWith("/kit/") || path.startsWith("/kit/lease")) {
+    return residentAnswer(403, "kit_resident_route_refused");
+  }
+  let authorization;
+  try {
+    authorization = await heldControl(session, signal);
+  } catch (error) {
+    return residentAnswer(error.status ?? 401, error.code ?? "kit_lease_rejected");
+  }
+  try {
+    const response = await (session.dependencies.fetch ?? fetch)(`${session.house}${path}`, {
+      method: request.method,
+      headers: { ...request.headers, authorization },
+      body: request.text ?? (request.bytes === undefined ? undefined : Buffer.from(request.bytes, "base64")),
+      signal,
+    });
+    const body = Buffer.from(await response.arrayBuffer());
+    return {
+      status: response.status,
+      type: response.headers.get("content-type"),
+      body: body.length === 0 ? null : body.toString("base64"),
+    };
+  } catch {
+    return residentAnswer(503, "kit_house_unavailable");
+  }
+}
+
+export async function serveResident(session, directory, signal) {
+  const socket = residentSocket(directory);
+  await mkdir(dirname(socket), { recursive: true, mode: 0o700 });
+  await rm(socket, { force: true });
+  let holder = null;
+  const server = createServer((connection) => {
+    connection.on("error", () => connection.destroy());
+    if (holder !== null) {
+      writeFrame(connection, { refused: LEASE_ACTIVE });
+      connection.end();
+      return;
+    }
+    holder = connection;
+    const relayed = new Map();
+    connection.on("close", () => {
+      if (holder === connection) holder = null;
+      for (const cancelled of relayed.values()) cancelled.abort();
+    });
+    renewControl(session, signal).then(
+      () => {
+        writeFrame(connection, { held: true });
+        readFrames(connection, {
+          onFrame: (request) => {
+            if (request.cancel === true) {
+              relayed.get(request.id)?.abort();
+              return;
+            }
+            const cancelled = new AbortController();
+            relayed.set(request.id, cancelled);
+            void residentRelay(session, request, AbortSignal.any([signal, cancelled.signal])).then(
+              (answer) => {
+                relayed.delete(request.id);
+                if (!connection.destroyed) writeFrame(connection, { id: request.id, ...answer });
+              },
+            );
+          },
+          onRefusal: () => connection.destroy(),
+        });
+      },
+      (error) => {
+        writeFrame(connection, { refused: error.code ?? "kit_lease_rejected" });
+        connection.end();
+      },
+    );
+  });
+  await new Promise((resolve, reject) => {
+    server.once("error", reject);
+    server.listen(socket, resolve);
+  });
+  await chmod(socket, 0o600);
+  return async () => {
+    holder?.destroy();
+    await new Promise((resolve) => server.close(resolve));
+    await rm(socket, { force: true });
+  };
+}
+
+async function residentDelegation(directory, fetcher) {
+  const connection = connect(residentSocket(directory));
+  const pending = new Map();
+  let greet;
+  const greeting = new Promise((resolve) => {
+    greet = resolve;
+  });
+  const unavailable = () => residentAnswer(503, "kit_resident_unavailable");
+  function settle(id, answer) {
+    pending.get(id)?.(answer);
+    pending.delete(id);
+    if (pending.size === 0) connection.unref();
+  }
+  connection.on("error", () => connection.destroy());
+  connection.on("close", () => {
+    greet(null);
+    for (const id of [...pending.keys()]) settle(id, unavailable());
+  });
+  readFrames(connection, {
+    onFrame: (frame) => (frame.id === undefined ? greet(frame) : settle(frame.id, frame)),
+    onRefusal: () => connection.destroy(),
+  });
+  const held = await greeting;
+  if (held === null) return null;
+  if (held.held !== true) {
+    connection.destroy();
+    throw new KitRefusal(held.refused);
+  }
+  connection.unref();
+  let next = 0;
+  async function delegated(url, init = {}) {
+    const { authorization, ...headers } = init.headers ?? {};
+    if (authorization !== RESIDENT) return fetcher(url, init);
+    init.signal?.throwIfAborted();
+    const address = new URL(url);
+    const answer = connection.destroyed
+      ? unavailable()
+      : await new Promise((resolve, reject) => {
+          const id = (next += 1);
+          const cancel = () => {
+            pending.delete(id);
+            settle(id);
+            if (!connection.destroyed) writeFrame(connection, { id, cancel: true });
+            reject(init.signal.reason);
+          };
+          init.signal?.addEventListener("abort", cancel, { once: true });
+          pending.set(id, (answer) => {
+            init.signal?.removeEventListener("abort", cancel);
+            resolve(answer);
+          });
+          connection.ref();
+          writeFrame(connection, {
+            id,
+            method: init.method ?? "GET",
+            path: `${address.pathname}${address.search}`,
+            headers,
+            ...(typeof init.body === "string"
+              ? { text: init.body }
+              : init.body === undefined
+                ? {}
+                : { bytes: Buffer.from(init.body).toString("base64") }),
+          });
+        });
+    return new Response(answer.body === null ? null : Buffer.from(answer.body, "base64"), {
+      status: answer.status,
+      headers: answer.type === null ? {} : { "content-type": answer.type },
+    });
+  }
+  return { connection, fetch: delegated };
+}
+
 export async function openSession(options, dependencies = {}) {
   const session = {
     house: options.house,
@@ -114,6 +284,15 @@ export async function openSession(options, dependencies = {}) {
     expiresAt: 0,
     signal: options.signal,
   };
+  const resident =
+    options.acquire === true
+      ? null
+      : await residentDelegation(options.directory, dependencies.fetch ?? fetch);
+  if (resident !== null) {
+    session.resident = resident.connection;
+    session.dependencies = { ...dependencies, fetch: resident.fetch };
+    return session;
+  }
   const credential = await readCredential(options.directory);
   Object.assign(
     session,
@@ -131,10 +310,7 @@ export async function openSession(options, dependencies = {}) {
   return session;
 }
 
-export async function heldControl(session, signal = session.signal) {
-  if (session.expiresAt - session.now().getTime() > LEASE_RENEWAL_MARGIN_MS) {
-    return `Bearer ${session.lease}`;
-  }
+async function renewControl(session, signal) {
   Object.assign(
     session,
     heldLease(
@@ -148,6 +324,13 @@ export async function heldControl(session, signal = session.signal) {
       ),
     ),
   );
+}
+
+export async function heldControl(session, signal = session.signal) {
+  if (session.resident !== undefined) return RESIDENT;
+  if (session.expiresAt - session.now().getTime() <= LEASE_RENEWAL_MARGIN_MS) {
+    await renewControl(session, signal);
+  }
   return `Bearer ${session.lease}`;
 }
 
@@ -195,6 +378,7 @@ export function mutatingCall(classification) {
 }
 
 export async function releaseControl(session, options = {}) {
+  session.resident?.end();
   if (session.lease === null) return;
   const held = session.lease;
   try {
@@ -229,6 +413,7 @@ export async function maintainControl(session, operation, signal, { release = tr
     signal?.addEventListener("abort", cancelOperation, { once: true });
   });
   const renewal = (async () => {
+    if (session.resident !== undefined) return;
     const sleep = session.dependencies.controlSleep ?? controlSleep;
     while (!stopped) {
       const delay = Math.max(
@@ -253,20 +438,14 @@ export async function maintainControl(session, operation, signal, { release = tr
     }
   })();
   const running = Promise.resolve().then(() => operation(operationSignal));
-  let ended = false;
   try {
-    const answer = await Promise.race([running, failure, cancellation]);
-    ended = true;
-    return answer;
-  } catch (error) {
-    ended = error instanceof KitRefusal && error.code === "kit_command_cancelled";
-    throw error;
+    return await Promise.race([running, failure, cancellation]);
   } finally {
     stopped = true;
     maintenance.abort();
     signal?.removeEventListener("abort", cancelOperation);
     await Promise.all([renewal, running.catch(() => undefined)]);
-    if (ended && release) await releaseControl(session);
+    if (release) await releaseControl(session);
   }
 }
 
@@ -480,6 +659,11 @@ export async function reportLogin(session, login, outcome, signal = session.sign
 
 const envelope = declarationReader("kit_acquire_rejected");
 
+function routineInput(value, declaration) {
+  if (typeof value !== "string" || Array.from(value).length > 20_000) envelope.refuse(declaration);
+  return value;
+}
+
 function acquiredRoute(value, declaration) {
   const route = envelope.plain(value, declaration);
   if (!Array.isArray(route.arguments)) envelope.refuse(`${declaration}.arguments`);
@@ -551,7 +735,7 @@ function acquiredRequest(acquired, declaration) {
       name: envelope.text(routine.name, `${declaration}.routine.name`, 256),
       version: envelope.positiveCount(routine.version, `${declaration}.routine.version`),
     },
-    payload: envelope.plain(request.payload, `${declaration}.request.payload`),
+    payload: routineInput(request.payload, `${declaration}.request.payload`),
     requested_by: envelope.text(
       request.requested_by,
       `${declaration}.request.requested_by`,

package/src/kit-work-stream.js

@@ -0,0 +1,84 @@
+import { setTimeout as delay } from "node:timers/promises";
+import WebSocket from "ws";
+import { KIT_CONTRACT_VERSION } from "./contract.js";
+import { heldControl } from "./house.js";
+import { KitRefusal } from "./refusal.js";
+
+const PROTOCOL = "house.kit.stream.1";
+
+function streamAddress(house) {
+  const url = new URL("/kit/stream", house);
+  if (url.protocol === "https:") url.protocol = "wss:";
+  else if (url.protocol === "http:") url.protocol = "ws:";
+  else throw new KitRefusal("kit_house_unavailable", "/kit/stream");
+  return url;
+}
+
+async function connected(session, signal, available, dependencies) {
+  const authorization = await heldControl(session, signal);
+  return new Promise((resolve, reject) => {
+    const Socket = dependencies.workSocket ?? WebSocket;
+    const socket = new Socket(streamAddress(session.house), PROTOCOL, {
+      headers: { authorization, "x-house-kit-version": KIT_CONTRACT_VERSION },
+      handshakeTimeout: 10_000,
+      maxPayload: 4096,
+      perMessageDeflate: false,
+    });
+    let finished = false;
+    const finish = (error = null) => {
+      if (finished) return;
+      finished = true;
+      signal.removeEventListener("abort", stop);
+      if (error === null) resolve();
+      else reject(error);
+    };
+    const stop = () => {
+      socket.terminate();
+      finish();
+    };
+    signal.addEventListener("abort", stop, { once: true });
+    socket.on("open", available);
+    socket.on("message", (data) => {
+      let frame;
+      try {
+        frame = JSON.parse(data.toString());
+      } catch {
+        return;
+      }
+      if (frame?.type === "work_available" && frame.subject === "agents") available();
+    });
+    socket.on("unexpected-response", (_request, response) => {
+      response.resume();
+      const refusal =
+        response.statusCode === 409 ? new KitRefusal("kit_upgrade_required") : null;
+      socket.terminate();
+      finish(refusal);
+    });
+    socket.on("error", () => {
+      socket.terminate();
+      finish();
+    });
+    socket.on("close", () => finish());
+    if (signal.aborted) stop();
+  });
+}
+
+export async function watchKitAgentWork(session, signal, available, dependencies = {}) {
+  let retryMs = 500;
+  while (!signal.aborted) {
+    const began = Date.now();
+    try {
+      await connected(session, signal, available, dependencies);
+    } catch (error) {
+      if (error instanceof KitRefusal && error.code === "kit_upgrade_required") throw error;
+      if (signal.aborted) return;
+    }
+    if (signal.aborted) return;
+    retryMs = Date.now() - began >= 30_000 ? 1000 : Math.min(retryMs * 2, 30_000);
+    try {
+      await delay(dependencies.workReconnectMs ?? retryMs, undefined, { signal });
+    } catch (error) {
+      if (!signal.aborted) throw error;
+    }
+  }
+}

package/src/mount.js

@@ -84,7 +84,6 @@ export function mountTarget(subtree) {
     refuse("kit_mount_subtree_invalid", subtree);
   }
   return {
-    kind: own ? "private" : "room",
     handle: own ? null : segments[1],
     root: own ? "/private" : `/rooms/${segments[1]}`,
     subtree,
@@ -107,10 +106,11 @@ function discoveryOf(value) {
 
 function authorityFor(discovery, target) {
   const found = discovery.authorities.find((entry) =>
-    target.kind === "private" ? entry.kind === "private" : entry.room_handle === target.handle,
+    entry.kind === "room" && entry.room_handle === target.handle,
   );
   if (found === undefined) refuse("kit_mount_authority_unknown", target.root);
   if (
+    typeof found.room_ref !== "string" ||
     !Array.isArray(found.protected_paths) ||
     found.operations === null ||
     typeof found.operations !== "object" ||
@@ -119,8 +119,8 @@ function authorityFor(discovery, target) {
     refuse("kit_mount_rejected", "discover");
   }
   return {
-    kind: found.kind,
-    authority: found.kind === "private" ? "private" : found.room_ref,
+    kind: "room",
+    authority: found.room_ref,
     path: target.root,
     protected_paths: found.protected_paths,
     operations: found.operations,
@@ -326,7 +326,7 @@ function pendingTransfer(held, save) {
   };
 }
 
-async function submitAttachments(session, held, target, authority, plan, signal) {
+async function submitAttachments(session, held, authority, plan, signal) {
   const transfers = [];
   for (const save of plan.saves) {
     const local = held.attachments.get(save.path);
@@ -341,7 +341,7 @@ async function submitAttachments(session, held, target, authority, plan, signal)
         version: save.version,
         name: declared.name,
         mediaType: declared.mediaType ?? undefined,
-        room: target.kind === "room" ? authority.authority : null,
+        room: authority.authority,
         body,
         signal,
       });
@@ -471,7 +471,7 @@ export async function mountPackage(held, options, dependencies = {}) {
     };
     const localReport = join(held.root, "MIGRATION.json");
     await writeFile(localReport, `${JSON.stringify(result, null, 2)}\n`);
-    result.transfers = await submitAttachments(session, held, target, authority, plan, signal);
+    result.transfers = await submitAttachments(session, held, authority, plan, signal);
 
     const report = `${target.subtree}/${MOUNT_REPORT_NAME}`;
     await writeFile(localReport, `${JSON.stringify(result, null, 2)}\n`);

package/src/setup.js

@@ -114,14 +114,14 @@ async function signedIn(module) {
   }
 }
 
-async function setUpModule(context, module) {
+async function setUpModule(context, module, install = true) {
   const summary = (status, reason = null) => ({
     category: module.category,
     name: module.name,
     status,
     reason,
   });
-  if (module.install !== null) {
+  if (install && module.install !== null) {
     try {
       await module.install();
     } catch (error) {
@@ -144,10 +144,15 @@ async function setUpModule(context, module) {
   return summary("ready");
 }
 
+export function signInWantedAgent(context, kind) {
+  return setUpModule(context, agentModule(context, kind), false);
+}
+
 export async function runSetup(context, request) {
   await prepareLaunchBase(launchBase(context.dependencies));
   const modules = [];
   for (const kind of context.configuration.agents) {
+    await context.waitAgentLogin?.(kind);
     modules.push(await setUpModule(context, agentModule(context, kind)));
   }
   for (const instance of context.configuration.instances) {

package/src/source.js

@@ -12,7 +12,7 @@ import {
   newCheckpoint,
   stallOf,
 } from "./delivery.js";
-import { captureBatch, openSession } from "./house.js";
+import { captureBatch, maintainControl, openSession } from "./house.js";
 import { googlePage } from "./google/source.js";
 import { houseAnswered, houseAnsweredFailure, houseFailed, KitRefusal } from "./refusal.js";
 import { recordAcceptedOriginals, sameReference, saveSourceOriginals } from "./source-originals.js";
@@ -204,18 +204,24 @@ export async function captureSource(options, dependencies = {}) {
   const googleOAuthClient =
     instance.preset === "google" ? dependencies.googleOAuthClient() : null;
   const session = await openSession(options, dependencies);
-  const context = {
-    instance,
+  return maintainControl(
     session,
-    directory: options.directory,
-    signal: options.signal,
-    now: session.now,
-    dependencies,
-    googleOAuthClient,
-  };
-  const delivered = [];
-  for (const service of services) delivered.push(await deliverService(context, service));
-  return { instance: instance.id, services: delivered };
+    async (signal) => {
+      const context = {
+        instance,
+        session,
+        directory: options.directory,
+        signal,
+        now: session.now,
+        dependencies,
+        googleOAuthClient,
+      };
+      const delivered = [];
+      for (const service of services) delivered.push(await deliverService(context, service));
+      return { instance: instance.id, services: delivered };
+    },
+    options.signal,
+  );
 }
 
 export async function sourceStatus(options) {
@@ -255,32 +261,40 @@ export async function retrySourceOriginals(options, dependencies = {}) {
   const configuration = await readConfiguration(options.configuration);
   const instance = sourceInstance(configuration, options.instance);
   const services = declaredServices(instance, options.service ?? null);
-  const context = { instance, directory: options.directory, signal: options.signal };
   const outstanding = [];
   for (const service of services) {
     outstanding.push((await readSourceOriginals(options.directory, instance.id, service.service)).length);
   }
-  if (outstanding.some((count) => count > 0)) context.session = await openSession(options, dependencies);
-  const retried = [];
-  for (const [position, service] of services.entries()) {
-    retried.push({
-      service: service.service,
-      originals:
-        outstanding[position] === 0
-          ? []
-          : await saveSourceOriginals(context, serviceBinding(context, service)),
-    });
-  }
-  return { instance: instance.id, services: retried };
+  const retry = async (context) => {
+    const retried = [];
+    for (const [position, service] of services.entries()) {
+      retried.push({
+        service: service.service,
+        originals:
+          outstanding[position] === 0
+            ? []
+            : await saveSourceOriginals(context, serviceBinding(context, service)),
+      });
+    }
+    return { instance: instance.id, services: retried };
+  };
+  const context = { instance, directory: options.directory, signal: options.signal };
+  if (outstanding.every((count) => count === 0)) return retry(context);
+  const session = await openSession(options, dependencies);
+  return maintainControl(session, (signal) => retry({ ...context, session, signal }), options.signal);
 }
 
 export async function resetSource(options, dependencies = {}) {
   const configuration = await readConfiguration(options.configuration);
   const instance = sourceInstance(configuration, options.instance);
   declaredServices(instance, options.service);
-  await openSession(options, dependencies);
-  return {
-    instance: instance.id,
-    services: await resetCheckpoints(options.directory, instance.id, options.service),
-  };
+  const session = await openSession(options, dependencies);
+  return maintainControl(
+    session,
+    async () => ({
+      instance: instance.id,
+      services: await resetCheckpoints(options.directory, instance.id, options.service),
+    }),
+    options.signal,
+  );
 }

package/src/sync-state.js

@@ -26,6 +26,9 @@ function syncFile(directory, key, kind = "") {
 function recordedAuthority(authority) {
   const operations = authority?.operations;
   return (
+    authority?.kind === "room" &&
+    typeof authority.authority === "string" &&
+    typeof authority.path === "string" &&
     typeof authority?.log_epoch === "string" &&
     operations !== null &&
     typeof operations === "object" &&
@@ -43,6 +46,7 @@ export async function readReplicaState(directory, replica) {
   }
   const state = await readNote(syncFile(directory, replicaKey(canonical)), null);
   if (state === null) return null;
+  if (state.version !== EXPLICIT_SYNC_VERSION) return null;
   if (!Array.isArray(state.authorities) || !state.authorities.every(recordedAuthority)) return null;
   return state;
 }

package/src/sync.js

@@ -105,9 +105,6 @@ async function bootstrapBundle(session, prefixes, signal) {
 }
 
 function matchedDiscovery(discovery, scope) {
-  if (scope.manifest?.scope === "private") {
-    return discovery.authorities.find((entry) => entry.kind === "private");
-  }
   return discovery.authorities.find(
     (entry) => entry.kind === "room" && entry.room_ref === scope.manifest?.anchor,
   );
@@ -173,11 +170,17 @@ function operationsOf(value) {
 function authorityRecord(scope, discovery, files) {
   const found = matchedDiscovery(discovery, scope);
   const operations = found === undefined ? null : operationsOf(found.operations);
-  if (found === undefined || !stringList(found.protected_paths) || operations === null) {
+  if (
+    scope.manifest?.scope !== "room" ||
+    typeof scope.manifest.anchor !== "string" ||
+    found === undefined ||
+    !stringList(found.protected_paths) ||
+    operations === null
+  ) {
     throw new KitRefusal("kit_bootstrap_rejected");
   }
   return {
-    kind: scope.manifest.scope === "private" ? "private" : "room",
+    kind: "room",
     authority: scope.manifest.anchor,
     room_handle: scope.manifest.room_handle,
     path: scope.manifest.path,
@@ -192,9 +195,10 @@ function authorityRecord(scope, discovery, files) {
 async function materialize(options, discovery, bundle, signal, previous = null) {
   const authorities = [];
   for (const scope of bundle.scopes) {
+    const record = authorityRecord(scope, discovery, []);
     const held = previous?.authorities.find((entry) => entry.authority === scope.manifest?.anchor);
     const files = await replaceScope(options.replica, held, scope, signal);
-    authorities.push(authorityRecord(scope, discovery, files));
+    authorities.push({ ...record, files });
   }
   const state = {
     version: EXPLICIT_SYNC_VERSION,
@@ -409,6 +413,7 @@ export async function bootstrap(options, dependencies = {}) {
   const canonical = await canonicalReplica(options.replica);
   const existing = await readReplicaState(options.directory, canonical);
   if (existing !== null) throw new KitRefusal("kit_sync_already_bootstrapped", canonical);
+  await clearPending(options.directory, canonical);
   return withSession(options, dependencies, async (session, signal) => {
     const discovery = discoveryOf(
       await door(session, KIT_DOOR_DISCOVER_PATH, { prefixes: options.prefixes }, signal),