@agentshouse/kit 0.1.0-alpha.26 → 0.1.0-alpha.28

package/src/kit-work-stream.js

@@ -1,11 +1,17 @@
 import { setTimeout as delay } from "node:timers/promises";
 import WebSocket from "ws";
-import { KIT_CONTRACT_VERSION } from "./contract.js";
+import {
+  KIT_CONTRACT_VERSION,
+  KIT_STREAM_RECONNECT_MAX_MS,
+  KIT_STREAM_RECONNECT_MS,
+  KIT_STREAM_REJOIN_MS,
+} from "./contract.js";
 import { heldControl } from "./house.js";
 import { KitRefusal } from "./refusal.js";
 
 const PROTOCOL = "house.kit.stream.1";
-const WATCHED_SUBJECTS = ["agents", "working_copy"];
+const WORK_SUBJECTS = ["runs", "commands", "agents", "working_copy", "sources"];
+const FINAL_REFUSALS = ["kit_upgrade_required", "kit_stream_replaced"];
 
 function streamAddress(house) {
   const url = new URL("/kit/stream", house);
@@ -15,72 +21,88 @@ function streamAddress(house) {
   return url;
 }
 
-async function connected(session, signal, available, dependencies) {
+function frameOf(data) {
+  try {
+    return JSON.parse(data.toString());
+  } catch {
+    return null;
+  }
+}
+
+async function held(session, signal, heard, dependencies) {
   const authorization = await heldControl(session, signal);
   return new Promise((resolve, reject) => {
     const Socket = dependencies.workSocket ?? WebSocket;
     const socket = new Socket(streamAddress(session.house), PROTOCOL, {
       headers: { authorization, "x-house-kit-version": KIT_CONTRACT_VERSION },
       handshakeTimeout: 10_000,
-      maxPayload: 4096,
       perMessageDeflate: false,
     });
     let finished = false;
-    const finish = (error = null) => {
+    const finish = (reason = null, refusal = null) => {
       if (finished) return;
       finished = true;
       signal.removeEventListener("abort", stop);
-      if (error === null) resolve();
-      else reject(error);
+      if (refusal === null) resolve(reason);
+      else reject(refusal);
     };
     const stop = () => {
       socket.terminate();
       finish();
     };
     signal.addEventListener("abort", stop, { once: true });
-    socket.on("open", () => available(null));
+    socket.on("open", () => heard.available(null));
     socket.on("message", (data) => {
-      let frame;
-      try {
-        frame = JSON.parse(data.toString());
-      } catch {
-        return;
-      }
-      if (frame?.type === "work_available" && WATCHED_SUBJECTS.includes(frame.subject)) {
-        available(frame.subject);
+      const frame = frameOf(data);
+      if (frame?.type === "work_available" && WORK_SUBJECTS.includes(frame.subject)) {
+        heard.available(frame.subject);
+      } else if (frame?.type === "entries") {
+        heard.changed();
       }
     });
     socket.on("unexpected-response", (_request, response) => {
       response.resume();
       if (response.statusCode === 409) session.dependencies.contractRefused?.();
-      const refusal =
-        response.statusCode === 409 ? new KitRefusal("kit_upgrade_required") : null;
+      finish(null, response.statusCode === 409 ? new KitRefusal("kit_upgrade_required") : null);
       socket.terminate();
-      finish(refusal);
     });
     socket.on("error", () => {
       socket.terminate();
       finish();
     });
-    socket.on("close", () => finish());
+    socket.on("close", (_code, reason) => {
+      const closed = reason?.toString() ?? "";
+      finish(closed, closed === "replaced" ? new KitRefusal("kit_stream_replaced") : null);
+    });
     if (signal.aborted) stop();
   });
 }
 
-export async function watchKitWork(session, signal, available, dependencies = {}) {
-  let retryMs = 500;
+export async function watchKitWork(session, signal, heard, dependencies = {}) {
+  const random = dependencies.random ?? Math.random;
+  const sleep = dependencies.streamSleep ?? ((milliseconds) => delay(milliseconds, undefined, { signal }));
+  let failures = 0;
   while (!signal.aborted) {
     const began = Date.now();
+    let reason = null;
     try {
-      await connected(session, signal, available, dependencies);
+      reason = await held(session, signal, heard, dependencies);
     } catch (error) {
-      if (error instanceof KitRefusal && error.code === "kit_upgrade_required") throw error;
-      if (signal.aborted) return;
+      if (error instanceof KitRefusal && FINAL_REFUSALS.includes(error.code)) throw error;
     }
     if (signal.aborted) return;
-    retryMs = Date.now() - began >= 30_000 ? 1000 : Math.min(retryMs * 2, 30_000);
+    let wait;
+    if (reason === "authority_changed") {
+      failures = 0;
+      wait = random() * KIT_STREAM_REJOIN_MS;
+    } else {
+      if (Date.now() - began >= KIT_STREAM_RECONNECT_MAX_MS) failures = 0;
+      const ceiling = Math.min(KIT_STREAM_RECONNECT_MAX_MS, KIT_STREAM_RECONNECT_MS * 2 ** failures);
+      failures += 1;
+      wait = ceiling / 2 + (random() * ceiling) / 2;
+    }
     try {
-      await delay(dependencies.workReconnectMs ?? retryMs, undefined, { signal });
+      await sleep(wait);
     } catch (error) {
       if (!signal.aborted) throw error;
     }

package/src/setup.js

@@ -6,7 +6,7 @@ import {
   updateDeclaredAgent,
 } from "./agents.js";
 import { readConfiguration } from "./configuration.js";
-import { connectorLogin } from "./connector-login.js";
+import { customAuthorizationStatus } from "./custom/authorization.js";
 import { GUIDED_LOGIN_TIMEOUT_MS } from "./contract.js";
 import { launchBase, prepareLaunchBase } from "./directory.js";
 import {
@@ -45,24 +45,14 @@ function agentModule(context, kind) {
 }
 
 function connectorModule(context, instance) {
-  const gateway = connectorLogin(instance, context.dependencies);
   return {
     category: "connector",
     name: instance.id,
     login: { integration: instance.preset, instance: instance.id, service: instance.id },
     install: null,
     signedIn: () =>
-      gateway.status(instance, { directory: context.directory, signal: context.signal }),
-    guided:
-      gateway.guided === undefined
-        ? null
-        : ({ present, signal }) =>
-            gateway.guided(instance, {
-              directory: context.directory,
-              dependencies: context.dependencies,
-              present,
-              signal,
-            }),
+      customAuthorizationStatus(instance, { directory: context.directory, signal: context.signal }),
+    guided: null,
   };
 }
 
@@ -144,8 +134,12 @@ async function setUpModule(context, module, install = true) {
   return summary("ready");
 }
 
+export function signInModule(context, module) {
+  return setUpModule(context, module, false);
+}
+
 export function signInWantedAgent(context, kind) {
-  return setUpModule(context, agentModule(context, kind), false);
+  return signInModule(context, agentModule(context, kind));
 }
 
 export async function runSetup(context, request) {

package/src/source.js

@@ -13,7 +13,8 @@ import {
   stallOf,
 } from "./delivery.js";
 import { captureBatch, maintainControl, openSession } from "./house.js";
-import { googlePage } from "./google/source.js";
+import { GOOGLE_SOURCES } from "./google/oauth.js";
+import { googlePage, recordAccountEmail } from "./google/source.js";
 import { houseAnswered, houseAnsweredFailure, houseFailed, KitRefusal } from "./refusal.js";
 import { recordAcceptedOriginals, sameReference, saveSourceOriginals } from "./source-originals.js";
 import {
@@ -29,14 +30,15 @@ import {
   writeCheckpoint,
 } from "./state.js";
 
+const GOOGLE = Object.freeze({ id: GOOGLE_SOURCES, preset: "google" });
+
 async function acquire(context, service, checkpoint, limit) {
-  if (context.instance.preset === "google") {
+  if (service.google !== undefined) {
     return googlePage(
       {
         directory: context.directory,
-        instance: context.instance.id,
-        client: context.googleOAuthClient,
-        service,
+        account: service.google.account,
+        service: service.google.service,
         cursor: checkpoint.cursor,
         limit,
         backfillFloor:
@@ -169,6 +171,7 @@ async function deliverService(context, service) {
       context.instance.preset === "custom"
         ? { ...context, environment: await resolveEnvironment(service.connector) }
         : context;
+    if (service.google !== undefined) await recordAccountEmail(context, service.google);
     await runDeliveryLoop(resolved, service, progress);
   } catch (refusal) {
     if (!(refusal instanceof KitRefusal) || refusal.code === "kit_command_cancelled") throw refusal;
@@ -197,12 +200,31 @@ async function deliverService(context, service) {
   };
 }
 
+function initialBackfill(initial) {
+  if (initial.new === true) return { kind: "days", bound: 0 };
+  if (initial.days !== undefined) return { kind: "days", bound: initial.days };
+  if (initial.items !== undefined) return { kind: "count", bound: initial.items };
+  return { kind: "all" };
+}
+
+export async function captureEntry(context, entry) {
+  const delivered = await deliverService(
+    { ...context, instance: GOOGLE },
+    {
+      service: entry.source,
+      sourceRef: entry.source,
+      rooms: entry.rooms,
+      backfill: initialBackfill(entry.initial),
+      google: { account: entry.account, service: entry.service },
+    },
+  );
+  return { ...delivered, service: entry.service };
+}
+
 export async function captureSource(options, dependencies = {}) {
   const configuration = await readConfiguration(options.configuration);
   const instance = sourceInstance(configuration, options.instance);
   const services = declaredServices(instance, options.service ?? null);
-  const googleOAuthClient =
-    instance.preset === "google" ? dependencies.googleOAuthClient() : null;
   const session = await openSession(options, dependencies);
   return maintainControl(
     session,
@@ -214,7 +236,6 @@ export async function captureSource(options, dependencies = {}) {
         signal,
         now: session.now,
         dependencies,
-        googleOAuthClient,
       };
       const delivered = [];
       for (const service of services) delivered.push(await deliverService(context, service));
@@ -224,6 +245,27 @@ export async function captureSource(options, dependencies = {}) {
   );
 }
 
+export async function localServiceState(directory, instance, service) {
+  return {
+    health: (await readHealth(directory, instance, service)) ?? {
+      state: "pending",
+      category: null,
+      observed_at: null,
+    },
+    checkpoint: await readCheckpoint(directory, instance, service),
+    gaps: await readCoverageGaps(directory, instance, service),
+    dropped: await readDroppedItems(directory, instance, service),
+    originals: (await readSourceOriginals(directory, instance, service)).map((reference) => ({
+      room: reference.room,
+      record: reference.record,
+      position: reference.position,
+      native_object_id: reference.native_object_id,
+      save: reference.save,
+      error: reference.error,
+    })),
+  };
+}
+
 export async function sourceStatus(options) {
   const configuration = await readConfiguration(options.configuration);
   const instance = sourceInstance(configuration, options.instance);
@@ -234,24 +276,7 @@ export async function sourceStatus(options) {
       service: service.service,
       source_ref: service.sourceRef,
       rooms: service.rooms,
-      health: (await readHealth(options.directory, instance.id, service.service)) ?? {
-        state: "pending",
-        category: null,
-        observed_at: null,
-      },
-      checkpoint: await readCheckpoint(options.directory, instance.id, service.service),
-      gaps: await readCoverageGaps(options.directory, instance.id, service.service),
-      dropped: await readDroppedItems(options.directory, instance.id, service.service),
-      originals: (await readSourceOriginals(options.directory, instance.id, service.service)).map(
-        (reference) => ({
-          room: reference.room,
-          record: reference.record,
-          position: reference.position,
-          native_object_id: reference.native_object_id,
-          save: reference.save,
-          error: reference.error,
-        }),
-      ),
+      ...(await localServiceState(options.directory, instance.id, service.service)),
     });
   }
   return { instance: instance.id, preset: instance.preset, services: reported };

package/src/state.js

@@ -9,10 +9,28 @@ function segment(name) {
   return name;
 }
 
-function notePath(directory, instance, service, kind) {
+export function notePath(directory, instance, service, kind) {
   return join(directory, "state", segment(instance), kind, `${segment(service)}.json`);
 }
 
+export async function noteNames(directory, instance, kind) {
+  let present;
+  try {
+    present = await readdir(join(directory, "state", segment(instance), kind));
+  } catch (error) {
+    if (error.code === "ENOENT") return [];
+    throw new KitRefusal("kit_state_unreadable", kind);
+  }
+  return present
+    .filter((name) => name.endsWith(".json"))
+    .map((name) => name.slice(0, -".json".length))
+    .sort();
+}
+
+export async function removeNotes(directory, instance, service, kinds) {
+  for (const kind of kinds) await rm(notePath(directory, instance, service, kind), { force: true });
+}
+
 function repositoryPath(directory, key) {
   if (!/^[a-f0-9]{64}$/.test(key)) throw new KitRefusal("kit_state_name_invalid", key);
   return join(directory, "state", "repositories", `${key}.json`);

package/src/tasks.js

@@ -0,0 +1,75 @@
+import { setTimeout as delay } from "node:timers/promises";
+
+export function coalescedTask(signal, fail, work) {
+  let pending = false;
+  let task = null;
+  const trigger = () => {
+    pending = true;
+    if (task !== null) return;
+    task = (async () => {
+      while (pending && !signal.aborted) {
+        pending = false;
+        await work();
+      }
+    })()
+      .catch((error) => {
+        if (signal.aborted) return;
+        fail(error);
+      })
+      .finally(() => {
+        task = null;
+        if (pending && !signal.aborted) trigger();
+      });
+  };
+  return { trigger, idle: () => task === null, settled: () => task };
+}
+
+export function residentWake() {
+  let pending = false;
+  let release = null;
+  return {
+    notify() {
+      pending = true;
+      release?.();
+    },
+    async wait(milliseconds, signal) {
+      if (!pending && !signal.aborted) {
+        const pause = new AbortController();
+        const paused = AbortSignal.any([signal, pause.signal]);
+        try {
+          await Promise.race([
+            new Promise((resolve) => {
+              release = resolve;
+            }),
+            Number.isFinite(milliseconds)
+              ? delay(milliseconds, undefined, { signal: paused })
+              : new Promise((resolve) => paused.addEventListener("abort", resolve, { once: true })),
+          ]);
+        } catch (error) {
+          if (!signal.aborted) throw error;
+        } finally {
+          release = null;
+          pause.abort();
+        }
+      }
+      const woken = pending;
+      pending = false;
+      return woken;
+    },
+  };
+}
+
+const turns = new Map();
+
+export function exclusively(path, work) {
+  const turn = (turns.get(path) ?? Promise.resolve()).then(work);
+  const settled = turn.then(
+    () => undefined,
+    () => undefined,
+  );
+  turns.set(path, settled);
+  void settled.then(() => {
+    if (turns.get(path) === settled) turns.delete(path);
+  });
+  return turn;
+}