@agentshouse/kit 0.1.0-alpha.26 → 0.1.0-alpha.27

package/src/source.js

@@ -13,7 +13,8 @@ import {
   stallOf,
 } from "./delivery.js";
 import { captureBatch, maintainControl, openSession } from "./house.js";
-import { googlePage } from "./google/source.js";
+import { GOOGLE_SOURCES } from "./google/oauth.js";
+import { googlePage, recordAccountEmail } from "./google/source.js";
 import { houseAnswered, houseAnsweredFailure, houseFailed, KitRefusal } from "./refusal.js";
 import { recordAcceptedOriginals, sameReference, saveSourceOriginals } from "./source-originals.js";
 import {
@@ -29,14 +30,15 @@ import {
   writeCheckpoint,
 } from "./state.js";
 
+const GOOGLE = Object.freeze({ id: GOOGLE_SOURCES, preset: "google" });
+
 async function acquire(context, service, checkpoint, limit) {
-  if (context.instance.preset === "google") {
+  if (service.google !== undefined) {
     return googlePage(
       {
         directory: context.directory,
-        instance: context.instance.id,
-        client: context.googleOAuthClient,
-        service,
+        account: service.google.account,
+        service: service.google.service,
         cursor: checkpoint.cursor,
         limit,
         backfillFloor:
@@ -169,6 +171,7 @@ async function deliverService(context, service) {
       context.instance.preset === "custom"
         ? { ...context, environment: await resolveEnvironment(service.connector) }
         : context;
+    if (service.google !== undefined) await recordAccountEmail(context, service.google);
     await runDeliveryLoop(resolved, service, progress);
   } catch (refusal) {
     if (!(refusal instanceof KitRefusal) || refusal.code === "kit_command_cancelled") throw refusal;
@@ -197,12 +200,31 @@ async function deliverService(context, service) {
   };
 }
 
+function initialBackfill(initial) {
+  if (initial.new === true) return { kind: "days", bound: 0 };
+  if (initial.days !== undefined) return { kind: "days", bound: initial.days };
+  if (initial.items !== undefined) return { kind: "count", bound: initial.items };
+  return { kind: "all" };
+}
+
+export async function captureEntry(context, entry) {
+  const delivered = await deliverService(
+    { ...context, instance: GOOGLE },
+    {
+      service: entry.source,
+      sourceRef: entry.source,
+      rooms: entry.rooms,
+      backfill: initialBackfill(entry.initial),
+      google: { account: entry.account, service: entry.service },
+    },
+  );
+  return { ...delivered, service: entry.service };
+}
+
 export async function captureSource(options, dependencies = {}) {
   const configuration = await readConfiguration(options.configuration);
   const instance = sourceInstance(configuration, options.instance);
   const services = declaredServices(instance, options.service ?? null);
-  const googleOAuthClient =
-    instance.preset === "google" ? dependencies.googleOAuthClient() : null;
   const session = await openSession(options, dependencies);
   return maintainControl(
     session,
@@ -214,7 +236,6 @@ export async function captureSource(options, dependencies = {}) {
         signal,
         now: session.now,
         dependencies,
-        googleOAuthClient,
       };
       const delivered = [];
       for (const service of services) delivered.push(await deliverService(context, service));
@@ -224,6 +245,27 @@ export async function captureSource(options, dependencies = {}) {
   );
 }
 
+export async function localServiceState(directory, instance, service) {
+  return {
+    health: (await readHealth(directory, instance, service)) ?? {
+      state: "pending",
+      category: null,
+      observed_at: null,
+    },
+    checkpoint: await readCheckpoint(directory, instance, service),
+    gaps: await readCoverageGaps(directory, instance, service),
+    dropped: await readDroppedItems(directory, instance, service),
+    originals: (await readSourceOriginals(directory, instance, service)).map((reference) => ({
+      room: reference.room,
+      record: reference.record,
+      position: reference.position,
+      native_object_id: reference.native_object_id,
+      save: reference.save,
+      error: reference.error,
+    })),
+  };
+}
+
 export async function sourceStatus(options) {
   const configuration = await readConfiguration(options.configuration);
   const instance = sourceInstance(configuration, options.instance);
@@ -234,24 +276,7 @@ export async function sourceStatus(options) {
       service: service.service,
       source_ref: service.sourceRef,
       rooms: service.rooms,
-      health: (await readHealth(options.directory, instance.id, service.service)) ?? {
-        state: "pending",
-        category: null,
-        observed_at: null,
-      },
-      checkpoint: await readCheckpoint(options.directory, instance.id, service.service),
-      gaps: await readCoverageGaps(options.directory, instance.id, service.service),
-      dropped: await readDroppedItems(options.directory, instance.id, service.service),
-      originals: (await readSourceOriginals(options.directory, instance.id, service.service)).map(
-        (reference) => ({
-          room: reference.room,
-          record: reference.record,
-          position: reference.position,
-          native_object_id: reference.native_object_id,
-          save: reference.save,
-          error: reference.error,
-        }),
-      ),
+      ...(await localServiceState(options.directory, instance.id, service.service)),
     });
   }
   return { instance: instance.id, preset: instance.preset, services: reported };

package/src/state.js

@@ -9,10 +9,28 @@ function segment(name) {
   return name;
 }
 
-function notePath(directory, instance, service, kind) {
+export function notePath(directory, instance, service, kind) {
   return join(directory, "state", segment(instance), kind, `${segment(service)}.json`);
 }
 
+export async function noteNames(directory, instance, kind) {
+  let present;
+  try {
+    present = await readdir(join(directory, "state", segment(instance), kind));
+  } catch (error) {
+    if (error.code === "ENOENT") return [];
+    throw new KitRefusal("kit_state_unreadable", kind);
+  }
+  return present
+    .filter((name) => name.endsWith(".json"))
+    .map((name) => name.slice(0, -".json".length))
+    .sort();
+}
+
+export async function removeNotes(directory, instance, service, kinds) {
+  for (const kind of kinds) await rm(notePath(directory, instance, service, kind), { force: true });
+}
+
 function repositoryPath(directory, key) {
   if (!/^[a-f0-9]{64}$/.test(key)) throw new KitRefusal("kit_state_name_invalid", key);
   return join(directory, "state", "repositories", `${key}.json`);

package/src/tasks.js

@@ -0,0 +1,72 @@
+import { setTimeout as delay } from "node:timers/promises";
+
+export function coalescedTask(signal, fail, work) {
+  let pending = false;
+  let task = null;
+  const trigger = () => {
+    pending = true;
+    if (task !== null) return;
+    task = (async () => {
+      while (pending && !signal.aborted) {
+        pending = false;
+        await work();
+      }
+    })()
+      .catch((error) => {
+        if (signal.aborted) return;
+        fail(error);
+      })
+      .finally(() => {
+        task = null;
+        if (pending && !signal.aborted) trigger();
+      });
+  };
+  return { trigger, idle: () => task === null, settled: () => task };
+}
+
+export function residentWake() {
+  let pending = false;
+  let release = null;
+  return {
+    notify() {
+      pending = true;
+      release?.();
+    },
+    async wait(milliseconds, signal) {
+      if (pending) {
+        pending = false;
+        return;
+      }
+      const pause = new AbortController();
+      try {
+        await Promise.race([
+          new Promise((resolve) => {
+            release = resolve;
+          }),
+          delay(milliseconds, undefined, { signal: AbortSignal.any([signal, pause.signal]) }),
+        ]);
+      } catch (error) {
+        if (!signal.aborted) throw error;
+      } finally {
+        release = null;
+        pending = false;
+        pause.abort();
+      }
+    },
+  };
+}
+
+const turns = new Map();
+
+export function exclusively(path, work) {
+  const turn = (turns.get(path) ?? Promise.resolve()).then(work);
+  const settled = turn.then(
+    () => undefined,
+    () => undefined,
+  );
+  turns.set(path, settled);
+  void settled.then(() => {
+    if (turns.get(path) === settled) turns.delete(path);
+  });
+  return turn;
+}

package/src/worker.js

@@ -20,7 +20,9 @@ import { localRequest, reconcileInbound } from "./inbound.js";
 import { localFiles } from "./local-files.js";
 import { answerText, answerYaml, editRefusal } from "./mcp.js";
 import { inheritedEnvironment } from "./command.js";
+import { sourceEntries } from "./google/entries.js";
 import { watchKitWork } from "./kit-work-stream.js";
+import { coalescedTask, residentWake } from "./tasks.js";
 import {
   applyWorkingCopies,
   deliverWorkingCopies,
@@ -955,62 +957,6 @@ async function reportedWorkingCopies(act, context, signal) {
   }
 }
 
-function coalescedTask(signal, fail, work) {
-  let pending = false;
-  let task = null;
-  const trigger = () => {
-    pending = true;
-    if (task !== null) return;
-    task = (async () => {
-      while (pending && !signal.aborted) {
-        pending = false;
-        await work();
-      }
-    })()
-      .catch((error) => {
-        if (signal.aborted) return;
-        fail(error);
-      })
-      .finally(() => {
-        task = null;
-        if (pending && !signal.aborted) trigger();
-      });
-  };
-  return { trigger, idle: () => task === null, settled: () => task };
-}
-
-function residentWake() {
-  let pending = false;
-  let release = null;
-  return {
-    notify() {
-      pending = true;
-      release?.();
-    },
-    async wait(milliseconds, signal) {
-      if (pending) {
-        pending = false;
-        return;
-      }
-      const pause = new AbortController();
-      try {
-        await Promise.race([
-          new Promise((resolve) => {
-            release = resolve;
-          }),
-          delay(milliseconds, undefined, { signal: AbortSignal.any([signal, pause.signal]) }),
-        ]);
-      } catch (error) {
-        if (!signal.aborted) throw error;
-      } finally {
-        release = null;
-        pending = false;
-        pause.abort();
-      }
-    },
-  };
-}
-
 function residentUpdate(context, wake) {
   let requested = existsSync(updateDrain(context.directory));
   let ready = null;
@@ -1078,6 +1024,7 @@ export async function runResident(options, dependencies = {}) {
   const wake = residentWake();
   const update = residentUpdate(context, wake);
   context.update = update;
+  let sources = null;
   const served = await serveResident(
     session,
     context.directory,
@@ -1087,6 +1034,8 @@ export async function runResident(options, dependencies = {}) {
       pushWorkingCopy({ directory: context.directory, session }, request, ownerCaller(session), signal).catch(
         (error) => ({ refused: true, text: formatCommandError(error) }),
       ),
+    (source) =>
+      sources === null ? Promise.reject(new KitRefusal("kit_resident_unavailable")) : sources.capture(source),
   );
   const serve = async (signal) => {
     context.busy = true;
@@ -1149,13 +1098,24 @@ export async function runResident(options, dependencies = {}) {
     const deliveries = coalescedTask(signal, unavailable, () =>
       reportedWorkingCopies(deliverWorkingCopies, context, signal),
     );
+    sources = sourceEntries({
+      session: context.session,
+      directory: context.directory,
+      dependencies,
+      signal: AbortSignal.any([signal, watcher.signal]),
+      now: session.now,
+      draining: () => update.requested,
+      unavailable: (error) => reported(`google sources unavailable: ${error?.message ?? error}`),
+    });
+    await sources.start();
     const pointer = () => {
       if (!update.requested) agentRefresh.trigger();
     };
-    const available = (subject) => {
+    const available = (subject = null) => {
       if (update.requested) return;
-      if (subject !== "working_copy") agentRefresh.trigger();
-      if (subject !== "agents") copies.trigger();
+      if (subject === null || subject === "agents") agentRefresh.trigger();
+      if (subject === null || subject === "working_copy") copies.trigger();
+      if (subject === null || subject === "sources") sources.reconcile();
     };
     context.onIdle = pointer;
     context.workingCopiesApplied = copies.settled;
@@ -1180,7 +1140,12 @@ export async function runResident(options, dependencies = {}) {
           if (sessions.length > 0) update.answer({ ready: false, sessions });
           else if (
             session.refused ||
-            (!served.serving() && logins.size === 0 && agentRefresh.idle() && copies.idle() && deliveries.idle())
+            (!served.serving() &&
+              logins.size === 0 &&
+              agentRefresh.idle() &&
+              copies.idle() &&
+              deliveries.idle() &&
+              sources.idle())
           ) {
             return;
           }
@@ -1194,6 +1159,7 @@ export async function runResident(options, dependencies = {}) {
       await agentRefresh.settled();
       await copies.settled();
       await deliveries.settled();
+      await sources.settled();
       for (const running of logins.values()) running.controller.abort();
       await Promise.all([...logins.values()].map((running) => running.task));
       await context.agentRefresh;

package/src/working-copy.js

@@ -23,6 +23,7 @@ import { readNote, writeNote, writeText } from "./state.js";
 import { bootstrapBundle, discoveryOf, door, withSession } from "./sync.js";
 import { isGeneratedProvenance, sidecarPath, structuredOrReplace } from "./sync-library.js";
 import { locate, removeReplicaFile, writeReplicaFile } from "./sync-state.js";
+import { exclusively } from "./tasks.js";
 
 const COMMITTER_NAME = "House Kit";
 const COMMITTER_EMAIL = "house-kit@localhost";
@@ -366,21 +367,6 @@ async function heldCopies(directory) {
   return held.filter((copy) => copy !== null);
 }
 
-const turns = new Map();
-
-function exclusively(path, work) {
-  const turn = (turns.get(path) ?? Promise.resolve()).then(work);
-  const settled = turn.then(
-    () => undefined,
-    () => undefined,
-  );
-  turns.set(path, settled);
-  void settled.then(() => {
-    if (turns.get(path) === settled) turns.delete(path);
-  });
-  return turn;
-}
-
 function received(scope, read, parents) {
   const files = read.files.filter((file) => file.writable);
   return { message: `House ${housePath(scope)} at position ${read.position}`, parents, files, whole: true };

package/src/google/declared.js

@@ -1,146 +0,0 @@
-import { ITEM_BODY_MAX_BYTES } from "../contract.js";
-import { KitRefusal } from "../refusal.js";
-import { googleJson } from "./gateway.js";
-
-function valueAt(value, path) {
-  let current = value;
-  for (const segment of path.split(".")) {
-    if (current === null || current === undefined || typeof current !== "object") return undefined;
-    current = current[segment];
-  }
-  return current;
-}
-
-function requiredText(value, declaration) {
-  if (typeof value !== "string" || value.length === 0) {
-    throw new KitRefusal("google_provider_response_invalid", declaration);
-  }
-  return value;
-}
-
-function optionalText(value, declaration) {
-  if (value === undefined || value === null) return null;
-  return requiredText(value, declaration);
-}
-
-function cursorOf(value) {
-  if (value === null) return { pageToken: null, checkpoint: null };
-  let cursor;
-  try {
-    cursor = JSON.parse(value);
-  } catch {
-    throw new KitRefusal("google_checkpoint_invalid", "declared service");
-  }
-  if (
-    typeof cursor !== "object" ||
-    cursor === null ||
-    (cursor.pageToken !== null && typeof cursor.pageToken !== "string") ||
-    (cursor.checkpoint !== null && typeof cursor.checkpoint !== "string")
-  ) {
-    throw new KitRefusal("google_checkpoint_invalid", "declared service");
-  }
-  return cursor;
-}
-
-function composedBody(item, paths) {
-  const parts = paths.map((path) => valueAt(item, path));
-  if (parts.some((part) => !["string", "number", "boolean"].includes(typeof part))) {
-    throw new KitRefusal("google_provider_response_invalid", "declared body");
-  }
-  const body = parts.map((part) => `${part}`).join("\n\n");
-  if (Buffer.byteLength(body, "utf8") > ITEM_BODY_MAX_BYTES) {
-    throw new KitRefusal("google_provider_response_invalid", "declared body");
-  }
-  return body;
-}
-
-function normalizedItem(service, item) {
-  const fields = service.declaration.fields;
-  const observedAt = requiredText(valueAt(item, fields.observedAt), "declared observed time");
-  if (Number.isNaN(Date.parse(observedAt))) {
-    throw new KitRefusal("google_provider_response_invalid", "declared observed time");
-  }
-  const container = requiredText(valueAt(item, fields.container), "declared container");
-  const nativeVersion =
-    fields.nativeVersion === null
-      ? null
-      : optionalText(valueAt(item, fields.nativeVersion), "declared native version");
-  const providerModifiedAt =
-    fields.providerModifiedAt === null
-      ? null
-      : optionalText(
-          valueAt(item, fields.providerModifiedAt),
-          "declared provider modification time",
-        );
-  if (providerModifiedAt !== null && Number.isNaN(Date.parse(providerModifiedAt))) {
-    throw new KitRefusal("google_provider_response_invalid", "declared provider modification time");
-  }
-  return {
-    namespace: `google/${service.service}`,
-    nativeObjectId: requiredText(valueAt(item, fields.nativeId), "declared native id"),
-    nativeVersion,
-    providerModifiedAt:
-      providerModifiedAt === null ? null : new Date(providerModifiedAt).toISOString(),
-    observedAt: new Date(observedAt).toISOString(),
-    body: composedBody(item, fields.body),
-    author:
-      fields.author === null
-        ? null
-        : optionalText(valueAt(item, fields.author), "declared author"),
-    url:
-      fields.url === null ? null : optionalText(valueAt(item, fields.url), "declared url"),
-    attachments: [],
-    container: `google/${service.service}/${container}`,
-  };
-}
-
-export async function declaredPage(options, dependencies = {}) {
-  const declared = options.service.declaration;
-  const cursor = cursorOf(options.cursor);
-  const url = new URL(declared.path, `https://${declared.host}`);
-  for (const [name, value] of Object.entries(declared.query)) url.searchParams.set(name, value);
-  url.searchParams.set(declared.pageSize, `${options.limit}`);
-  if (cursor.pageToken !== null) {
-    url.searchParams.set(declared.pageToken.request, cursor.pageToken);
-  }
-  if (cursor.checkpoint !== null) {
-    url.searchParams.set(declared.pageToken.checkpointRequest, cursor.checkpoint);
-  }
-  const answer = await googleJson(
-    {
-      directory: options.directory,
-      instance: options.instance,
-      client: options.client,
-      requiredScopes: declared.scopes,
-      url: url.toString(),
-      signal: options.signal,
-    },
-    dependencies,
-  );
-  const entries = valueAt(answer, declared.items);
-  if (!Array.isArray(entries) || entries.length > options.limit) {
-    throw new KitRefusal("google_provider_response_invalid", "declared items");
-  }
-  const pageToken = valueAt(answer, declared.pageToken.response);
-  let next;
-  if (pageToken !== undefined && pageToken !== null) {
-    next = {
-      pageToken: requiredText(pageToken, "declared page token"),
-      checkpoint: cursor.checkpoint,
-    };
-  } else {
-    next = {
-      pageToken: null,
-      checkpoint: requiredText(
-        valueAt(answer, declared.pageToken.checkpointResponse),
-        "declared checkpoint",
-      ),
-    };
-  }
-  return {
-    items: entries.map((entry) => normalizedItem(options.service, entry)),
-    dropped: [],
-    cursor: JSON.stringify(next),
-    more: pageToken !== undefined && pageToken !== null,
-  };
-}

package/src/google/id.js

@@ -1 +0,0 @@
-export const GOOGLE_DRIVE_ID = /^[A-Za-z0-9_-]{1,512}$/;

package/src/instance/google.js

@@ -1 +0,0 @@
-export const GOOGLE_OAUTH_CLIENT = Object.freeze({"id":"911965338518-867gdvuddb7kdkjo3qvpv6gds369alb7.apps.googleusercontent.com","secret":"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"});