@agentshouse/kit 0.1.0-alpha.25 → 0.1.0-alpha.27

package/src/google/entries.js

@@ -0,0 +1,346 @@
+import { SOURCE_CAPTURE_INTERVAL_MS } from "../contract.js";
+import { desiredSources, maintainControl, openSession, reportSources } from "../house.js";
+import { KitRefusal } from "../refusal.js";
+import { signInModule } from "../setup.js";
+import { captureEntry, localServiceState } from "../source.js";
+import { noteNames, notePath, readHealth, readNote, removeNotes, writeNote } from "../state.js";
+import { coalescedTask, residentWake } from "../tasks.js";
+import {
+  authorizeGoogle,
+  forgetGoogleAccount,
+  googleAccount,
+  googleSignedIn,
+  GOOGLE_SCOPES,
+  GOOGLE_SOURCES,
+} from "./oauth.js";
+import { recordAccountEmail } from "./source.js";
+
+const ENTRY_NOTES = ["checkpoints", "health", "gaps", "dropped", "originals", "entries"];
+
+export const GOOGLE_APIS = Object.freeze({
+  gmail: "Gmail API",
+  calendar: "Google Calendar API",
+  drive: "Google Drive API",
+});
+
+async function heldEntries(directory) {
+  const entries = [];
+  for (const source of await noteNames(directory, GOOGLE_SOURCES, "entries")) {
+    entries.push(await readNote(notePath(directory, GOOGLE_SOURCES, source, "entries"), null));
+  }
+  return entries;
+}
+
+export async function accountEntries(directory, account) {
+  return (await heldEntries(directory)).filter((entry) => entry.account === account);
+}
+
+async function entryReport(directory, entry) {
+  const account = await googleAccount(directory, entry.account);
+  const email = account?.email ?? null;
+  if (account?.scopes.includes(GOOGLE_SCOPES[entry.service]) !== true) {
+    return { source: entry.source, state: "signed_out", email };
+  }
+  const health = await readHealth(directory, GOOGLE_SOURCES, entry.source);
+  if (health?.state === "failed") {
+    return { source: entry.source, state: "failed", email, cause: health.cause ?? null };
+  }
+  return { source: entry.source, state: "ready", email };
+}
+
+async function reportEntries(context, entries) {
+  if (entries.length === 0) return;
+  const sources = [];
+  for (const entry of entries) sources.push(await entryReport(context.directory, entry));
+  await reportSources(context.session, sources, context.signal);
+}
+
+async function captureOccurrence(context, entry) {
+  const delivered = await captureEntry(context, entry);
+  await reportEntries(context, [entry]);
+  return delivered;
+}
+
+async function collectedClient(entry, present, signal) {
+  const collected = await present(
+    {
+      steps: [
+        {
+          kind: "collect",
+          label: "Client ID of your Google OAuth client",
+          name: "client_id",
+          description: `In your own Google Cloud project, enable the ${GOOGLE_APIS[entry.service]} and create an OAuth client of type Desktop app. Paste its client ID here.`,
+        },
+        {
+          kind: "collect",
+          label: "Client secret of that OAuth client",
+          name: "client_secret",
+          description: `Paste the same client's secret. It stays on this Environment beside the ${entry.account} account's sign-in.`,
+        },
+      ],
+    },
+    signal,
+  );
+  return { id: collected.client_id.trim(), secret: collected.client_secret.trim() };
+}
+
+function consent(entry, present, signal) {
+  return async ({ url }) => {
+    const collected = await present(
+      {
+        steps: [
+          {
+            kind: "visit",
+            label: `Sign in to Google for ${entry.account} and allow access for ${entry.service}`,
+            url,
+          },
+          {
+            kind: "collect",
+            label: "The address your browser ended on",
+            name: "address",
+            description:
+              "After you allow access, Google sends your browser to a 127.0.0.1 address that does not open. Copy that whole address from the address bar and paste it here.",
+          },
+        ],
+      },
+      signal,
+    );
+    return collected.address;
+  };
+}
+
+async function guidedSignIn(context, entry, present, signal) {
+  try {
+    const account = await googleAccount(context.directory, entry.account);
+    await authorizeGoogle(
+      {
+        directory: context.directory,
+        account: entry.account,
+        client: account?.client ?? (await collectedClient(entry, present, signal)),
+        scopes: [GOOGLE_SCOPES[entry.service]],
+        approve: consent(entry, present, signal),
+      },
+      context.dependencies,
+    );
+    return "ready";
+  } catch (error) {
+    if (signal.aborted) return "cancelled";
+    if (error instanceof KitRefusal && error.code === "google_authorization_declined") {
+      return "cancelled";
+    }
+    return "failed";
+  }
+}
+
+async function signIn(context, entry) {
+  const outcome = await signInModule(context, {
+    category: "connector",
+    name: entry.source,
+    login: { integration: "google", instance: entry.account, service: entry.service },
+    install: null,
+    signedIn: () => googleSignedIn(context.directory, entry.account, entry.service),
+    guided: ({ present, signal }) => guidedSignIn(context, entry, present, signal),
+  });
+  if (outcome.status !== "ready") return false;
+  await removeNotes(context.directory, GOOGLE_SOURCES, entry.source, ["health"]);
+  try {
+    await recordAccountEmail(context, { account: entry.account, service: entry.service });
+  } catch (error) {
+    if (!(error instanceof KitRefusal) || error.code === "kit_command_cancelled") throw error;
+  }
+  await reportEntries(context, [entry]);
+  return true;
+}
+
+export function sourceEntries(context) {
+  const { directory, session, dependencies } = context;
+  const interval = dependencies.sourceCaptureIntervalMs ?? SOURCE_CAPTURE_INTERVAL_MS;
+  const held = new Map();
+  const wake = residentWake();
+  let attempted = new Set();
+  let signingIn = null;
+  let cadence = Promise.resolve();
+
+  function occasion(record) {
+    return {
+      session,
+      directory,
+      dependencies,
+      now: context.now,
+      signal: AbortSignal.any([context.signal, record.stop.signal]),
+    };
+  }
+
+  function occur(record) {
+    const current = (record.running ?? Promise.resolve())
+      .catch(() => {})
+      .then(() => {
+        record.last = context.now().getTime();
+        return captureOccurrence(occasion(record), record.entry);
+      });
+    record.running = current;
+    current
+      .finally(() => {
+        if (record.running === current) record.running = null;
+        signIns.trigger();
+        wake.notify();
+      })
+      .catch(() => {});
+    return current;
+  }
+
+  function track(entry) {
+    const record = held.get(entry.source);
+    if (record !== undefined) record.entry = entry;
+    else held.set(entry.source, { entry, last: null, running: null, stop: new AbortController() });
+  }
+
+  async function drop(source, record) {
+    record.stop.abort();
+    held.delete(source);
+    if (signingIn?.source === source) await signingIn.task.catch(() => {});
+    await record.running?.catch(() => {});
+    await removeNotes(directory, GOOGLE_SOURCES, source, ENTRY_NOTES);
+  }
+
+  const signIns = coalescedTask(context.signal, context.unavailable, async () => {
+    for (const record of [...held.values()]) {
+      if (context.draining()) return;
+      const { entry } = record;
+      if (record.stop.signal.aborted || attempted.has(entry.source)) continue;
+      if (await googleSignedIn(directory, entry.account, entry.service)) continue;
+      attempted.add(entry.source);
+      const task = signIn(occasion(record), entry);
+      signingIn = { source: entry.source, task };
+      try {
+        if (await task) record.last = null;
+      } catch (error) {
+        if (!record.stop.signal.aborted) throw error;
+      } finally {
+        signingIn = null;
+      }
+      wake.notify();
+    }
+  });
+
+  const reconciling = coalescedTask(context.signal, context.unavailable, async () => {
+    const desired = await desiredSources(session, context.signal);
+    const named = new Set(desired.map((entry) => entry.source));
+    for (const [source, record] of [...held]) {
+      if (!named.has(source)) await drop(source, record);
+    }
+    for (const entry of desired) {
+      await writeNote(notePath(directory, GOOGLE_SOURCES, entry.source, "entries"), entry);
+      track(entry);
+    }
+    await reportEntries({ session, directory, signal: context.signal }, desired);
+    attempted = new Set();
+    signIns.trigger();
+    wake.notify();
+    const kept = [...new Set(desired.map((entry) => entry.account))];
+    for (const account of await noteNames(directory, GOOGLE_SOURCES, "credentials")) {
+      if (!kept.includes(account)) await forgetGoogleAccount(directory, account, kept, dependencies);
+    }
+  });
+
+  async function startDue() {
+    let next = interval;
+    const now = context.now().getTime();
+    for (const record of [...held.values()]) {
+      if (record.running !== null) continue;
+      const due = record.last === null ? 0 : record.last + interval - now;
+      if (due > 0) {
+        next = Math.min(next, due);
+        continue;
+      }
+      try {
+        if (!(await googleSignedIn(directory, record.entry.account, record.entry.service))) continue;
+      } catch (error) {
+        context.unavailable(error);
+        continue;
+      }
+      if (record.stop.signal.aborted || record.running !== null) continue;
+      occur(record).catch((error) => {
+        if (!record.stop.signal.aborted && !context.signal.aborted) context.unavailable(error);
+      });
+    }
+    return next;
+  }
+
+  return {
+    async start() {
+      try {
+        for (const entry of await heldEntries(directory)) track(entry);
+      } catch (error) {
+        context.unavailable(error);
+      }
+      cadence = (async () => {
+        while (!context.signal.aborted) {
+          await wake.wait(context.draining() ? interval : await startDue(), context.signal);
+        }
+      })();
+    },
+    reconcile: () => reconciling.trigger(),
+    capture(source) {
+      const record = held.get(source);
+      if (record === undefined) return Promise.reject(new KitRefusal("kit_instance_unknown", source));
+      return occur(record);
+    },
+    idle: () =>
+      reconciling.idle() &&
+      signIns.idle() &&
+      [...held.values()].every((record) => record.running === null),
+    async settled() {
+      await reconciling.settled();
+      await signIns.settled();
+      await cadence;
+      await Promise.all([...held.values()].map((record) => record.running?.catch(() => {})));
+    },
+  };
+}
+
+function namedEntries(entries, service) {
+  if (service === null) return entries;
+  const named = entries.filter((entry) => entry.service === service);
+  if (named.length === 0) throw new KitRefusal("kit_service_unknown", service);
+  return named;
+}
+
+export async function captureAccount(options, dependencies = {}) {
+  const entries = namedEntries(options.entries, options.service);
+  const session = await openSession(options, dependencies);
+  return maintainControl(
+    session,
+    async (signal) => {
+      const services = [];
+      for (const entry of entries) {
+        services.push(
+          session.resident === undefined
+            ? await captureOccurrence(
+                { session, directory: options.directory, dependencies, signal, now: session.now },
+                entry,
+              )
+            : await session.capture(entry.source, signal),
+        );
+      }
+      return { instance: options.account, services };
+    },
+    options.signal,
+  );
+}
+
+export async function accountStatus(options) {
+  const services = [];
+  for (const entry of namedEntries(options.entries, options.service)) {
+    const local = await localServiceState(options.directory, GOOGLE_SOURCES, entry.source);
+    services.push({
+      service: entry.service,
+      source_ref: entry.source,
+      rooms: entry.rooms,
+      signed_in: await googleSignedIn(options.directory, entry.account, entry.service),
+      ...local,
+      enable: local.health.cause === "api_disabled" ? GOOGLE_APIS[entry.service] : null,
+    });
+  }
+  return { instance: options.account, preset: "google", services };
+}

package/src/google/gateway.js

@@ -84,7 +84,7 @@ async function googleResponse(options, dependencies) {
         attempt -= 1;
         continue;
       }
-      await revokeGoogleAuthorization(options.directory, options.instance, options.requiredScopes);
+      await revokeGoogleAuthorization(options.directory, options.account, options.requiredScopes);
       throw new KitRefusal("google_authority_revoked");
     }
     if (response.status === 403) {
@@ -98,13 +98,10 @@ async function googleResponse(options, dependencies) {
         throw new KitRefusal("google_provider_unavailable");
       }
       if (reasons.includes("insufficientPermissions")) {
-        await revokeGoogleAuthorization(
-          options.directory,
-          options.instance,
-          options.requiredScopes,
-        );
+        await revokeGoogleAuthorization(options.directory, options.account, options.requiredScopes);
         throw new KitRefusal("google_scope_revoked");
       }
+      if (reasons.includes("accessNotConfigured")) throw new KitRefusal("api_disabled");
       throw new KitRefusal("google_provider_unavailable");
     }
     if (RETRYABLE_STATUS.has(response.status) && attempt + 1 < RETRY_LIMIT) {

package/src/google/gmail.js

@@ -4,6 +4,7 @@ import { googleJson } from "./gateway.js";
 import { GOOGLE_SCOPES } from "./oauth.js";
 
 const GMAIL_ORIGIN = "https://gmail.googleapis.com";
+const EXCLUDED_LABELS = ["SPAM", "TRASH"];
 
 function cursorOf(value) {
   if (value === null) return { mode: "messages", pageToken: null, historyId: null };
@@ -88,18 +89,36 @@ function normalizedMessage(message) {
   };
 }
 
-async function currentHistoryId(common, dependencies) {
-  const profile = await googleJson(
-    { ...common, url: `${GMAIL_ORIGIN}/gmail/v1/users/me/profile` },
-    dependencies,
-  );
+function common(options) {
+  return {
+    directory: options.directory,
+    account: options.account,
+    requiredScopes: [GOOGLE_SCOPES.gmail],
+    signal: options.signal,
+  };
+}
+
+async function profileOf(shared, dependencies) {
+  return googleJson({ ...shared, url: `${GMAIL_ORIGIN}/gmail/v1/users/me/profile` }, dependencies);
+}
+
+async function currentHistoryId(shared, dependencies) {
+  const profile = await profileOf(shared, dependencies);
   if (typeof profile.historyId !== "string" || !/^\d+$/.test(profile.historyId)) {
     throw new KitRefusal("google_provider_response_invalid", "gmail profile");
   }
   return profile.historyId;
 }
 
-function listUrl(selection, cursor, limit, backfillFloor) {
+export async function gmailEmail(options, dependencies = {}) {
+  const profile = await profileOf(common(options), dependencies);
+  if (typeof profile.emailAddress !== "string") {
+    throw new KitRefusal("google_provider_response_invalid", "gmail profile");
+  }
+  return profile.emailAddress;
+}
+
+function listUrl(cursor, limit, backfillFloor) {
   const url = new URL(
     cursor.mode === "messages"
       ? "/gmail/v1/users/me/messages"
@@ -109,7 +128,6 @@ function listUrl(selection, cursor, limit, backfillFloor) {
   url.searchParams.set("maxResults", `${limit}`);
   if (cursor.pageToken !== null) url.searchParams.set("pageToken", cursor.pageToken);
   if (cursor.mode === "messages") {
-    for (const label of selection.labels) url.searchParams.append("labelIds", label);
     if (backfillFloor !== null && backfillFloor !== undefined) {
       url.searchParams.set("q", `after:${Math.floor(Date.parse(backfillFloor) / 1000)}`);
     }
@@ -151,22 +169,16 @@ function referencesOf(answer, mode) {
 
 export async function gmailPage(options, dependencies = {}) {
   let cursor = cursorOf(options.cursor);
-  const common = {
-    directory: options.directory,
-    instance: options.instance,
-    client: options.client,
-    requiredScopes: [GOOGLE_SCOPES.gmail],
-    signal: options.signal,
-  };
+  const shared = common(options);
   if (cursor.mode === "messages" && cursor.historyId === null) {
-    cursor = { ...cursor, historyId: await currentHistoryId(common, dependencies) };
+    cursor = { ...cursor, historyId: await currentHistoryId(shared, dependencies) };
   }
   let answer;
   try {
     answer = await googleJson(
       {
-        ...common,
-        url: listUrl(options.selection, cursor, options.limit, options.backfillFloor),
+        ...shared,
+        url: listUrl(cursor, options.limit, options.backfillFloor),
         ...(cursor.mode === "history" ? { notFoundCode: "google_history_expired" } : {}),
       },
       dependencies,
@@ -176,10 +188,10 @@ export async function gmailPage(options, dependencies = {}) {
     cursor = {
       mode: "messages",
       pageToken: null,
-      historyId: await currentHistoryId(common, dependencies),
+      historyId: await currentHistoryId(shared, dependencies),
     };
     answer = await googleJson(
-      { ...common, url: listUrl(options.selection, cursor, options.limit, options.backfillFloor) },
+      { ...shared, url: listUrl(cursor, options.limit, options.backfillFloor) },
       dependencies,
     );
   }
@@ -197,16 +209,14 @@ export async function gmailPage(options, dependencies = {}) {
     try {
       detail = await googleJson(
         {
-          ...common,
+          ...shared,
           notFoundCode: "google_gmail_item_missing",
           url: `${GMAIL_ORIGIN}/gmail/v1/users/me/messages/${encodeURIComponent(identifier)}?format=full`,
         },
         dependencies,
       );
       const normalized = normalizedMessage(detail);
-      if (options.selection.labels.every((label) => detail.labelIds.includes(label))) {
-        items.push(normalized);
-      }
+      if (!detail.labelIds.some((label) => EXCLUDED_LABELS.includes(label))) items.push(normalized);
     } catch (error) {
       if (
         error instanceof KitRefusal &&