@aspalagin/openclaw-max 0.7.2

package/src/webhook-queue.ts

@@ -0,0 +1,248 @@
+/**
+ * Webhook update queue: the gateway route only enqueues, the account task
+ * consumes and dispatches.
+ *
+ * Why the hop matters: the gateway runs every plugin HTTP handler under a
+ * request-scoped root-work admission (AsyncLocalStorage) and releases it as
+ * soon as the handler returns. Agent runs and session work started from that
+ * async context later see a released admission and fail with
+ * `GatewayDrainingError: Gateway is draining` although the gateway is healthy.
+ * The account task is started outside any request root, so work it spawns is
+ * admitted like the polling loop's. Every dispatch therefore has to start
+ * from the consumer (`consume()`), never from `push()`.
+ */
+
+import type { MaxUpdate } from './api.js';
+
+/** Updates handled at once across chats; one chat is always sequential. */
+export const MAX_WEBHOOK_CHAT_CONCURRENCY = 4;
+/** Accepted-but-unprocessed updates kept per account before new ones are dropped. */
+export const MAX_WEBHOOK_PENDING_LIMIT = 5000;
+/** How long a stopping consumer waits for in-flight dispatches (gateway stop budget is 5 s). */
+export const MAX_WEBHOOK_STOP_GRACE_MS = 3000;
+/** How long a restarted consumer waits for its predecessor's in-flight dispatches. */
+export const MAX_WEBHOOK_HANDOVER_WAIT_MS = 60_000;
+
+/** Serialization lane: the chat an update belongs to. */
+export function maxWebhookUpdateLane(update: MaxUpdate): string {
+  return String(update.message?.recipient?.chat_id ?? update.chat_id ?? 'global');
+}
+
+type Handover = { updates: MaxUpdate[]; settled: Promise<void> };
+
+/**
+ * Updates acked to MAX but not dispatched when an account task stopped
+ * (config reload, channel restart). MAX will not redeliver them, so the next
+ * task of the same account in this process picks them up.
+ */
+const handovers = new Map<string, Handover>();
+/** Open queues per account: a stopping task hands leftovers to an overlapping successor. */
+const openQueues = new Map<string, Set<(update: MaxUpdate) => void>>();
+
+export type MaxWebhookUpdateQueue = {
+  /** Called by the route handler (request context): enqueue only, never dispatch. */
+  push: (update: MaxUpdate) => void;
+  /**
+   * Runs in the account task until abortSignal fires: dispatches updates in
+   * order per chat, up to `concurrency` chats at once. Resolves after stop.
+   */
+  consume: (params: {
+    dispatch: (update: MaxUpdate) => Promise<void>;
+    abortSignal: AbortSignal;
+    onError: (err: unknown, update: MaxUpdate) => void;
+    onWarn?: (message: string) => void;
+  }) => Promise<void>;
+  /** Start failed before consume(): release the queue, keeping taken-over updates. */
+  close: () => void;
+};
+
+export function createMaxWebhookUpdateQueue(params: {
+  accountId: string;
+  concurrency?: number;
+  pendingLimit?: number;
+  stopGraceMs?: number;
+  handoverWaitMs?: number;
+}): MaxWebhookUpdateQueue {
+  const { accountId } = params;
+  const concurrency = Math.max(1, params.concurrency ?? MAX_WEBHOOK_CHAT_CONCURRENCY);
+  const pendingLimit = params.pendingLimit ?? MAX_WEBHOOK_PENDING_LIMIT;
+  const stopGraceMs = params.stopGraceMs ?? MAX_WEBHOOK_STOP_GRACE_MS;
+  const handoverWaitMs = params.handoverWaitMs ?? MAX_WEBHOOK_HANDOVER_WAIT_MS;
+
+  // Taken over at creation: updates pushed later must queue behind them.
+  const previous = handovers.get(accountId);
+  handovers.delete(accountId);
+  const inbox: MaxUpdate[] = previous ? [...previous.updates] : [];
+  let pending = inbox.length;
+  let closed = false;
+  let wake: (() => void) | undefined;
+  let warn: ((message: string) => void) | undefined;
+
+  const push = (update: MaxUpdate): void => {
+    if (closed) {
+      // Route still selected this target while the task was stopping.
+      const successor = [...siblings].at(-1);
+      if (successor) return successor(update);
+      const handover = handovers.get(accountId);
+      if (handover) handover.updates.push(update);
+      else handovers.set(accountId, { updates: [update], settled: Promise.resolve() });
+      return;
+    }
+    if (pending >= pendingLimit) {
+      warn?.(
+        `[${accountId}] MAX webhook queue full (${pendingLimit}); dropping ${update.update_type}`,
+      );
+      return;
+    }
+    pending += 1;
+    inbox.push(update);
+    wake?.();
+  };
+  const siblings = openQueues.get(accountId) ?? new Set();
+  siblings.add(push);
+  openQueues.set(accountId, siblings);
+
+  /** Stop accepting and pass undispatched updates (lanes first, then inbox) on. */
+  const close = (fromLanes: MaxUpdate[], settled: Promise<void>): void => {
+    if (closed) return;
+    closed = true;
+    wake = undefined;
+    siblings.delete(push);
+    if (siblings.size === 0 && openQueues.get(accountId) === siblings) {
+      openQueues.delete(accountId);
+    }
+    const leftover = [...fromLanes, ...inbox];
+    inbox.length = 0;
+    pending = 0;
+    if (leftover.length === 0) return;
+    const successor = [...siblings].at(-1);
+    if (successor) {
+      // A newer task of this account is already consuming: queue there.
+      for (const update of leftover) successor(update);
+      warn?.(
+        `[${accountId}] MAX webhook stopped with ${leftover.length} update(s) not dispatched; moved to the running successor`,
+      );
+      return;
+    }
+    const late = handovers.get(accountId)?.updates ?? [];
+    handovers.set(accountId, { updates: [...leftover, ...late], settled });
+    warn?.(
+      `[${accountId}] MAX webhook stopped with ${leftover.length} update(s) not dispatched; handed over to the next start`,
+    );
+  };
+
+  const consume: MaxWebhookUpdateQueue['consume'] = async ({
+    dispatch,
+    abortSignal,
+    onError,
+    onWarn,
+  }) => {
+    warn = onWarn;
+    const lanes = new Map<string, MaxUpdate[]>();
+    const ready: string[] = [];
+    const active = new Set<string>();
+    const inflight = new Set<Promise<void>>();
+
+    const route = (update: MaxUpdate): void => {
+      const lane = maxWebhookUpdateLane(update);
+      const queued = lanes.get(lane);
+      if (queued) queued.push(update);
+      else lanes.set(lane, [update]);
+      if (!active.has(lane) && !ready.includes(lane)) ready.push(lane);
+    };
+
+    const runLane = async (lane: string): Promise<void> => {
+      try {
+        for (;;) {
+          if (abortSignal.aborted) return;
+          const update = lanes.get(lane)?.shift();
+          if (!update) return;
+          pending -= 1;
+          try {
+            await dispatch(update);
+          } catch (err) {
+            onError(err, update);
+          }
+        }
+      } finally {
+        active.delete(lane);
+        if (lanes.get(lane)?.length === 0) lanes.delete(lane);
+        else if (lanes.has(lane) && !ready.includes(lane)) ready.push(lane);
+        pump();
+      }
+    };
+
+    // Called only from this consumer's async chain (loop or runLane), so every
+    // dispatch inherits the account task's context.
+    const pump = (): void => {
+      if (abortSignal.aborted) return;
+      while (active.size < concurrency && ready.length > 0) {
+        const lane = ready.shift() as string;
+        active.add(lane);
+        const run = runLane(lane);
+        inflight.add(run);
+        void run.finally(() => inflight.delete(run));
+      }
+    };
+
+    const nextWake = (): Promise<void> =>
+      new Promise<void>((resolve) => {
+        wake = resolve;
+        if (inbox.length > 0 || abortSignal.aborted) resolve();
+      });
+    const onAbort = (): void => wake?.();
+    abortSignal.addEventListener('abort', onAbort, { once: true });
+
+    try {
+      if (previous && !abortSignal.aborted) {
+        // Keep per-chat order across a restart: the stopped task may still be
+        // finishing an update of a chat whose next updates we now hold.
+        await waitAtMost(previous.settled, handoverWaitMs, abortSignal);
+      }
+      while (!abortSignal.aborted) {
+        while (inbox.length > 0) route(inbox.shift() as MaxUpdate);
+        pump();
+        await nextWake();
+        wake = undefined;
+      }
+    } finally {
+      abortSignal.removeEventListener('abort', onAbort);
+      const leftover: MaxUpdate[] = [];
+      for (const queued of lanes.values()) leftover.push(...queued);
+      lanes.clear();
+      const settled = Promise.allSettled([...inflight]).then(() => undefined);
+      close(leftover, settled);
+      await waitAtMost(settled, stopGraceMs);
+    }
+  };
+
+  return { push, consume, close: () => close([], Promise.resolve()) };
+}
+
+/** Resolves when `promise` settles, `ms` elapse or `signal` aborts — whichever is first. */
+async function waitAtMost(promise: Promise<void>, ms: number, signal?: AbortSignal): Promise<void> {
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  let onAbort: (() => void) | undefined;
+  try {
+    await Promise.race([
+      promise,
+      new Promise<void>((resolve) => {
+        timer = setTimeout(resolve, ms);
+        timer.unref?.();
+        if (signal) {
+          onAbort = resolve;
+          signal.addEventListener('abort', onAbort, { once: true });
+        }
+      }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+    if (signal && onAbort) signal.removeEventListener('abort', onAbort);
+  }
+}
+
+/** @internal test helper: forget handed-over updates. */
+export function resetMaxWebhookHandoversForTest(): void {
+  handovers.clear();
+  openQueues.clear();
+}

package/src/webhook-runner.ts

@@ -0,0 +1,296 @@
+/**
+ * Webhook transport: secret resolution, MAX subscription lifecycle (sync on
+ * start, periodic re-check), the gateway route/target registration and the
+ * account-task consumer of the ack-first per-chat update queue.
+ */
+
+import { randomBytes } from 'node:crypto';
+
+import type { ChannelLogSink } from 'openclaw/plugin-sdk/channel-contract';
+import { channelReadyPatch } from 'openclaw/plugin-sdk/gateway-runtime';
+
+import { readSecretFile, type ResolvedMaxAccount } from './accounts.js';
+import type { MaxApi, MaxSubscription, MaxUpdate } from './api.js';
+import { dispatchUpdate } from './dispatch.js';
+import {
+  MAX_SUBSCRIBED_UPDATE_TYPES,
+  type MaxMonitorOptions,
+  type MaxStatusPatch,
+} from './monitor-types.js';
+import type { MaxStateStore } from './state.js';
+import {
+  type MaxWebhookTarget,
+  registerMaxWebhookRoute,
+  registerMaxWebhookTarget,
+  resolveMaxWebhookPath,
+  subscribeMaxWebhook,
+} from './webhook.js';
+import { createMaxWebhookUpdateQueue } from './webhook-queue.js';
+
+/** MAX secret format (SubscriptionRequestBody.secret): 5–256 of [A-Za-z0-9_-]. */
+const MAX_WEBHOOK_SECRET_PATTERN = /^[\w-]{5,256}$/;
+
+/**
+ * Webhook secret: `webhookSecret`, else `webhookSecretFile`, else one generated
+ * once and kept in the account state file so restarts re-subscribe with the
+ * same value (a fresh secret per start would reject MAX retries of updates
+ * sent before the restart).
+ * @internal exported for testing.
+ */
+export async function resolveMaxWebhookSecret(
+  account: ResolvedMaxAccount,
+  state?: MaxStateStore,
+): Promise<string> {
+  const validate = (secret: string, source: string): string => {
+    if (!MAX_WEBHOOK_SECRET_PATTERN.test(secret)) {
+      throw new Error(`MAX ${source} must be 5–256 characters of A-Z, a-z, 0-9, _ and -`);
+    }
+    return secret;
+  };
+
+  const configured = account.config.webhookSecret?.trim();
+  if (configured) return validate(configured, 'webhookSecret');
+
+  const secretFile = account.config.webhookSecretFile?.trim();
+  if (secretFile) {
+    const fromFile = readSecretFile(secretFile);
+    if (!fromFile)
+      throw new Error(
+        `MAX webhookSecretFile ${secretFile} is missing, empty or not a regular file`,
+      );
+    return validate(fromFile, 'webhookSecretFile');
+  }
+
+  const stored = state?.webhookSecret;
+  if (stored && MAX_WEBHOOK_SECRET_PATTERN.test(stored)) return stored;
+
+  const generated = generateWebhookSecret();
+  if (state) {
+    state.setWebhookSecret(generated);
+    await state.flush();
+  }
+  return generated;
+}
+
+/**
+ * Make the MAX side point at exactly one URL: drop this bot's subscriptions to
+ * other URLs, then (re)subscribe ours. POST is repeated on every start so the
+ * secret and update_types always match the running config.
+ * @internal exported for testing.
+ */
+export async function syncMaxWebhookSubscription(params: {
+  api: MaxApi;
+  accountId: string;
+  webhookUrl: string;
+  secret: string;
+  log?: ChannelLogSink;
+}): Promise<void> {
+  const { api, accountId, webhookUrl, secret, log } = params;
+  let existing: MaxSubscription[] = [];
+  try {
+    existing = (await api.getSubscriptions()).subscriptions ?? [];
+  } catch (err) {
+    log?.warn(`[${accountId}] MAX subscriptions check failed, subscribing anyway: ${String(err)}`);
+  }
+  for (const subscription of existing) {
+    if (subscription.url === webhookUrl) continue;
+    log?.warn(
+      `[${accountId}] MAX webhook: removing subscription to another URL ${subscription.url}`,
+    );
+    await api.unsubscribe(subscription.url);
+  }
+  await subscribeMaxWebhook({
+    api,
+    webhookUrl,
+    secret,
+    updateTypes: MAX_SUBSCRIBED_UPDATE_TYPES,
+  });
+}
+
+/** How often webhook mode re-checks that MAX still holds our subscription. */
+export const MAX_SUBSCRIPTION_CHECK_INTERVAL_MS = 12 * 60 * 1000;
+
+/**
+ * MAX drops a subscription after 8 hours of failed deliveries (tunnel or
+ * gateway outage), and then nothing arrives until the next restart. Re-check
+ * periodically and re-subscribe when our URL is gone. Network/API errors on
+ * the check are only logged; the next tick tries again.
+ * Returns a stop function; the timer is also cleared when abortSignal fires.
+ * @internal exported for testing.
+ */
+export function startMaxSubscriptionWatch(params: {
+  api: MaxApi;
+  accountId: string;
+  webhookUrl: string;
+  secret: string;
+  abortSignal: AbortSignal;
+  log?: ChannelLogSink;
+  intervalMs?: number;
+}): () => void {
+  const { api, accountId, webhookUrl, secret, abortSignal, log } = params;
+  let running = false;
+
+  const check = async (): Promise<void> => {
+    if (running || abortSignal.aborted) return;
+    running = true;
+    try {
+      let subscriptions: MaxSubscription[];
+      try {
+        subscriptions = (await api.getSubscriptions()).subscriptions ?? [];
+      } catch (err) {
+        log?.warn(`[${accountId}] MAX subscription check failed: ${String(err)}`);
+        return;
+      }
+      if (abortSignal.aborted || subscriptions.some((s) => s.url === webhookUrl)) return;
+      log?.warn(
+        `[${accountId}] MAX webhook subscription to ${webhookUrl} is gone (MAX unsubscribes after 8 h of failed deliveries); re-subscribing`,
+      );
+      try {
+        await subscribeMaxWebhook({
+          api,
+          webhookUrl,
+          secret,
+          updateTypes: MAX_SUBSCRIBED_UPDATE_TYPES,
+        });
+        log?.info(`[${accountId}] MAX webhook re-subscribed: ${webhookUrl}`);
+      } catch (err) {
+        log?.error(`[${accountId}] MAX webhook re-subscribe failed: ${String(err)}`);
+      }
+    } finally {
+      running = false;
+    }
+  };
+
+  const timer = setInterval(
+    () => void check(),
+    params.intervalMs ?? MAX_SUBSCRIPTION_CHECK_INTERVAL_MS,
+  );
+  timer.unref?.();
+  const stop = (): void => {
+    clearInterval(timer);
+    abortSignal.removeEventListener('abort', stop);
+  };
+  abortSignal.addEventListener('abort', stop, { once: true });
+  return stop;
+}
+
+export async function startMaxWebhook(opts: MaxMonitorOptions): Promise<void> {
+  const { api, account, config, abortSignal, log, statusSink } = opts;
+
+  const webhookUrl = account.config.webhookUrl?.trim();
+  if (!webhookUrl) {
+    throw new Error(`MAX transport "webhook" requires webhookUrl (account ${account.accountId})`);
+  }
+  if (!webhookUrl.startsWith('https://')) {
+    log?.warn(
+      `[${account.accountId}] MAX accepts only HTTPS webhook URLs on port 443; got ${webhookUrl}`,
+    );
+  }
+
+  const webhookPath = resolveMaxWebhookPath(account.config.webhookPath, webhookUrl);
+  const webhookSecret = await resolveMaxWebhookSecret(account, opts.state);
+
+  log?.info(`[${account.accountId}] MAX webhook mode: ${webhookUrl} (path: ${webhookPath})`);
+
+  // Like Telegram webhook mode: no transport-activity timestamp, so the health
+  // policy never flags a quiet (but healthy) webhook as a stale socket.
+  statusSink?.({
+    mode: 'webhook',
+    connected: false,
+    lastConnectedAt: null,
+    lastEventAt: null,
+    lastTransportActivityAt: null,
+  } as MaxStatusPatch);
+
+  // MAX requires HTTP 200 within 30s while agent runs regularly take minutes.
+  // The route handler acks and only enqueues; this account task dispatches
+  // (see webhook-queue.ts: dispatching from the request context fails with
+  // GatewayDrainingError once the request's work admission is released).
+  const queue = createMaxWebhookUpdateQueue({ accountId: account.accountId });
+  const onUpdate = (update: MaxUpdate): Promise<void> => {
+    const at = Date.now();
+    statusSink?.(
+      channelReadyPatch({
+        lastConnectedAt: at,
+        lastEventAt: at,
+        mode: 'webhook',
+      }) as MaxStatusPatch,
+    );
+    queue.push(update);
+    return Promise.resolve();
+  };
+
+  const target: MaxWebhookTarget = {
+    account,
+    config,
+    path: webhookPath,
+    secret: webhookSecret,
+    onUpdate,
+    log: (msg) => log?.debug?.(msg),
+    error: (msg) => log?.error(msg),
+  };
+
+  const unregisterTarget = registerMaxWebhookTarget(target);
+  let unregisterRoute: () => void = () => {};
+  try {
+    // Route first: MAX may deliver the moment the subscription exists.
+    unregisterRoute = registerMaxWebhookRoute({
+      path: webhookPath,
+      accountId: account.accountId,
+      log: (msg) => log?.warn(`[${account.accountId}] ${msg}`),
+      register: opts.registerWebhookRoute,
+    });
+    await syncMaxWebhookSubscription({
+      api,
+      accountId: account.accountId,
+      webhookUrl,
+      secret: webhookSecret,
+      log,
+    });
+    log?.info(`[${account.accountId}] MAX webhook subscribed: ${webhookUrl}`);
+  } catch (err) {
+    log?.error(`[${account.accountId}] MAX webhook start failed: ${String(err)}`);
+    unregisterRoute();
+    unregisterTarget();
+    queue.close();
+    statusSink?.({ mode: 'webhook', connected: false, lastError: String(err) } as MaxStatusPatch);
+    throw err;
+  }
+
+  statusSink?.(channelReadyPatch({ mode: 'webhook' }) as MaxStatusPatch);
+
+  const stopSubscriptionWatch = startMaxSubscriptionWatch({
+    api,
+    accountId: account.accountId,
+    webhookUrl,
+    secret: webhookSecret,
+    abortSignal,
+    log,
+  });
+
+  // Runs until abort; every dispatch starts here, under the account task.
+  await queue.consume({
+    dispatch: (update) => dispatchUpdate(update, opts),
+    abortSignal,
+    onError: (err, update) =>
+      log?.error(
+        `[${account.accountId}] Webhook update dispatch failed (${update.update_type}): ${String(err)}`,
+      ),
+    onWarn: (message) => log?.warn(message),
+  });
+  stopSubscriptionWatch();
+
+  // The subscription is kept on purpose: a restart or config reload must not
+  // lose updates, and MAX retries undelivered ones while the route is back.
+  // To leave webhook mode, switch transport to "polling" (the polling start
+  // removes it) or call DELETE /subscriptions?url=… by hand.
+  unregisterRoute();
+  unregisterTarget();
+  statusSink?.({ mode: 'webhook', connected: false } as MaxStatusPatch);
+  log?.info(`[${account.accountId}] MAX webhook mode stopped (subscription kept)`);
+}
+
+/** MAX webhook secret: 5–256 chars of [A-Za-z0-9-]. */
+function generateWebhookSecret(): string {
+  return randomBytes(24).toString('base64url').replace(/_/g, '-');
+}