@opengeni/api-router 0.30.1 → 1.0.0

package/src/workspace-webhook-dispatch.ts

@@ -0,0 +1,190 @@
+import { randomUUID } from "node:crypto";
+import { environmentsEncryptionKeyBytes, type Settings } from "@opengeni/config";
+import {
+  OPENGENI_DELIVERY_ID_HEADER,
+  OPENGENI_EVENT_ID_HEADER,
+  OPENGENI_SIGNATURE_HEADER,
+  signOpenGeniPayload,
+} from "@opengeni/contracts";
+import {
+  claimWorkspaceWebhookDeliveries,
+  claimOrganizationWebhookDeliveries,
+  decryptEnvironmentValue,
+  maintainWorkspaceAllowances,
+  pruneOrganizationWebhookDeliveries,
+  pruneWorkspaceWebhookDeliveries,
+  settleOrganizationWebhookDelivery,
+  settleWorkspaceWebhookDelivery,
+  type ClaimedWorkspaceWebhookDelivery,
+  type Database,
+} from "@opengeni/db";
+import { pinnedFetch } from "@opengeni/network";
+import type { Observability } from "@opengeni/observability";
+
+export const WORKSPACE_WEBHOOK_DISPATCH_BATCH_SIZE = 32;
+const CLAIM_SECONDS = 60;
+const POLL_INTERVAL_MS = 2_000;
+const DELIVERY_TIMEOUT_MS = 10_000;
+const PRUNE_EVERY_TICKS = 300;
+
+export type WorkspaceWebhookDispatchDeps = {
+  db: Database;
+  settings: Settings;
+  observability?: Pick<Observability, "info" | "warn" | "error">;
+  fetch?: typeof pinnedFetch;
+};
+
+export type WorkspaceWebhookBatchResult = { claimed: number; delivered: number; failed: number };
+
+function errorText(error: unknown): string {
+  if (error instanceof Error) {
+    if (error.name === "TimeoutError" || error.name === "AbortError") return "timeout";
+    return `${error.name}: ${error.message}`.slice(0, 500);
+  }
+  return String(error).slice(0, 500);
+}
+
+async function deliverOne(
+  deps: WorkspaceWebhookDispatchDeps,
+  key: Uint8Array,
+  claimId: string,
+  delivery: ClaimedWorkspaceWebhookDelivery,
+  organization = false,
+): Promise<"delivered" | "failed"> {
+  let status: number | null = null;
+  let error: string | null = null;
+  try {
+    const body = JSON.stringify(delivery.payload);
+    const secret = decryptEnvironmentValue(key, delivery.secretEncrypted);
+    const response = await (deps.fetch ?? pinnedFetch)(
+      delivery.url,
+      {
+        method: "POST",
+        headers: {
+          "content-type": "application/json",
+          "user-agent": "OpenGeni-Webhooks/1",
+          [OPENGENI_SIGNATURE_HEADER]: await signOpenGeniPayload(secret, body),
+          [OPENGENI_EVENT_ID_HEADER]: delivery.eventId,
+          [OPENGENI_DELIVERY_ID_HEADER]: delivery.deliveryId,
+        },
+        body,
+        signal: AbortSignal.timeout(DELIVERY_TIMEOUT_MS),
+      },
+      deps.settings,
+      {
+        label: organization ? "Organization webhook" : "Workspace webhook",
+        requireHttpsOutsideLocalTest: true,
+      },
+    );
+    status = response.status;
+    await response.body?.cancel().catch(() => undefined);
+    if (status < 200 || status >= 300) error = `HTTP ${status}`;
+  } catch (caught) {
+    error = errorText(caught);
+  }
+  await (organization ? settleOrganizationWebhookDelivery : settleWorkspaceWebhookDelivery)(
+    deps.db,
+    {
+      deliveryId: delivery.deliveryId,
+      claimId,
+      status,
+      error,
+    },
+  );
+  return error === null ? "delivered" : "failed";
+}
+
+/** Claim and deliver one globally bounded batch. At-least-once: receivers dedupe on event id. */
+export async function drainWorkspaceWebhookDeliveries(
+  deps: WorkspaceWebhookDispatchDeps,
+): Promise<WorkspaceWebhookBatchResult> {
+  // Allowance rollover/expiry does not depend on webhook secrets, usage
+  // readers, inference or having any pending delivery.
+  await maintainWorkspaceAllowances(deps.db);
+  const key = environmentsEncryptionKeyBytes(deps.settings);
+  if (!key) return { claimed: 0, delivered: 0, failed: 0 };
+  const claimId = randomUUID();
+  const claims = await claimWorkspaceWebhookDeliveries(deps.db, {
+    claimId,
+    limit: WORKSPACE_WEBHOOK_DISPATCH_BATCH_SIZE,
+    claimSeconds: CLAIM_SECONDS,
+  });
+  // Each lane is bounded; the shared pump does not starve organization queues
+  // behind a sustained workspace backlog.
+  const organizationClaims = await claimOrganizationWebhookDeliveries(deps.db, {
+    claimId,
+    limit: WORKSPACE_WEBHOOK_DISPATCH_BATCH_SIZE,
+    claimSeconds: CLAIM_SECONDS,
+  });
+  const outcomes = await Promise.all(
+    [
+      ...claims.map((delivery) => ({ delivery, organization: false })),
+      ...organizationClaims.map((delivery) => ({ delivery, organization: true })),
+    ].map(async ({ delivery, organization }) => {
+      try {
+        return await deliverOne(deps, key, claimId, delivery, organization);
+      } catch (error) {
+        deps.observability?.warn("Workspace webhook settlement failed", {
+          deliveryId: delivery.deliveryId,
+          error: errorText(error),
+        });
+        return "failed" as const;
+      }
+    }),
+  );
+  return {
+    claimed: claims.length + organizationClaims.length,
+    delivered: outcomes.filter((outcome) => outcome === "delivered").length,
+    failed: outcomes.filter((outcome) => outcome === "failed").length,
+  };
+}
+
+/**
+ * One non-overlapping loop per API replica. PostgreSQL claims coordinate
+ * replicas; a crash only delays a delivery until its claim expires.
+ */
+export function startWorkspaceWebhookDispatchPump(
+  deps: WorkspaceWebhookDispatchDeps,
+  options: {
+    intervalMs?: number;
+    drain?: (deps: WorkspaceWebhookDispatchDeps) => Promise<WorkspaceWebhookBatchResult>;
+  } = {},
+): () => Promise<void> {
+  const intervalMs = Math.max(100, options.intervalMs ?? POLL_INTERVAL_MS);
+  const drain = options.drain ?? drainWorkspaceWebhookDeliveries;
+  let stopped = false;
+  let running: Promise<void> | undefined;
+  let ticks = 0;
+
+  const tick = (): void => {
+    if (stopped || running) return;
+    ticks += 1;
+    // A full batch means a backlog: drain it now instead of one batch per interval.
+    let backlog = false;
+    running = (async () => {
+      const result = await drain(deps);
+      backlog = result.claimed >= WORKSPACE_WEBHOOK_DISPATCH_BATCH_SIZE;
+      if (result.claimed > 0) deps.observability?.info("Workspace webhook batch settled", result);
+      if (ticks % PRUNE_EVERY_TICKS === 0) {
+        await pruneWorkspaceWebhookDeliveries(deps.db);
+        await pruneOrganizationWebhookDeliveries(deps.db);
+      }
+    })()
+      .catch((error: unknown) => {
+        deps.observability?.error("Workspace webhook dispatch failed", { error: errorText(error) });
+      })
+      .finally(() => {
+        running = undefined;
+        if (backlog && !stopped) queueMicrotask(tick);
+      });
+  };
+
+  const timer = setInterval(tick, intervalMs);
+  queueMicrotask(tick);
+
+  return async () => {
+    stopped = true;
+    clearInterval(timer);
+    await running;
+  };
+}

package/src/xai-subscription-auth.ts

@@ -77,14 +77,20 @@ export async function buildXaiSubscriptionAuthorization(input: {
       userId: credential.providerAccountId,
     };
   };
-  return {
+  const authorization: XaiSubscriptionAuthorization = {
     credentialId,
     authoritySnapshot,
     rotationEnabled: selected.rotationEnabled,
     context: {
       clientVersion: XAI_CLIENT_VERSION,
       ...(input.sessionId ? { sessionId: input.sessionId } : {}),
-      getToken: async () => tokenSnapshot(),
+      getToken: async () => {
+        const expiresAt = xaiAccessTokenExpiry(credential.secret.accessToken ?? "");
+        if (expiresAt && expiresAt.getTime() <= Date.now() + 60_000) {
+          return await authorization.context.refresh();
+        }
+        return tokenSnapshot();
+      },
       refresh: async () => {
         const observedAccessToken = credential.secret.accessToken;
         const observedRefreshToken = credential.secret.refreshToken;
@@ -129,4 +135,5 @@ export async function buildXaiSubscriptionAuthorization(input: {
       },
     },
   };
+  return authorization;
 }