@opengeni/api-router 0.30.1 → 2.1.0-canary.0

package/src/http/api-error.ts

@@ -1,4 +1,5 @@
 import type { ErrorCode } from "@opengeni/contracts";
+import { WorkspaceControlBusyError } from "@opengeni/db";
 import type { ContentfulStatusCode } from "hono/utils/http-status";
 import { HTTPException } from "hono/http-exception";
 
@@ -6,6 +7,7 @@ type ApiHttpErrorOptions = {
   code: ErrorCode;
   message: string;
   retryable?: boolean;
+  outcomeUnknown?: boolean;
   details?: Record<string, unknown>;
 };
 
@@ -13,6 +15,7 @@ type ApiHttpErrorOptions = {
 export class ApiHttpError extends HTTPException {
   readonly code: ErrorCode;
   readonly retryable: boolean | undefined;
+  readonly outcomeUnknown: boolean | undefined;
   readonly details: Record<string, unknown> | undefined;
 
   constructor(status: number, options: ApiHttpErrorOptions) {
@@ -20,6 +23,31 @@ export class ApiHttpError extends HTTPException {
     this.name = "ApiHttpError";
     this.code = options.code;
     this.retryable = options.retryable;
+    this.outcomeUnknown = options.outcomeUnknown;
     this.details = options.details;
   }
 }
+
+/**
+ * A request-scoped session/workspace mutation could not enter the workspace
+ * control prefix within its bounded wait. The transaction rolled back before
+ * any write, so the outcome is known and the client may retry. `app.onError`
+ * applies this once for every route; Slack interactions keep the raw typed
+ * error so their retry classifier treats it as transient.
+ */
+export function workspaceControlBusyHttpError(error: unknown): ApiHttpError | null {
+  const busy =
+    error instanceof WorkspaceControlBusyError
+      ? error
+      : error instanceof HTTPException && error.cause instanceof WorkspaceControlBusyError
+        ? error.cause
+        : null;
+  if (!busy) return null;
+  return new ApiHttpError(503, {
+    code: "upstream_unavailable",
+    message: "The workspace is busy applying other session commands; retry shortly.",
+    retryable: true,
+    outcomeUnknown: false,
+    details: { code: busy.code, lockTimeoutMs: busy.lockTimeoutMs },
+  });
+}

package/src/http/auth.ts

@@ -40,6 +40,9 @@ function isAuthExempt(c: Context, settings: Settings): boolean {
   if (path === "/v1/webhooks/stripe") {
     return true;
   }
+  if (c.req.method === "POST" && path.startsWith("/v1/webhooks/automations/")) {
+    return true;
+  }
   if (
     path === "/v1/github/setup" ||
     path === "/v1/github/install/callback" ||
@@ -51,6 +54,7 @@ function isAuthExempt(c: Context, settings: Settings): boolean {
   if (
     path === "/v1/integrations/oauth/callback" ||
     path === "/v1/integrations/provider-oauth/callback" ||
+    path === "/v1/integrations/github-personal/oauth/callback" ||
     path === "/v1/integrations/google-drive/callback" ||
     path === "/v1/integrations/oauth/client-metadata.json" ||
     path === "/v1/integrations/slack/callback" ||

package/src/http/cors.ts

@@ -1,3 +1,38 @@
+import { HTTPException } from "hono/http-exception";
+
 export function allowedCorsOrigin(pattern: string, origin: string): boolean {
   return new RegExp(`^(?:${pattern})$`).test(origin);
 }
+
+export function validateInteractionRequestOrigin(
+  value: string | undefined,
+  input: {
+    corsAllowOriginRegex: string;
+    publicBaseUrl?: string | undefined;
+    webBaseUrl?: string | undefined;
+  },
+): string | null {
+  if (!value) return null;
+  let url: URL;
+  try {
+    url = new URL(value);
+  } catch {
+    throw new HTTPException(400, { message: "invalid request origin" });
+  }
+  if (
+    url.origin === "null" ||
+    (url.protocol !== "http:" && url.protocol !== "https:") ||
+    url.origin !== value
+  ) {
+    throw new HTTPException(400, { message: "invalid request origin" });
+  }
+  if (
+    allowedCorsOrigin(input.corsAllowOriginRegex, url.origin) ||
+    [input.publicBaseUrl, input.webBaseUrl].some((baseUrl) =>
+      baseUrl ? new URL(baseUrl).origin === url.origin : false,
+    )
+  ) {
+    return url.origin;
+  }
+  throw new HTTPException(403, { message: "request origin is not allowed" });
+}

package/src/http/interaction-control-error.ts

@@ -0,0 +1,164 @@
+import { ErrorCode as AgentErrorCode } from "@opengeni/agent-proto";
+import {
+  InteractionControlFailureDetails,
+  type ErrorCode,
+  type InteractionControlFailureCode,
+} from "@opengeni/contracts";
+import { SelfhostedControlError } from "@opengeni/runtime/sandbox";
+import { HTTPException } from "hono/http-exception";
+import { ApiHttpError } from "./api-error";
+
+export type InteractionControlSurface = "browser" | "computer";
+
+/**
+ * Projects an internal connected-machine control fault into the public API
+ * vocabulary. This is deliberately an allowlist: the agent's free-form message
+ * and detail map may contain paths, process output, or provider diagnostics and
+ * never cross this boundary. The exact inner request id is safe opaque
+ * correlation and links API logs back to the machine operation.
+ */
+export function interactionControlApiError(
+  error: unknown,
+  surface: InteractionControlSurface,
+): ApiHttpError | null {
+  const controlError =
+    error instanceof SelfhostedControlError
+      ? error
+      : error instanceof HTTPException && error.cause instanceof SelfhostedControlError
+        ? error.cause
+        : null;
+  if (!controlError) return null;
+  const controlFailureCode = publicControlFailureCode(controlError.code);
+  const details = InteractionControlFailureDetails.parse({
+    interactionLayer: "connected_machine",
+    interactionSurface: surface,
+    controlFailureCode,
+    ...(safeControlRequestId(controlError.controlRequestId)
+      ? { controlRequestId: controlError.controlRequestId }
+      : {}),
+  });
+  const ruling = publicControlFailureRuling(controlError, surface, controlFailureCode);
+  return new ApiHttpError(ruling.status, {
+    code: ruling.code,
+    message: ruling.message,
+    retryable: controlError.retryable,
+    outcomeUnknown:
+      controlError.code === AgentErrorCode.ERROR_CODE_TIMEOUT && !controlError.neverSent,
+    details,
+  });
+}
+
+export function publicControlFailureCode(code: AgentErrorCode): InteractionControlFailureCode {
+  switch (code) {
+    case AgentErrorCode.ERROR_CODE_UNSUPPORTED:
+      return "unsupported";
+    case AgentErrorCode.ERROR_CODE_OS:
+      return "os";
+    case AgentErrorCode.ERROR_CODE_NOT_FOUND:
+      return "not_found";
+    case AgentErrorCode.ERROR_CODE_CONSENT_REQUIRED:
+      return "consent_required";
+    case AgentErrorCode.ERROR_CODE_TIMEOUT:
+      return "timeout";
+    case AgentErrorCode.ERROR_CODE_DRAINING:
+      return "draining";
+    case AgentErrorCode.ERROR_CODE_PROTOCOL:
+      return "protocol";
+    case AgentErrorCode.ERROR_CODE_STREAM:
+      return "stream";
+    case AgentErrorCode.ERROR_CODE_AGENT_OFFLINE:
+      return "agent_offline";
+    case AgentErrorCode.ERROR_CODE_FENCED:
+      return "fenced";
+    case AgentErrorCode.ERROR_CODE_PAYLOAD_TOO_LARGE:
+      return "payload_too_large";
+    default:
+      return "unknown";
+  }
+}
+
+function publicControlFailureRuling(
+  error: SelfhostedControlError,
+  surface: InteractionControlSurface,
+  code: InteractionControlFailureCode,
+): { status: number; code: ErrorCode; message: string } {
+  const liveView = surface === "browser" ? "browser live view" : "computer live view";
+  switch (code) {
+    case "consent_required":
+      return {
+        status: 403,
+        code: "forbidden",
+        message: `Screen access is not enabled for this connected machine's ${liveView}.`,
+      };
+    case "unsupported":
+      return {
+        status: 409,
+        code: "conflict",
+        message: `This connected machine does not support the ${liveView}.`,
+      };
+    case "not_found":
+      return {
+        status: 404,
+        code: "not_found",
+        message: `The connected-machine resource for this ${liveView} no longer exists.`,
+      };
+    case "fenced":
+      return {
+        status: 409,
+        code: "conflict",
+        message: `The connected-machine ${liveView} changed while it was opening.`,
+      };
+    case "agent_offline":
+      return {
+        status: 503,
+        code: "upstream_unavailable",
+        message: `The connected machine is offline, so its ${liveView} is unavailable.`,
+      };
+    case "draining":
+      return {
+        status: 503,
+        code: "upstream_unavailable",
+        message: `The connected machine is busy and could not open the ${liveView}.`,
+      };
+    case "timeout":
+      return {
+        status: 504,
+        code: "upstream_unavailable",
+        message: `The connected machine did not finish opening the ${liveView} in time.`,
+      };
+    case "payload_too_large":
+      return {
+        status: 502,
+        code: "upstream_unavailable",
+        message: `The connected machine could not deliver the ${liveView} within its transport limit.`,
+      };
+    case "stream":
+      return {
+        status: error.retryable ? 503 : 502,
+        code: "upstream_unavailable",
+        message: `The connected machine could not open the ${liveView} stream.`,
+      };
+    case "protocol":
+      return {
+        status: 502,
+        code: "upstream_unavailable",
+        message: `The connected machine returned an invalid ${liveView} response.`,
+      };
+    case "os":
+      return {
+        status: 502,
+        code: "upstream_unavailable",
+        message: `The connected machine could not prepare the ${liveView}.`,
+      };
+    case "unknown":
+      return {
+        status: 502,
+        code: "upstream_unavailable",
+        message: `The connected machine could not open the ${liveView}.`,
+      };
+  }
+}
+
+function safeControlRequestId(value: string | null): value is string {
+  return value !== null && value.length <= 128 && /^[A-Za-z0-9._:-]+$/u.test(value);
+}

package/src/http/sse.ts

@@ -10,9 +10,11 @@ import {
   type Database,
 } from "@opengeni/db";
 import {
+  coalesceSessionEventDeltas,
   formatSessionEventSse,
   formatWorkspaceControlEventSse,
   SESSION_EVENT_SSE_FRAME_MAX_BYTES,
+  sessionEventResumeSequence,
   type EventBus,
 } from "@opengeni/events";
 import type { Observability } from "@opengeni/observability";
@@ -268,31 +270,19 @@ export async function sseSessionStream(
   options: SessionSseDeliveryOptions = {},
 ): Promise<Response> {
   const heartbeatIntervalMs = resolveHeartbeatInterval(options.heartbeatIntervalMs);
-  if (
-    options.reauthorize &&
-    options.reauthorizeAfterMs !== undefined &&
-    (!Number.isSafeInteger(options.reauthorizeAfterMs) ||
-      options.reauthorizeAfterMs < 1_000 ||
-      options.reauthorizeAfterMs > 60_000)
-  ) {
-    throw new RangeError("session stream reauthorization must be between 1000 and 60000ms");
-  }
   let lastSent = after;
   let bootstrapping = true;
   let newestBuffered: SessionEvent | null = null;
   let unsubscribe: (() => void) | null = null;
   let delivery: LatestWinsDelivery<SessionEvent> | null = null;
-  let reauthorizationTimer: ReturnType<typeof setTimeout> | null = null;
+  let stopReauthorization = () => {};
   let heartbeatTimer: ReturnType<typeof setTimeout> | null = null;
   let detachAbortListener = () => {};
   let closeMetrics = () => {};
   const stopUpstream = () => {
     closeMetrics();
     detachAbortListener();
-    if (reauthorizationTimer) {
-      clearTimeout(reauthorizationTimer);
-      reauthorizationTimer = null;
-    }
+    stopReauthorization();
     if (heartbeatTimer) {
       clearTimeout(heartbeatTimer);
       heartbeatTimer = null;
@@ -313,20 +303,14 @@ export async function sseSessionStream(
   const fail = (error: unknown) => {
     channel.fail(retryableSseFailure("session event stream delivery failed", error));
   };
-  const scheduleReauthorization = () => {
-    if (!options.reauthorize || channel.stopped()) return;
-    const interval = options.reauthorizeAfterMs ?? 15_000;
-    reauthorizationTimer = setTimeout(() => {
-      reauthorizationTimer = null;
-      void options.reauthorize!()
-        .then(scheduleReauthorization)
-        .catch((error) => fail(error));
-    }, interval);
-  };
-  scheduleReauthorization();
+  stopReauthorization = startSseReauthorization(options, channel.stopped, fail);
   let writeTail = Promise.resolve();
   const writeFrame = (frame: string): Promise<void> => {
     const write = writeTail.then(async () => {
+      // A periodic check closes an idle stream, while this exact pre-delivery
+      // check prevents a buffered/replayed event from crossing a revocation
+      // boundary merely because its timer has not fired yet.
+      await options.reauthorize?.();
       if (!(await channel.write(frame))) throw new SseStreamStoppedError();
     });
     writeTail = write.catch(() => {});
@@ -343,36 +327,47 @@ export async function sseSessionStream(
         });
     }, heartbeatIntervalMs);
   };
-  const send = async (event: SessionEvent) => {
-    if (event.sequence <= lastSent) return;
-    if (event.sequence > lastSent + 1) {
-      while (lastSent + 1 < event.sequence) {
-        const previousLastSent = lastSent;
-        const missing = await listSessionEvents(db, workspaceId, sessionId, {
-          after: lastSent,
-          limit: Math.min(SESSION_REPLAY_PAGE_SIZE, event.sequence - lastSent - 1),
-        });
-        if (missing.length === 0) {
-          throw new Error(
-            `Session event replay stalled before sequence ${event.sequence}; last sent ${lastSent}`,
-          );
-        }
-        for (const missed of missing) {
-          if (missed.sequence >= event.sequence) break;
-          if (missed.sequence > lastSent) {
-            await writeFrame(formatSessionEventSse(missed));
-            lastSent = missed.sequence;
-          }
-        }
-        if (lastSent === previousLastSent) {
-          throw new Error(
-            `Session event replay made no progress before sequence ${event.sequence}; last sent ${lastSent}`,
-          );
-        }
+  const deliverDurableThrough = async (targetSequence?: number) => {
+    while (true) {
+      if (targetSequence !== undefined && lastSent >= targetSequence) return;
+      const previousLastSent = lastSent;
+      const limit =
+        targetSequence === undefined
+          ? SESSION_REPLAY_PAGE_SIZE
+          : Math.min(SESSION_REPLAY_PAGE_SIZE, targetSequence - lastSent);
+      const page = await listSessionEvents(db, workspaceId, sessionId, {
+        after: lastSent,
+        limit,
+      });
+      const eligible =
+        targetSequence === undefined
+          ? page
+          : page.filter((event) => event.sequence <= targetSequence);
+      if (eligible.length === 0) {
+        if (targetSequence === undefined) return;
+        throw new Error(
+          `Session event replay stalled before sequence ${targetSequence}; last sent ${lastSent}`,
+        );
+      }
+      // The durable audit log remains exact. The browser transport combines
+      // adjacent text deltas into bounded frames carrying `coalescedUntil`, so
+      // a long answer cannot create thousands of React renders and starve
+      // command acknowledgements behind its own token stream.
+      for (const projected of coalesceSessionEventDeltas(eligible)) {
+        await writeFrame(formatSessionEventSse(projected));
+        lastSent = sessionEventResumeSequence(projected);
       }
+      if (lastSent <= previousLastSent) {
+        throw new Error(`Session event replay made no progress after sequence ${lastSent}`);
+      }
+      if (targetSequence !== undefined && lastSent >= targetSequence) return;
+      if (targetSequence === undefined && page.length < limit) return;
     }
-    await writeFrame(formatSessionEventSse(event));
-    lastSent = event.sequence;
+  };
+  const send = async (event: SessionEvent) => {
+    const targetSequence = sessionEventResumeSequence(event);
+    if (targetSequence <= lastSent) return;
+    await deliverDurableThrough(targetSequence);
   };
   delivery = createLatestWinsDelivery(send, fail);
 
@@ -394,12 +389,7 @@ export async function sseSessionStream(
     }
     unsubscribe = release;
 
-    await replaySessionEvents(
-      (cursor, limit) => listSessionEvents(db, workspaceId, sessionId, cursor, limit),
-      send,
-      after,
-      SESSION_REPLAY_PAGE_SIZE,
-    );
+    await deliverDurableThrough();
     await writeFrame(": connected\n\n");
     scheduleHeartbeat();
     bootstrapping = false;
@@ -471,12 +461,14 @@ export async function sseWorkspaceControlStream(
   let newestBuffered: WorkspaceControlEvent | null = null;
   let unsubscribe: (() => void) | null = null;
   let delivery: LatestWinsDelivery<WorkspaceControlEvent> | null = null;
+  let stopReauthorization = () => {};
   let heartbeatTimer: ReturnType<typeof setTimeout> | null = null;
   let detachAbortListener = () => {};
   let closeMetrics = () => {};
   const stopUpstream = () => {
     closeMetrics();
     detachAbortListener();
+    stopReauthorization();
     if (heartbeatTimer) {
       clearTimeout(heartbeatTimer);
       heartbeatTimer = null;
@@ -497,9 +489,11 @@ export async function sseWorkspaceControlStream(
   const fail = (error: unknown) => {
     channel.fail(retryableSseFailure("workspace control stream delivery failed", error));
   };
+  stopReauthorization = startSseReauthorization(options, channel.stopped, fail);
   let writeTail = Promise.resolve();
   const writeFrame = (frame: string): Promise<void> => {
     const write = writeTail.then(async () => {
+      await options.reauthorize?.();
       if (!(await channel.write(frame))) throw new SseStreamStoppedError();
     });
     writeTail = write.catch(() => {});
@@ -619,11 +613,13 @@ export async function sseWorkspaceLiveStream(
   options: WorkspaceInteractionSseOptions = {},
 ): Promise<Response> {
   const upstream = new AbortController();
+  let stopReauthorization = () => {};
   const channel = createByteBoundedSseStream({
     maxQueuedBytes: options.maxQueuedBytes ?? SESSION_EVENT_SSE_FRAME_MAX_BYTES,
     ...(options.stallTimeoutMs === undefined ? {} : { stallTimeoutMs: options.stallTimeoutMs }),
     onObservation: sseObservationReporter("workspace_interaction", options),
     onStop: () => {
+      stopReauthorization();
       signal.removeEventListener("abort", abort);
       upstream.abort();
     },
@@ -634,10 +630,22 @@ export async function sseWorkspaceLiveStream(
   };
   if (signal.aborted) abort();
   else signal.addEventListener("abort", abort, { once: true });
+  stopReauthorization = startSseReauthorization(options, channel.stopped, (error) => {
+    channel.fail(retryableSseFailure("workspace live stream authorization failed", error));
+  });
+
+  const upstreamOptions: WorkspaceInteractionSseOptions = {
+    ...options,
+    reauthorize: undefined,
+    reauthorizeAfterMs: undefined,
+  };
 
   let writeTail = Promise.resolve(true);
   const write = (frame: string): Promise<boolean> => {
-    const pending = writeTail.then(() => channel.write(frame));
+    const pending = writeTail.then(async () => {
+      await options.reauthorize?.();
+      return await channel.write(frame);
+    });
     writeTail = pending.catch(() => false);
     return pending;
   };
@@ -665,14 +673,21 @@ export async function sseWorkspaceLiveStream(
   void (async () => {
     try {
       const [control, interaction] = await Promise.all([
-        sseWorkspaceControlStream(db, bus, workspaceId, controlAfter, upstream.signal, options),
+        sseWorkspaceControlStream(
+          db,
+          bus,
+          workspaceId,
+          controlAfter,
+          upstream.signal,
+          upstreamOptions,
+        ),
         sseWorkspaceInteractionRevisionStream(
           db,
           accountId,
           workspaceId,
           interactionAfter,
           upstream.signal,
-          options,
+          upstreamOptions,
         ),
       ]);
       const pumps = [pump(control), pump(interaction)];
@@ -721,6 +736,7 @@ export async function sseWorkspaceInteractionRevisionStream(
   let lastSent = after;
   let lastWriteAt = Date.now();
   let stopRequested = false;
+  let stopReauthorization = () => {};
   let detachAbortListener = () => {};
   let closeMetrics = () => {};
   const channel = createByteBoundedSseStream({
@@ -729,13 +745,18 @@ export async function sseWorkspaceInteractionRevisionStream(
     onObservation: sseObservationReporter("workspace_interaction", options),
     onStop: () => {
       stopRequested = true;
+      stopReauthorization();
       closeMetrics();
       detachAbortListener();
     },
   });
   closeMetrics = observeSseConnection("workspace_interaction", after, options.observability);
+  stopReauthorization = startSseReauthorization(options, channel.stopped, (error) => {
+    channel.fail(retryableSseFailure("workspace interaction stream authorization failed", error));
+  });
 
   const write = async (frame: string): Promise<boolean> => {
+    await options.reauthorize?.();
     const accepted = await channel.write(frame);
     if (accepted) lastWriteAt = Date.now();
     return accepted;
@@ -849,14 +870,13 @@ export type SseDeliveryOptions = {
   heartbeatIntervalMs?: number;
   observability?: Observability | undefined;
   onObservation?: ((observation: SseDeliveryBoundObservation) => void) | undefined;
-};
-
-export type SessionSseDeliveryOptions = SseDeliveryOptions & {
-  /** Host ACL re-check, run even while the event stream is otherwise idle. */
+  /** Current ACL re-check, run even while the event stream is idle. */
   reauthorize?: (() => Promise<void>) | undefined;
   reauthorizeAfterMs?: number | undefined;
 };
 
+export type SessionSseDeliveryOptions = SseDeliveryOptions;
+
 function sseObservationReporter(
   stream: SseStreamKind,
   options: SseDeliveryOptions,
@@ -900,6 +920,32 @@ function resolveInteractionPollInterval(value: number | undefined): number {
   return interval;
 }
 
+function startSseReauthorization(
+  options: SseDeliveryOptions,
+  stopped: () => boolean,
+  fail: (error: unknown) => void,
+): () => void {
+  if (!options.reauthorize) return () => {};
+  const interval = options.reauthorizeAfterMs ?? 15_000;
+  if (!Number.isSafeInteger(interval) || interval < 1_000 || interval > 60_000) {
+    throw new RangeError("SSE reauthorization must be between 1000 and 60000ms");
+  }
+  let timer: ReturnType<typeof setTimeout> | null = null;
+  const schedule = () => {
+    if (stopped()) return;
+    timer = setTimeout(() => {
+      timer = null;
+      void options.reauthorize!().then(schedule).catch(fail);
+    }, interval);
+  };
+  schedule();
+  return () => {
+    if (!timer) return;
+    clearTimeout(timer);
+    timer = null;
+  };
+}
+
 function formatWorkspaceInteractionRevisionSse(event: WorkspaceInteractionRevisionEvent): string {
   return `id: ${event.sequence}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`;
 }