@opengeni/api-router 0.23.1 → 0.30.0

package/src/http/sse.ts

@@ -1,5 +1,14 @@
-import type { SessionEvent, WorkspaceControlEvent } from "@opengeni/contracts";
-import { listSessionEvents, listWorkspaceControlEvents, type Database } from "@opengeni/db";
+import {
+  WorkspaceInteractionRevisionEvent,
+  type SessionEvent,
+  type WorkspaceControlEvent,
+} from "@opengeni/contracts";
+import {
+  getWorkspaceInteractionRevisionState,
+  listSessionEvents,
+  listWorkspaceControlEvents,
+  type Database,
+} from "@opengeni/db";
 import {
   formatSessionEventSse,
   formatWorkspaceControlEventSse,
@@ -13,9 +22,11 @@ const WORKSPACE_CONTROL_REPLAY_PAGE_SIZE = 100;
 export const SSE_QUEUED_FRAME_MAX_COUNT = 1;
 export const SSE_WRITE_STALL_TIMEOUT_MS = 30_000;
 export const SSE_HEARTBEAT_INTERVAL_MS = 15_000;
-const activeSseStreams: Record<"session" | "workspace_control", number> = {
+type SseStreamKind = "session" | "workspace_control" | "workspace_interaction";
+const activeSseStreams: Record<SseStreamKind, number> = {
   session: 0,
   workspace_control: 0,
+  workspace_interaction: 0,
 };
 
 export type SseDeliveryBoundObservation = {
@@ -586,8 +597,196 @@ export async function sseWorkspaceControlStream(
   });
 }
 
+export type WorkspaceInteractionSseOptions = SseDeliveryOptions & {
+  pollIntervalMs?: number | undefined;
+};
+
+/**
+ * One HTTP connection for the two workspace-wide invalidation domains used by
+ * every visible OpenGeni surface. Keeping these as separate HTTP/1 streams
+ * consumes all six per-origin browser connections with only two windows and
+ * starves ordinary mutations/terminal grants. The durable cursors remain
+ * independent; this function only multiplexes their already-bounded SSE frames.
+ */
+export async function sseWorkspaceLiveStream(
+  db: Database,
+  bus: EventBus,
+  accountId: string,
+  workspaceId: string,
+  controlAfter: number,
+  interactionAfter: number,
+  signal: AbortSignal,
+  options: WorkspaceInteractionSseOptions = {},
+): Promise<Response> {
+  const upstream = new AbortController();
+  const channel = createByteBoundedSseStream({
+    maxQueuedBytes: options.maxQueuedBytes ?? SESSION_EVENT_SSE_FRAME_MAX_BYTES,
+    ...(options.stallTimeoutMs === undefined ? {} : { stallTimeoutMs: options.stallTimeoutMs }),
+    onObservation: sseObservationReporter("workspace_interaction", options),
+    onStop: () => {
+      signal.removeEventListener("abort", abort);
+      upstream.abort();
+    },
+  });
+  const abort = () => {
+    upstream.abort();
+    channel.close();
+  };
+  if (signal.aborted) abort();
+  else signal.addEventListener("abort", abort, { once: true });
+
+  let writeTail = Promise.resolve(true);
+  const write = (frame: string): Promise<boolean> => {
+    const pending = writeTail.then(() => channel.write(frame));
+    writeTail = pending.catch(() => false);
+    return pending;
+  };
+  const pump = async (response: Response): Promise<void> => {
+    if (!response.body) throw new TypeError("workspace live upstream omitted its body");
+    const reader = response.body.getReader();
+    const decoder = new TextDecoder();
+    try {
+      for (;;) {
+        const next = await reader.read();
+        if (next.done) {
+          const tail = decoder.decode();
+          if (tail) await write(tail);
+          return;
+        }
+        const frame = decoder.decode(next.value, { stream: true });
+        if (frame && !(await write(frame))) return;
+      }
+    } finally {
+      await reader.cancel().catch(() => undefined);
+      reader.releaseLock();
+    }
+  };
+
+  void (async () => {
+    try {
+      const [control, interaction] = await Promise.all([
+        sseWorkspaceControlStream(db, bus, workspaceId, controlAfter, upstream.signal, options),
+        sseWorkspaceInteractionRevisionStream(
+          db,
+          accountId,
+          workspaceId,
+          interactionAfter,
+          upstream.signal,
+          options,
+        ),
+      ]);
+      const pumps = [pump(control), pump(interaction)];
+      let termination: unknown = new TypeError("workspace live upstream ended unexpectedly");
+      try {
+        await Promise.race(pumps);
+      } catch (error) {
+        termination = error;
+      } finally {
+        // Either durable domain ending makes the multiplexed connection stale.
+        // Stop its sibling immediately so the SDK reconnects both independent
+        // cursors instead of silently losing one domain forever.
+        upstream.abort();
+        await Promise.allSettled(pumps);
+      }
+      if (!signal.aborted && !channel.stopped()) channel.fail(termination);
+    } catch (error) {
+      if (!upstream.signal.aborted && !channel.stopped()) channel.fail(error);
+    }
+  })();
+
+  return new Response(channel.stream, {
+    headers: {
+      "Content-Type": "text/event-stream; charset=utf-8",
+      "Cache-Control": "no-cache, no-transform",
+      Connection: "keep-alive",
+    },
+  });
+}
+
+/**
+ * Latest-wins interaction invalidation stream. The durable truth is one
+ * monotonic workspace row, not an ever-growing event log. Each poll reads only
+ * that row; reconnect immediately projects the newest revision after `after`.
+ */
+export async function sseWorkspaceInteractionRevisionStream(
+  db: Database,
+  accountId: string,
+  workspaceId: string,
+  after: number,
+  signal: AbortSignal,
+  options: WorkspaceInteractionSseOptions = {},
+): Promise<Response> {
+  const heartbeatIntervalMs = resolveHeartbeatInterval(options.heartbeatIntervalMs);
+  const pollIntervalMs = resolveInteractionPollInterval(options.pollIntervalMs);
+  let lastSent = after;
+  let lastWriteAt = Date.now();
+  let stopRequested = false;
+  let detachAbortListener = () => {};
+  let closeMetrics = () => {};
+  const channel = createByteBoundedSseStream({
+    maxQueuedBytes: options.maxQueuedBytes ?? SESSION_EVENT_SSE_FRAME_MAX_BYTES,
+    ...(options.stallTimeoutMs === undefined ? {} : { stallTimeoutMs: options.stallTimeoutMs }),
+    onObservation: sseObservationReporter("workspace_interaction", options),
+    onStop: () => {
+      stopRequested = true;
+      closeMetrics();
+      detachAbortListener();
+    },
+  });
+  closeMetrics = observeSseConnection("workspace_interaction", after, options.observability);
+
+  const write = async (frame: string): Promise<boolean> => {
+    const accepted = await channel.write(frame);
+    if (accepted) lastWriteAt = Date.now();
+    return accepted;
+  };
+
+  void (async () => {
+    try {
+      if (!(await write(": connected\n\n"))) return;
+      for (;;) {
+        if (stopRequested || signal.aborted || channel.stopped()) return;
+        const state = await getWorkspaceInteractionRevisionState(db, { accountId, workspaceId });
+        if (state.revision > lastSent) {
+          const event = WorkspaceInteractionRevisionEvent.parse({
+            workspaceId,
+            sequence: state.revision,
+            revision: state.revision,
+            type: "workspace.interaction.changed",
+            occurredAt: (state.updatedAt ?? new Date()).toISOString(),
+          });
+          if (!(await write(formatWorkspaceInteractionRevisionSse(event)))) return;
+          lastSent = event.revision;
+        } else if (Date.now() - lastWriteAt >= heartbeatIntervalMs) {
+          if (!(await write(": heartbeat\n\n"))) return;
+        }
+        await abortableDelay(pollIntervalMs, signal);
+      }
+    } catch (error) {
+      if (!signal.aborted && !channel.stopped()) {
+        channel.fail(retryableSseFailure("workspace interaction stream delivery failed", error));
+      }
+    }
+  })();
+
+  const abort = () => channel.close();
+  if (signal.aborted) abort();
+  else {
+    signal.addEventListener("abort", abort, { once: true });
+    detachAbortListener = () => signal.removeEventListener("abort", abort);
+  }
+
+  return new Response(channel.stream, {
+    headers: {
+      "Content-Type": "text/event-stream; charset=utf-8",
+      "Cache-Control": "no-cache, no-transform",
+      Connection: "keep-alive",
+    },
+  });
+}
+
 function observeSseConnection(
-  stream: "session" | "workspace_control",
+  stream: SseStreamKind,
   after: number,
   observability: Observability | undefined,
 ): () => void {
@@ -659,7 +858,7 @@ export type SessionSseDeliveryOptions = SseDeliveryOptions & {
 };
 
 function sseObservationReporter(
-  stream: "session" | "workspace_control",
+  stream: SseStreamKind,
   options: SseDeliveryOptions,
 ): (observation: SseDeliveryBoundObservation) => void {
   return (observation) => {
@@ -692,3 +891,28 @@ function resolveHeartbeatInterval(value: number | undefined): number {
   }
   return interval;
 }
+
+function resolveInteractionPollInterval(value: number | undefined): number {
+  const interval = value ?? 1_000;
+  if (!Number.isSafeInteger(interval) || interval < 100 || interval > 60_000) {
+    throw new RangeError("interaction revision poll interval must be between 100 and 60000ms");
+  }
+  return interval;
+}
+
+function formatWorkspaceInteractionRevisionSse(event: WorkspaceInteractionRevisionEvent): string {
+  return `id: ${event.sequence}\nevent: ${event.type}\ndata: ${JSON.stringify(event)}\n\n`;
+}
+
+async function abortableDelay(delayMs: number, signal: AbortSignal): Promise<void> {
+  if (signal.aborted) return;
+  await new Promise<void>((resolve) => {
+    const timer = setTimeout(done, delayMs);
+    function done() {
+      clearTimeout(timer);
+      signal.removeEventListener("abort", done);
+      resolve();
+    }
+    signal.addEventListener("abort", done, { once: true });
+  });
+}

package/src/index.ts

@@ -21,6 +21,8 @@ import {
 } from "@opengeni/db";
 import { createNatsEventBus, type ResponderConnection } from "@opengeni/events";
 import { createObservability, logStartupDependencyRetry } from "@opengeni/observability";
+import { createObjectStorage } from "@opengeni/storage";
+import { isArtifactRuntimeConfigured } from "@opengeni/artifact-tool/runtime/development";
 import { SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID } from "@opengeni/core";
 import {
   Connection,
@@ -35,6 +37,17 @@ import { observabilityEventLogger } from "./observability";
 import { startAuthCalloutResponder } from "./sandbox/auth-callout";
 import { startHelloIngestion, startMetricsIngestion } from "./sandbox/metrics-ingestion";
 import { startSlackInteractionPump } from "./integrations/slack-interactions";
+import { startMemorySlackPublicationPump } from "./memory-slack-delivery";
+import { startTemporalScheduleCleanupPump } from "./temporal-schedule-cleanup";
+import {
+  EDITABLE_ARTIFACT_LIVE_WEBSOCKET_MAX_MESSAGE_BYTES,
+  EditableArtifactWebSocketTransport,
+  type EditableArtifactWebSocketConnection,
+} from "./editable-artifact-websocket";
+import {
+  createStandaloneEditableArtifactApplication,
+  type StandaloneEditableArtifactApplication,
+} from "./editable-artifact-production";
 
 /**
  * A REJECT_DUPLICATE start collides on the deterministic workflowId when the
@@ -156,6 +169,14 @@ export async function createTemporalWorkflowClient(
     },
     syncScheduledTask: async ({ task }) => {
       const schedule = temporal.schedule.getHandle(task.temporalScheduleId);
+      if (task.schedule.type === "manual") {
+        try {
+          await schedule.delete();
+        } catch (error) {
+          if (!(error instanceof ScheduleNotFoundError)) throw error;
+        }
+        return;
+      }
       const options = temporalScheduleOptions(task, settings.temporalTaskQueue);
       try {
         await schedule.update(() => temporalScheduleUpdateOptions(options));
@@ -167,16 +188,21 @@ export async function createTemporalWorkflowClient(
       }
     },
     deleteScheduledTaskSchedule: async ({ temporalScheduleId }) => {
-      await temporal.schedule
-        .getHandle(temporalScheduleId)
-        .delete()
-        .catch(() => undefined);
+      try {
+        await temporal.withDeadline(Date.now() + 5_000, async () => {
+          await temporal.schedule.getHandle(temporalScheduleId).delete();
+        });
+      } catch (error) {
+        if (error instanceof ScheduleNotFoundError) return;
+        throw error;
+      }
     },
     triggerScheduledTask: async ({
       task,
       agentRunUsageIdempotencyKey,
       triggerWorkflowId,
       initiator,
+      triggerType = "manual",
     }) => {
       // Deterministic workflowId (derived from the trigger token by the
       // caller) + REJECT_DUPLICATE makes a retried manual trigger idempotent:
@@ -193,7 +219,7 @@ export async function createTemporalWorkflowClient(
               accountId: task.accountId,
               workspaceId: task.workspaceId,
               taskId: task.id,
-              triggerType: "manual",
+              triggerType,
               agentRunUsageIdempotencyKey,
               initiator,
             },
@@ -213,18 +239,25 @@ export async function createTemporalWorkflowClient(
       if (!targetId) {
         throw new Error("rig verification requires changeId or versionId");
       }
-      await temporal.workflow.start("rigVerificationWorkflow", {
-        taskQueue: settings.temporalTaskQueue,
-        workflowId: workflowId ?? `rig-verification-${targetId}-${crypto.randomUUID()}`,
-        workflowIdReusePolicy: "ALLOW_DUPLICATE",
-        args: [
-          {
-            workspaceId,
-            ...(changeId ? { changeId } : {}),
-            ...(versionId ? { versionId } : {}),
-          },
-        ],
-      });
+      try {
+        await temporal.workflow.start("rigVerificationWorkflow", {
+          taskQueue: settings.temporalTaskQueue,
+          workflowId: workflowId ?? `rig-verification-${targetId}-${crypto.randomUUID()}`,
+          workflowIdReusePolicy: "REJECT_DUPLICATE",
+          args: [
+            {
+              workspaceId,
+              ...(changeId ? { changeId } : {}),
+              ...(versionId ? { versionId } : {}),
+            },
+          ],
+        });
+      } catch (error) {
+        // A lost start acknowledgement is indistinguishable from a retry. The
+        // caller-owned workflow id makes both cases the same successful start.
+        if (isWorkflowAlreadyStarted(error)) return;
+        throw error;
+      }
     },
     check: async () => {
       await connection.workflowService.getSystemInfo({});
@@ -312,26 +345,77 @@ export async function startApi() {
     await dbClient.close();
     throw new Error("OpenGeni API startup dependencies were not initialized");
   }
+  const objectStorage = createObjectStorage(settings);
+  let editableArtifactComposition: StandaloneEditableArtifactApplication | undefined;
+  if (objectStorage && isArtifactRuntimeConfigured()) {
+    try {
+      editableArtifactComposition = await createStandaloneEditableArtifactApplication({
+        db: dbClient.db,
+        bus,
+        objectStorage,
+      });
+    } catch (error) {
+      await Promise.allSettled([bus.close(), workflowClient.close(), dbClient.close()]);
+      throw error;
+    }
+  }
   const { app, routeDeps } = createAppComposition({
     settings,
     db: dbClient.db,
     bus,
     workflowClient: workflowClient.client,
     documentIndexer: workflowClient.documentIndexer,
+    objectStorage,
+    ...(editableArtifactComposition
+      ? {
+          editableArtifacts: editableArtifactComposition.application,
+          editableArtifactExports: editableArtifactComposition.durableExports,
+          editableArtifactAgent: editableArtifactComposition.agent,
+          editableArtifactOfficeImports: editableArtifactComposition.officeImports,
+        }
+      : {}),
     observability,
     readinessChecks: {
       db: runtimeDatabaseReadyCheck(dbClient.db, databasePosture),
     },
   });
-  const server = Bun.serve({
+  if (!routeDeps.editableArtifacts) {
+    observability.warn("editable artifact engine is not composed", {
+      subsystem: "editable_artifacts",
+      behavior: "http_and_websocket_fail_closed",
+    });
+  }
+  const artifactWebSockets = new EditableArtifactWebSocketTransport(routeDeps.editableArtifacts);
+  const server = Bun.serve<EditableArtifactWebSocketConnection>({
     hostname: settings.apiHost,
     port: settings.apiPort,
     idleTimeout: 255,
-    fetch: app.fetch,
+    fetch: (request, bunServer) => {
+      if (artifactWebSockets.handles(request)) {
+        return artifactWebSockets.upgrade(request, bunServer);
+      }
+      return app.fetch(request);
+    },
+    websocket: {
+      maxPayloadLength: EDITABLE_ARTIFACT_LIVE_WEBSOCKET_MAX_MESSAGE_BYTES,
+      backpressureLimit: 16 * 1024 * 1024,
+      closeOnBackpressureLimit: true,
+      open: (socket) => artifactWebSockets.websocket.open(socket),
+      message: (socket, message) => artifactWebSockets.websocket.message(socket, message),
+      close: (socket) => artifactWebSockets.websocket.close(socket),
+    },
   });
   const stopSlackInteractionPump = settings.slackSigningSecret
     ? startSlackInteractionPump(routeDeps)
     : undefined;
+  const stopMemorySlackPublicationPump = startMemorySlackPublicationPump(routeDeps);
+  const stopTemporalScheduleCleanupPump = startTemporalScheduleCleanupPump({
+    db: dbClient.db,
+    deleteSchedule: async (temporalScheduleId) => {
+      await workflowClient.client.deleteScheduledTaskSchedule({ temporalScheduleId });
+    },
+    observability,
+  });
   // M10 — start the metrics-ingestion consumer (agent heartbeats → DB last-sample
   // + downsampled series), gated on the selfhosted flag. A no-op when disabled.
   let stopMetricsIngestion: (() => void) | undefined;
@@ -391,9 +475,12 @@ export async function startApi() {
     close: async () => {
       server.stop(true);
       stopSlackInteractionPump?.();
+      await stopMemorySlackPublicationPump();
       stopMetricsIngestion?.();
       stopHelloIngestion?.();
+      await stopTemporalScheduleCleanupPump();
       await Promise.allSettled([
+        Promise.resolve(editableArtifactComposition?.close()),
         authCalloutResponder?.close(),
         bus.close(),
         workflowClient.close(),
@@ -422,6 +509,9 @@ export function shouldCreateScheduleAfterUpdateError(error: unknown): boolean {
 }
 
 export function temporalScheduleSpec(schedule: ScheduledTaskScheduleSpec): ScheduleSpec {
+  if (schedule.type === "manual") {
+    throw new Error("manual scheduled tasks do not have a Temporal Schedule spec");
+  }
   if (schedule.type === "interval") {
     return {
       intervals: [temporalIntervalSpec(schedule)],
@@ -508,7 +598,7 @@ function temporalScheduleOptions(task: ScheduledTask, taskQueue: string): Schedu
       pauseOnFailure: false,
     },
     state: {
-      paused: task.status === "paused",
+      paused: scheduledTaskEffectivelyPaused(task),
       ...(task.schedule.type === "once" ? { remainingActions: 1 } : {}),
     },
     memo: {
@@ -520,6 +610,15 @@ function temporalScheduleOptions(task: ScheduledTask, taskQueue: string): Schedu
   };
 }
 
+function scheduledTaskEffectivelyPaused(task: ScheduledTask): boolean {
+  if (task.status === "paused") return true;
+  if (task.action.kind !== "knowledge_source_sync") return false;
+  const control = task.metadata.knowledgeSourceSync;
+  if (!control || typeof control !== "object") return false;
+  const record = control as Record<string, unknown>;
+  return record.sourceEnabled === false || record.connectionPaused === true;
+}
+
 function temporalScheduleUpdateOptions(options: ScheduleOptions): ScheduleUpdateOptions {
   return {
     spec: options.spec,