@opengeni/api-router 0.22.2 → 0.26.1

package/src/http/sse.ts

@@ -12,6 +12,11 @@ const SESSION_REPLAY_PAGE_SIZE = 100;
 const WORKSPACE_CONTROL_REPLAY_PAGE_SIZE = 100;
 export const SSE_QUEUED_FRAME_MAX_COUNT = 1;
 export const SSE_WRITE_STALL_TIMEOUT_MS = 30_000;
+export const SSE_HEARTBEAT_INTERVAL_MS = 15_000;
+const activeSseStreams: Record<"session" | "workspace_control", number> = {
+  session: 0,
+  workspace_control: 0,
+};
 
 export type SseDeliveryBoundObservation = {
   reason: "desired_size_non_positive" | "stall_timeout" | "frame_too_large";
@@ -251,6 +256,7 @@ export async function sseSessionStream(
   signal: AbortSignal,
   options: SessionSseDeliveryOptions = {},
 ): Promise<Response> {
+  const heartbeatIntervalMs = resolveHeartbeatInterval(options.heartbeatIntervalMs);
   if (
     options.reauthorize &&
     options.reauthorizeAfterMs !== undefined &&
@@ -266,13 +272,20 @@ export async function sseSessionStream(
   let unsubscribe: (() => void) | null = null;
   let delivery: LatestWinsDelivery<SessionEvent> | null = null;
   let reauthorizationTimer: ReturnType<typeof setTimeout> | null = null;
+  let heartbeatTimer: ReturnType<typeof setTimeout> | null = null;
   let detachAbortListener = () => {};
+  let closeMetrics = () => {};
   const stopUpstream = () => {
+    closeMetrics();
     detachAbortListener();
     if (reauthorizationTimer) {
       clearTimeout(reauthorizationTimer);
       reauthorizationTimer = null;
     }
+    if (heartbeatTimer) {
+      clearTimeout(heartbeatTimer);
+      heartbeatTimer = null;
+    }
     delivery?.stop();
     const release = unsubscribe;
     unsubscribe = null;
@@ -284,6 +297,7 @@ export async function sseSessionStream(
     onObservation: sseObservationReporter("session", options),
     onStop: stopUpstream,
   });
+  closeMetrics = observeSseConnection("session", after, options.observability);
 
   const fail = (error: unknown) => {
     channel.fail(retryableSseFailure("session event stream delivery failed", error));
@@ -299,10 +313,24 @@ export async function sseSessionStream(
     }, interval);
   };
   scheduleReauthorization();
-  const writeFrame = async (frame: string) => {
-    if (!(await channel.write(frame))) {
-      throw new SseStreamStoppedError();
-    }
+  let writeTail = Promise.resolve();
+  const writeFrame = (frame: string): Promise<void> => {
+    const write = writeTail.then(async () => {
+      if (!(await channel.write(frame))) throw new SseStreamStoppedError();
+    });
+    writeTail = write.catch(() => {});
+    return write;
+  };
+  const scheduleHeartbeat = () => {
+    if (channel.stopped()) return;
+    heartbeatTimer = setTimeout(() => {
+      heartbeatTimer = null;
+      void writeFrame(": heartbeat\n\n")
+        .then(scheduleHeartbeat)
+        .catch((error) => {
+          if (!(error instanceof SseStreamStoppedError)) fail(error);
+        });
+    }, heartbeatIntervalMs);
   };
   const send = async (event: SessionEvent) => {
     if (event.sequence <= lastSent) return;
@@ -362,6 +390,7 @@ export async function sseSessionStream(
       SESSION_REPLAY_PAGE_SIZE,
     );
     await writeFrame(": connected\n\n");
+    scheduleHeartbeat();
     bootstrapping = false;
     const buffered = newestBuffered;
     newestBuffered = null;
@@ -425,14 +454,22 @@ export async function sseWorkspaceControlStream(
   signal: AbortSignal,
   options: SseDeliveryOptions = {},
 ): Promise<Response> {
+  const heartbeatIntervalMs = resolveHeartbeatInterval(options.heartbeatIntervalMs);
   let lastSent = after;
   let bootstrapping = true;
   let newestBuffered: WorkspaceControlEvent | null = null;
   let unsubscribe: (() => void) | null = null;
   let delivery: LatestWinsDelivery<WorkspaceControlEvent> | null = null;
+  let heartbeatTimer: ReturnType<typeof setTimeout> | null = null;
   let detachAbortListener = () => {};
+  let closeMetrics = () => {};
   const stopUpstream = () => {
+    closeMetrics();
     detachAbortListener();
+    if (heartbeatTimer) {
+      clearTimeout(heartbeatTimer);
+      heartbeatTimer = null;
+    }
     delivery?.stop();
     const release = unsubscribe;
     unsubscribe = null;
@@ -444,12 +481,29 @@ export async function sseWorkspaceControlStream(
     onObservation: sseObservationReporter("workspace_control", options),
     onStop: stopUpstream,
   });
+  closeMetrics = observeSseConnection("workspace_control", after, options.observability);
 
   const fail = (error: unknown) => {
     channel.fail(retryableSseFailure("workspace control stream delivery failed", error));
   };
-  const writeFrame = async (frame: string) => {
-    if (!(await channel.write(frame))) throw new SseStreamStoppedError();
+  let writeTail = Promise.resolve();
+  const writeFrame = (frame: string): Promise<void> => {
+    const write = writeTail.then(async () => {
+      if (!(await channel.write(frame))) throw new SseStreamStoppedError();
+    });
+    writeTail = write.catch(() => {});
+    return write;
+  };
+  const scheduleHeartbeat = () => {
+    if (channel.stopped()) return;
+    heartbeatTimer = setTimeout(() => {
+      heartbeatTimer = null;
+      void writeFrame(": heartbeat\n\n")
+        .then(scheduleHeartbeat)
+        .catch((error) => {
+          if (!(error instanceof SseStreamStoppedError)) fail(error);
+        });
+    }, heartbeatIntervalMs);
   };
   const send = async (event: WorkspaceControlEvent) => {
     if (event.sequence <= lastSent) return;
@@ -505,6 +559,7 @@ export async function sseWorkspaceControlStream(
       WORKSPACE_CONTROL_REPLAY_PAGE_SIZE,
     );
     await writeFrame(": connected\n\n");
+    scheduleHeartbeat();
     bootstrapping = false;
     const buffered = newestBuffered;
     newestBuffered = null;
@@ -531,6 +586,37 @@ export async function sseWorkspaceControlStream(
   });
 }
 
+function observeSseConnection(
+  stream: "session" | "workspace_control",
+  after: number,
+  observability: Observability | undefined,
+): () => void {
+  activeSseStreams[stream] += 1;
+  observability?.incrementCounter({
+    name: "opengeni_sse_connections_total",
+    help: "SSE connections opened at the API boundary.",
+    labels: { stream, resume: after > 0 ? "resumed" : "fresh" },
+  });
+  observability?.setGauge?.({
+    name: "opengeni_sse_connections_active",
+    help: "Currently active SSE connections at the API boundary.",
+    labels: { stream },
+    value: activeSseStreams[stream],
+  });
+  let closed = false;
+  return () => {
+    if (closed) return;
+    closed = true;
+    activeSseStreams[stream] = Math.max(0, activeSseStreams[stream] - 1);
+    observability?.setGauge?.({
+      name: "opengeni_sse_connections_active",
+      help: "Currently active SSE connections at the API boundary.",
+      labels: { stream },
+      value: activeSseStreams[stream],
+    });
+  };
+}
+
 async function replayWorkspaceControlEvents(
   loadPage: (after: number, limit: number) => Promise<WorkspaceControlEvent[]>,
   send: (event: WorkspaceControlEvent) => Promise<void>,
@@ -561,6 +647,7 @@ class SseStreamStoppedError extends Error {}
 export type SseDeliveryOptions = {
   maxQueuedBytes?: number;
   stallTimeoutMs?: number;
+  heartbeatIntervalMs?: number;
   observability?: Observability | undefined;
   onObservation?: ((observation: SseDeliveryBoundObservation) => void) | undefined;
 };
@@ -597,3 +684,11 @@ function sseObservationReporter(
 function retryableSseFailure(message: string, error: unknown): TypeError {
   return error instanceof TypeError ? error : new TypeError(message, { cause: error });
 }
+
+function resolveHeartbeatInterval(value: number | undefined): number {
+  const interval = value ?? SSE_HEARTBEAT_INTERVAL_MS;
+  if (!Number.isSafeInteger(interval) || interval < 1_000 || interval > 60_000) {
+    throw new RangeError("SSE heartbeat interval must be between 1000 and 60000ms");
+  }
+  return interval;
+}

package/src/index.ts

@@ -21,6 +21,8 @@ import {
 } from "@opengeni/db";
 import { createNatsEventBus, type ResponderConnection } from "@opengeni/events";
 import { createObservability, logStartupDependencyRetry } from "@opengeni/observability";
+import { createObjectStorage } from "@opengeni/storage";
+import { isArtifactRuntimeConfigured } from "@opengeni/artifact-tool/runtime/development";
 import { SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID } from "@opengeni/core";
 import {
   Connection,
@@ -35,6 +37,17 @@ import { observabilityEventLogger } from "./observability";
 import { startAuthCalloutResponder } from "./sandbox/auth-callout";
 import { startHelloIngestion, startMetricsIngestion } from "./sandbox/metrics-ingestion";
 import { startSlackInteractionPump } from "./integrations/slack-interactions";
+import { startMemorySlackPublicationPump } from "./memory-slack-delivery";
+import { startTemporalScheduleCleanupPump } from "./temporal-schedule-cleanup";
+import {
+  EDITABLE_ARTIFACT_LIVE_WEBSOCKET_MAX_MESSAGE_BYTES,
+  EditableArtifactWebSocketTransport,
+  type EditableArtifactWebSocketConnection,
+} from "./editable-artifact-websocket";
+import {
+  createStandaloneEditableArtifactApplication,
+  type StandaloneEditableArtifactApplication,
+} from "./editable-artifact-production";
 
 /**
  * A REJECT_DUPLICATE start collides on the deterministic workflowId when the
@@ -156,6 +169,14 @@ export async function createTemporalWorkflowClient(
     },
     syncScheduledTask: async ({ task }) => {
       const schedule = temporal.schedule.getHandle(task.temporalScheduleId);
+      if (task.schedule.type === "manual") {
+        try {
+          await schedule.delete();
+        } catch (error) {
+          if (!(error instanceof ScheduleNotFoundError)) throw error;
+        }
+        return;
+      }
       const options = temporalScheduleOptions(task, settings.temporalTaskQueue);
       try {
         await schedule.update(() => temporalScheduleUpdateOptions(options));
@@ -167,16 +188,21 @@ export async function createTemporalWorkflowClient(
       }
     },
     deleteScheduledTaskSchedule: async ({ temporalScheduleId }) => {
-      await temporal.schedule
-        .getHandle(temporalScheduleId)
-        .delete()
-        .catch(() => undefined);
+      try {
+        await temporal.withDeadline(Date.now() + 5_000, async () => {
+          await temporal.schedule.getHandle(temporalScheduleId).delete();
+        });
+      } catch (error) {
+        if (error instanceof ScheduleNotFoundError) return;
+        throw error;
+      }
     },
     triggerScheduledTask: async ({
       task,
       agentRunUsageIdempotencyKey,
       triggerWorkflowId,
       initiator,
+      triggerType = "manual",
     }) => {
       // Deterministic workflowId (derived from the trigger token by the
       // caller) + REJECT_DUPLICATE makes a retried manual trigger idempotent:
@@ -193,7 +219,7 @@ export async function createTemporalWorkflowClient(
               accountId: task.accountId,
               workspaceId: task.workspaceId,
               taskId: task.id,
-              triggerType: "manual",
+              triggerType,
               agentRunUsageIdempotencyKey,
               initiator,
             },
@@ -213,18 +239,25 @@ export async function createTemporalWorkflowClient(
       if (!targetId) {
         throw new Error("rig verification requires changeId or versionId");
       }
-      await temporal.workflow.start("rigVerificationWorkflow", {
-        taskQueue: settings.temporalTaskQueue,
-        workflowId: workflowId ?? `rig-verification-${targetId}-${crypto.randomUUID()}`,
-        workflowIdReusePolicy: "ALLOW_DUPLICATE",
-        args: [
-          {
-            workspaceId,
-            ...(changeId ? { changeId } : {}),
-            ...(versionId ? { versionId } : {}),
-          },
-        ],
-      });
+      try {
+        await temporal.workflow.start("rigVerificationWorkflow", {
+          taskQueue: settings.temporalTaskQueue,
+          workflowId: workflowId ?? `rig-verification-${targetId}-${crypto.randomUUID()}`,
+          workflowIdReusePolicy: "REJECT_DUPLICATE",
+          args: [
+            {
+              workspaceId,
+              ...(changeId ? { changeId } : {}),
+              ...(versionId ? { versionId } : {}),
+            },
+          ],
+        });
+      } catch (error) {
+        // A lost start acknowledgement is indistinguishable from a retry. The
+        // caller-owned workflow id makes both cases the same successful start.
+        if (isWorkflowAlreadyStarted(error)) return;
+        throw error;
+      }
     },
     check: async () => {
       await connection.workflowService.getSystemInfo({});
@@ -312,26 +345,77 @@ export async function startApi() {
     await dbClient.close();
     throw new Error("OpenGeni API startup dependencies were not initialized");
   }
+  const objectStorage = createObjectStorage(settings);
+  let editableArtifactComposition: StandaloneEditableArtifactApplication | undefined;
+  if (objectStorage && isArtifactRuntimeConfigured()) {
+    try {
+      editableArtifactComposition = await createStandaloneEditableArtifactApplication({
+        db: dbClient.db,
+        bus,
+        objectStorage,
+      });
+    } catch (error) {
+      await Promise.allSettled([bus.close(), workflowClient.close(), dbClient.close()]);
+      throw error;
+    }
+  }
   const { app, routeDeps } = createAppComposition({
     settings,
     db: dbClient.db,
     bus,
     workflowClient: workflowClient.client,
     documentIndexer: workflowClient.documentIndexer,
+    objectStorage,
+    ...(editableArtifactComposition
+      ? {
+          editableArtifacts: editableArtifactComposition.application,
+          editableArtifactExports: editableArtifactComposition.durableExports,
+          editableArtifactAgent: editableArtifactComposition.agent,
+          editableArtifactOfficeImports: editableArtifactComposition.officeImports,
+        }
+      : {}),
     observability,
     readinessChecks: {
       db: runtimeDatabaseReadyCheck(dbClient.db, databasePosture),
     },
   });
-  const server = Bun.serve({
+  if (!routeDeps.editableArtifacts) {
+    observability.warn("editable artifact engine is not composed", {
+      subsystem: "editable_artifacts",
+      behavior: "http_and_websocket_fail_closed",
+    });
+  }
+  const artifactWebSockets = new EditableArtifactWebSocketTransport(routeDeps.editableArtifacts);
+  const server = Bun.serve<EditableArtifactWebSocketConnection>({
     hostname: settings.apiHost,
     port: settings.apiPort,
     idleTimeout: 255,
-    fetch: app.fetch,
+    fetch: (request, bunServer) => {
+      if (artifactWebSockets.handles(request)) {
+        return artifactWebSockets.upgrade(request, bunServer);
+      }
+      return app.fetch(request);
+    },
+    websocket: {
+      maxPayloadLength: EDITABLE_ARTIFACT_LIVE_WEBSOCKET_MAX_MESSAGE_BYTES,
+      backpressureLimit: 16 * 1024 * 1024,
+      closeOnBackpressureLimit: true,
+      open: (socket) => artifactWebSockets.websocket.open(socket),
+      message: (socket, message) => artifactWebSockets.websocket.message(socket, message),
+      close: (socket) => artifactWebSockets.websocket.close(socket),
+    },
   });
   const stopSlackInteractionPump = settings.slackSigningSecret
     ? startSlackInteractionPump(routeDeps)
     : undefined;
+  const stopMemorySlackPublicationPump = startMemorySlackPublicationPump(routeDeps);
+  const stopTemporalScheduleCleanupPump = startTemporalScheduleCleanupPump({
+    db: dbClient.db,
+    deleteSchedule: async (temporalScheduleId) => {
+      await workflowClient.client.deleteScheduledTaskSchedule({ temporalScheduleId });
+    },
+    observability,
+  });
   // M10 — start the metrics-ingestion consumer (agent heartbeats → DB last-sample
   // + downsampled series), gated on the selfhosted flag. A no-op when disabled.
   let stopMetricsIngestion: (() => void) | undefined;
@@ -366,11 +450,13 @@ export async function startApi() {
           { db: dbClient.db, settings, callout, observability },
           settings.natsUrl,
         );
-      } catch (error) {
+      } catch {
         // A responder start failure must not crash the API (other planes work); log
         // loudly — selfhosted agents will fail to connect until it is up.
         observability.error("OpenGeni NATS auth-callout responder failed to start", {
-          error: error instanceof Error ? error.message : String(error),
+          errorClass: "NatsAuthCalloutOperationError",
+          errorCode: "nats_auth_callout_start_failed",
+          origin: "api",
         });
       }
     } else {
@@ -389,9 +475,12 @@ export async function startApi() {
     close: async () => {
       server.stop(true);
       stopSlackInteractionPump?.();
+      await stopMemorySlackPublicationPump();
       stopMetricsIngestion?.();
       stopHelloIngestion?.();
+      await stopTemporalScheduleCleanupPump();
       await Promise.allSettled([
+        Promise.resolve(editableArtifactComposition?.close()),
         authCalloutResponder?.close(),
         bus.close(),
         workflowClient.close(),
@@ -420,9 +509,12 @@ export function shouldCreateScheduleAfterUpdateError(error: unknown): boolean {
 }
 
 export function temporalScheduleSpec(schedule: ScheduledTaskScheduleSpec): ScheduleSpec {
+  if (schedule.type === "manual") {
+    throw new Error("manual scheduled tasks do not have a Temporal Schedule spec");
+  }
   if (schedule.type === "interval") {
     return {
-      intervals: [{ every: `${schedule.everySeconds}s` }],
+      intervals: [temporalIntervalSpec(schedule)],
       ...(schedule.startAt ? { startAt: new Date(schedule.startAt) } : {}),
       ...(schedule.endAt ? { endAt: new Date(schedule.endAt) } : {}),
     };
@@ -456,6 +548,29 @@ export function temporalScheduleSpec(schedule: ScheduledTaskScheduleSpec): Sched
   };
 }
 
+function temporalIntervalSpec(
+  schedule: Extract<ScheduledTaskScheduleSpec, { type: "interval" }>,
+): NonNullable<ScheduleSpec["intervals"]>[number] {
+  const every = `${schedule.everySeconds}s` as `${number}s`;
+  if (!schedule.startAt) {
+    return { every };
+  }
+
+  // Temporal interval schedules match Epoch + (n * every) + offset. Its
+  // top-level startAt only filters matching times before that boundary, so it
+  // does not itself anchor the cadence. Derive the phase from startAt to make
+  // the stored OpenGeni timestamp the first interval boundary rather than the
+  // next epoch-aligned match.
+  const everyMilliseconds = BigInt(schedule.everySeconds) * 1_000n;
+  const startMilliseconds = BigInt(new Date(schedule.startAt).getTime());
+  const offsetMilliseconds =
+    ((startMilliseconds % everyMilliseconds) + everyMilliseconds) % everyMilliseconds;
+  return {
+    every,
+    ...(offsetMilliseconds === 0n ? {} : { offset: `${offsetMilliseconds}ms` as `${number}ms` }),
+  };
+}
+
 function temporalMonth(monthIndex: number) {
   return TEMPORAL_MONTHS[monthIndex]!;
 }
@@ -483,7 +598,7 @@ function temporalScheduleOptions(task: ScheduledTask, taskQueue: string): Schedu
       pauseOnFailure: false,
     },
     state: {
-      paused: task.status === "paused",
+      paused: scheduledTaskEffectivelyPaused(task),
       ...(task.schedule.type === "once" ? { remainingActions: 1 } : {}),
     },
     memo: {
@@ -495,6 +610,15 @@ function temporalScheduleOptions(task: ScheduledTask, taskQueue: string): Schedu
   };
 }
 
+function scheduledTaskEffectivelyPaused(task: ScheduledTask): boolean {
+  if (task.status === "paused") return true;
+  if (task.action.kind !== "knowledge_source_sync") return false;
+  const control = task.metadata.knowledgeSourceSync;
+  if (!control || typeof control !== "object") return false;
+  const record = control as Record<string, unknown>;
+  return record.sourceEnabled === false || record.connectionPaused === true;
+}
+
 function temporalScheduleUpdateOptions(options: ScheduleOptions): ScheduleUpdateOptions {
   return {
     spec: options.spec,