@opengeni/api-router 0.5.3 → 0.5.5

package/dist/index.d.ts

@@ -1,9 +1,10 @@
 import { getSettings } from '@opengeni/config';
 import { ScheduledTaskOverlapPolicy, ScheduledTaskScheduleSpec } from '@opengeni/contracts';
+import { Database } from '@opengeni/db';
 import { ScheduleOverlapPolicy, ScheduleSpec } from '@temporalio/client';
 import { SessionWorkflowClient, DocumentIndexClient } from '@opengeni/core';
 
-declare function createTemporalWorkflowClient(settings: ReturnType<typeof getSettings>): Promise<{
+declare function createTemporalWorkflowClient(settings: ReturnType<typeof getSettings>, db: Database): Promise<{
     client: SessionWorkflowClient;
     documentIndexer: DocumentIndexClient;
     close: () => Promise<void>;

package/dist/index.js

@@ -1,13 +1,27 @@
 import {
   createApp
-} from "./chunk-3HIA43CC.js";
+} from "./chunk-HBEJMWD3.js";
 
 // src/index.ts
-import { dbSearchPath, getSettings, resolveNatsCalloutConfig, resolveNatsControlPlaneAuth, retryStartupDependency, startupRetryOptions } from "@opengeni/config";
-import { createDb } from "@opengeni/db";
+import {
+  dbSearchPath,
+  getSettings,
+  resolveNatsCalloutConfig,
+  resolveNatsControlPlaneAuth,
+  retryStartupDependency,
+  startupRetryOptions
+} from "@opengeni/config";
+import { createDb, markSessionWorkflowWakeDelivered } from "@opengeni/db";
 import { createNatsEventBus } from "@opengeni/events";
 import { createObservability, logStartupDependencyRetry } from "@opengeni/observability";
-import { Connection, Client as TemporalClient, ScheduleNotFoundError, ScheduleOverlapPolicy, WorkflowExecutionAlreadyStartedError } from "@temporalio/client";
+import { SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID } from "@opengeni/core";
+import {
+  Connection,
+  Client as TemporalClient,
+  ScheduleNotFoundError,
+  ScheduleOverlapPolicy,
+  WorkflowExecutionAlreadyStartedError
+} from "@temporalio/client";
 
 // src/observability.ts
 function observabilityEventLogger(observability) {
@@ -38,7 +52,9 @@ function eventAttributeValue(value) {
 }
 
 // src/sandbox/auth-callout.ts
-import { resolveEnrollmentSigningSecret } from "@opengeni/config";
+import {
+  resolveEnrollmentSigningSecret
+} from "@opengeni/config";
 import { verifyEnrollmentBearer } from "@opengeni/contracts";
 import { getEnrollment } from "@opengeni/db";
 import {
@@ -135,12 +151,22 @@ async function startAuthCalloutResponder(deps, natsUrl) {
 
 // src/sandbox/metrics-ingestion.ts
 import {
+  clearEnrollmentWentOffline,
   getEnrollment as getEnrollment2,
   ingestMachineMetricsSample,
+  sessionsWithActiveOpOnEnrollment,
   setEnrollmentDisplayState,
+  setEnrollmentOpStreamState,
+  setEnrollmentWentOffline,
   touchEnrollmentLastSeen
 } from "@opengeni/db";
-import { AgentEvent, Hello } from "@opengeni/agent-proto";
+import { appendAndPublishEvents } from "@opengeni/events";
+import {
+  AgentEvent,
+  GoingOfflineReason,
+  Hello,
+  goingOfflineReasonToJSON
+} from "@opengeni/agent-proto";
 var AGENT_EVENTS_SUBJECT = "agent.*.*.events";
 var AGENT_HELLO_SUBJECT = "agent.*.*.hello";
 function parseAgentSubject(subject, tail) {
@@ -196,7 +222,27 @@ async function ingestHeartbeat(db, input) {
   });
   return { ingested: true, seriesAppended: result.seriesAppended };
 }
-async function handleAgentEventPayload(db, observability, payload, subject) {
+async function fanOutMachineLinkEvents(db, bus, observability, workspaceId, enrollmentId, build) {
+  const sessions = await sessionsWithActiveOpOnEnrollment(db, { workspaceId, enrollmentId });
+  for (const session of sessions) {
+    try {
+      await appendAndPublishEvents(
+        db,
+        bus,
+        workspaceId,
+        session.sessionId,
+        build(session.activeTurnId)
+      );
+    } catch (error) {
+      observability?.warn?.("Failed to fan out a machine-link event to a session", {
+        workspaceId,
+        sessionId: session.sessionId,
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+  }
+}
+async function handleAgentEventPayload(db, observability, payload, subject, bus) {
   const ids = parseAgentEventSubject(subject);
   if (!ids) {
     return;
@@ -211,6 +257,54 @@ async function handleAgentEventPayload(db, observability, payload, subject) {
     });
     return;
   }
+  if (event.event?.$case === "goingOffline") {
+    const reason = goingOfflineReasonToJSON(event.event.goingOffline.reason);
+    observability?.incrementCounter({
+      name: "opengeni_machine_going_offline_total",
+      help: "Total Connected Machine clean GoingOffline signals by typed reason.",
+      labels: { reason }
+    });
+    try {
+      const enrollment = await getEnrollment2(db, ids.workspaceId, ids.agentId);
+      if (enrollment) {
+        await setEnrollmentWentOffline(db, {
+          accountId: enrollment.accountId,
+          workspaceId: ids.workspaceId,
+          enrollmentId: ids.agentId,
+          reason
+        });
+        if (bus) {
+          const isSelfUpdate = event.event.goingOffline.reason === GoingOfflineReason.GOING_OFFLINE_REASON_UPDATE;
+          await fanOutMachineLinkEvents(
+            db,
+            bus,
+            observability,
+            ids.workspaceId,
+            ids.agentId,
+            (activeTurnId) => {
+              const events = [
+                { type: "machine.link.lost", turnId: activeTurnId, payload: { reason } }
+              ];
+              if (isSelfUpdate) {
+                events.push({
+                  type: "machine.runner.restarted",
+                  turnId: activeTurnId,
+                  payload: {}
+                });
+              }
+              return events;
+            }
+          );
+        }
+      }
+    } catch (error) {
+      observability?.warn?.("Failed to record a machine clean going-offline", {
+        subject,
+        error: error instanceof Error ? error.message : String(error)
+      });
+    }
+    return;
+  }
   if (event.event?.$case !== "heartbeat") {
     return;
   }
@@ -219,7 +313,11 @@ async function handleAgentEventPayload(db, observability, payload, subject) {
     return;
   }
   try {
-    await ingestHeartbeat(db, { workspaceId: ids.workspaceId, agentId: ids.agentId, sample: metrics });
+    await ingestHeartbeat(db, {
+      workspaceId: ids.workspaceId,
+      agentId: ids.agentId,
+      sample: metrics
+    });
   } catch (error) {
     observability?.warn?.("Failed to ingest a machine metrics heartbeat", {
       subject,
@@ -230,7 +328,7 @@ async function handleAgentEventPayload(db, observability, payload, subject) {
 function startMetricsIngestion(deps) {
   return deps.bus.subscribeAgentEvents(
     AGENT_EVENTS_SUBJECT,
-    (payload, subject) => handleAgentEventPayload(deps.db, deps.observability, payload, subject)
+    (payload, subject) => handleAgentEventPayload(deps.db, deps.observability, payload, subject, deps.bus)
   );
 }
 function helloReportsDisplay(hello) {
@@ -247,6 +345,9 @@ function helloDesktopUnavailableReason(hello) {
   const reason = hello.capabilities?.desktopUnavailableReason;
   return reason ? reason : null;
 }
+function helloReportsOpStream(hello) {
+  return hello.capabilities?.opStream === true;
+}
 async function refreshEnrollmentDisplay(db, input) {
   const desktopUnavailableReason = input.desktopUnavailableReason ?? null;
   const enrollment = await getEnrollment2(db, input.workspaceId, input.agentId);
@@ -264,7 +365,22 @@ async function refreshEnrollmentDisplay(db, input) {
     desktopUnavailableReason
   });
 }
-async function handleHelloPayload(db, observability, payload, subject) {
+async function refreshEnrollmentOpStream(db, input) {
+  const enrollment = await getEnrollment2(db, input.workspaceId, input.agentId);
+  if (!enrollment) {
+    return { updated: false };
+  }
+  if (enrollment.opStream === input.opStream) {
+    return { updated: false };
+  }
+  return await setEnrollmentOpStreamState(db, {
+    accountId: enrollment.accountId,
+    workspaceId: input.workspaceId,
+    enrollmentId: input.agentId,
+    opStream: input.opStream
+  });
+}
+async function handleHelloPayload(db, observability, payload, subject, bus) {
   const ids = parseAgentHelloSubject(subject);
   if (!ids) {
     return;
@@ -286,8 +402,38 @@ async function handleHelloPayload(db, observability, payload, subject) {
       hasDisplay: helloReportsDisplay(hello),
       desktopUnavailableReason: helloDesktopUnavailableReason(hello)
     });
+    await refreshEnrollmentOpStream(db, {
+      workspaceId: ids.workspaceId,
+      agentId: ids.agentId,
+      opStream: helloReportsOpStream(hello)
+    });
   } catch (error) {
-    observability?.warn?.("Failed to refresh an enrollment's display from a Hello", {
+    observability?.warn?.("Failed to refresh an enrollment's capabilities from a Hello", {
+      subject,
+      error: error instanceof Error ? error.message : String(error)
+    });
+  }
+  try {
+    const enrollment = await getEnrollment2(db, ids.workspaceId, ids.agentId);
+    if (enrollment) {
+      const { cleared } = await clearEnrollmentWentOffline(db, {
+        accountId: enrollment.accountId,
+        workspaceId: ids.workspaceId,
+        enrollmentId: ids.agentId
+      });
+      if (cleared && bus) {
+        await fanOutMachineLinkEvents(
+          db,
+          bus,
+          observability,
+          ids.workspaceId,
+          ids.agentId,
+          (activeTurnId) => [{ type: "machine.link.restored", turnId: activeTurnId, payload: {} }]
+        );
+      }
+    }
+  } catch (error) {
+    observability?.warn?.("Failed to clear a machine going-offline marker on a Hello", {
       subject,
       error: error instanceof Error ? error.message : String(error)
     });
@@ -296,7 +442,7 @@ async function handleHelloPayload(db, observability, payload, subject) {
 function startHelloIngestion(deps) {
   return deps.bus.subscribeAgentEvents(
     AGENT_HELLO_SUBJECT,
-    (payload, subject) => handleHelloPayload(deps.db, deps.observability, payload, subject)
+    (payload, subject) => handleHelloPayload(deps.db, deps.observability, payload, subject, deps.bus)
   );
 }
 
@@ -318,7 +464,7 @@ var TEMPORAL_MONTHS = [
   "NOVEMBER",
   "DECEMBER"
 ];
-async function createTemporalWorkflowClient(settings) {
+async function createTemporalWorkflowClient(settings, db) {
   const connection = await Connection.connect({ address: settings.temporalHost });
   const temporal = new TemporalClient({
     connection,
@@ -328,27 +474,79 @@ async function createTemporalWorkflowClient(settings) {
     signalUserMessage: async ({ eventId, workflowId }) => {
       await temporal.workflow.getHandle(workflowId).signal("userMessage", eventId);
     },
-    wakeSessionWorkflow: async ({ accountId, workspaceId, sessionId, workflowId }) => {
+    wakeSessionWorkflow: async ({
+      accountId,
+      workspaceId,
+      sessionId,
+      workflowId,
+      wakeRevision,
+      interruptionRequested
+    }) => {
       await temporal.workflow.signalWithStart("sessionWorkflow", {
         taskQueue: settings.temporalTaskQueue,
         workflowId,
         workflowIdReusePolicy: "ALLOW_DUPLICATE",
         args: [{ accountId, workspaceId, sessionId }],
-        signal: "queueChanged"
+        signal: interruptionRequested ? "sessionControl" : "queueChanged"
+      });
+      await markSessionWorkflowWakeDelivered(db, {
+        accountId,
+        workspaceId,
+        sessionId,
+        temporalWorkflowId: workflowId,
+        wakeRevision
       });
     },
-    signalApprovalDecision: async ({ eventId, workflowId }) => {
-      await temporal.workflow.getHandle(workflowId).signal("approvalDecision", eventId);
+    requestSessionWorkflowWakeDispatch: async () => {
+      await temporal.schedule.getHandle(SESSION_WORKFLOW_WAKE_DISPATCHER_SCHEDULE_ID).trigger(ScheduleOverlapPolicy.BUFFER_ONE);
     },
-    signalInterrupt: async ({ accountId, workspaceId, sessionId, eventId, workflowId }) => {
+    signalCodexCapacity: async ({
+      accountId,
+      workspaceId,
+      sessionId,
+      workflowId,
+      wakeRevision,
+      workflowWakeRevision
+    }) => {
       await temporal.workflow.signalWithStart("sessionWorkflow", {
         taskQueue: settings.temporalTaskQueue,
         workflowId,
         workflowIdReusePolicy: "ALLOW_DUPLICATE",
         args: [{ accountId, workspaceId, sessionId }],
-        signal: "interrupt",
+        signal: "codexCapacityChanged",
+        signalArgs: [wakeRevision]
+      });
+      await markSessionWorkflowWakeDelivered(db, {
+        accountId,
+        workspaceId,
+        sessionId,
+        temporalWorkflowId: workflowId,
+        wakeRevision: workflowWakeRevision
+      });
+    },
+    signalApprovalDecision: async ({
+      accountId,
+      workspaceId,
+      sessionId,
+      eventId,
+      workflowId,
+      workflowWakeRevision
+    }) => {
+      await temporal.workflow.signalWithStart("sessionWorkflow", {
+        taskQueue: settings.temporalTaskQueue,
+        workflowId,
+        workflowIdReusePolicy: "ALLOW_DUPLICATE",
+        args: [{ accountId, workspaceId, sessionId }],
+        signal: "approvalDecision",
         signalArgs: [eventId]
       });
+      await markSessionWorkflowWakeDelivered(db, {
+        accountId,
+        workspaceId,
+        sessionId,
+        temporalWorkflowId: workflowId,
+        wakeRevision: workflowWakeRevision
+      });
     },
     syncScheduledTask: async ({ task }) => {
       const schedule = temporal.schedule.getHandle(task.temporalScheduleId);
@@ -372,13 +570,15 @@ async function createTemporalWorkflowClient(settings) {
           taskQueue: settings.temporalTaskQueue,
           workflowId,
           workflowIdReusePolicy: "REJECT_DUPLICATE",
-          args: [{
-            accountId: task.accountId,
-            workspaceId: task.workspaceId,
-            taskId: task.id,
-            triggerType: "manual",
-            agentRunUsageIdempotencyKey
-          }]
+          args: [
+            {
+              accountId: task.accountId,
+              workspaceId: task.workspaceId,
+              taskId: task.id,
+              triggerType: "manual",
+              agentRunUsageIdempotencyKey
+            }
+          ]
         });
       } catch (error) {
         if (isWorkflowAlreadyStarted(error)) {
@@ -387,6 +587,24 @@ async function createTemporalWorkflowClient(settings) {
         throw error;
       }
     },
+    startRigVerification: async ({ workspaceId, changeId, versionId, workflowId }) => {
+      const targetId = changeId ?? versionId;
+      if (!targetId) {
+        throw new Error("rig verification requires changeId or versionId");
+      }
+      await temporal.workflow.start("rigVerificationWorkflow", {
+        taskQueue: settings.temporalTaskQueue,
+        workflowId: workflowId ?? `rig-verification-${targetId}-${crypto.randomUUID()}`,
+        workflowIdReusePolicy: "ALLOW_DUPLICATE",
+        args: [
+          {
+            workspaceId,
+            ...changeId ? { changeId } : {},
+            ...versionId ? { versionId } : {}
+          }
+        ]
+      });
+    },
     check: async () => {
       await connection.workflowService.getSystemInfo({});
     }
@@ -435,16 +653,16 @@ async function startApi() {
         onRetry
       }
     );
-    workflowClient = await retryStartupDependency("Temporal", () => createTemporalWorkflowClient(settings), {
-      ...retryOptions,
-      onRetry
-    });
+    workflowClient = await retryStartupDependency(
+      "Temporal",
+      () => createTemporalWorkflowClient(settings, dbClient.db),
+      {
+        ...retryOptions,
+        onRetry
+      }
+    );
   } catch (error) {
-    await Promise.allSettled([
-      bus?.close(),
-      workflowClient?.close(),
-      dbClient.close()
-    ]);
+    await Promise.allSettled([bus?.close(), workflowClient?.close(), dbClient.close()]);
     throw error;
   }
   if (!bus || !workflowClient) {
@@ -535,25 +753,29 @@ function temporalScheduleSpec(schedule) {
   }
   if (schedule.type === "calendar") {
     return {
-      calendars: [{
-        hour: schedule.hour,
-        minute: schedule.minute,
-        second: 0,
-        ...schedule.daysOfWeek ? { dayOfWeek: schedule.daysOfWeek } : {}
-      }],
+      calendars: [
+        {
+          hour: schedule.hour,
+          minute: schedule.minute,
+          second: 0,
+          ...schedule.daysOfWeek ? { dayOfWeek: schedule.daysOfWeek } : {}
+        }
+      ],
       timezone: schedule.timeZone
     };
   }
   const runAt = new Date(schedule.runAt);
   return {
-    calendars: [{
-      year: runAt.getUTCFullYear(),
-      month: temporalMonth(runAt.getUTCMonth()),
-      dayOfMonth: runAt.getUTCDate(),
-      hour: runAt.getUTCHours(),
-      minute: runAt.getUTCMinutes(),
-      second: runAt.getUTCSeconds()
-    }],
+    calendars: [
+      {
+        year: runAt.getUTCFullYear(),
+        month: temporalMonth(runAt.getUTCMonth()),
+        dayOfMonth: runAt.getUTCDate(),
+        hour: runAt.getUTCHours(),
+        minute: runAt.getUTCMinutes(),
+        second: runAt.getUTCSeconds()
+      }
+    ],
     timezone: "UTC"
   };
 }
@@ -568,12 +790,14 @@ function temporalScheduleOptions(task, taskQueue) {
       type: "startWorkflow",
       workflowType: "scheduledTaskFireWorkflow",
       taskQueue,
-      args: [{
-        accountId: task.accountId,
-        workspaceId: task.workspaceId,
-        taskId: task.id,
-        triggerType: "scheduled"
-      }]
+      args: [
+        {
+          accountId: task.accountId,
+          workspaceId: task.workspaceId,
+          taskId: task.id,
+          triggerType: "scheduled"
+        }
+      ]
     },
     policies: {
       overlap: temporalOverlapPolicy(task.overlapPolicy),