deepline 0.3.213 → 0.3.215

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts

@@ -1,5 +1,8 @@
 import { STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS } from '../../../plays/sandbox-runtime-limits';
 import { RUNTIME_SANDBOX_MEMORY_ABORT_CODE } from '../../run-failure';
+import { PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER } from '../../runtime-contract';
+import { RUNTIME_CLAIM_HEARTBEAT_DIVISOR } from '../../runtime-constants';
+import { RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS } from '../../runner-app/runtime-heartbeat';
 import {
   RUNTIME_MEMORY_LEDGER_PATH_ENV,
   RUNTIME_MEMORY_PRESSURE_PATH_ENV,
@@ -116,11 +119,11 @@ export const RUNTIME_SANDBOX_MEMORY_ABORT_REASON =
   RUNTIME_SANDBOX_MEMORY_ABORT_CODE;
 
 /**
- * Build the tiny parent process that owns the hard runtime deadline and the
- * sandbox memory envelope outside the customer-code Node event loop. A
- * synchronous infinite loop can block the runner's own timer, but it cannot
- * block this separate process; nor can a runner whose heap is thrashing run
- * its own memory watch reliably.
+ * Build the parent process that owns runner liveness, the hard runtime
+ * deadline, and the sandbox memory envelope outside the customer-code Node
+ * event loop. A synchronous infinite loop can block the runner's timers, but
+ * it cannot block this separate process; nor can a runner whose heap is
+ * thrashing run its own memory watch reliably.
  *
  * argv (after `node -e <source>`):
  *   1 runnerPath, 2 configPath, 3 runtimeStartedPath, 4 runtimeCompletedPath,
@@ -185,6 +188,178 @@ let settled = false;
 let forceKillTimer = null;
 let runtimeTimer = null;
 let terminalGraceTimer = null;
+let heartbeatTimer = null;
+let heartbeatController = null;
+let heartbeatInFlight = false;
+let heartbeatStopped = false;
+let heartbeatConfigRead = false;
+let heartbeatConfig = null;
+let heartbeatLastSuccessAtMs = 0;
+let heartbeatAttempt = 0;
+let heartbeatLastErrorCode = null;
+let heartbeatStopBeforeLeaseMs = 0;
+let stopRequested = false;
+
+function readRunnerHeartbeatConfig() {
+  if (heartbeatConfigRead) return heartbeatConfig;
+  heartbeatConfigRead = true;
+  try {
+    const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    const context = config && config.context;
+    const push = context && context.runnerPushExecution;
+    const gatewayBaseUrl = String(context && context.receiptGatewayBaseUrl || '').replace(/\\/$/, '');
+    const executorToken = context && context.executorToken;
+    const runId = push && push.runId;
+    const leaseSeconds = Number(push && push.leaseSeconds);
+    if (
+      !gatewayBaseUrl ||
+      typeof executorToken !== 'string' ||
+      !executorToken.trim() ||
+      typeof runId !== 'string' ||
+      !runId.trim() ||
+      !Number.isFinite(leaseSeconds) ||
+      leaseSeconds <= 0
+    ) {
+      return null;
+    }
+    heartbeatConfig = {
+      gatewayBaseUrl,
+      executorToken,
+      runId,
+      leaseMs: Math.max(1_000, Math.floor(leaseSeconds * 1_000)),
+    };
+  } catch {}
+  return heartbeatConfig;
+}
+
+function stopRunnerHeartbeat() {
+  heartbeatStopped = true;
+  if (heartbeatTimer) {
+    clearInterval(heartbeatTimer);
+    heartbeatTimer = null;
+  }
+  if (heartbeatController) {
+    heartbeatController.abort();
+    heartbeatController = null;
+  }
+}
+
+function runnerHeartbeatErrorCode(error) {
+  const code = error && typeof error.code === 'string' ? error.code : '';
+  if (/^(?:E[A-Z0-9_]+|ERR_[A-Z0-9_]+|UND_ERR_[A-Z0-9_]+)$/.test(code)) return code;
+  const name = error && typeof error.name === 'string' ? error.name : '';
+  if (name === 'AbortError' || name === 'TimeoutError') return name;
+  return 'unknown';
+}
+
+function startRunnerHeartbeatSupervisor() {
+  if (heartbeatTimer || heartbeatStopped || settled) return;
+  const config = readRunnerHeartbeatConfig();
+  if (!config) return;
+  let handoff = null;
+  try {
+    handoff = JSON.parse(fs.readFileSync(runtimeStartedPath, 'utf8'));
+  } catch {}
+  const handoffAt = Number(handoff && handoff.lastHeartbeatAtMs);
+  heartbeatLastSuccessAtMs = Number.isFinite(handoffAt) ? handoffAt : Date.now();
+  const cadenceMs = Math.max(250, Math.floor(config.leaseMs / ${RUNTIME_CLAIM_HEARTBEAT_DIVISOR}));
+  heartbeatStopBeforeLeaseMs = Math.max(
+    0,
+    config.leaseMs - cadenceMs - ${RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS},
+  );
+  process.stderr.write(
+    '[play-runner.heartbeat] watchdog_owner_started ' +
+      JSON.stringify({ runId: config.runId, cadenceMs, handoffHeartbeatAgeMs: Math.max(0, Date.now() - heartbeatLastSuccessAtMs) }) +
+      '\\n',
+  );
+  void runnerHeartbeatTick();
+  heartbeatTimer = setInterval(runnerHeartbeatTick, cadenceMs);
+  heartbeatTimer.unref?.();
+}
+
+async function runnerHeartbeatTick() {
+  if (heartbeatStopped || heartbeatInFlight || settled || !child || child.exitCode !== null) return;
+  const config = readRunnerHeartbeatConfig();
+  if (!config) return;
+  heartbeatInFlight = true;
+  heartbeatAttempt += 1;
+  const startedAt = Date.now();
+  const controller = new AbortController();
+  heartbeatController = controller;
+  const timeout = setTimeout(() => controller.abort(), ${RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS});
+  try {
+    const response = await fetch(
+      config.gatewayBaseUrl + '/api/v2/plays/runtime',
+      {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          authorization: 'Bearer ' + config.executorToken,
+          ${JSON.stringify(PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER)}: require('node:crypto').randomUUID(),
+        },
+        body: JSON.stringify({ action: 'runner_heartbeat', runId: config.runId }),
+        redirect: 'manual',
+        signal: controller.signal,
+      },
+    );
+    if (response.status >= 300 && response.status < 400) {
+      await response.body?.cancel().catch(() => undefined);
+      throw Object.assign(new Error('gateway_redirect'), { code: 'redirect_' + response.status });
+    }
+    if (!response.ok) {
+      await response.body?.cancel().catch(() => undefined);
+      throw Object.assign(new Error('gateway_http_error'), { code: 'http_' + response.status });
+    }
+    const body = await response.json().catch(() => null);
+    if (body && body.ok === false && ['cancelled', 'failed', 'completed'].includes(body.terminal)) {
+      process.stderr.write(
+        '[play-runner.heartbeat] watchdog_owner_terminal ' +
+          JSON.stringify({ runId: config.runId, terminal: body.terminal, attempt: heartbeatAttempt }) +
+          '\\n',
+      );
+      stopRunnerHeartbeat();
+      stopChild('runner_scheduler_terminal', { terminal: body.terminal, heartbeatAttempt });
+      return;
+    }
+    if (!body || body.ok !== true || (body.schedulerStatus !== undefined && !['queued', 'running', 'waiting'].includes(body.schedulerStatus))) {
+      throw Object.assign(new Error('gateway_protocol_error'), { code: 'protocol_error' });
+    }
+    heartbeatLastSuccessAtMs = Date.now();
+    heartbeatLastErrorCode = null;
+  } catch (error) {
+    if (heartbeatStopped || settled) return;
+    heartbeatLastErrorCode = controller.signal.aborted
+      ? 'TimeoutError'
+      : runnerHeartbeatErrorCode(error);
+    const heartbeatAgeMs = Math.max(0, Date.now() - heartbeatLastSuccessAtMs);
+    process.stderr.write(
+      '[play-runner.heartbeat] watchdog_owner_retrying ' +
+        JSON.stringify({ runId: config.runId, attempt: heartbeatAttempt, durationMs: Math.max(0, Date.now() - startedAt), heartbeatAgeMs, errorCode: heartbeatLastErrorCode }) +
+        '\\n',
+    );
+    // Stop before the scheduler's lease expires. A failed request can consume
+    // its full timeout, and the next cadence would otherwise let the scheduler
+    // win the terminal race and report an ambiguous runner loss.
+    if (heartbeatAgeMs >= heartbeatStopBeforeLeaseMs) {
+      stopRunnerHeartbeat();
+      const diagnostic = {
+        heartbeatAttempt,
+        heartbeatAgeMs,
+        leaseMs: config.leaseMs,
+        stopBeforeLeaseMs: heartbeatStopBeforeLeaseMs,
+        lastErrorCode: heartbeatLastErrorCode,
+      };
+      process.stderr.write(
+        'RUNTIME_CONTROL_PLANE_UNAVAILABLE: ' + JSON.stringify(diagnostic) + '\\n',
+      );
+      stopChild('runner_heartbeat_unreachable', diagnostic);
+    }
+  } finally {
+    clearTimeout(timeout);
+    if (heartbeatController === controller) heartbeatController = null;
+    heartbeatInFlight = false;
+  }
+}
 
 function writeTerminationDiagnostic(reason, details = {}) {
   if (!terminationDiagnosticPath) return;
@@ -204,6 +379,8 @@ function writeTerminationDiagnostic(reason, details = {}) {
 }
 
 function stopChild(reason, details) {
+  if (settled || stopRequested) return;
+  stopRequested = true;
   writeTerminationDiagnostic(reason, details);
   child.kill('SIGTERM');
   forceKillTimer = setTimeout(() => child.kill('SIGKILL'), ${RUNNER_TERMINATION_KILL_GRACE_MS});
@@ -391,7 +568,10 @@ function armTerminalGrace() {
 }
 
 const runtimeBoundaryPoll = setInterval(() => {
-  if (fs.existsSync(runtimeStartedPath)) armRuntimeDeadline();
+  if (fs.existsSync(runtimeStartedPath)) {
+    armRuntimeDeadline();
+    startRunnerHeartbeatSupervisor();
+  }
   if (fs.existsSync(runtimeCompletedPath)) armTerminalGrace();
 }, 25);
 const startupTimer = setTimeout(() => {
@@ -413,6 +593,7 @@ function finish(code) {
   settled = true;
   clearInterval(runtimeBoundaryPoll);
   clearInterval(memoryWatch);
+  stopRunnerHeartbeat();
   clearTimeout(startupTimer);
   if (runtimeTimer) clearTimeout(runtimeTimer);
   if (terminalGraceTimer) clearTimeout(terminalGraceTimer);

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -13891,24 +13891,33 @@ export async function upsertPostgresSchedulerObservation(
 /** Read every observation for a run/attempt, ordered for deterministic folds. */
 export async function getPostgresSchedulerObservations(
   client: PostgresSchedulerQueryClient,
-  input: { runId: string; attempt?: number | null },
+  input: {
+    runId: string;
+    attempt?: number | null;
+    /** Return only the newest settled invocation for each node. */
+    latestSettledPerNode?: boolean;
+  },
   options: PostgresSchedulerReceiptOptions,
 ): Promise<DocflowObservationRecord[]> {
   const hasAttempt = input.attempt != null;
+  const latestSettledPerNode = input.latestSettledPerNode === true;
   const params: unknown[] = [receiptOrgId(options), input.runId];
   if (hasAttempt) {
     params.push(normalizePostgresSchedulerReceiptRunAttempt(input.attempt));
   }
   const result = await client.query<ObservationRow>(
     `
-      SELECT run_id, attempt, node_id, invocation_seq, invocation_id, status,
+      SELECT ${latestSettledPerNode ? 'DISTINCT ON (node_id)' : ''}
+             run_id, attempt, node_id, invocation_seq, invocation_id, status,
              inputs_json, outputs_json, inputs_truncated, outputs_truncated,
              error, started_at, settled_at
         FROM ${tableName('observations', options)}
        WHERE org_id = $1
          AND run_id = $2
          ${hasAttempt ? 'AND attempt = $3' : ''}
-       ORDER BY node_id, attempt, invocation_seq
+         ${latestSettledPerNode ? "AND status IN ('completed', 'failed')" : ''}
+       ORDER BY node_id, attempt ${latestSettledPerNode ? 'DESC' : ''},
+                invocation_seq ${latestSettledPerNode ? 'DESC' : ''}
     `,
     params,
   );

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -219,11 +219,14 @@ export function runtimeSheetPageTailHoldCompletedEvent(
   tableNamespace: string,
 ): PlayRunnerEvent | null {
   if (!Number.isInteger(holdMs) || (holdMs ?? 0) <= 0) return null;
+  const at = new Date().toISOString();
   return {
     type: 'log',
-    at: new Date().toISOString(),
+    at,
     source: 'play',
-    line: `[runtime.sheet-page-tail-hold] phase=finish table=${tableNamespace} hold_ms=${holdMs}`,
+    // The status projection keeps the line but drops the event envelope's
+    // timestamp, so include the emission time for runtime-preview diagnostics.
+    line: `[${at}] [runtime.sheet-page-tail-hold] phase=finish table=${tableNamespace} hold_ms=${holdMs}`,
   };
 }
 

package/dist/bundling-sources/shared_libs/plays/deploy/api-client.ts

@@ -91,6 +91,23 @@ export type DeploymentPlayState = {
   liveRevision: DeploymentPlayState['workingRevision'];
 };
 
+export type DeploymentMonitorEntry = {
+  /** Repository-relative source path, including the linked target's root. */
+  file: string;
+  /** The monitor API owns definition validation. */
+  definition: unknown;
+};
+
+export type DeploymentMonitorOutcome = {
+  key: string;
+  file: string;
+  status: 'planned' | 'applying' | 'applied' | 'stopped' | 'failed';
+  jobId?: string;
+  warning?: string;
+  error?: string;
+  credits?: number | null;
+};
+
 export class DeployApiClient {
   constructor(
     private readonly config: {
@@ -223,6 +240,29 @@ export class DeployApiClient {
     });
   }
 
+  monitors(
+    action: 'plan' | 'apply',
+    entries: DeploymentMonitorEntry[],
+  ): Promise<{ monitors: DeploymentMonitorOutcome[] }> {
+    return this.request({
+      method: 'POST',
+      path: this.deploymentPath('/monitors'),
+      body: { action, entries },
+    });
+  }
+
+  monitorJob(
+    jobId: string,
+    monitorKey: string,
+  ): Promise<{ monitors: DeploymentMonitorOutcome[] }> {
+    return this.request({
+      method: 'GET',
+      path: this.deploymentPath(
+        `/monitors?jobId=${encodeURIComponent(jobId)}&monitorKey=${encodeURIComponent(monitorKey)}`,
+      ),
+    });
+  }
+
   compileManifest(input: {
     name: string;
     sourceCode: string;

package/dist/bundling-sources/shared_libs/plays/deploy/manifest.ts

@@ -4,19 +4,22 @@ import picomatch from 'picomatch';
 
 /**
  * deepline.json — the repo-owned half of deploy configuration. The repo
- * describes what plays exist and where; Convex (playProjects) decides what
+ * describes what plays and monitors exist and where; Convex (playProjects) decides what
  * happens when they change (production branch, PR checks, enable/disable).
  * No field lives in both places.
  */
 export type DeeplineRepoManifest = {
   /** Globs relative to the project root directory. */
   plays: string[];
+  /** Monitor JSON files to discover, relative to the project root directory. */
+  monitors: string[];
   /** Optional tenant guard: the workspace slug this repo expects. */
   org?: string;
 };
 
 export const DEFAULT_PLAY_GLOBS = ['**/*.play.ts'];
-const MAX_DISCOVERED_PLAY_FILES = 200;
+export const DEFAULT_MONITOR_GLOBS = ['**/*.monitor.json'];
+const MAX_DISCOVERED_FILES = 200;
 const IGNORED_DIRECTORIES = new Set([
   'node_modules',
   '.git',
@@ -32,11 +35,33 @@ export class DeeplineManifestError extends Error {
   }
 }
 
+function readGlobs(value: unknown, field: 'plays' | 'monitors'): string[] {
+  if (
+    !Array.isArray(value) ||
+    (field === 'plays' && value.length === 0) ||
+    !value.every((glob) => typeof glob === 'string' && glob.trim().length > 0)
+  ) {
+    throw new DeeplineManifestError(
+      `deepline.json "${field}" must be ${field === 'plays' ? 'a non-empty' : 'an'} array of glob strings.`,
+    );
+  }
+  const globs = value.map((glob: string) => glob.trim().replace(/^\.\//, ''));
+  if (globs.some((glob) => glob.startsWith('/') || glob.includes('..'))) {
+    throw new DeeplineManifestError(
+      `deepline.json "${field}" globs must be relative and must not contain "..".`,
+    );
+  }
+  return globs;
+}
+
 export function parseDeeplineManifest(
   raw: string | null,
 ): DeeplineRepoManifest {
   if (raw === null) {
-    return { plays: DEFAULT_PLAY_GLOBS };
+    return {
+      plays: DEFAULT_PLAY_GLOBS,
+      monitors: DEFAULT_MONITOR_GLOBS,
+    };
   }
   let parsed: unknown;
   try {
@@ -49,32 +74,28 @@ export function parseDeeplineManifest(
   if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
     throw new DeeplineManifestError('deepline.json must be a JSON object.');
   }
-  const record = parsed as { plays?: unknown; org?: unknown };
-  let plays = DEFAULT_PLAY_GLOBS;
-  if (record.plays !== undefined) {
-    if (
-      !Array.isArray(record.plays) ||
-      record.plays.length === 0 ||
-      !record.plays.every(
-        (glob) => typeof glob === 'string' && glob.trim().length > 0,
-      )
-    ) {
-      throw new DeeplineManifestError(
-        'deepline.json "plays" must be a non-empty array of glob strings, e.g. ["plays/**/*.play.ts"].',
-      );
-    }
-    plays = record.plays.map((glob) => glob.trim().replace(/^\.\//, ''));
-  }
-  if (plays.some((glob) => glob.startsWith('/') || glob.includes('..'))) {
-    throw new DeeplineManifestError(
-      'deepline.json "plays" globs must be relative and must not contain "..".',
-    );
-  }
+  const record = parsed as {
+    plays?: unknown;
+    monitors?: unknown;
+    org?: unknown;
+  };
+  const plays =
+    record.plays === undefined
+      ? DEFAULT_PLAY_GLOBS
+      : readGlobs(record.plays, 'plays');
+  const monitors =
+    record.monitors === undefined
+      ? DEFAULT_MONITOR_GLOBS
+      : readGlobs(record.monitors, 'monitors');
   const org =
     typeof record.org === 'string' && record.org.trim()
       ? record.org.trim()
       : undefined;
-  return { plays, ...(org ? { org } : {}) };
+  return {
+    plays,
+    monitors,
+    ...(org ? { org } : {}),
+  };
 }
 
 export const DEFAULT_MANIFEST_FILE = 'deepline.json';
@@ -121,7 +142,7 @@ export async function readDeeplineManifest(
  * node_modules or fixtures publishes garbage, so vendored/build directories
  * are always skipped and discovery is capped loudly.
  */
-export async function discoverPlayFiles(input: {
+export async function discoverDeploymentFiles(input: {
   projectRoot: string;
   globs: string[];
 }): Promise<string[]> {
@@ -142,9 +163,9 @@ export async function discoverPlayFiles(input: {
       const relativePath = relative(input.projectRoot, absolute);
       if (matchers.some((match) => match(relativePath))) {
         found.push(relativePath);
-        if (found.length > MAX_DISCOVERED_PLAY_FILES) {
+        if (found.length > MAX_DISCOVERED_FILES) {
           throw new DeeplineManifestError(
-            `deepline.json globs matched more than ${MAX_DISCOVERED_PLAY_FILES} play files. Tighten the globs.`,
+            `deepline.json globs matched more than ${MAX_DISCOVERED_FILES} files. Tighten the globs.`,
           );
         }
       }
@@ -160,3 +181,5 @@ export async function discoverPlayFiles(input: {
   await walk(input.projectRoot);
   return found.sort();
 }
+
+export const discoverPlayFiles = discoverDeploymentFiles;

package/dist/bundling-sources/shared_libs/plays/deploy/monitor-files.ts

@@ -0,0 +1,52 @@
+import { readFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import type {
+  DeploymentMonitorEntry,
+  DeploymentMonitorOutcome,
+} from './api-client';
+import { DeeplineManifestError, discoverDeploymentFiles } from './manifest';
+
+export async function readMonitorFiles(input: {
+  projectRoot: string;
+  rootDirectory: string;
+  globs: string[];
+}): Promise<DeploymentMonitorEntry[]> {
+  const files = await discoverDeploymentFiles(input);
+  if (files.length > 100) {
+    throw new DeeplineManifestError(
+      'A deployment may contain at most 100 monitor definitions. Split the repository into multiple linked targets.',
+    );
+  }
+  return Promise.all(
+    files.map(async (file) => {
+      const sourcePath = input.rootDirectory
+        ? `${input.rootDirectory}/${file}`
+        : file;
+      const raw = await readFile(join(input.projectRoot, file), 'utf8');
+      try {
+        return { file: sourcePath, definition: JSON.parse(raw) as unknown };
+      } catch {
+        throw new DeeplineManifestError(
+          `${sourcePath} must contain a monitor definition as valid JSON.`,
+        );
+      }
+    }),
+  );
+}
+
+export function monitorDeploymentSummary(
+  outcomes: DeploymentMonitorOutcome[],
+): string {
+  return outcomes
+    .map(
+      (outcome) =>
+        `- Monitor \`${outcome.key}\` (\`${outcome.file}\`): ${outcome.status}${
+          outcome.credits != null
+            ? ` (${outcome.credits} Deepline credits)`
+            : ''
+        }${outcome.jobId ? ` — job \`${outcome.jobId}\`` : ''}${
+          outcome.error ? `\n  - ${outcome.error}` : ''
+        }${outcome.warning ? `\n  - :warning: ${outcome.warning}` : ''}`,
+    )
+    .join('\n');
+}