deepline 0.3.213 → 0.3.214

package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts

@@ -1,5 +1,8 @@
 import { STANDARD_PLAY_SANDBOX_RUNTIME_LIMITS } from '../../../plays/sandbox-runtime-limits';
 import { RUNTIME_SANDBOX_MEMORY_ABORT_CODE } from '../../run-failure';
+import { PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER } from '../../runtime-contract';
+import { RUNTIME_CLAIM_HEARTBEAT_DIVISOR } from '../../runtime-constants';
+import { RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS } from '../../runner-app/runtime-heartbeat';
 import {
   RUNTIME_MEMORY_LEDGER_PATH_ENV,
   RUNTIME_MEMORY_PRESSURE_PATH_ENV,
@@ -116,11 +119,11 @@ export const RUNTIME_SANDBOX_MEMORY_ABORT_REASON =
   RUNTIME_SANDBOX_MEMORY_ABORT_CODE;
 
 /**
- * Build the tiny parent process that owns the hard runtime deadline and the
- * sandbox memory envelope outside the customer-code Node event loop. A
- * synchronous infinite loop can block the runner's own timer, but it cannot
- * block this separate process; nor can a runner whose heap is thrashing run
- * its own memory watch reliably.
+ * Build the parent process that owns runner liveness, the hard runtime
+ * deadline, and the sandbox memory envelope outside the customer-code Node
+ * event loop. A synchronous infinite loop can block the runner's timers, but
+ * it cannot block this separate process; nor can a runner whose heap is
+ * thrashing run its own memory watch reliably.
  *
  * argv (after `node -e <source>`):
  *   1 runnerPath, 2 configPath, 3 runtimeStartedPath, 4 runtimeCompletedPath,
@@ -185,6 +188,178 @@ let settled = false;
 let forceKillTimer = null;
 let runtimeTimer = null;
 let terminalGraceTimer = null;
+let heartbeatTimer = null;
+let heartbeatController = null;
+let heartbeatInFlight = false;
+let heartbeatStopped = false;
+let heartbeatConfigRead = false;
+let heartbeatConfig = null;
+let heartbeatLastSuccessAtMs = 0;
+let heartbeatAttempt = 0;
+let heartbeatLastErrorCode = null;
+let heartbeatStopBeforeLeaseMs = 0;
+let stopRequested = false;
+
+function readRunnerHeartbeatConfig() {
+  if (heartbeatConfigRead) return heartbeatConfig;
+  heartbeatConfigRead = true;
+  try {
+    const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
+    const context = config && config.context;
+    const push = context && context.runnerPushExecution;
+    const gatewayBaseUrl = String(context && context.receiptGatewayBaseUrl || '').replace(/\\/$/, '');
+    const executorToken = context && context.executorToken;
+    const runId = push && push.runId;
+    const leaseSeconds = Number(push && push.leaseSeconds);
+    if (
+      !gatewayBaseUrl ||
+      typeof executorToken !== 'string' ||
+      !executorToken.trim() ||
+      typeof runId !== 'string' ||
+      !runId.trim() ||
+      !Number.isFinite(leaseSeconds) ||
+      leaseSeconds <= 0
+    ) {
+      return null;
+    }
+    heartbeatConfig = {
+      gatewayBaseUrl,
+      executorToken,
+      runId,
+      leaseMs: Math.max(1_000, Math.floor(leaseSeconds * 1_000)),
+    };
+  } catch {}
+  return heartbeatConfig;
+}
+
+function stopRunnerHeartbeat() {
+  heartbeatStopped = true;
+  if (heartbeatTimer) {
+    clearInterval(heartbeatTimer);
+    heartbeatTimer = null;
+  }
+  if (heartbeatController) {
+    heartbeatController.abort();
+    heartbeatController = null;
+  }
+}
+
+function runnerHeartbeatErrorCode(error) {
+  const code = error && typeof error.code === 'string' ? error.code : '';
+  if (/^(?:E[A-Z0-9_]+|ERR_[A-Z0-9_]+|UND_ERR_[A-Z0-9_]+)$/.test(code)) return code;
+  const name = error && typeof error.name === 'string' ? error.name : '';
+  if (name === 'AbortError' || name === 'TimeoutError') return name;
+  return 'unknown';
+}
+
+function startRunnerHeartbeatSupervisor() {
+  if (heartbeatTimer || heartbeatStopped || settled) return;
+  const config = readRunnerHeartbeatConfig();
+  if (!config) return;
+  let handoff = null;
+  try {
+    handoff = JSON.parse(fs.readFileSync(runtimeStartedPath, 'utf8'));
+  } catch {}
+  const handoffAt = Number(handoff && handoff.lastHeartbeatAtMs);
+  heartbeatLastSuccessAtMs = Number.isFinite(handoffAt) ? handoffAt : Date.now();
+  const cadenceMs = Math.max(250, Math.floor(config.leaseMs / ${RUNTIME_CLAIM_HEARTBEAT_DIVISOR}));
+  heartbeatStopBeforeLeaseMs = Math.max(
+    0,
+    config.leaseMs - cadenceMs - ${RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS},
+  );
+  process.stderr.write(
+    '[play-runner.heartbeat] watchdog_owner_started ' +
+      JSON.stringify({ runId: config.runId, cadenceMs, handoffHeartbeatAgeMs: Math.max(0, Date.now() - heartbeatLastSuccessAtMs) }) +
+      '\\n',
+  );
+  void runnerHeartbeatTick();
+  heartbeatTimer = setInterval(runnerHeartbeatTick, cadenceMs);
+  heartbeatTimer.unref?.();
+}
+
+async function runnerHeartbeatTick() {
+  if (heartbeatStopped || heartbeatInFlight || settled || !child || child.exitCode !== null) return;
+  const config = readRunnerHeartbeatConfig();
+  if (!config) return;
+  heartbeatInFlight = true;
+  heartbeatAttempt += 1;
+  const startedAt = Date.now();
+  const controller = new AbortController();
+  heartbeatController = controller;
+  const timeout = setTimeout(() => controller.abort(), ${RUNNER_HEARTBEAT_REQUEST_TIMEOUT_MS});
+  try {
+    const response = await fetch(
+      config.gatewayBaseUrl + '/api/v2/plays/runtime',
+      {
+        method: 'POST',
+        headers: {
+          'content-type': 'application/json',
+          authorization: 'Bearer ' + config.executorToken,
+          ${JSON.stringify(PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER)}: require('node:crypto').randomUUID(),
+        },
+        body: JSON.stringify({ action: 'runner_heartbeat', runId: config.runId }),
+        redirect: 'manual',
+        signal: controller.signal,
+      },
+    );
+    if (response.status >= 300 && response.status < 400) {
+      await response.body?.cancel().catch(() => undefined);
+      throw Object.assign(new Error('gateway_redirect'), { code: 'redirect_' + response.status });
+    }
+    if (!response.ok) {
+      await response.body?.cancel().catch(() => undefined);
+      throw Object.assign(new Error('gateway_http_error'), { code: 'http_' + response.status });
+    }
+    const body = await response.json().catch(() => null);
+    if (body && body.ok === false && ['cancelled', 'failed', 'completed'].includes(body.terminal)) {
+      process.stderr.write(
+        '[play-runner.heartbeat] watchdog_owner_terminal ' +
+          JSON.stringify({ runId: config.runId, terminal: body.terminal, attempt: heartbeatAttempt }) +
+          '\\n',
+      );
+      stopRunnerHeartbeat();
+      stopChild('runner_scheduler_terminal', { terminal: body.terminal, heartbeatAttempt });
+      return;
+    }
+    if (!body || body.ok !== true || (body.schedulerStatus !== undefined && !['queued', 'running', 'waiting'].includes(body.schedulerStatus))) {
+      throw Object.assign(new Error('gateway_protocol_error'), { code: 'protocol_error' });
+    }
+    heartbeatLastSuccessAtMs = Date.now();
+    heartbeatLastErrorCode = null;
+  } catch (error) {
+    if (heartbeatStopped || settled) return;
+    heartbeatLastErrorCode = controller.signal.aborted
+      ? 'TimeoutError'
+      : runnerHeartbeatErrorCode(error);
+    const heartbeatAgeMs = Math.max(0, Date.now() - heartbeatLastSuccessAtMs);
+    process.stderr.write(
+      '[play-runner.heartbeat] watchdog_owner_retrying ' +
+        JSON.stringify({ runId: config.runId, attempt: heartbeatAttempt, durationMs: Math.max(0, Date.now() - startedAt), heartbeatAgeMs, errorCode: heartbeatLastErrorCode }) +
+        '\\n',
+    );
+    // Stop before the scheduler's lease expires. A failed request can consume
+    // its full timeout, and the next cadence would otherwise let the scheduler
+    // win the terminal race and report an ambiguous runner loss.
+    if (heartbeatAgeMs >= heartbeatStopBeforeLeaseMs) {
+      stopRunnerHeartbeat();
+      const diagnostic = {
+        heartbeatAttempt,
+        heartbeatAgeMs,
+        leaseMs: config.leaseMs,
+        stopBeforeLeaseMs: heartbeatStopBeforeLeaseMs,
+        lastErrorCode: heartbeatLastErrorCode,
+      };
+      process.stderr.write(
+        'RUNTIME_CONTROL_PLANE_UNAVAILABLE: ' + JSON.stringify(diagnostic) + '\\n',
+      );
+      stopChild('runner_heartbeat_unreachable', diagnostic);
+    }
+  } finally {
+    clearTimeout(timeout);
+    if (heartbeatController === controller) heartbeatController = null;
+    heartbeatInFlight = false;
+  }
+}
 
 function writeTerminationDiagnostic(reason, details = {}) {
   if (!terminationDiagnosticPath) return;
@@ -204,6 +379,8 @@ function writeTerminationDiagnostic(reason, details = {}) {
 }
 
 function stopChild(reason, details) {
+  if (settled || stopRequested) return;
+  stopRequested = true;
   writeTerminationDiagnostic(reason, details);
   child.kill('SIGTERM');
   forceKillTimer = setTimeout(() => child.kill('SIGKILL'), ${RUNNER_TERMINATION_KILL_GRACE_MS});
@@ -391,7 +568,10 @@ function armTerminalGrace() {
 }
 
 const runtimeBoundaryPoll = setInterval(() => {
-  if (fs.existsSync(runtimeStartedPath)) armRuntimeDeadline();
+  if (fs.existsSync(runtimeStartedPath)) {
+    armRuntimeDeadline();
+    startRunnerHeartbeatSupervisor();
+  }
   if (fs.existsSync(runtimeCompletedPath)) armTerminalGrace();
 }, 25);
 const startupTimer = setTimeout(() => {
@@ -413,6 +593,7 @@ function finish(code) {
   settled = true;
   clearInterval(runtimeBoundaryPoll);
   clearInterval(memoryWatch);
+  stopRunnerHeartbeat();
   clearTimeout(startupTimer);
   if (runtimeTimer) clearTimeout(runtimeTimer);
   if (terminalGraceTimer) clearTimeout(terminalGraceTimer);

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts

@@ -13891,24 +13891,33 @@ export async function upsertPostgresSchedulerObservation(
 /** Read every observation for a run/attempt, ordered for deterministic folds. */
 export async function getPostgresSchedulerObservations(
   client: PostgresSchedulerQueryClient,
-  input: { runId: string; attempt?: number | null },
+  input: {
+    runId: string;
+    attempt?: number | null;
+    /** Return only the newest settled invocation for each node. */
+    latestSettledPerNode?: boolean;
+  },
   options: PostgresSchedulerReceiptOptions,
 ): Promise<DocflowObservationRecord[]> {
   const hasAttempt = input.attempt != null;
+  const latestSettledPerNode = input.latestSettledPerNode === true;
   const params: unknown[] = [receiptOrgId(options), input.runId];
   if (hasAttempt) {
     params.push(normalizePostgresSchedulerReceiptRunAttempt(input.attempt));
   }
   const result = await client.query<ObservationRow>(
     `
-      SELECT run_id, attempt, node_id, invocation_seq, invocation_id, status,
+      SELECT ${latestSettledPerNode ? 'DISTINCT ON (node_id)' : ''}
+             run_id, attempt, node_id, invocation_seq, invocation_id, status,
              inputs_json, outputs_json, inputs_truncated, outputs_truncated,
              error, started_at, settled_at
         FROM ${tableName('observations', options)}
        WHERE org_id = $1
          AND run_id = $2
          ${hasAttempt ? 'AND attempt = $3' : ''}
-       ORDER BY node_id, attempt, invocation_seq
+         ${latestSettledPerNode ? "AND status IN ('completed', 'failed')" : ''}
+       ORDER BY node_id, attempt ${latestSettledPerNode ? 'DESC' : ''},
+                invocation_seq ${latestSettledPerNode ? 'DESC' : ''}
     `,
     params,
   );

package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts

@@ -219,11 +219,14 @@ export function runtimeSheetPageTailHoldCompletedEvent(
   tableNamespace: string,
 ): PlayRunnerEvent | null {
   if (!Number.isInteger(holdMs) || (holdMs ?? 0) <= 0) return null;
+  const at = new Date().toISOString();
   return {
     type: 'log',
-    at: new Date().toISOString(),
+    at,
     source: 'play',
-    line: `[runtime.sheet-page-tail-hold] phase=finish table=${tableNamespace} hold_ms=${holdMs}`,
+    // The status projection keeps the line but drops the event envelope's
+    // timestamp, so include the emission time for runtime-preview diagnostics.
+    line: `[${at}] [runtime.sheet-page-tail-hold] phase=finish table=${tableNamespace} hold_ms=${holdMs}`,
   };
 }
 

package/dist/bundling-sources/shared_libs/plays/settings-contract.ts

@@ -0,0 +1,176 @@
+import { z } from 'zod';
+
+/** Keep schema diagnostics readable in both interactive settings and the CLI. */
+export function formatPlaySettingsError(error: unknown): string {
+  if (error instanceof z.ZodError)
+    return [...new Set(error.issues.map((issue) => issue.message))].join(' ');
+  return error instanceof Error ? error.message : String(error);
+}
+
+/** Location organizes Plays. Existing workspace permissions still apply. */
+export const PLAY_LOCATION_ROOTS = [
+  {
+    value: 'person',
+    label: 'Personal',
+    description:
+      'The creator’s personal root, displayed as their name followed by “’s team”.',
+  },
+  {
+    value: 'team',
+    label: 'Team library',
+    description:
+      'The shared team library. Publishing moves personal locations here, keeping their folders.',
+  },
+] as const;
+
+const folder = z
+  .string()
+  .min(1)
+  .max(128)
+  .refine(
+    (value) =>
+      value === value.trim() &&
+      value !== '.' &&
+      value !== '..' &&
+      !/[\\/\u0000-\u001f\u007f]/u.test(value),
+    'Folder names must be trimmed, cannot be . or .., and cannot contain slashes or control characters.',
+  );
+export const playPathSchema = z
+  .union([
+    z.null(),
+    z
+      .object({
+        root: z.enum(['person', 'team']),
+        folders: z.array(folder).max(16).default([]),
+      })
+      .strict(),
+  ])
+  .transform((path) =>
+    path?.root === 'person' && path.folders.length === 0 ? null : path,
+  );
+export type PlayPath = z.output<typeof playPathSchema>;
+
+export function parsePlayPath(value: string): PlayPath {
+  const [root, ...parts] = value.split('/');
+  if (!PLAY_LOCATION_ROOTS.some((option) => option.value === root)) {
+    throw new Error(
+      'Path must start with person or team. Example: person/Research/ICP.',
+    );
+  }
+  let folders: string[];
+  try {
+    folders = parts.map(decodeURIComponent);
+  } catch {
+    throw new Error(
+      'Path contains an invalid percent escape. Encode a literal % as %25.',
+    );
+  }
+  return playPathSchema.parse({ root, folders });
+}
+export function formatPlayPath(path: PlayPath): string {
+  return path === null
+    ? 'person'
+    : [path.root, ...path.folders.map(encodeURIComponent)].join('/');
+}
+export function personalPlayRootLabel(ownerName?: string | null): string {
+  const name = ownerName?.trim();
+  return name ? `${name.split(/\s+/)[0]}’s team` : 'Personal';
+}
+export function displayPlayPath(
+  path: PlayPath,
+  ownerName?: string | null,
+): string {
+  return [
+    path?.root === 'team' ? 'Team library' : personalPlayRootLabel(ownerName),
+    ...(path?.folders ?? []),
+  ].join(' / ');
+}
+export function publishedPlayPath(path: PlayPath): PlayPath {
+  return { root: 'team', folders: path?.folders ?? [] };
+}
+
+/** One registry drives validation, defaults, CLI keys/help and settings controls. */
+export const PLAY_SETTING_FIELDS = {
+  path: {
+    schema: playPathSchema
+      .default(null)
+      .describe(
+        'Play location. Null means the creator’s personal root. Publishing moves it into Team library, preserving folders.',
+      ),
+    label: 'Location',
+    description:
+      'Organize this Play in the creator’s personal root or Team library. Publishing moves it to Team library and keeps its folders. Workspace sharing permissions apply in both roots.',
+    get defaultValue(): PlayPath {
+      return this.schema.parse(undefined);
+    },
+    cli: {
+      valueSyntax: 'person[/folder/…] | team[/folder/…] | null',
+      examples: [
+        'person',
+        'person/Research/ICP',
+        'team',
+        'team/Research',
+        'null',
+      ],
+      parse: (value: string) =>
+        value === 'null' ? null : parsePlayPath(value),
+      format: formatPlayPath,
+    },
+    ui: { control: 'play-location-picker' as const },
+  },
+} as const;
+export const playSettingsSchema = z.object(
+  Object.fromEntries(
+    Object.entries(PLAY_SETTING_FIELDS).map(([key, field]) => [
+      key,
+      field.schema,
+    ]),
+  ) as {
+    [K in keyof typeof PLAY_SETTING_FIELDS]: (typeof PLAY_SETTING_FIELDS)[K]['schema'];
+  },
+).passthrough();
+export const playSettingsPatchSchema = z
+  .object(
+    Object.fromEntries(
+      Object.entries(PLAY_SETTING_FIELDS).map(([key, field]) => [
+        key,
+        field.schema.unwrap().optional(),
+      ]),
+    ) as {
+      [K in keyof typeof PLAY_SETTING_FIELDS]: z.ZodOptional<
+        ReturnType<(typeof PLAY_SETTING_FIELDS)[K]['schema']['unwrap']>
+      >;
+    },
+  )
+  .strict()
+  .refine(
+    (patch) => Object.keys(patch).length > 0,
+    'Supply at least one setting.',
+  );
+export type PlaySettings = {
+  [K in keyof typeof PLAY_SETTING_FIELDS]: z.output<
+    (typeof PLAY_SETTING_FIELDS)[K]['schema']
+  >;
+};
+export type PlaySettingsPatch = z.output<typeof playSettingsPatchSchema>;
+/** Validate registered keys while preserving future keys in stored objects. */
+export function parsePlaySettings(value: unknown): PlaySettings {
+  return playSettingsSchema.parse(value) as PlaySettings;
+}
+export const DEFAULT_PLAY_SETTINGS: PlaySettings = parsePlaySettings({});
+export type PlaySettingsResult = {
+  playDefinitionId: string;
+  ownerUserId: string | null;
+  ownerName: string | null;
+  settings: PlaySettings;
+};
+export type PlaySettingsPathsPage = {
+  paths: Array<{
+    path: PlayPath;
+    ownerUserId: string | null;
+    ownerName: string | null;
+    playName: string;
+  }>;
+  cursor: string | null;
+  hasMore: boolean;
+};