deepline 0.3.229 → 0.3.231

package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts

@@ -2,7 +2,7 @@
  * One capacity policy for the Absurd runtime.
  *
  * Postgres owns backlog. The active lane adds workers from durable claimable
- * demand and oldest-claimable age; historical lanes stay cold unless they own
+ * demand; historical lanes stay cold unless they own
  * runnable work. Keep these values together so the worker, scaler, gateway,
  * and tests cannot independently invent concurrency or timeout budgets.
  */
@@ -19,14 +19,20 @@ export const RUNTIME_CAPACITY_POLICY = {
     /** Warm floor for availability and deploy rollovers. */
     activeLaneMachines: 2,
     /**
-     * Keep the fleet below the PlanetScale direct-session budget. Each worker
+     * Bound active-lane burst demand. Each worker
      * has one elected direct wake listener and a two-client pooled scheduler
      * budget; scaling past this point amplifies connection pressure faster
      * than it adds useful control-plane throughput.
      */
     maxActiveLaneMachines: 8,
+    /**
+     * Shared started/pending worker budget across every release lane. Existing
+     * claim owners and rollback holds are retained; restoring the active warm
+     * floor may temporarily exceed this budget without forcing any worker off.
+     */
+    maxFleetMachines: 8,
     workerSlotsPerMachine: 8,
-    /** React within one control interval when a due backlog is not draining. */
+    /** Queue-age diagnostic threshold; age alone never grants worker slots. */
     scaleUpQueueAgeMs: 1_000,
     /** Bound one Fly reconciliation without turning backlog into a stampede. */
     maxScaleUpMachinesPerReconcile: 4,
@@ -75,9 +81,9 @@ export const ABSURD_GLOBAL_RUN_CONCURRENCY =
   RUNTIME_CAPACITY_POLICY.absurd.workerSlotsPerMachine;
 
 /**
- * Active releases use pg-boss-style burst semantics: claimable count sets the
- * target, an aged claimable queue forces continued growth, and growth per
- * observation is bounded. A short/empty observation ends burst growth; the
+ * Active releases use claimable plus running count to set the target, with
+ * growth per observation bounded. Queue age is diagnostic, not additional
+ * runnable demand. A short/empty observation ends burst growth; the
  * caller supplies idleForMs to hold excess capacity through a quiet window.
  *
  * Historical releases stay cold until they have immediately runnable or
@@ -105,14 +111,6 @@ export function desiredAbsurdLaneMachines(input: {
   );
   let desired = Math.max(policy.activeLaneMachines, demandedMachines);
 
-  if (
-    claimableRuns > 0 &&
-    nonNegativeNumber(input.oldestClaimableAgeMs ?? 0) >=
-      policy.scaleUpQueueAgeMs
-  ) {
-    desired = Math.max(desired, currentMachines + 1);
-  }
-
   desired = Math.min(
     desired,
     currentMachines + policy.maxScaleUpMachinesPerReconcile,
@@ -129,6 +127,172 @@ export function desiredAbsurdLaneMachines(input: {
   return desired;
 }
 
+/**
+ * Allocate only requested starts/creates against the physical worker fleet.
+ * The caller counts all started and pending Machines, including drainers,
+ * unhealthy workers, fresh claim owners, and rollback holds. Planned stops do
+ * not free budget until Fly confirms them. This function never removes workers.
+ *
+ * Restore the active warm floor first, even when that reports budget debt.
+ * Give freed budget to historical runnable lanes in round-robin order before
+ * active burst growth. Persist nextCursor between reconciliations so repeated
+ * single-slot openings do not always favor the first historical release.
+ */
+export function planAbsurdWorkerFleetGrowth(input: {
+  existingMachines: number;
+  maxFleetMachines?: number;
+  activeWarmFloorMissing: number;
+  demands: readonly {
+    releaseId: string;
+    active: boolean;
+    startMachineIds: readonly string[];
+    createCount: number;
+  }[];
+  cursor?: string;
+}): {
+  allocations: {
+    releaseId: string;
+    startMachineIds: string[];
+    createCount: number;
+    deferredCount: number;
+  }[];
+  nextCursor: string;
+  physicalCount: number;
+  projectedPhysicalCount: number;
+  budgetDebt: number;
+  deferredCount: number;
+  warmFloorGrowth: number;
+  maxFleetMachines: number;
+} {
+  const physicalCount = requireMachineCount(input.existingMachines);
+  const maxFleetMachines = requireMachineCount(
+    input.maxFleetMachines ?? RUNTIME_CAPACITY_POLICY.absurd.maxFleetMachines,
+  );
+  const warmFloorMissing = requireMachineCount(input.activeWarmFloorMissing);
+  const cursor = input.cursor ?? '';
+  if (typeof cursor !== 'string') {
+    throw new Error('Worker fleet cursor must be a release ID.');
+  }
+  const releases = new Set<string>();
+  const machines = new Set<string>();
+  const lanes = input.demands.map((demand) => {
+    if (!demand.releaseId || releases.has(demand.releaseId)) {
+      throw new Error('Worker fleet demands must have unique release IDs.');
+    }
+    releases.add(demand.releaseId);
+    for (const machineId of demand.startMachineIds) {
+      if (!machineId || machines.has(machineId)) {
+        throw new Error('Worker fleet starts must have unique Machine IDs.');
+      }
+      machines.add(machineId);
+    }
+    const createCount = requireMachineCount(demand.createCount);
+    return {
+      demand,
+      remaining: requireMachineCount(
+        demand.startMachineIds.length + createCount,
+      ),
+      allocation: {
+        releaseId: demand.releaseId,
+        startMachineIds: [] as string[],
+        createCount: 0,
+        deferredCount: 0,
+      },
+    };
+  });
+  const activeLanes = lanes.filter((lane) => lane.demand.active);
+  if (
+    activeLanes.length > 1 ||
+    (warmFloorMissing > 0 && activeLanes.length !== 1)
+  ) {
+    throw new Error('Worker fleet warm capacity requires one active release.');
+  }
+  const activeLane = activeLanes[0];
+  const grant = (lane: (typeof lanes)[number]) => {
+    const startIndex = lane.allocation.startMachineIds.length;
+    if (startIndex < lane.demand.startMachineIds.length) {
+      lane.allocation.startMachineIds.push(
+        lane.demand.startMachineIds[startIndex],
+      );
+    } else {
+      lane.allocation.createCount += 1;
+    }
+    lane.remaining -= 1;
+  };
+  const warmFloorGrowth = Math.min(
+    warmFloorMissing,
+    activeLane?.remaining ?? 0,
+  );
+  for (let seat = 0; seat < warmFloorGrowth; seat += 1) grant(activeLane!);
+  let available = Math.max(
+    0,
+    maxFleetMachines - physicalCount - warmFloorGrowth,
+  );
+  const historical = lanes
+    .filter((lane) => !lane.demand.active && lane.remaining > 0)
+    .sort((a, b) =>
+      a.demand.releaseId < b.demand.releaseId
+        ? -1
+        : a.demand.releaseId > b.demand.releaseId
+          ? 1
+          : 0,
+    );
+  let nextCursor = cursor;
+  let historicalRemaining = historical.reduce(
+    (sum, lane) => sum + lane.remaining,
+    0,
+  );
+  // The requester set changes as lanes run out of work or disappear. A
+  // positional cursor would silently move across releases and can repeatedly
+  // skip a continuously waiting lane. Resume after the last granted identity.
+  const afterCursor = historical.findIndex(
+    (lane) => lane.demand.releaseId > cursor,
+  );
+  let index = afterCursor >= 0 ? afterCursor : 0;
+  while (available > 0 && historicalRemaining > 0) {
+    const lane = historical[index];
+    if (lane.remaining > 0) {
+      grant(lane);
+      available -= 1;
+      historicalRemaining -= 1;
+      nextCursor = lane.demand.releaseId;
+    }
+    index = (index + 1) % historical.length;
+  }
+  while (available > 0 && activeLane && activeLane.remaining > 0) {
+    grant(activeLane);
+    available -= 1;
+  }
+  let allocatedCount = 0;
+  let deferredCount = 0;
+  const allocations = lanes.map((lane) => {
+    allocatedCount +=
+      lane.allocation.startMachineIds.length + lane.allocation.createCount;
+    deferredCount += lane.remaining;
+    return { ...lane.allocation, deferredCount: lane.remaining };
+  });
+  const projectedPhysicalCount = requireMachineCount(
+    physicalCount + allocatedCount,
+  );
+  return {
+    allocations,
+    nextCursor,
+    physicalCount,
+    projectedPhysicalCount,
+    budgetDebt: Math.max(0, projectedPhysicalCount - maxFleetMachines),
+    deferredCount,
+    warmFloorGrowth,
+    maxFleetMachines,
+  };
+}
+
+function requireMachineCount(value: number): number {
+  if (!Number.isSafeInteger(value) || value < 0) {
+    throw new Error('Worker fleet counts must be nonnegative safe integers.');
+  }
+  return value;
+}
+
 /**
  * Execution-relay demand is the exact upstream admission state, not request
  * count. Active permits consume capacity and waiting requests are an early
@@ -246,6 +410,15 @@ export function assertRuntimeCapacityPolicy(): void {
       'Active-lane minimum Machines cannot exceed the active-lane maximum.',
     );
   }
+  if (
+    !Number.isSafeInteger(RUNTIME_CAPACITY_POLICY.absurd.maxFleetMachines) ||
+    RUNTIME_CAPACITY_POLICY.absurd.maxFleetMachines <
+      RUNTIME_CAPACITY_POLICY.absurd.activeLaneMachines
+  ) {
+    throw new Error(
+      'Worker fleet budget must cover the active-lane warm floor.',
+    );
+  }
   const relay = RUNTIME_CAPACITY_POLICY.executionRelay;
   if (
     relay.minimumMachines > relay.maxMachines ||

package/dist/bundling-sources/shared_libs/play-runtime/runtime-work-counters.ts

@@ -2,8 +2,25 @@
 export type RuntimeWorkCounters = {
   logicalCalls: number;
   providerRequests: number;
+  /** Older resource snapshots did not retain this scalar. */
+  provider429s?: number;
 };
 
+/** Sequence actual runner invocations within one scheduler generation. */
+export function nextWorkCounterInvocationSeq(
+  checkpoint: unknown,
+  attempt: number,
+): number {
+  const previous = record(checkpoint);
+  if (previous?.workCounterInvocationAttempt !== attempt) return 1;
+  const previousSeq = count(previous.workCounterInvocationSeq);
+  if (previousSeq === null) return 1;
+  if (previousSeq === Number.MAX_SAFE_INTEGER) {
+    throw new Error('Work counter invocation sequence exhausted.');
+  }
+  return previousSeq + 1;
+}
+
 function record(value: unknown): Record<string, unknown> | null {
   return value !== null && typeof value === 'object' && !Array.isArray(value)
     ? (value as Record<string, unknown>)
@@ -16,6 +33,26 @@ function count(value: unknown): number | null {
     : null;
 }
 
+export function runtimeWorkCountersFromStats(
+  stats: unknown,
+): RuntimeWorkCounters | null {
+  const root = record(stats);
+  const compact = record(root?.runtimeWorkCounters);
+  if (compact) {
+    const logicalCalls = count(compact.logicalCalls);
+    const providerRequests = count(compact.providerRequests);
+    const provider429s = count(compact.provider429s);
+    if (
+      logicalCalls !== null &&
+      providerRequests !== null &&
+      provider429s !== null
+    ) {
+      return { logicalCalls, providerRequests, provider429s };
+    }
+  }
+  return null;
+}
+
 export function runtimeWorkCountersFromResources(
   resources: unknown,
 ): RuntimeWorkCounters | null {
@@ -26,12 +63,25 @@ export function runtimeWorkCountersFromResources(
   if (logicalCalls === null || !providers) return null;
 
   let providerRequests = 0;
+  let provider429s = 0;
+  let provider429sKnown = true;
   for (const [key, value] of Object.entries(providers)) {
     if (!key.startsWith('tool:')) continue;
     const requests = count(record(record(value)?.residence)?.count);
     if (requests === null) return null;
     providerRequests += requests;
-    if (!Number.isSafeInteger(providerRequests)) return null;
+    const rejected = count(record(value)?.provider429Count);
+    if (rejected === null) provider429sKnown = false;
+    else provider429s += rejected;
+    if (
+      !Number.isSafeInteger(providerRequests) ||
+      !Number.isSafeInteger(provider429s)
+    )
+      return null;
   }
-  return { logicalCalls, providerRequests };
+  return {
+    logicalCalls,
+    providerRequests,
+    ...(provider429sKnown ? { provider429s } : {}),
+  };
 }

package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts

@@ -12,6 +12,7 @@ import type {
 } from '../ctx-types';
 import type {
   PlayRunnerExecutionConfig,
+  PlayRunnerEvent,
   PlayRunnerLogEvent,
   PlayRunnerResult,
 } from '../protocol';
@@ -37,6 +38,9 @@ export type SandboxRunnerProgress = {
   onCheckpoint?: (checkpoint: PlayCheckpoint) => void;
   onRowUpdate?: (update: PlayRowUpdate) => void;
   onExecutionEvent?: (event: PlayExecutionEvent) => void;
+  onWorkCounters?: (
+    event: Extract<PlayRunnerEvent, { type: 'work_counters' }>,
+  ) => void;
 };
 
 /**
@@ -147,6 +151,7 @@ function createRunnerCallbacks(input: {
     onCheckpoint: input.progress?.onCheckpoint,
     onRowUpdate: input.progress?.onRowUpdate,
     onExecutionEvent: input.progress?.onExecutionEvent,
+    onWorkCounters: input.progress?.onWorkCounters,
     readDetachedRunnerReadiness: async (baselineHeartbeatAt) =>
       await input.readiness({ baselineHeartbeatAt }),
     onRuntimeResourceAcquired: async (resource) => {

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts

@@ -2764,8 +2764,34 @@ async function executeAssignedAbsurdAttempt(input: {
       replayAllTerminalLogs: input.fromDetachedRunner,
       terminalLogProducerAttempt: input.terminalLogProducerAttempt,
     });
-    const suspendedCheckpoint =
-      input.runnerResult.checkpoint ?? checkpoint ?? null;
+    const suspendedCheckpoint = absurdCheckpointBase(
+      input.runnerResult.checkpoint ?? checkpoint ?? null,
+    );
+    const segment = input.runnerResult.stats?.runtimeWorkCounterSegment as
+      | { invocationSeq?: unknown; segmentId?: unknown }
+      | undefined;
+    if (
+      typeof segment?.invocationSeq === 'number' &&
+      Number.isSafeInteger(segment.invocationSeq) &&
+      segment.invocationSeq >= 1 &&
+      typeof segment.segmentId === 'string' &&
+      segment.segmentId.length > 0
+    ) {
+      // The checkpoint is persisted by the existing suspension transaction.
+      // A resumed runner in this generation gets the next segment sequence;
+      // a crash before that commit cannot silently reuse a finalized segment.
+      const previous = suspendedCheckpoint.workCounterInvocationSeq;
+      suspendedCheckpoint.workCounterInvocationSeq = Math.max(
+        suspendedCheckpoint.workCounterInvocationAttempt === generation &&
+          typeof previous === 'number' &&
+          Number.isSafeInteger(previous) &&
+          previous >= 0
+          ? previous
+          : 0,
+        segment.invocationSeq,
+      );
+      suspendedCheckpoint.workCounterInvocationAttempt = generation;
+    }
     const engine = createAbsurdEngineSuspensionAdapter({
       ctx,
       runId,
@@ -3439,6 +3465,8 @@ async function executeAssignedAbsurdAttempt(input: {
                       type: 'execution_event',
                       event,
                     }),
+                  onWorkCounters: (event) =>
+                    progressReporter.onRunnerEvent(event),
                 },
                 interpretResult: ({ result }) => {
                   throwRetryableRuntimeInitializationFailure(result);

package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts

@@ -26,6 +26,7 @@ import type {
 } from '../protocol';
 import type { PlaySchedulerSubmitInput } from '../scheduler-backend';
 import type { DocflowObservationRecord } from '../docflow-observation';
+import { runtimeWorkCountersFromStats } from '../runtime-work-counters';
 import {
   buildPlayRunLedgerEventsFromStatusPatch,
   createEmptyPlayRunLedgerSnapshot,
@@ -231,8 +232,7 @@ export type PostgresSchedulerProgressReporter = {
 };
 
 export type PostgresSchedulerDurableLogMode =
-  | 'progress_reporter'
-  | 'direct_worker';
+  'progress_reporter' | 'direct_worker';
 
 function shouldPublishDurableLogEvent(
   mode: PostgresSchedulerDurableLogMode,
@@ -387,7 +387,10 @@ export function createPostgresSchedulerProgressReporter(input: {
   let liveLogs: string[] = [];
   let liveLogsTotalCount = input.initialLiveLogTotalCount ?? 0;
   let liveLogsDirty = false;
+  let workCounters: RuntimeStatusUpdate['workCounters'];
+  let workCountersDirty = false;
   let lastFlushAt = 0;
+  let delayedFlushTimer: ReturnType<typeof setTimeout> | null = null;
   let totalRows = 0;
   let activeMapNodeId: string | null = null;
   let pendingDatasetLifecycleEvents: Array<
@@ -464,9 +467,11 @@ export function createPostgresSchedulerProgressReporter(input: {
           const nextActivityObservations = [
             ...pendingActivityObservations.values(),
           ];
+          const nextWorkCounters = workCountersDirty ? workCounters : undefined;
           pendingActivityObservations.clear();
           liveNodeProgressDirty = false;
           liveLogsDirty = false;
+          workCountersDirty = false;
 
           try {
             if (await isCurrentAttempt()) {
@@ -479,6 +484,7 @@ export function createPostgresSchedulerProgressReporter(input: {
                 liveLogProducerAttempt: producerAttempt,
                 datasetLifecycleEvents: nextDatasetLifecycleEvents,
                 activityObservations: nextActivityObservations,
+                workCounters: nextWorkCounters,
                 terminalLogReplay: request.terminalLogReplay,
                 lastCheckpointAt: Date.now(),
               });
@@ -489,6 +495,7 @@ export function createPostgresSchedulerProgressReporter(input: {
               datasetLifecycleEvents: nextDatasetLifecycleEvents,
               activityObservations: nextActivityObservations,
             });
+            workCountersDirty = true;
             console.warn('[postgres-scheduler.progress_flush_failed]', {
               runId: launch.runId,
               error: error instanceof Error ? error.message : String(error),
@@ -520,18 +527,35 @@ export function createPostgresSchedulerProgressReporter(input: {
       !force &&
       !liveNodeProgressDirty &&
       !liveLogsDirty &&
+      !workCountersDirty &&
       pendingDatasetLifecycleEvents.length === 0 &&
       pendingActivityObservations.size === 0
     ) {
+      if (delayedFlushTimer) {
+        clearTimeout(delayedFlushTimer);
+        delayedFlushTimer = null;
+      }
       return activeProgressUpload ?? Promise.resolve();
     }
     const now = Date.now();
-    if (
-      !force &&
-      now - lastFlushAt < getAdaptiveRunEventIntervalMs(totalRows)
-    ) {
+    const intervalMs = getAdaptiveRunEventIntervalMs(totalRows);
+    if (!force && now - lastFlushAt < intervalMs) {
+      if (!delayedFlushTimer) {
+        delayedFlushTimer = setTimeout(
+          () => {
+            delayedFlushTimer = null;
+            void flush().catch(() => undefined);
+          },
+          Math.max(1, intervalMs - (now - lastFlushAt)),
+        );
+        delayedFlushTimer.unref?.();
+      }
       return activeProgressUpload ?? Promise.resolve();
     }
+    if (delayedFlushTimer) {
+      clearTimeout(delayedFlushTimer);
+      delayedFlushTimer = null;
+    }
     lastFlushAt = now;
     const requested = new Promise<void>((resolve, reject) => {
       pendingProgressUpload = mergeProgressFlushRequest(
@@ -1057,6 +1081,62 @@ export function createPostgresSchedulerProgressReporter(input: {
       return;
     }
     const type = (event as { type?: unknown }).type;
+    if (type === 'work_counters') {
+      const observed = event as {
+        attempt?: unknown;
+        invocationSeq?: unknown;
+        segmentId?: unknown;
+        logicalCalls?: unknown;
+        providerRequests?: unknown;
+        provider429s?: unknown;
+      };
+      if (
+        typeof observed.attempt !== 'number' ||
+        !Number.isSafeInteger(observed.attempt) ||
+        observed.attempt < 0 ||
+        typeof observed.invocationSeq !== 'number' ||
+        !Number.isSafeInteger(observed.invocationSeq) ||
+        observed.invocationSeq < 1 ||
+        typeof observed.segmentId !== 'string' ||
+        observed.segmentId.length === 0 ||
+        (producerAttempt !== undefined &&
+          observed.attempt !== producerAttempt) ||
+        ![
+          observed.logicalCalls,
+          observed.providerRequests,
+          observed.provider429s,
+        ].every(
+          (value) =>
+            typeof value === 'number' &&
+            Number.isSafeInteger(value) &&
+            value >= 0,
+        )
+      )
+        return;
+      const next = {
+        attempt: observed.attempt,
+        invocationSeq: observed.invocationSeq,
+        segmentId: observed.segmentId,
+        logicalCalls: observed.logicalCalls as number,
+        providerRequests: observed.providerRequests as number,
+        provider429s: observed.provider429s as number,
+        final: false,
+      };
+      if (
+        workCounters &&
+        workCounters.attempt === next.attempt &&
+        workCounters.invocationSeq === next.invocationSeq &&
+        workCounters.segmentId === next.segmentId &&
+        workCounters.logicalCalls >= next.logicalCalls &&
+        workCounters.providerRequests >= next.providerRequests &&
+        workCounters.provider429s >= next.provider429s
+      )
+        return;
+      workCounters = next;
+      workCountersDirty = true;
+      scheduleFlush();
+      return;
+    }
     if (type === 'log') {
       const logEvent = event as { line?: unknown; source?: unknown };
       const rawLine = logEvent.line;
@@ -1291,6 +1371,29 @@ export function createPostgresSchedulerProgressReporter(input: {
       skipTerminalLogReplay?: boolean;
     } = {},
   ): Promise<void> => {
+    const terminalCounters = runtimeWorkCountersFromStats(result?.stats);
+    const terminalSegment = result?.stats?.runtimeWorkCounterSegment as
+      { invocationSeq?: unknown; segmentId?: unknown } | undefined;
+    if (
+      terminalCounters &&
+      terminalCounters.provider429s !== undefined &&
+      typeof terminalSegment?.invocationSeq === 'number' &&
+      Number.isSafeInteger(terminalSegment.invocationSeq) &&
+      terminalSegment.invocationSeq >= 1 &&
+      typeof terminalSegment.segmentId === 'string' &&
+      terminalSegment.segmentId.length > 0
+    ) {
+      workCounters = {
+        attempt: producerAttempt ?? workCounters?.attempt ?? 0,
+        invocationSeq: terminalSegment.invocationSeq,
+        segmentId: terminalSegment.segmentId,
+        logicalCalls: terminalCounters.logicalCalls,
+        providerRequests: terminalCounters.providerRequests,
+        provider429s: terminalCounters.provider429s,
+        final: true,
+      };
+      workCountersDirty = true;
+    }
     // Mark started steps terminal with real timestamps. Static branches that
     // never started remain non-executed; suspended runs stay in-flight.
     if (result?.status === 'failed') {
@@ -1409,6 +1512,15 @@ export async function collectGatewayRunnerTerminalProgressLedgerEvents(input: {
     }),
     source: 'worker',
   });
+  if (finalUpdate.workCounters) {
+    progressEvents.push({
+      type: 'work.counters',
+      runId: input.launch.runId,
+      source: 'worker',
+      occurredAt: finalUpdate.lastCheckpointAt ?? Date.now(),
+      ...finalUpdate.workCounters,
+    });
+  }
   const datasetEvents: PlayRunLedgerEvent[] = (
     finalUpdate.datasetLifecycleEvents ?? []
   ).map((event) => ({