deepline 0.3.127 → 0.3.129

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +34 -9
  2. package/dist/bundling-sources/sdk/src/http.ts +22 -0
  3. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  4. package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +22 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +265 -2
  6. package/dist/bundling-sources/shared_libs/play-runtime/bounded-batch-function.ts +682 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +21 -5
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5390 -1732
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +107 -1
  10. package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +7 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +229 -7
  12. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +25 -4
  13. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts +71 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +1 -1
  15. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +4 -1
  16. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +694 -28
  17. package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts +167 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +103 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts +34 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts +27 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +66 -1
  22. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +425 -71
  23. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +210 -32
  24. package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +27 -13
  25. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +19 -6
  26. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +61 -2
  27. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +33 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +42 -1
  29. package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +11 -11
  30. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +49 -16
  31. package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +1653 -98
  32. package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +807 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/memory-pressure-signal.ts +138 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +85 -3
  35. package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +60 -2
  36. package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +46 -1
  37. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +2 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
  40. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +1 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +70 -27
  42. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +56 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/adaptive-provider-capacity-controller.ts +192 -0
  44. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/epoch-aimd-rate-controller.ts +194 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/fixture-capacity-simulator.ts +220 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts +214 -0
  47. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts +43 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts +30 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-rate-policy.ts +95 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-throttle-classifier.ts +40 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +1535 -11
  52. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +49 -2
  53. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +6 -1
  54. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +55 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +221 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +339 -29
  57. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +28 -1
  58. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +192 -165
  59. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +6 -0
  60. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +90 -5
  61. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +281 -6
  62. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +27 -1
  63. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +19 -0
  64. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +108 -0
  65. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +40 -0
  66. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +11 -5
  67. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +30 -0
  68. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-compression.ts +276 -0
  69. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +723 -115
  70. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +98 -2
  71. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +3 -2
  72. package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +41 -4
  73. package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +448 -0
  74. package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts +215 -0
  75. package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +11 -0
  76. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +161 -14
  77. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts +184 -0
  78. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +607 -121
  79. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +4 -0
  80. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +21 -0
  81. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +15 -2
  82. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +71 -14
  83. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +2 -2
  84. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +288 -119
  85. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -6
  86. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +28 -3
  87. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts +179 -0
  88. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +80 -2
  89. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +31 -2
  90. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts +104 -0
  91. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts +47 -0
  92. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts +193 -0
  93. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts +420 -0
  94. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +41 -0
  95. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +134 -3
  96. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +1657 -95
  97. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +885 -73
  98. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts +452 -0
  99. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +601 -42
  100. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +1 -0
  101. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +371 -0
  102. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +86 -7
  103. package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +153 -22
  104. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
  105. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +188 -38
  106. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +28 -13
  107. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +44 -1
  108. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +56 -4
  109. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +2 -0
  110. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +16 -10
  111. package/dist/bundling-sources/shared_libs/plays/dataset.ts +4 -1
  112. package/dist/cli/index.js +254 -38
  113. package/dist/cli/index.mjs +254 -38
  114. package/dist/{compiler-manifest-DfSnbCTW.d.mts → compiler-manifest-BIqRyj5m.d.mts} +1 -1
  115. package/dist/{compiler-manifest-DfSnbCTW.d.ts → compiler-manifest-BIqRyj5m.d.ts} +1 -1
  116. package/dist/index.d.mts +4 -2
  117. package/dist/index.d.ts +4 -2
  118. package/dist/index.js +231 -35
  119. package/dist/index.mjs +231 -35
  120. package/dist/install-integrity.json +31 -2
  121. package/dist/plays/bundle-play-file.d.mts +2 -2
  122. package/dist/plays/bundle-play-file.d.ts +2 -2
  123. package/dist/plays/bundle-play-file.mjs +65 -1
  124. package/dist/release.d.mts +1 -1
  125. package/dist/release.d.ts +1 -1
  126. package/dist/release.js +1 -1
  127. package/dist/release.mjs +1 -1
  128. package/package.json +1 -1
@@ -0,0 +1,138 @@
1
+ import { readFileSync, renameSync, writeFileSync } from 'node:fs';
2
+
3
+ /**
4
+ * Runner half of the sandbox memory envelope.
5
+ *
6
+ * The watchdog parent (see `runner-backends/backends/daytona-runtime-watchdog.ts`)
7
+ * samples the cgroup every 250 ms from outside this process. At the pressure
8
+ * ratio it writes the observed sample to `DEEPLINE_RUNTIME_MEMORY_PRESSURE_PATH`
9
+ * and raises SIGUSR2 here; at the abort ratio it terminates this process and
10
+ * attaches whatever `reportMemoryLedger` last wrote to
11
+ * `DEEPLINE_RUNTIME_MEMORY_LEDGER_PATH` to the termination diagnostic.
12
+ *
13
+ * Node's default SIGUSR2 disposition terminates the process, so the handler
14
+ * must exist before the watchdog can ever signal. Install it at module load of
15
+ * the runner entry, not lazily at first map admission.
16
+ *
17
+ * Dependency-free on purpose: this runs inside the customer runner bundle, on
18
+ * a process that is by construction already short of memory.
19
+ */
20
+
21
+ /**
22
+ * Path this process writes its latest byte-ledger snapshot to. Per-run, so an
23
+ * env var rather than a constant; the watchdog passes it to the child.
24
+ */
25
+ export const RUNTIME_MEMORY_LEDGER_PATH_ENV =
26
+ 'DEEPLINE_RUNTIME_MEMORY_LEDGER_PATH';
27
+
28
+ /**
29
+ * Path the watchdog writes the observed usage/limit to immediately before it
30
+ * raises SIGUSR2, so this process reports real cgroup numbers instead of
31
+ * guessing from inside its own heap.
32
+ */
33
+ export const RUNTIME_MEMORY_PRESSURE_PATH_ENV =
34
+ 'DEEPLINE_RUNTIME_MEMORY_PRESSURE_PATH';
35
+
36
+ export type MemoryPressureSample = {
37
+ /** Which cgroup interface the watchdog read, when it reported one. */
38
+ source: string | null;
39
+ /** Observed cgroup usage in bytes, or null when the watchdog wrote nothing. */
40
+ usageBytes: number | null;
41
+ /** Observed cgroup limit in bytes, or null when the watchdog wrote nothing. */
42
+ limitBytes: number | null;
43
+ /** usageBytes / limitBytes, or null when either is unknown. */
44
+ ratio: number | null;
45
+ };
46
+
47
+ export type MemoryPressureListener = (sample: MemoryPressureSample) => void;
48
+
49
+ const EMPTY_SAMPLE: MemoryPressureSample = {
50
+ source: null,
51
+ usageBytes: null,
52
+ limitBytes: null,
53
+ ratio: null,
54
+ };
55
+
56
+ const listeners = new Set<MemoryPressureListener>();
57
+ let handlerInstalled = false;
58
+
59
+ function finiteOrNull(value: unknown): number | null {
60
+ return typeof value === 'number' && Number.isFinite(value) ? value : null;
61
+ }
62
+
63
+ /**
64
+ * Read the sample the watchdog wrote alongside the signal. A missing or
65
+ * unreadable file is not a failure: the signal itself is the instruction, and
66
+ * these numbers are evidence for the log line.
67
+ */
68
+ export function readMemoryPressureSample(): MemoryPressureSample {
69
+ const path = process.env[RUNTIME_MEMORY_PRESSURE_PATH_ENV];
70
+ if (!path) return EMPTY_SAMPLE;
71
+ try {
72
+ const parsed: unknown = JSON.parse(readFileSync(path, 'utf8'));
73
+ if (!parsed || typeof parsed !== 'object') return EMPTY_SAMPLE;
74
+ const record = parsed as Record<string, unknown>;
75
+ return {
76
+ source: typeof record.source === 'string' ? record.source : null,
77
+ usageBytes: finiteOrNull(record.usageBytes),
78
+ limitBytes: finiteOrNull(record.limitBytes),
79
+ ratio: finiteOrNull(record.ratio),
80
+ };
81
+ } catch {
82
+ return EMPTY_SAMPLE;
83
+ }
84
+ }
85
+
86
+ /**
87
+ * Deliver a sample to every subscriber. A throwing subscriber must not stop
88
+ * the others: a memory hold that half-runs is worse than one that is noisy.
89
+ */
90
+ export function emitMemoryPressure(sample: MemoryPressureSample): void {
91
+ for (const listener of Array.from(listeners)) {
92
+ try {
93
+ listener(sample);
94
+ } catch {}
95
+ }
96
+ }
97
+
98
+ /**
99
+ * Install the SIGUSR2 handler. Idempotent, and safe to call before any
100
+ * listener exists — installing it is what stops Node's default disposition
101
+ * from killing the runner on the first pressure signal.
102
+ */
103
+ export function installMemoryPressureHandler(): void {
104
+ if (handlerInstalled) return;
105
+ handlerInstalled = true;
106
+ process.on('SIGUSR2', () => {
107
+ emitMemoryPressure(readMemoryPressureSample());
108
+ });
109
+ }
110
+
111
+ /** Subscribe to watchdog memory-pressure signals. Returns an unsubscribe. */
112
+ export function onMemoryPressure(listener: MemoryPressureListener): () => void {
113
+ installMemoryPressureHandler();
114
+ listeners.add(listener);
115
+ return () => {
116
+ listeners.delete(listener);
117
+ };
118
+ }
119
+
120
+ /**
121
+ * Publish the runtime's latest byte-ledger snapshot for the watchdog to attach
122
+ * to an abort diagnostic. The snapshot is opaque here: the ledger owner
123
+ * decides its shape. Written to a sibling temp file and renamed, so the
124
+ * watchdog never parses a half-written object.
125
+ *
126
+ * Never throws. A failed ledger write must not fail a run.
127
+ */
128
+ export function reportMemoryLedger(snapshot: unknown): void {
129
+ const path = process.env[RUNTIME_MEMORY_LEDGER_PATH_ENV];
130
+ if (!path) return;
131
+ try {
132
+ const serialized = JSON.stringify(snapshot);
133
+ if (serialized === undefined) return;
134
+ const temporaryPath = `${path}.tmp`;
135
+ writeFileSync(temporaryPath, serialized);
136
+ renameSync(temporaryPath, path);
137
+ } catch {}
138
+ }
@@ -38,6 +38,18 @@ export const CUSTOMER_OUTPUT_TOTAL_MAX_BYTES = 5 * 1024 * 1024;
38
38
  export const RUNTIME_RECEIPT_OUTPUT_MAX_BYTES = 10 * 1024 * 1024;
39
39
  /** One completion request may contain multiple receipts up to 32 MiB total. */
40
40
  export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_BYTES = 32 * 1024 * 1024;
41
+ /**
42
+ * A Context can retain at most one 1,000-command logical request for each of
43
+ * its sixteen completion flights. This bounds closures and tiny-result bursts
44
+ * independently of the byte budget above.
45
+ */
46
+ export const RUNTIME_RECEIPT_COMPLETION_BUFFER_MAX_ITEMS = 16_000;
47
+ /**
48
+ * Controls stay small but need their own retained envelope during a full
49
+ * completion burst; this is a reservation, not added completion throughput.
50
+ */
51
+ export const RUNTIME_RECEIPT_CONTROL_BUFFER_MAX_BYTES = 2 * 1024 * 1024;
52
+ export const RUNTIME_RECEIPT_CONTROL_BUFFER_MAX_ITEMS = 2_000;
41
53
  /**
42
54
  * The Fly receipt gateway accepts one runner-terminal control request up to
43
55
  * this size. This is a transport ceiling, not a customer-output contract:
@@ -58,7 +70,49 @@ export const RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_BYTES = 12 * 1024 * 1024;
58
70
  * prevents several multi-megabyte results from monopolizing the serialized
59
71
  * gateway writer behind one request.
60
72
  */
61
- export const RUNTIME_RECEIPT_WRITER_TARGET_BATCH_BYTES = 4 * 1024 * 1024;
73
+ export const RUNTIME_RECEIPT_WRITER_TARGET_BATCH_BYTES = 8 * 1024 * 1024;
74
+ /**
75
+ * Batching window for runtime step receipt completions.
76
+ *
77
+ * A completion writer with no window sends whatever settled in the current
78
+ * event-loop turn, so an idle writer sends singletons: one measured 1k-row
79
+ * fixture run spent 1,479 gateway completion requests on 17,307 receipts, 691
80
+ * of them carrying a single receipt. Holding results for this window lets the
81
+ * results of one provider wave share one request, one gateway heavy-admission
82
+ * slot, and one storage statement. Size and byte triggers still flush at once,
83
+ * so the window bounds latency without bounding throughput.
84
+ *
85
+ * The window is charged to `resultDelivery`, the interval between a parsed
86
+ * provider body and the row that resumes on it. On run 34088584407 (1000-row
87
+ * modal) that interval was 1.57 s mean / 1.61 s p50 against a 0.16 s p50
88
+ * gateway round trip, so the window is a small but real slice of a delivery
89
+ * that is otherwise transport and queue wait. Ten milliseconds still coalesces
90
+ * a provider wave settling in the same tick; zero would restore the
91
+ * singleton-request behaviour measured above.
92
+ */
93
+ export const RUNTIME_RECEIPT_COMPLETION_DEBOUNCE_MS = 10;
94
+ /**
95
+ * Completion batches the runner may hold open against the receipt gateway.
96
+ *
97
+ * Receipts are keyed, so completion order across batches carries no meaning;
98
+ * in-flight batches overlap queue wait with flight. Two were not enough for
99
+ * body-heavy shapes: with ~500 ms per request the runner drained at most two
100
+ * target batches per round trip, and jai (agent and enformion bodies of
101
+ * megabytes) queued 59 s per receipt while jacob queued 14 s (run
102
+ * 34066820247). The gateway coalesces concurrent requests of one org into a
103
+ * single statement under one heavy slot, so a wider runner pipeline no longer
104
+ * multiplies gateway statements. Sixteen, not twelve: a direct 1k-row jai
105
+ * Preview run filled all twelve flights, queued further receipts for 8.8 s at
106
+ * p90, and retained 31.2 MiB of the same 32 MiB byte budget while gateway pool
107
+ * wait p90 was below 1 ms and R2 persistence p90 was 1.3 ms. Sixteen turns
108
+ * retained queued receipts into active bounded flights; it does not enlarge
109
+ * that byte budget or add another persistence path. The temporary transport
110
+ * copy is ledgered independently, so map admission still reacts to the actual
111
+ * resident-memory increase.
112
+ */
113
+ export const RUNTIME_RECEIPT_COMPLETION_MAX_IN_FLIGHT_BATCHES = 16;
114
+ /** Control traffic keeps two physical transports during a completion burst. */
115
+ export const RUNTIME_RECEIPT_CONTROL_MAX_IN_FLIGHT_BATCHES = 2;
62
116
  /**
63
117
  * The terminal `result` line is the crash-recovery record in Daytona stdout.
64
118
  * Once that line is written, retries may still emit runner-owned diagnostics.
@@ -910,6 +964,9 @@ export function assertCustomerPlayReturnWithinLimit(input: {
910
964
  });
911
965
  }
912
966
 
967
+ const RUNTIME_RECEIPT_OUTPUT_SIZE_ADVICE =
968
+ 'This receipt result is too large for durable call caching. This usually means raw search results, HTML, or a large list is being cached as one cell. Parse it before returning it from the tool boundary.';
969
+
913
970
  export function assertRuntimeReceiptOutputWithinLimit(input: {
914
971
  output: unknown;
915
972
  path: string;
@@ -919,7 +976,32 @@ export function assertRuntimeReceiptOutputWithinLimit(input: {
919
976
  value: input.output,
920
977
  path: input.path,
921
978
  limitBytes: input.limitBytes ?? RUNTIME_RECEIPT_OUTPUT_MAX_BYTES,
922
- advice:
923
- 'This receipt result is too large for durable call caching. This usually means raw search results, HTML, or a large list is being cached as one cell. Parse it before returning it from the tool boundary.',
979
+ advice: RUNTIME_RECEIPT_OUTPUT_SIZE_ADVICE,
924
980
  });
925
981
  }
982
+
983
+ /**
984
+ * Enforce the same receipt contract against an already-serialized value.
985
+ *
986
+ * The accepted completion path serializes a receipt output exactly once (the
987
+ * canonical Postgres JSON text that is persisted). Measuring that text instead
988
+ * of re-walking the value keeps the limit exact while removing the duplicate
989
+ * `JSON.stringify` passes that dominated gateway CPU on large results.
990
+ */
991
+ export function assertSerializedRuntimeReceiptOutputWithinLimit(input: {
992
+ serialized: string;
993
+ path: string;
994
+ limitBytes?: number;
995
+ }): number {
996
+ const limitBytes = input.limitBytes ?? RUNTIME_RECEIPT_OUTPUT_MAX_BYTES;
997
+ const bytes = utf8ByteLength(input.serialized);
998
+ if (bytes > limitBytes) {
999
+ throw new OutputTooLargeError({
1000
+ path: input.path,
1001
+ bytes,
1002
+ limitBytes,
1003
+ advice: RUNTIME_RECEIPT_OUTPUT_SIZE_ADVICE,
1004
+ });
1005
+ }
1006
+ return bytes;
1007
+ }
@@ -1,4 +1,5 @@
1
1
  import { isAppRuntimeApiCapacityError } from './app-runtime-api';
2
+ import { RuntimeReceiptWriterCapacityError } from './runtime-receipt-writer';
2
3
 
3
4
  /**
4
5
  * Runtime persistence-failure circuit breaker.
@@ -107,18 +108,75 @@ function formatLatchCause(error: unknown): string {
107
108
  return message.length > 1_000 ? `${message.slice(0, 1_000)}…` : message;
108
109
  }
109
110
 
111
+ type RuntimePersistenceLatchListener = (cause: unknown) => void;
112
+
113
+ // Kept beside the latch rather than on it: the latch is a plain data object
114
+ // that tests compare structurally, and the error object / listener set are
115
+ // process-local state that must not travel with a serialized latch.
116
+ const latchCauseErrors = new WeakMap<RuntimePersistenceLatch, unknown>();
117
+ const latchTripListeners = new WeakMap<
118
+ RuntimePersistenceLatch,
119
+ Set<RuntimePersistenceLatchListener>
120
+ >();
121
+
110
122
  /**
111
123
  * Trip the latch on the first persistence failure. Idempotent: later trips are
112
- * no-ops so `cause` keeps the first, root failure.
124
+ * no-ops so `cause` keeps the first, root failure. Listeners registered with
125
+ * `onRuntimePersistenceLatchTripped` run once, synchronously, on that first
126
+ * trip, so a map can fail fast and its writers can reject their pending work
127
+ * with this same cause instead of waiting for it to reach them by accident.
113
128
  */
114
129
  export function tripRuntimePersistenceLatch(
115
130
  latch: RuntimePersistenceLatch,
116
131
  error: unknown,
117
132
  ): void {
118
- if (isAppRuntimeApiCapacityError(error)) return;
133
+ if (
134
+ isAppRuntimeApiCapacityError(error) ||
135
+ error instanceof RuntimeReceiptWriterCapacityError
136
+ ) {
137
+ return;
138
+ }
119
139
  if (latch.tripped) return;
120
140
  latch.tripped = true;
121
141
  latch.cause = formatLatchCause(error);
142
+ latchCauseErrors.set(latch, error);
143
+ const listeners = latchTripListeners.get(latch);
144
+ if (!listeners) return;
145
+ for (const listener of [...listeners]) listener(error);
146
+ }
147
+
148
+ /**
149
+ * The error object behind `latch.cause`. A latch tripped through this module
150
+ * always has one; a latch constructed elsewhere and marked tripped by hand gets
151
+ * an Error carrying its formatted cause so callers can still reject with it.
152
+ */
153
+ export function runtimePersistenceLatchCauseError(
154
+ latch: RuntimePersistenceLatch,
155
+ ): unknown {
156
+ if (latchCauseErrors.has(latch)) return latchCauseErrors.get(latch);
157
+ return new Error(latch.cause ?? 'Runtime persistence failed.');
158
+ }
159
+
160
+ /**
161
+ * Observe the latch's first trip. The listener does NOT run for a latch that
162
+ * is already tripped when it subscribes: a caller that cares about that state
163
+ * reads `latch.tripped` itself, because "tripped before I started" and
164
+ * "tripped while I was running" call for different reactions (a map started
165
+ * after a trip still dispatches so the circuit breaker names the refusal).
166
+ */
167
+ export function onRuntimePersistenceLatchTripped(
168
+ latch: RuntimePersistenceLatch,
169
+ listener: RuntimePersistenceLatchListener,
170
+ ): () => void {
171
+ let listeners = latchTripListeners.get(latch);
172
+ if (!listeners) {
173
+ listeners = new Set();
174
+ latchTripListeners.set(latch, listeners);
175
+ }
176
+ listeners.add(listener);
177
+ return () => {
178
+ listeners.delete(listener);
179
+ };
122
180
  }
123
181
 
124
182
  /**
@@ -42,6 +42,7 @@ export type PlayRunRecoveryReason =
42
42
  | 'modal_sandbox_unavailable_before_execution'
43
43
  | 'modal_runner_start_failed_before_execution'
44
44
  | 'daytona_sandbox_transition_pending_before_execution'
45
+ | 'daytona_sandbox_acquisition_unavailable_before_execution'
45
46
  | 'runtime_initialization_transport_before_execution'
46
47
  | 'runtime_traffic_hold_before_execution'
47
48
  | 'runtime_traffic_throttled_before_execution';
@@ -231,6 +232,49 @@ export function recoveryForDaytonaSandboxTransitionPending(
231
232
  });
232
233
  }
233
234
 
235
+ /**
236
+ * Daytona proved that a fresh sandbox was not acquired, but the provider's
237
+ * bounded create ladder was exhausted on a shared capacity/rate/startup
238
+ * condition. Keep this distinct from an ordinary runner failure: no customer
239
+ * code could have started, so the durable scheduler must defer the same run
240
+ * instead of consuming the engine-attempt budget and terminalizing it.
241
+ *
242
+ * This classifier intentionally uses the serialized error shape rather than
243
+ * importing the Daytona backend. The app-owned scheduler and the provider
244
+ * adapter must remain separate, and placement backends may still consume the
245
+ * raw error for an explicitly configured fallback provider before it reaches
246
+ * this boundary.
247
+ */
248
+ export function recoveryForDaytonaSandboxAcquisitionUnavailable(
249
+ error: unknown,
250
+ ): PlayRunRecoveryError | null {
251
+ if (!error || typeof error !== 'object') return null;
252
+ const candidate = error as {
253
+ name?: unknown;
254
+ reason?: unknown;
255
+ };
256
+ if (
257
+ candidate.name !== 'DaytonaSandboxAcquisitionUnavailableError' ||
258
+ (candidate.reason !== 'daytona_total_cpu_limit_exceeded' &&
259
+ candidate.reason !== 'daytona_acquisition_rate_limited' &&
260
+ candidate.reason !== 'daytona_sandbox_start_timeout')
261
+ ) {
262
+ return null;
263
+ }
264
+ return new PlayRunRecoveryError({
265
+ decision: {
266
+ stage: 'pre_code',
267
+ action: 'defer',
268
+ reason: 'daytona_sandbox_acquisition_unavailable_before_execution',
269
+ dependency: 'runtime',
270
+ },
271
+ message:
272
+ `Daytona sandbox acquisition is temporarily unavailable before customer ` +
273
+ `execution began (${candidate.reason}).`,
274
+ cause: error,
275
+ });
276
+ }
277
+
234
278
  /**
235
279
  * This is constructed only by the detached-runner readiness check. That
236
280
  * check has not parked the runner or observed scheduler liveness, so it is a
@@ -264,7 +308,8 @@ export function recoveryForRuntimeInitializationTransportFailure(
264
308
  | 'runtime_api_transport'
265
309
  | 'runtime_receipt_gateway_transport'
266
310
  | 'daytona_public_status'
267
- | 'daytona_runner_startup_transport',
311
+ | 'daytona_runner_startup_transport'
312
+ | 'daytona_infrastructure',
268
313
  ): PlayRunRecoveryError {
269
314
  return new PlayRunRecoveryError({
270
315
  decision: {
@@ -53,6 +53,8 @@ export type PlayRunTerminalInput = {
53
53
  attempt: number;
54
54
  authority?: PlayRunTerminalAuthority | null;
55
55
  result: unknown;
56
+ /** Completed-run progress already folded by the receipt gateway. */
57
+ terminalProgressEvents?: readonly unknown[];
56
58
  };
57
59
 
58
60
  export type PlayRunTerminalJob = {
@@ -34,6 +34,7 @@ export function createPlayRunTerminalJob(
34
34
  attempt: input.attempt,
35
35
  authority,
36
36
  result: input.result,
37
+ terminalProgressEvents: input.terminalProgressEvents,
37
38
  queueName: input.queueName,
38
39
  // An authority exists only for the one terminal contract this
39
40
  // build recognizes, and that contract is what proves the durable
@@ -22,6 +22,7 @@ export type DirectTerminalCommitter = {
22
22
  attempt: number;
23
23
  authority: PlayRunTerminalAuthority;
24
24
  result: unknown;
25
+ terminalProgressEvents?: readonly unknown[];
25
26
  queueName: string;
26
27
  allowCappedCompletion: boolean;
27
28
  }): Promise<DirectTerminalCommitOutcome>;
@@ -29,6 +29,7 @@ export function createPostgresPlayRunTerminalPorts(input: {
29
29
  attempt: terminal.attempt,
30
30
  leaseToken: terminal.authority.leaseToken,
31
31
  result: terminal.result,
32
+ terminalProgressEvents: terminal.terminalProgressEvents,
32
33
  terminalLogOutbox: 'ctx_log_tail',
33
34
  allowCappedCompletion: terminal.allowCappedCompletion,
34
35
  engineWake: {
@@ -34,11 +34,24 @@ export type PlayRuntimeSemantics = Readonly<{
34
34
  */
35
35
  receiptIdentityEdition: PlayAuthoringContractEdition | null;
36
36
  }>;
37
+ inlineChild: Readonly<{
38
+ /**
39
+ * Whether a scalar child may rely solely on its execution-time
40
+ * capabilities instead of a stored static pipeline. The runtime still
41
+ * rejects lifecycle-owning APIs at their own effect boundary.
42
+ */
43
+ runtimeOnly: boolean;
44
+ }>;
37
45
  slackHitl: Readonly<{
38
46
  defaultTimeoutMs: number;
39
47
  }>;
40
48
  }>;
41
49
 
50
+ type PlayRuntimeSemanticsTemplate = Omit<
51
+ PlayRuntimeSemantics,
52
+ 'authoringContractEdition'
53
+ >;
54
+
42
55
  function definePlayRuntimeSemantics(
43
56
  semantics: PlayRuntimeSemantics,
44
57
  ): PlayRuntimeSemantics {
@@ -46,6 +59,7 @@ function definePlayRuntimeSemantics(
46
59
  ...semantics,
47
60
  secrets: Object.freeze({ ...semantics.secrets }),
48
61
  fetch: Object.freeze({ ...semantics.fetch }),
62
+ inlineChild: Object.freeze({ ...semantics.inlineChild }),
49
63
  slackHitl: Object.freeze({ ...semantics.slackHitl }),
50
64
  });
51
65
  }
@@ -58,6 +72,7 @@ const LEGACY_SEMANTICS = {
58
72
  persistHttpFailure: true,
59
73
  receiptIdentityEdition: null,
60
74
  },
75
+ inlineChild: { runtimeOnly: false },
61
76
  } as const;
62
77
 
63
78
  const STRICT_SEMANTICS = {
@@ -78,6 +93,13 @@ const WEEK_LONG_SLACK_HITL_SEMANTICS = {
78
93
  defaultTimeoutMs: 7 * 24 * 60 * 60 * 1000,
79
94
  } as const;
80
95
 
96
+ // These are historical receipt-key partitions, named so later authoring
97
+ // changes can explicitly retain them instead of creating another wire tag.
98
+ const FETCH_RECEIPT_IDENTITIES = {
99
+ firstTypedHttpFailure: 5 as PlayAuthoringContractEdition,
100
+ typedHttpFailure: 6 as PlayAuthoringContractEdition,
101
+ } as const;
102
+
81
103
  /**
82
104
  * Exhaustive compatibility registry for worker-visible Play behavior.
83
105
  *
@@ -85,39 +107,33 @@ const WEEK_LONG_SLACK_HITL_SEMANTICS = {
85
107
  * declared here. Entries are fully materialized and frozen so every context
86
108
  * using the same edition shares one cached contract projection.
87
109
  */
88
- const PLAY_RUNTIME_SEMANTICS_BY_EDITION = {
89
- 1: definePlayRuntimeSemantics({
90
- authoringContractEdition: 1,
110
+ const PLAY_RUNTIME_SEMANTICS_BY_AUTHORING = {
111
+ 1: {
91
112
  ...LEGACY_SEMANTICS,
92
113
  slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
93
- }),
94
- 2: definePlayRuntimeSemantics({
95
- authoringContractEdition: 2,
114
+ },
115
+ 2: {
96
116
  ...STRICT_SEMANTICS,
97
117
  slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
98
- }),
99
- 3: definePlayRuntimeSemantics({
100
- authoringContractEdition: 3,
118
+ },
119
+ 3: {
101
120
  ...STRICT_SEMANTICS,
102
121
  slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
103
- }),
104
- 4: definePlayRuntimeSemantics({
105
- authoringContractEdition: 4,
122
+ },
123
+ 4: {
106
124
  ...PLAINTEXT_SECRET_SEMANTICS,
107
125
  slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
108
- }),
109
- 5: definePlayRuntimeSemantics({
110
- authoringContractEdition: 5,
126
+ },
127
+ 5: {
111
128
  ...PLAINTEXT_SECRET_SEMANTICS,
112
129
  slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
113
130
  fetch: {
114
131
  httpFailure: 'throw_typed_error',
115
132
  persistHttpFailure: false,
116
- receiptIdentityEdition: 5,
133
+ receiptIdentityEdition: FETCH_RECEIPT_IDENTITIES.firstTypedHttpFailure,
117
134
  },
118
- }),
119
- 6: definePlayRuntimeSemantics({
120
- authoringContractEdition: 6,
135
+ },
136
+ 6: {
121
137
  ...PLAINTEXT_SECRET_SEMANTICS,
122
138
  slackHitl: LEGACY_SLACK_HITL_SEMANTICS,
123
139
  fetch: {
@@ -125,20 +141,47 @@ const PLAY_RUNTIME_SEMANTICS_BY_EDITION = {
125
141
  persistHttpFailure: false,
126
142
  // Preserve the existing edition-6 receipt identity. Future unrelated
127
143
  // editions may retain this epoch rather than partitioning the cache.
128
- receiptIdentityEdition: 6,
144
+ receiptIdentityEdition: FETCH_RECEIPT_IDENTITIES.typedHttpFailure,
129
145
  },
130
- }),
131
- 7: definePlayRuntimeSemantics({
132
- authoringContractEdition: 7,
146
+ },
147
+ 7: {
133
148
  ...PLAINTEXT_SECRET_SEMANTICS,
149
+ fetch: {
150
+ httpFailure: 'throw_typed_error',
151
+ persistHttpFailure: false,
152
+ receiptIdentityEdition: FETCH_RECEIPT_IDENTITIES.typedHttpFailure,
153
+ },
134
154
  slackHitl: WEEK_LONG_SLACK_HITL_SEMANTICS,
155
+ },
156
+ 8: {
157
+ ...PLAINTEXT_SECRET_SEMANTICS,
135
158
  fetch: {
136
159
  httpFailure: 'throw_typed_error',
137
160
  persistHttpFailure: false,
138
- receiptIdentityEdition: 6,
161
+ // Inline-child admission does not change fetch receipt identity.
162
+ receiptIdentityEdition: FETCH_RECEIPT_IDENTITIES.typedHttpFailure,
163
+ },
164
+ inlineChild: { runtimeOnly: true },
165
+ slackHitl: WEEK_LONG_SLACK_HITL_SEMANTICS,
166
+ },
167
+ } satisfies Record<PlayAuthoringContractEdition, PlayRuntimeSemanticsTemplate>;
168
+
169
+ const PLAY_RUNTIME_SEMANTICS = Object.fromEntries(
170
+ Object.entries(PLAY_RUNTIME_SEMANTICS_BY_AUTHORING).map(
171
+ ([storedEdition, semantics]) => {
172
+ const authoringContractEdition = normalizePlayAuthoringContractEdition(
173
+ Number(storedEdition),
174
+ );
175
+ return [
176
+ authoringContractEdition,
177
+ definePlayRuntimeSemantics({
178
+ authoringContractEdition,
179
+ ...semantics,
180
+ }),
181
+ ];
139
182
  },
140
- }),
141
- } satisfies Record<PlayAuthoringContractEdition, PlayRuntimeSemantics>;
183
+ ),
184
+ ) as Record<PlayAuthoringContractEdition, PlayRuntimeSemantics>;
142
185
 
143
186
  export function resolvePlayRuntimeSemantics(
144
187
  authoringContractEdition: unknown,
@@ -146,5 +189,5 @@ export function resolvePlayRuntimeSemantics(
146
189
  const edition = normalizePlayAuthoringContractEdition(
147
190
  authoringContractEdition,
148
191
  );
149
- return PLAY_RUNTIME_SEMANTICS_BY_EDITION[edition];
192
+ return PLAY_RUNTIME_SEMANTICS[edition];
150
193
  }