deepline 0.3.128 → 0.3.129

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +34 -9
  2. package/dist/bundling-sources/sdk/src/http.ts +22 -0
  3. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  4. package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +22 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +265 -2
  6. package/dist/bundling-sources/shared_libs/play-runtime/bounded-batch-function.ts +682 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +21 -5
  8. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5390 -1732
  9. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +107 -1
  10. package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +7 -0
  11. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +229 -7
  12. package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +25 -4
  13. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts +71 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +1 -1
  15. package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +4 -1
  16. package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +694 -28
  17. package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts +167 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +103 -0
  19. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts +34 -0
  20. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts +27 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +66 -1
  22. package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +425 -71
  23. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +210 -32
  24. package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +27 -13
  25. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +19 -6
  26. package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +61 -2
  27. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +33 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +42 -1
  29. package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +11 -11
  30. package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +49 -16
  31. package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +1653 -98
  32. package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +807 -0
  33. package/dist/bundling-sources/shared_libs/play-runtime/memory-pressure-signal.ts +138 -0
  34. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +85 -3
  35. package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +60 -2
  36. package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +46 -1
  37. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +2 -0
  38. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
  39. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
  40. package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +1 -0
  41. package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +70 -27
  42. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +56 -0
  43. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/adaptive-provider-capacity-controller.ts +192 -0
  44. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/epoch-aimd-rate-controller.ts +194 -0
  45. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/fixture-capacity-simulator.ts +220 -0
  46. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts +214 -0
  47. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts +43 -0
  48. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts +30 -0
  49. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-rate-policy.ts +95 -0
  50. package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-throttle-classifier.ts +40 -0
  51. package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +1535 -11
  52. package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +49 -2
  53. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +6 -1
  54. package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +55 -0
  55. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +221 -0
  56. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +339 -29
  57. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +28 -1
  58. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +192 -165
  59. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +6 -0
  60. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +90 -5
  61. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +281 -6
  62. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +27 -1
  63. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +19 -0
  64. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +108 -0
  65. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +40 -0
  66. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +11 -5
  67. package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +30 -0
  68. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-compression.ts +276 -0
  69. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +723 -115
  70. package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +98 -2
  71. package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +3 -2
  72. package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +41 -4
  73. package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +448 -0
  74. package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts +215 -0
  75. package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +11 -0
  76. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +161 -14
  77. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts +184 -0
  78. package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +607 -121
  79. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +4 -0
  80. package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +21 -0
  81. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +15 -2
  82. package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +71 -14
  83. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +2 -2
  84. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +288 -119
  85. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -6
  86. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +28 -3
  87. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts +179 -0
  88. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +80 -2
  89. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +31 -2
  90. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts +104 -0
  91. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts +47 -0
  92. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts +193 -0
  93. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts +420 -0
  94. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +41 -0
  95. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +134 -3
  96. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +1657 -95
  97. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +885 -73
  98. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts +452 -0
  99. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +601 -42
  100. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +1 -0
  101. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +371 -0
  102. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +86 -7
  103. package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +153 -22
  104. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
  105. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +188 -38
  106. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +28 -13
  107. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +44 -1
  108. package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +56 -4
  109. package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +2 -0
  110. package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +16 -10
  111. package/dist/bundling-sources/shared_libs/plays/dataset.ts +4 -1
  112. package/dist/cli/index.js +250 -38
  113. package/dist/cli/index.mjs +250 -38
  114. package/dist/{compiler-manifest-DfSnbCTW.d.mts → compiler-manifest-BIqRyj5m.d.mts} +1 -1
  115. package/dist/{compiler-manifest-DfSnbCTW.d.ts → compiler-manifest-BIqRyj5m.d.ts} +1 -1
  116. package/dist/index.d.mts +4 -2
  117. package/dist/index.d.ts +4 -2
  118. package/dist/index.js +231 -35
  119. package/dist/index.mjs +231 -35
  120. package/dist/install-integrity.json +31 -2
  121. package/dist/plays/bundle-play-file.d.mts +2 -2
  122. package/dist/plays/bundle-play-file.d.ts +2 -2
  123. package/dist/plays/bundle-play-file.mjs +65 -1
  124. package/dist/release.d.mts +1 -1
  125. package/dist/release.d.ts +1 -1
  126. package/dist/release.js +1 -1
  127. package/dist/release.mjs +1 -1
  128. package/package.json +1 -1
@@ -31,6 +31,23 @@ const SENSITIVE_FIELD_NAME_RE = new RegExp(
31
31
  `(^|[._-])(?:${SENSITIVE_FIELD_NAME})$`,
32
32
  'i',
33
33
  );
34
+
35
+ // Receipt payloads repeat the same handful of field names once per row, so the
36
+ // per-key verdict is memoized. The cap keeps a pathological payload of unique
37
+ // keys from turning the cache into a leak.
38
+ const SENSITIVE_FIELD_NAME_CACHE_MAX_ENTRIES = 4_096;
39
+ const sensitiveFieldNameCache = new Map<string, boolean>();
40
+
41
+ function isSensitiveFieldName(key: string): boolean {
42
+ const cached = sensitiveFieldNameCache.get(key);
43
+ if (cached !== undefined) return cached;
44
+ const sensitive = SENSITIVE_FIELD_NAME_RE.test(key);
45
+ if (sensitiveFieldNameCache.size >= SENSITIVE_FIELD_NAME_CACHE_MAX_ENTRIES) {
46
+ sensitiveFieldNameCache.clear();
47
+ }
48
+ sensitiveFieldNameCache.set(key, sensitive);
49
+ return sensitive;
50
+ }
34
51
  const SERIALIZED_SECRET_FIELD_RE = new RegExp(
35
52
  `"(${SENSITIVE_FIELD_NAME})"\\s*:\\s*(?:"[^"\\\\]*"|\\{[^{}]*\\})`,
36
53
  'gi',
@@ -40,7 +57,61 @@ function escapeRegExp(value: string): string {
40
57
  return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
41
58
  }
42
59
 
60
+ /**
61
+ * Cheap superset gate for every rule `redactSecretLikeString` applies.
62
+ *
63
+ * Each rewrite above needs one of these markers to fire, so a string matching
64
+ * none of them is returned byte-identical by the full chain. Running one test
65
+ * instead of ten replaces is what makes receipt sanitization affordable: a
66
+ * 600 KiB provider result is thousands of strings, and almost none of them
67
+ * carry a credential.
68
+ *
69
+ * Extra matches only cost the full chain, while a missed marker would persist
70
+ * a live credential, so keep every entry looser than the rule it guards
71
+ * wherever JSON escaping or whitespace could hide one.
72
+ */
73
+ const REDACTION_TRIGGER_RE = new RegExp(
74
+ [
75
+ // Sensitive field names, in object keys and in `name=`/`"name":` text.
76
+ 'token',
77
+ 'secret',
78
+ 'password',
79
+ 'authorization',
80
+ 'cookie',
81
+ 'credential',
82
+ // One `key` branch instead of `api[_-]?key`, `private[_-]?key` and `key_`.
83
+ // A regex alternation costs roughly one substring scan per branch, and a
84
+ // broader branch can only send more payloads down the exact path.
85
+ 'key',
86
+ // Self-identifying credential prefixes.
87
+ 'bearer',
88
+ 'basic',
89
+ '-----BEGIN',
90
+ 'xox[baprs]-',
91
+ '[prs]k_',
92
+ 'pat_',
93
+ 'ghp_',
94
+ // URL credentials: `//user:pass@host` userinfo and sensitive query params.
95
+ // The userinfo class excludes `/` exactly as the rule it guards does.
96
+ '//[^\\s/]{0,256}@',
97
+ '[?#&]sig',
98
+ // JWTs carry no keyword at all, so the shape itself is the marker. Anchor
99
+ // on the separating dot and drop JWT_RE's leading `{8,}` segment: a dot is
100
+ // rare enough for the engine to skip between candidates, which makes this
101
+ // gate roughly ten times cheaper than re-running JWT_RE itself, and
102
+ // dropping a required segment can only widen what it accepts.
103
+ '\\.[A-Za-z0-9_-]{1,16384}\\.[A-Za-z0-9_-]{8}',
104
+ ].join('|'),
105
+ 'i',
106
+ );
107
+
108
+ /** True when any redaction rule could rewrite `value`. Never a false negative. */
109
+ export function mightContainRedactableSecret(value: string): boolean {
110
+ return REDACTION_TRIGGER_RE.test(value);
111
+ }
112
+
43
113
  export function redactSecretLikeString(value: string): string {
114
+ if (!mightContainRedactableSecret(value)) return value;
44
115
  const serializedRedacted = value.replace(
45
116
  SERIALIZED_SECRET_FIELD_RE,
46
117
  `"$1":"${SECRET_REDACTION_PLACEHOLDER}"`,
@@ -91,11 +162,39 @@ export function createSecretRedactionContext(
91
162
  initialValues: readonly string[] = [],
92
163
  ): SecretRedactionContext {
93
164
  const exactSecrets = new Set<string>();
165
+ // One compiled matcher per registered secret, built on first use. Rebuilding
166
+ // `new RegExp(escapeRegExp(secret))` for every string of every receipt was
167
+ // pure waste once a run registers more than a couple of credentials.
168
+ const exactSecretMatchers = new Map<
169
+ string,
170
+ { readonly literal: string; readonly matcher: RegExp }[]
171
+ >();
94
172
 
95
173
  function register(value: string): void {
96
174
  if (value.length > 0) exactSecrets.add(value);
97
175
  }
98
176
 
177
+ function matchersForSecret(
178
+ secret: string,
179
+ ): { readonly literal: string; readonly matcher: RegExp }[] {
180
+ const cached = exactSecretMatchers.get(secret);
181
+ if (cached) return cached;
182
+ const literals = [secret];
183
+ try {
184
+ const encoded = encodeURIComponent(secret);
185
+ if (encoded !== secret) literals.push(encoded);
186
+ } catch {
187
+ // encodeURIComponent only fails on malformed surrogate pairs. Exact
188
+ // redaction of the raw value still applies in that rare case.
189
+ }
190
+ const built = literals.map((literal) => ({
191
+ literal,
192
+ matcher: new RegExp(escapeRegExp(literal), 'g'),
193
+ }));
194
+ exactSecretMatchers.set(secret, built);
195
+ return built;
196
+ }
197
+
99
198
  for (const value of initialValues) register(value);
100
199
 
101
200
  function containsRegisteredSecret(
@@ -125,24 +224,20 @@ export function createSecretRedactionContext(
125
224
  return exactSecrets.has(value);
126
225
  }
127
226
 
227
+ /**
228
+ * Replace registered secrets in registration order, exactly as a per-secret
229
+ * replace chain would. `includes` is a native substring scan, so a value
230
+ * that carries no secret pays one scan per secret instead of compiling and
231
+ * running a fresh regex per secret.
232
+ */
128
233
  function redactRegisteredSecrets(value: string): string {
234
+ if (exactSecrets.size === 0) return value;
129
235
  let output = value;
130
236
  for (const secret of exactSecrets) {
131
- output = output.replace(
132
- new RegExp(escapeRegExp(secret), 'g'),
133
- SECRET_REDACTION_PLACEHOLDER,
134
- );
135
- try {
136
- const encoded = encodeURIComponent(secret);
137
- if (encoded !== secret) {
138
- output = output.replace(
139
- new RegExp(escapeRegExp(encoded), 'g'),
140
- SECRET_REDACTION_PLACEHOLDER,
141
- );
142
- }
143
- } catch {
144
- // encodeURIComponent only fails on malformed surrogate pairs. Exact
145
- // redaction above still applies in that rare case.
237
+ for (const { literal, matcher } of matchersForSecret(secret)) {
238
+ if (!output.includes(literal)) continue;
239
+ matcher.lastIndex = 0;
240
+ output = output.replace(matcher, SECRET_REDACTION_PLACEHOLDER);
146
241
  }
147
242
  }
148
243
  return output;
@@ -153,17 +248,53 @@ export function createSecretRedactionContext(
153
248
  return redactSecretLikeString(output);
154
249
  }
155
250
 
251
+ /**
252
+ * Redact in place-preserving form: a plain object or array with nothing to
253
+ * redact is returned by reference instead of being rebuilt.
254
+ *
255
+ * This is what keeps receipt sanitization off the deep-clone path. A 600 KiB
256
+ * provider result almost never contains a credential, and cloning it only to
257
+ * re-serialize the identical value dominated gateway CPU.
258
+ *
259
+ * Non-plain objects (Date, Map, class instances) still go through the
260
+ * `Object.entries` rebuild, because collapsing them to plain objects is the
261
+ * long-standing persisted shape and changing it would rewrite stored data.
262
+ */
156
263
  function redact<T>(value: T): T {
157
264
  if (typeof value === 'string') return redactString(value) as T;
158
- if (Array.isArray(value)) return value.map((entry) => redact(entry)) as T;
265
+ if (Array.isArray(value)) {
266
+ let changed = false;
267
+ const next = new Array(value.length);
268
+ for (let index = 0; index < value.length; index += 1) {
269
+ const entry = value[index];
270
+ const redacted = redact(entry);
271
+ next[index] = redacted;
272
+ if (redacted !== entry) changed = true;
273
+ }
274
+ return changed ? (next as T) : value;
275
+ }
159
276
  if (value && typeof value === 'object') {
277
+ const prototype = Object.getPrototypeOf(value);
278
+ // Only a plain object can be returned by reference; anything else must
279
+ // keep its historical projection to a plain object.
280
+ let changed = prototype !== Object.prototype && prototype !== null;
281
+ const source = value as Record<string, unknown>;
282
+ const keys = Object.keys(source);
283
+ const redactedValues = new Array<unknown>(keys.length);
284
+ for (let index = 0; index < keys.length; index += 1) {
285
+ const key = keys[index];
286
+ const entry = source[key];
287
+ const redacted = isSensitiveFieldName(key)
288
+ ? SECRET_REDACTION_PLACEHOLDER
289
+ : redact(entry);
290
+ redactedValues[index] = redacted;
291
+ if (redacted !== entry) changed = true;
292
+ }
293
+ if (!changed) return value;
294
+ // `Object.fromEntries` defines own properties, so a `__proto__` key stays
295
+ // data instead of reassigning the prototype of the rebuilt object.
160
296
  return Object.fromEntries(
161
- Object.entries(value).map(([key, entry]) => [
162
- key,
163
- SENSITIVE_FIELD_NAME_RE.test(key)
164
- ? SECRET_REDACTION_PLACEHOLDER
165
- : redact(entry),
166
- ]),
297
+ keys.map((key, index) => [key, redactedValues[index]]),
167
298
  ) as T;
168
299
  }
169
300
  return value;
@@ -20,6 +20,7 @@ export {
20
20
  type ToolCallReceiptCohortMember,
21
21
  type ToolCallReceiptCohortMemberState,
22
22
  type ToolCallReceiptCohortStore,
23
+ type ToolCallReceiptCohortTimings,
23
24
  type ToolCallReceiptRecord,
24
25
  } from './receipt-cohort';
25
26
  export {
@@ -20,10 +20,27 @@ export type ToolCallReceiptCohortMember = {
20
20
  waitMaxAttempts?: number;
21
21
  };
22
22
 
23
+ /**
24
+ * One receipt's own durable outcome, reported as it settles rather than when
25
+ * the batch carrying it does.
26
+ */
27
+ export type ToolCallReceiptCohortSettlement =
28
+ | { resultReceiptKey: string; status: 'stored' }
29
+ | { resultReceiptKey: string; status: 'failed'; error: unknown };
30
+
23
31
  export type ToolCallReceiptCohortStore = {
24
32
  read(input: {
25
33
  resultReceiptKeys: string[];
26
34
  }): Promise<ToolCallReceiptRecord[]>;
35
+ /**
36
+ * Publish these results to the completed-result cache.
37
+ *
38
+ * Deliberately returns nothing. A completed-result insert is immutable, so a
39
+ * caller that already executed keeps its own result whether or not it won
40
+ * the insert (see `flushPublications`). Materializing the stored rows here
41
+ * only bought a second deserialize-and-wrap of every result, inside the
42
+ * delivery interval a row is blocked on.
43
+ */
27
44
  complete(input: {
28
45
  ownerRunId: string;
29
46
  ownerAttempt: number;
@@ -32,7 +49,23 @@ export type ToolCallReceiptCohortStore = {
32
49
  lease: ToolCallReceiptLease;
33
50
  result: ToolExecuteResult;
34
51
  }>;
35
- }): Promise<ToolCallReceiptRecord[]>;
52
+ /**
53
+ * Report each receipt the instant its OWN durable write settles.
54
+ *
55
+ * The cohort hands the store a coalesced wave, but the store batches by
56
+ * arrival underneath: two receipts of one wave can land in different
57
+ * batches, and the earlier one is durable a whole flight before the later
58
+ * one. Settling the wave together made every row in it wait for the last
59
+ * batch, so a store that can separate them calls this and each publication
60
+ * resolves on its own receipt.
61
+ *
62
+ * Optional. A store that cannot separate them simply resolves or rejects
63
+ * the returned promise, and every receipt it did not report settles at
64
+ * that point — successfully when the promise resolves, with its error
65
+ * when it rejects.
66
+ */
67
+ onReceiptSettled?(settlement: ToolCallReceiptCohortSettlement): void;
68
+ }): Promise<void>;
36
69
  };
37
70
 
38
71
  export type ToolCallReceiptCohortMemberState =
@@ -70,6 +103,21 @@ export type ToolCallReceiptCohort = {
70
103
  settle(): Promise<void>;
71
104
  };
72
105
 
106
+ /**
107
+ * Measurement stamp for the end of a publication, which is otherwise invisible
108
+ * from outside: a coalesced `store.complete` settles here, and only the cohort
109
+ * knows which members it carried. A caller uses it to open its own `write
110
+ * settled -> row resumed` interval. Stamps only: nothing here is read by the
111
+ * cohort, and it fires on a failed write too, so the interval it opens is
112
+ * closed on both outcomes.
113
+ */
114
+ export type ToolCallReceiptCohortTimings = {
115
+ onCompleteSettled?(input: {
116
+ memberIds: readonly string[];
117
+ atMs: number;
118
+ }): void;
119
+ };
120
+
73
121
  export type ToolCallReceiptCohortJob = {
74
122
  acquire(input: {
75
123
  owner: { runId: string; attempt: number };
@@ -86,6 +134,7 @@ export type ToolCallReceiptCohortJob = {
86
134
  */
87
135
  export function createToolCallReceiptCohortJob(input: {
88
136
  store: ToolCallReceiptCohortStore;
137
+ timings?: ToolCallReceiptCohortTimings;
89
138
  }): ToolCallReceiptCohortJob {
90
139
  return {
91
140
  async acquire({ owner, members }) {
@@ -102,12 +151,20 @@ export function createToolCallReceiptCohortJob(input: {
102
151
  }
103
152
 
104
153
  const groups = groupMembers(members);
154
+ // A forced member deliberately discards any completed cache value and
155
+ // executes a fresh provider operation. Reading it first is not merely
156
+ // redundant: a large persisted result is fully hydrated and wrapped
157
+ // before the force branch below ignores it. Keep completed-result cache
158
+ // reads scoped to groups that can actually consume their values. This
159
+ // also leaves a mixed group correct: `groupMembers` marks it forced when
160
+ // any duplicate member explicitly requires fresh execution.
161
+ const cacheReadKeys = [...groups.values()]
162
+ .filter((group) => !group.force)
163
+ .map((group) => group.resultReceiptKey);
105
164
  const records = byKey(
106
- await input.store.read({
107
- resultReceiptKeys: [...groups.values()].map(
108
- (group) => group.resultReceiptKey,
109
- ),
110
- }),
165
+ cacheReadKeys.length > 0
166
+ ? await input.store.read({ resultReceiptKeys: cacheReadKeys })
167
+ : [],
111
168
  );
112
169
  const states = new Map<string, ToolCallReceiptCohortMemberState>();
113
170
  const deferred = new Map<string, Deferred<ToolExecuteResult>>();
@@ -168,49 +225,128 @@ export function createToolCallReceiptCohortJob(input: {
168
225
  publicationFlushScheduled = false;
169
226
  const pending = pendingPublications.splice(0);
170
227
  if (pending.length === 0) return;
228
+ // Which receipts each publication is waiting on, resolved before the
229
+ // write so a publication can be settled the moment ITS receipts are
230
+ // durable rather than when the whole wave is. A lease lost between the
231
+ // queue and this flush is a pre-write fault of the wave, so it still
232
+ // rejects every publication in it.
233
+ const claims: PublicationClaim[] = [];
234
+ const requestedByKey = new Map<string, RequestedReceipt>();
171
235
  try {
172
- const requested = ownedEntries(
173
- pending.flatMap((publication) => publication.entries),
174
- states,
175
- owned,
176
- );
177
- const persisted = byKey(
178
- await input.store.complete({
179
- ownerRunId,
180
- ownerAttempt: owner.attempt,
181
- receipts: requested.map(({ group, lease, result }) => ({
182
- resultReceiptKey: group.resultReceiptKey,
183
- lease,
184
- result,
185
- })),
186
- }),
187
- );
188
- const results = new Map<string, ToolExecuteResult>();
189
- for (const entry of requested) {
190
- // A completed-result insert is immutable: another independent
191
- // cold miss may have published first. That does not retroactively
192
- // turn this caller into a cache follower. It already executed, so
193
- // return its own result and use the persisted entry only for later
194
- // cache reads.
195
- const result = entry.result;
196
- owned.delete(entry.group.resultReceiptKey);
197
- resolveGroup(entry.group, result);
198
- for (const member of entry.group.members) {
199
- results.set(member.memberId, result);
236
+ for (const publication of pending) {
237
+ const entries = ownedEntries(publication.entries, states, owned);
238
+ for (const entry of entries) {
239
+ const key = entry.group.resultReceiptKey;
240
+ if (!requestedByKey.has(key)) requestedByKey.set(key, entry);
200
241
  }
242
+ claims.push({
243
+ publication,
244
+ resultReceiptKeys: entries.map(
245
+ (entry) => entry.group.resultReceiptKey,
246
+ ),
247
+ settled: false,
248
+ });
201
249
  }
202
- for (const publication of pending) {
203
- publication.resolve(
250
+ } catch (error) {
251
+ for (const publication of pending) publication.reject(error);
252
+ return;
253
+ }
254
+ const requested = [...requestedByKey.values()];
255
+ const settledByKey = new Map<string, ReceiptSettlement>();
256
+ const results = new Map<string, ToolExecuteResult>();
257
+
258
+ const settleReadyClaims = () => {
259
+ for (const claim of claims) {
260
+ if (claim.settled) continue;
261
+ const settlements = claim.resultReceiptKeys.map((key) =>
262
+ settledByKey.get(key),
263
+ );
264
+ if (settlements.some((settlement) => settlement === undefined)) {
265
+ continue;
266
+ }
267
+ claim.settled = true;
268
+ const failed = settlements.find(
269
+ (settlement) => settlement?.failed === true,
270
+ );
271
+ if (failed) {
272
+ // Only this publication: a receipt that failed to persist says
273
+ // nothing about the other receipts of the same wave.
274
+ claim.publication.reject(failed.error);
275
+ continue;
276
+ }
277
+ claim.publication.resolve(
204
278
  new Map(
205
- publication.entries.flatMap((entry) => {
279
+ claim.publication.entries.flatMap((entry) => {
206
280
  const result = results.get(entry.memberId);
207
281
  return result ? [[entry.memberId, result] as const] : [];
208
282
  }),
209
283
  ),
210
284
  );
211
285
  }
286
+ };
287
+
288
+ const settleReceipt = (
289
+ resultReceiptKey: string,
290
+ settlement: ReceiptSettlement,
291
+ ) => {
292
+ if (settledByKey.has(resultReceiptKey)) return;
293
+ const entry = requestedByKey.get(resultReceiptKey);
294
+ if (!entry) return;
295
+ settledByKey.set(resultReceiptKey, settlement);
296
+ // Per receipt, not per wave: the interval a caller opens here is
297
+ // "this row's write settled -> this row resumed", and stamping it
298
+ // when the last receipt of the wave landed charged an earlier row
299
+ // for a later row's flight. Still stamped on failure too, so the
300
+ // interval it opens is closed on both outcomes.
301
+ input.timings?.onCompleteSettled?.({
302
+ memberIds: entry.group.members.map((member) => member.memberId),
303
+ atMs: Date.now(),
304
+ });
305
+ if (!settlement.failed) {
306
+ // A completed-result insert is immutable: another independent
307
+ // cold miss may have published first. That does not retroactively
308
+ // turn this caller into a cache follower. It already executed, so
309
+ // it returns its own result, and the stored row matters only to a
310
+ // later cache read.
311
+ owned.delete(resultReceiptKey);
312
+ resolveGroup(entry.group, entry.result);
313
+ for (const member of entry.group.members) {
314
+ results.set(member.memberId, entry.result);
315
+ }
316
+ }
317
+ settleReadyClaims();
318
+ };
319
+
320
+ try {
321
+ await input.store.complete({
322
+ ownerRunId,
323
+ ownerAttempt: owner.attempt,
324
+ receipts: requested.map(({ group, lease, result }) => ({
325
+ resultReceiptKey: group.resultReceiptKey,
326
+ lease,
327
+ result,
328
+ })),
329
+ onReceiptSettled: (settlement) =>
330
+ settleReceipt(
331
+ settlement.resultReceiptKey,
332
+ settlement.status === 'failed'
333
+ ? { failed: true, error: settlement.error }
334
+ : { failed: false, error: undefined },
335
+ ),
336
+ });
337
+ for (const entry of requested) {
338
+ settleReceipt(entry.group.resultReceiptKey, {
339
+ failed: false,
340
+ error: undefined,
341
+ });
342
+ }
212
343
  } catch (error) {
213
- for (const publication of pending) publication.reject(error);
344
+ for (const entry of requested) {
345
+ settleReceipt(entry.group.resultReceiptKey, {
346
+ failed: true,
347
+ error,
348
+ });
349
+ }
214
350
  }
215
351
  };
216
352
 
@@ -353,6 +489,20 @@ type PendingPublication = {
353
489
  reject(error: unknown): void;
354
490
  };
355
491
 
492
+ type RequestedReceipt = {
493
+ memberId: string;
494
+ result: ToolExecuteResult;
495
+ } & OwnedGroup;
496
+
497
+ /** One queued publication and the receipts it is waiting to become durable. */
498
+ type PublicationClaim = {
499
+ publication: PendingPublication;
500
+ resultReceiptKeys: string[];
501
+ settled: boolean;
502
+ };
503
+
504
+ type ReceiptSettlement = { failed: boolean; error: unknown };
505
+
356
506
  function groupMembers(
357
507
  members: ToolCallReceiptCohortMember[],
358
508
  ): Map<string, Group> {
@@ -424,23 +424,38 @@ export function createRuntimeStepToolResultReceipts(
424
424
  if (record.force || !input.store.enabled) {
425
425
  return { kind: 'stored', result: candidate.result };
426
426
  }
427
- const persisted = await hydrate(
428
- async () =>
429
- await input.store.complete(
430
- record.resultReceiptKey,
431
- ownerRunId,
432
- serializeToolExecuteResult(candidate.result),
433
- COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
434
- ),
427
+ // A completion is a write, not a read. The bounded hydration window
428
+ // exists to cap how many stored payloads expand at once; holding one
429
+ // of its slots across this gateway round trip only queued unrelated
430
+ // rows behind this row's write, so the mutation runs outside it.
431
+ const persisted = await input.store.complete(
432
+ record.resultReceiptKey,
433
+ ownerRunId,
434
+ serializeToolExecuteResult(candidate.result),
435
+ COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
435
436
  );
436
- // Completion transports may acknowledge only status/ownership. The
437
- // durable row already has the full serialized result, so read it back
438
- // below instead of mistaking an intentionally compact reply for a
439
- // malformed completed receipt.
437
+ // Completion transports may acknowledge only status/ownership. A
438
+ // compact acknowledgement that says this insert won is complete
439
+ // information: the immutable stored result is the one just written, so
440
+ // the caller's own result is it. Nothing is read back and nothing
441
+ // hydrates.
442
+ if (
443
+ (persisted?.status === 'completed' ||
444
+ persisted?.status === 'skipped') &&
445
+ persisted.output === undefined &&
446
+ persisted.completionDisposition === 'persisted'
447
+ ) {
448
+ return { kind: 'stored', result: candidate.result };
449
+ }
450
+ // Anything else — a concurrent winner, or a gateway too old to say who
451
+ // won — really does need the durable row, so it decodes under the gate
452
+ // or is read back below.
440
453
  const result =
441
454
  persisted?.output === undefined
442
455
  ? null
443
- : completedResult(persisted, record.resultReceiptKey);
456
+ : await hydrate(async () =>
457
+ completedResult(persisted, record.resultReceiptKey),
458
+ );
444
459
  if (result) {
445
460
  await input.onStored?.({
446
461
  result,
@@ -50,6 +50,45 @@ export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_CODE =
50
50
  'GATEWAY_INVOCATION_IN_PROGRESS';
51
51
  export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_MAX_ATTEMPTS = 65;
52
52
 
53
+ /**
54
+ * Runner-to-gateway capability advertisement. A runner sends the outcomes it
55
+ * can absorb; a gateway may only emit an outcome the caller in front of it
56
+ * understands. This exists because the runner ships inside a Play artifact, so
57
+ * a gateway is always talking to a mixture of runner generations and can never
58
+ * infer the caller's behavior from its own deploy.
59
+ */
60
+ export const TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER = 'x-deepline-runner-accepts';
61
+ /**
62
+ * "I will pace on a Deepline concurrency shed instead of failing the row."
63
+ * Only a runner whose retry gate admits `concurrency_backpressure` for every
64
+ * tool — not just a transient-HTTP-safe or read-only one — may advertise this.
65
+ */
66
+ export const TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE =
67
+ 'concurrency-backpressure';
68
+
69
+ /** The capability tokens this runner build advertises on provider execute. */
70
+ export const TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE =
71
+ TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE;
72
+
73
+ /**
74
+ * Parse the advertisement as a comma-separated token list so a later runner can
75
+ * add a capability without a second header. An absent or unparsable value means
76
+ * "assume nothing": the gateway falls back to the behavior that is safe for the
77
+ * oldest deployed artifact.
78
+ */
79
+ export function runnerAcceptsConcurrencyBackpressure(
80
+ headerValue: string | null | undefined,
81
+ ): boolean {
82
+ if (typeof headerValue !== 'string') return false;
83
+ return headerValue
84
+ .split(',')
85
+ .some(
86
+ (token) =>
87
+ token.trim().toLowerCase() ===
88
+ TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE,
89
+ );
90
+ }
91
+
53
92
  export class ToolExecuteAuthScopeChangedError extends Error {
54
93
  readonly code = TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE;
55
94
  readonly status = 409;
@@ -503,6 +542,10 @@ export function classifyToolExecuteHttpFailure(input: {
503
542
  retryDecision.reason !== 'gateway_invocation_in_progress' &&
504
543
  retryDecision.reason !== 'concurrency_backpressure' &&
505
544
  retryDecision.reason !== 'runtime_scheduler_capacity',
506
- knownNotDispatched: runtimeSchedulerCapacity,
545
+ // Scheduler capacity and the relay's explicit concurrency shed both occur
546
+ // before a provider request is dispatched. Carry that proof in the shared
547
+ // classification so the retry boundary can safely pace either response
548
+ // without asking a non-idempotent provider call to be replayed.
549
+ knownNotDispatched: runtimeSchedulerCapacity || concurrencyBackpressure,
507
550
  };
508
551
  }