deepline 0.3.128 → 0.3.129
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +34 -9
- package/dist/bundling-sources/sdk/src/http.ts +22 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-data-plane/r2.ts +22 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +265 -2
- package/dist/bundling-sources/shared_libs/play-runtime/bounded-batch-function.ts +682 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +21 -5
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +5390 -1732
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +107 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/commands/roll-execution-relay.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-machines.ts +229 -7
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/fly-registry.ts +25 -4
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-capacity.ts +71 -0
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-gateway-drain.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/deployment/runtime-topology.ts +4 -1
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +694 -28
- package/dist/bundling-sources/shared_libs/play-runtime/egress-bandwidth-probe.ts +167 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execute-server-timing.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-receipt-output-reference.ts +34 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-response-size.ts +27 -0
- package/dist/bundling-sources/shared_libs/play-runtime/gateway-postgres-admission.ts +66 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +425 -71
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +210 -32
- package/dist/bundling-sources/shared_libs/play-runtime/governor/in-memory-rate-state-backend.ts +27 -13
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +19 -6
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +61 -2
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/contracts.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/integration-operation-reader-fence/job.ts +42 -1
- package/dist/bundling-sources/shared_libs/play-runtime/launch-source-policy.ts +11 -11
- package/dist/bundling-sources/shared_libs/play-runtime/ledger-safe-payload.ts +49 -16
- package/dist/bundling-sources/shared_libs/play-runtime/map-memory-limits.ts +1653 -98
- package/dist/bundling-sources/shared_libs/play-runtime/memory-ledger.ts +807 -0
- package/dist/bundling-sources/shared_libs/play-runtime/memory-pressure-signal.ts +138 -0
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +85 -3
- package/dist/bundling-sources/shared_libs/play-runtime/persistence-latch.ts +60 -2
- package/dist/bundling-sources/shared_libs/play-runtime/play-run-recovery-policy.ts +46 -1
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/contracts.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/job.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/ports.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runs/terminal/postgres-adapter.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/play-runtime-semantics.ts +70 -27
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +56 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/adaptive-provider-capacity-controller.ts +192 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/epoch-aimd-rate-controller.ts +194 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/fixture-capacity-simulator.ts +220 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/gradient2-concurrency-controller.ts +214 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-capacity-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-concurrency-policy.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-rate-policy.ts +95 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-capacity/provider-throttle-classifier.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +1535 -11
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +49 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-log-buffer.ts +55 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/egress-probe.ts +221 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +339 -29
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-deadline.ts +28 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-progress-push.ts +192 -165
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +6 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +90 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-runtime-watchdog.ts +281 -6
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +27 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +19 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/modal.ts +108 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/runtime-sandbox-reconciliation.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/types.ts +11 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +30 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-compression.ts +276 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +723 -115
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-capacity-policy.ts +98 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-environment.ts +41 -4
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +448 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-map-row-feed.ts +215 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-store-adapter.ts +161 -14
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-value.ts +184 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-receipt-writer.ts +607 -121
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sandbox-placement-policy.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-row-writer.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/contract.ts +15 -2
- package/dist/bundling-sources/shared_libs/play-runtime/sandbox-runs/runner-backend-adapter.ts +71 -14
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-play-deploy-task.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +288 -119
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-shared.ts +8 -6
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-worker-required-env.ts +28 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/canonical-integration-operation-store.ts +179 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-heavy-request-admission.ts +80 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/gateway-progress-registry.ts +31 -2
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-store-contract.ts +104 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-reference.ts +47 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-retirement.ts +193 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/integration-operation-value-storage.ts +420 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts +134 -3
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +1657 -95
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +885 -73
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts +452 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +601 -42
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/run-terminal-transition.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +371 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/worker-resource-metrics.ts +86 -7
- package/dist/bundling-sources/shared_libs/play-runtime/secret-redaction.ts +153 -22
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/receipt-cohort.ts +188 -38
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts +28 -13
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +44 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +56 -4
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +2 -0
- package/dist/bundling-sources/shared_libs/plays/authoring-contract.ts +16 -10
- package/dist/bundling-sources/shared_libs/plays/dataset.ts +4 -1
- package/dist/cli/index.js +250 -38
- package/dist/cli/index.mjs +250 -38
- package/dist/{compiler-manifest-DfSnbCTW.d.mts → compiler-manifest-BIqRyj5m.d.mts} +1 -1
- package/dist/{compiler-manifest-DfSnbCTW.d.ts → compiler-manifest-BIqRyj5m.d.ts} +1 -1
- package/dist/index.d.mts +4 -2
- package/dist/index.d.ts +4 -2
- package/dist/index.js +231 -35
- package/dist/index.mjs +231 -35
- package/dist/install-integrity.json +31 -2
- package/dist/plays/bundle-play-file.d.mts +2 -2
- package/dist/plays/bundle-play-file.d.ts +2 -2
- package/dist/plays/bundle-play-file.mjs +65 -1
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -31,6 +31,23 @@ const SENSITIVE_FIELD_NAME_RE = new RegExp(
|
|
|
31
31
|
`(^|[._-])(?:${SENSITIVE_FIELD_NAME})$`,
|
|
32
32
|
'i',
|
|
33
33
|
);
|
|
34
|
+
|
|
35
|
+
// Receipt payloads repeat the same handful of field names once per row, so the
|
|
36
|
+
// per-key verdict is memoized. The cap keeps a pathological payload of unique
|
|
37
|
+
// keys from turning the cache into a leak.
|
|
38
|
+
const SENSITIVE_FIELD_NAME_CACHE_MAX_ENTRIES = 4_096;
|
|
39
|
+
const sensitiveFieldNameCache = new Map<string, boolean>();
|
|
40
|
+
|
|
41
|
+
function isSensitiveFieldName(key: string): boolean {
|
|
42
|
+
const cached = sensitiveFieldNameCache.get(key);
|
|
43
|
+
if (cached !== undefined) return cached;
|
|
44
|
+
const sensitive = SENSITIVE_FIELD_NAME_RE.test(key);
|
|
45
|
+
if (sensitiveFieldNameCache.size >= SENSITIVE_FIELD_NAME_CACHE_MAX_ENTRIES) {
|
|
46
|
+
sensitiveFieldNameCache.clear();
|
|
47
|
+
}
|
|
48
|
+
sensitiveFieldNameCache.set(key, sensitive);
|
|
49
|
+
return sensitive;
|
|
50
|
+
}
|
|
34
51
|
const SERIALIZED_SECRET_FIELD_RE = new RegExp(
|
|
35
52
|
`"(${SENSITIVE_FIELD_NAME})"\\s*:\\s*(?:"[^"\\\\]*"|\\{[^{}]*\\})`,
|
|
36
53
|
'gi',
|
|
@@ -40,7 +57,61 @@ function escapeRegExp(value: string): string {
|
|
|
40
57
|
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
41
58
|
}
|
|
42
59
|
|
|
60
|
+
/**
|
|
61
|
+
* Cheap superset gate for every rule `redactSecretLikeString` applies.
|
|
62
|
+
*
|
|
63
|
+
* Each rewrite above needs one of these markers to fire, so a string matching
|
|
64
|
+
* none of them is returned byte-identical by the full chain. Running one test
|
|
65
|
+
* instead of ten replaces is what makes receipt sanitization affordable: a
|
|
66
|
+
* 600 KiB provider result is thousands of strings, and almost none of them
|
|
67
|
+
* carry a credential.
|
|
68
|
+
*
|
|
69
|
+
* Extra matches only cost the full chain, while a missed marker would persist
|
|
70
|
+
* a live credential, so keep every entry looser than the rule it guards
|
|
71
|
+
* wherever JSON escaping or whitespace could hide one.
|
|
72
|
+
*/
|
|
73
|
+
const REDACTION_TRIGGER_RE = new RegExp(
|
|
74
|
+
[
|
|
75
|
+
// Sensitive field names, in object keys and in `name=`/`"name":` text.
|
|
76
|
+
'token',
|
|
77
|
+
'secret',
|
|
78
|
+
'password',
|
|
79
|
+
'authorization',
|
|
80
|
+
'cookie',
|
|
81
|
+
'credential',
|
|
82
|
+
// One `key` branch instead of `api[_-]?key`, `private[_-]?key` and `key_`.
|
|
83
|
+
// A regex alternation costs roughly one substring scan per branch, and a
|
|
84
|
+
// broader branch can only send more payloads down the exact path.
|
|
85
|
+
'key',
|
|
86
|
+
// Self-identifying credential prefixes.
|
|
87
|
+
'bearer',
|
|
88
|
+
'basic',
|
|
89
|
+
'-----BEGIN',
|
|
90
|
+
'xox[baprs]-',
|
|
91
|
+
'[prs]k_',
|
|
92
|
+
'pat_',
|
|
93
|
+
'ghp_',
|
|
94
|
+
// URL credentials: `//user:pass@host` userinfo and sensitive query params.
|
|
95
|
+
// The userinfo class excludes `/` exactly as the rule it guards does.
|
|
96
|
+
'//[^\\s/]{0,256}@',
|
|
97
|
+
'[?#&]sig',
|
|
98
|
+
// JWTs carry no keyword at all, so the shape itself is the marker. Anchor
|
|
99
|
+
// on the separating dot and drop JWT_RE's leading `{8,}` segment: a dot is
|
|
100
|
+
// rare enough for the engine to skip between candidates, which makes this
|
|
101
|
+
// gate roughly ten times cheaper than re-running JWT_RE itself, and
|
|
102
|
+
// dropping a required segment can only widen what it accepts.
|
|
103
|
+
'\\.[A-Za-z0-9_-]{1,16384}\\.[A-Za-z0-9_-]{8}',
|
|
104
|
+
].join('|'),
|
|
105
|
+
'i',
|
|
106
|
+
);
|
|
107
|
+
|
|
108
|
+
/** True when any redaction rule could rewrite `value`. Never a false negative. */
|
|
109
|
+
export function mightContainRedactableSecret(value: string): boolean {
|
|
110
|
+
return REDACTION_TRIGGER_RE.test(value);
|
|
111
|
+
}
|
|
112
|
+
|
|
43
113
|
export function redactSecretLikeString(value: string): string {
|
|
114
|
+
if (!mightContainRedactableSecret(value)) return value;
|
|
44
115
|
const serializedRedacted = value.replace(
|
|
45
116
|
SERIALIZED_SECRET_FIELD_RE,
|
|
46
117
|
`"$1":"${SECRET_REDACTION_PLACEHOLDER}"`,
|
|
@@ -91,11 +162,39 @@ export function createSecretRedactionContext(
|
|
|
91
162
|
initialValues: readonly string[] = [],
|
|
92
163
|
): SecretRedactionContext {
|
|
93
164
|
const exactSecrets = new Set<string>();
|
|
165
|
+
// One compiled matcher per registered secret, built on first use. Rebuilding
|
|
166
|
+
// `new RegExp(escapeRegExp(secret))` for every string of every receipt was
|
|
167
|
+
// pure waste once a run registers more than a couple of credentials.
|
|
168
|
+
const exactSecretMatchers = new Map<
|
|
169
|
+
string,
|
|
170
|
+
{ readonly literal: string; readonly matcher: RegExp }[]
|
|
171
|
+
>();
|
|
94
172
|
|
|
95
173
|
function register(value: string): void {
|
|
96
174
|
if (value.length > 0) exactSecrets.add(value);
|
|
97
175
|
}
|
|
98
176
|
|
|
177
|
+
function matchersForSecret(
|
|
178
|
+
secret: string,
|
|
179
|
+
): { readonly literal: string; readonly matcher: RegExp }[] {
|
|
180
|
+
const cached = exactSecretMatchers.get(secret);
|
|
181
|
+
if (cached) return cached;
|
|
182
|
+
const literals = [secret];
|
|
183
|
+
try {
|
|
184
|
+
const encoded = encodeURIComponent(secret);
|
|
185
|
+
if (encoded !== secret) literals.push(encoded);
|
|
186
|
+
} catch {
|
|
187
|
+
// encodeURIComponent only fails on malformed surrogate pairs. Exact
|
|
188
|
+
// redaction of the raw value still applies in that rare case.
|
|
189
|
+
}
|
|
190
|
+
const built = literals.map((literal) => ({
|
|
191
|
+
literal,
|
|
192
|
+
matcher: new RegExp(escapeRegExp(literal), 'g'),
|
|
193
|
+
}));
|
|
194
|
+
exactSecretMatchers.set(secret, built);
|
|
195
|
+
return built;
|
|
196
|
+
}
|
|
197
|
+
|
|
99
198
|
for (const value of initialValues) register(value);
|
|
100
199
|
|
|
101
200
|
function containsRegisteredSecret(
|
|
@@ -125,24 +224,20 @@ export function createSecretRedactionContext(
|
|
|
125
224
|
return exactSecrets.has(value);
|
|
126
225
|
}
|
|
127
226
|
|
|
227
|
+
/**
|
|
228
|
+
* Replace registered secrets in registration order, exactly as a per-secret
|
|
229
|
+
* replace chain would. `includes` is a native substring scan, so a value
|
|
230
|
+
* that carries no secret pays one scan per secret instead of compiling and
|
|
231
|
+
* running a fresh regex per secret.
|
|
232
|
+
*/
|
|
128
233
|
function redactRegisteredSecrets(value: string): string {
|
|
234
|
+
if (exactSecrets.size === 0) return value;
|
|
129
235
|
let output = value;
|
|
130
236
|
for (const secret of exactSecrets) {
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
try {
|
|
136
|
-
const encoded = encodeURIComponent(secret);
|
|
137
|
-
if (encoded !== secret) {
|
|
138
|
-
output = output.replace(
|
|
139
|
-
new RegExp(escapeRegExp(encoded), 'g'),
|
|
140
|
-
SECRET_REDACTION_PLACEHOLDER,
|
|
141
|
-
);
|
|
142
|
-
}
|
|
143
|
-
} catch {
|
|
144
|
-
// encodeURIComponent only fails on malformed surrogate pairs. Exact
|
|
145
|
-
// redaction above still applies in that rare case.
|
|
237
|
+
for (const { literal, matcher } of matchersForSecret(secret)) {
|
|
238
|
+
if (!output.includes(literal)) continue;
|
|
239
|
+
matcher.lastIndex = 0;
|
|
240
|
+
output = output.replace(matcher, SECRET_REDACTION_PLACEHOLDER);
|
|
146
241
|
}
|
|
147
242
|
}
|
|
148
243
|
return output;
|
|
@@ -153,17 +248,53 @@ export function createSecretRedactionContext(
|
|
|
153
248
|
return redactSecretLikeString(output);
|
|
154
249
|
}
|
|
155
250
|
|
|
251
|
+
/**
|
|
252
|
+
* Redact in place-preserving form: a plain object or array with nothing to
|
|
253
|
+
* redact is returned by reference instead of being rebuilt.
|
|
254
|
+
*
|
|
255
|
+
* This is what keeps receipt sanitization off the deep-clone path. A 600 KiB
|
|
256
|
+
* provider result almost never contains a credential, and cloning it only to
|
|
257
|
+
* re-serialize the identical value dominated gateway CPU.
|
|
258
|
+
*
|
|
259
|
+
* Non-plain objects (Date, Map, class instances) still go through the
|
|
260
|
+
* `Object.entries` rebuild, because collapsing them to plain objects is the
|
|
261
|
+
* long-standing persisted shape and changing it would rewrite stored data.
|
|
262
|
+
*/
|
|
156
263
|
function redact<T>(value: T): T {
|
|
157
264
|
if (typeof value === 'string') return redactString(value) as T;
|
|
158
|
-
if (Array.isArray(value))
|
|
265
|
+
if (Array.isArray(value)) {
|
|
266
|
+
let changed = false;
|
|
267
|
+
const next = new Array(value.length);
|
|
268
|
+
for (let index = 0; index < value.length; index += 1) {
|
|
269
|
+
const entry = value[index];
|
|
270
|
+
const redacted = redact(entry);
|
|
271
|
+
next[index] = redacted;
|
|
272
|
+
if (redacted !== entry) changed = true;
|
|
273
|
+
}
|
|
274
|
+
return changed ? (next as T) : value;
|
|
275
|
+
}
|
|
159
276
|
if (value && typeof value === 'object') {
|
|
277
|
+
const prototype = Object.getPrototypeOf(value);
|
|
278
|
+
// Only a plain object can be returned by reference; anything else must
|
|
279
|
+
// keep its historical projection to a plain object.
|
|
280
|
+
let changed = prototype !== Object.prototype && prototype !== null;
|
|
281
|
+
const source = value as Record<string, unknown>;
|
|
282
|
+
const keys = Object.keys(source);
|
|
283
|
+
const redactedValues = new Array<unknown>(keys.length);
|
|
284
|
+
for (let index = 0; index < keys.length; index += 1) {
|
|
285
|
+
const key = keys[index];
|
|
286
|
+
const entry = source[key];
|
|
287
|
+
const redacted = isSensitiveFieldName(key)
|
|
288
|
+
? SECRET_REDACTION_PLACEHOLDER
|
|
289
|
+
: redact(entry);
|
|
290
|
+
redactedValues[index] = redacted;
|
|
291
|
+
if (redacted !== entry) changed = true;
|
|
292
|
+
}
|
|
293
|
+
if (!changed) return value;
|
|
294
|
+
// `Object.fromEntries` defines own properties, so a `__proto__` key stays
|
|
295
|
+
// data instead of reassigning the prototype of the rebuilt object.
|
|
160
296
|
return Object.fromEntries(
|
|
161
|
-
|
|
162
|
-
key,
|
|
163
|
-
SENSITIVE_FIELD_NAME_RE.test(key)
|
|
164
|
-
? SECRET_REDACTION_PLACEHOLDER
|
|
165
|
-
: redact(entry),
|
|
166
|
-
]),
|
|
297
|
+
keys.map((key, index) => [key, redactedValues[index]]),
|
|
167
298
|
) as T;
|
|
168
299
|
}
|
|
169
300
|
return value;
|
|
@@ -20,10 +20,27 @@ export type ToolCallReceiptCohortMember = {
|
|
|
20
20
|
waitMaxAttempts?: number;
|
|
21
21
|
};
|
|
22
22
|
|
|
23
|
+
/**
|
|
24
|
+
* One receipt's own durable outcome, reported as it settles rather than when
|
|
25
|
+
* the batch carrying it does.
|
|
26
|
+
*/
|
|
27
|
+
export type ToolCallReceiptCohortSettlement =
|
|
28
|
+
| { resultReceiptKey: string; status: 'stored' }
|
|
29
|
+
| { resultReceiptKey: string; status: 'failed'; error: unknown };
|
|
30
|
+
|
|
23
31
|
export type ToolCallReceiptCohortStore = {
|
|
24
32
|
read(input: {
|
|
25
33
|
resultReceiptKeys: string[];
|
|
26
34
|
}): Promise<ToolCallReceiptRecord[]>;
|
|
35
|
+
/**
|
|
36
|
+
* Publish these results to the completed-result cache.
|
|
37
|
+
*
|
|
38
|
+
* Deliberately returns nothing. A completed-result insert is immutable, so a
|
|
39
|
+
* caller that already executed keeps its own result whether or not it won
|
|
40
|
+
* the insert (see `flushPublications`). Materializing the stored rows here
|
|
41
|
+
* only bought a second deserialize-and-wrap of every result, inside the
|
|
42
|
+
* delivery interval a row is blocked on.
|
|
43
|
+
*/
|
|
27
44
|
complete(input: {
|
|
28
45
|
ownerRunId: string;
|
|
29
46
|
ownerAttempt: number;
|
|
@@ -32,7 +49,23 @@ export type ToolCallReceiptCohortStore = {
|
|
|
32
49
|
lease: ToolCallReceiptLease;
|
|
33
50
|
result: ToolExecuteResult;
|
|
34
51
|
}>;
|
|
35
|
-
|
|
52
|
+
/**
|
|
53
|
+
* Report each receipt the instant its OWN durable write settles.
|
|
54
|
+
*
|
|
55
|
+
* The cohort hands the store a coalesced wave, but the store batches by
|
|
56
|
+
* arrival underneath: two receipts of one wave can land in different
|
|
57
|
+
* batches, and the earlier one is durable a whole flight before the later
|
|
58
|
+
* one. Settling the wave together made every row in it wait for the last
|
|
59
|
+
* batch, so a store that can separate them calls this and each publication
|
|
60
|
+
* resolves on its own receipt.
|
|
61
|
+
*
|
|
62
|
+
* Optional. A store that cannot separate them simply resolves or rejects
|
|
63
|
+
* the returned promise, and every receipt it did not report settles at
|
|
64
|
+
* that point — successfully when the promise resolves, with its error
|
|
65
|
+
* when it rejects.
|
|
66
|
+
*/
|
|
67
|
+
onReceiptSettled?(settlement: ToolCallReceiptCohortSettlement): void;
|
|
68
|
+
}): Promise<void>;
|
|
36
69
|
};
|
|
37
70
|
|
|
38
71
|
export type ToolCallReceiptCohortMemberState =
|
|
@@ -70,6 +103,21 @@ export type ToolCallReceiptCohort = {
|
|
|
70
103
|
settle(): Promise<void>;
|
|
71
104
|
};
|
|
72
105
|
|
|
106
|
+
/**
|
|
107
|
+
* Measurement stamp for the end of a publication, which is otherwise invisible
|
|
108
|
+
* from outside: a coalesced `store.complete` settles here, and only the cohort
|
|
109
|
+
* knows which members it carried. A caller uses it to open its own `write
|
|
110
|
+
* settled -> row resumed` interval. Stamps only: nothing here is read by the
|
|
111
|
+
* cohort, and it fires on a failed write too, so the interval it opens is
|
|
112
|
+
* closed on both outcomes.
|
|
113
|
+
*/
|
|
114
|
+
export type ToolCallReceiptCohortTimings = {
|
|
115
|
+
onCompleteSettled?(input: {
|
|
116
|
+
memberIds: readonly string[];
|
|
117
|
+
atMs: number;
|
|
118
|
+
}): void;
|
|
119
|
+
};
|
|
120
|
+
|
|
73
121
|
export type ToolCallReceiptCohortJob = {
|
|
74
122
|
acquire(input: {
|
|
75
123
|
owner: { runId: string; attempt: number };
|
|
@@ -86,6 +134,7 @@ export type ToolCallReceiptCohortJob = {
|
|
|
86
134
|
*/
|
|
87
135
|
export function createToolCallReceiptCohortJob(input: {
|
|
88
136
|
store: ToolCallReceiptCohortStore;
|
|
137
|
+
timings?: ToolCallReceiptCohortTimings;
|
|
89
138
|
}): ToolCallReceiptCohortJob {
|
|
90
139
|
return {
|
|
91
140
|
async acquire({ owner, members }) {
|
|
@@ -102,12 +151,20 @@ export function createToolCallReceiptCohortJob(input: {
|
|
|
102
151
|
}
|
|
103
152
|
|
|
104
153
|
const groups = groupMembers(members);
|
|
154
|
+
// A forced member deliberately discards any completed cache value and
|
|
155
|
+
// executes a fresh provider operation. Reading it first is not merely
|
|
156
|
+
// redundant: a large persisted result is fully hydrated and wrapped
|
|
157
|
+
// before the force branch below ignores it. Keep completed-result cache
|
|
158
|
+
// reads scoped to groups that can actually consume their values. This
|
|
159
|
+
// also leaves a mixed group correct: `groupMembers` marks it forced when
|
|
160
|
+
// any duplicate member explicitly requires fresh execution.
|
|
161
|
+
const cacheReadKeys = [...groups.values()]
|
|
162
|
+
.filter((group) => !group.force)
|
|
163
|
+
.map((group) => group.resultReceiptKey);
|
|
105
164
|
const records = byKey(
|
|
106
|
-
|
|
107
|
-
resultReceiptKeys:
|
|
108
|
-
|
|
109
|
-
),
|
|
110
|
-
}),
|
|
165
|
+
cacheReadKeys.length > 0
|
|
166
|
+
? await input.store.read({ resultReceiptKeys: cacheReadKeys })
|
|
167
|
+
: [],
|
|
111
168
|
);
|
|
112
169
|
const states = new Map<string, ToolCallReceiptCohortMemberState>();
|
|
113
170
|
const deferred = new Map<string, Deferred<ToolExecuteResult>>();
|
|
@@ -168,49 +225,128 @@ export function createToolCallReceiptCohortJob(input: {
|
|
|
168
225
|
publicationFlushScheduled = false;
|
|
169
226
|
const pending = pendingPublications.splice(0);
|
|
170
227
|
if (pending.length === 0) return;
|
|
228
|
+
// Which receipts each publication is waiting on, resolved before the
|
|
229
|
+
// write so a publication can be settled the moment ITS receipts are
|
|
230
|
+
// durable rather than when the whole wave is. A lease lost between the
|
|
231
|
+
// queue and this flush is a pre-write fault of the wave, so it still
|
|
232
|
+
// rejects every publication in it.
|
|
233
|
+
const claims: PublicationClaim[] = [];
|
|
234
|
+
const requestedByKey = new Map<string, RequestedReceipt>();
|
|
171
235
|
try {
|
|
172
|
-
const
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
const persisted = byKey(
|
|
178
|
-
await input.store.complete({
|
|
179
|
-
ownerRunId,
|
|
180
|
-
ownerAttempt: owner.attempt,
|
|
181
|
-
receipts: requested.map(({ group, lease, result }) => ({
|
|
182
|
-
resultReceiptKey: group.resultReceiptKey,
|
|
183
|
-
lease,
|
|
184
|
-
result,
|
|
185
|
-
})),
|
|
186
|
-
}),
|
|
187
|
-
);
|
|
188
|
-
const results = new Map<string, ToolExecuteResult>();
|
|
189
|
-
for (const entry of requested) {
|
|
190
|
-
// A completed-result insert is immutable: another independent
|
|
191
|
-
// cold miss may have published first. That does not retroactively
|
|
192
|
-
// turn this caller into a cache follower. It already executed, so
|
|
193
|
-
// return its own result and use the persisted entry only for later
|
|
194
|
-
// cache reads.
|
|
195
|
-
const result = entry.result;
|
|
196
|
-
owned.delete(entry.group.resultReceiptKey);
|
|
197
|
-
resolveGroup(entry.group, result);
|
|
198
|
-
for (const member of entry.group.members) {
|
|
199
|
-
results.set(member.memberId, result);
|
|
236
|
+
for (const publication of pending) {
|
|
237
|
+
const entries = ownedEntries(publication.entries, states, owned);
|
|
238
|
+
for (const entry of entries) {
|
|
239
|
+
const key = entry.group.resultReceiptKey;
|
|
240
|
+
if (!requestedByKey.has(key)) requestedByKey.set(key, entry);
|
|
200
241
|
}
|
|
242
|
+
claims.push({
|
|
243
|
+
publication,
|
|
244
|
+
resultReceiptKeys: entries.map(
|
|
245
|
+
(entry) => entry.group.resultReceiptKey,
|
|
246
|
+
),
|
|
247
|
+
settled: false,
|
|
248
|
+
});
|
|
201
249
|
}
|
|
202
|
-
|
|
203
|
-
|
|
250
|
+
} catch (error) {
|
|
251
|
+
for (const publication of pending) publication.reject(error);
|
|
252
|
+
return;
|
|
253
|
+
}
|
|
254
|
+
const requested = [...requestedByKey.values()];
|
|
255
|
+
const settledByKey = new Map<string, ReceiptSettlement>();
|
|
256
|
+
const results = new Map<string, ToolExecuteResult>();
|
|
257
|
+
|
|
258
|
+
const settleReadyClaims = () => {
|
|
259
|
+
for (const claim of claims) {
|
|
260
|
+
if (claim.settled) continue;
|
|
261
|
+
const settlements = claim.resultReceiptKeys.map((key) =>
|
|
262
|
+
settledByKey.get(key),
|
|
263
|
+
);
|
|
264
|
+
if (settlements.some((settlement) => settlement === undefined)) {
|
|
265
|
+
continue;
|
|
266
|
+
}
|
|
267
|
+
claim.settled = true;
|
|
268
|
+
const failed = settlements.find(
|
|
269
|
+
(settlement) => settlement?.failed === true,
|
|
270
|
+
);
|
|
271
|
+
if (failed) {
|
|
272
|
+
// Only this publication: a receipt that failed to persist says
|
|
273
|
+
// nothing about the other receipts of the same wave.
|
|
274
|
+
claim.publication.reject(failed.error);
|
|
275
|
+
continue;
|
|
276
|
+
}
|
|
277
|
+
claim.publication.resolve(
|
|
204
278
|
new Map(
|
|
205
|
-
publication.entries.flatMap((entry) => {
|
|
279
|
+
claim.publication.entries.flatMap((entry) => {
|
|
206
280
|
const result = results.get(entry.memberId);
|
|
207
281
|
return result ? [[entry.memberId, result] as const] : [];
|
|
208
282
|
}),
|
|
209
283
|
),
|
|
210
284
|
);
|
|
211
285
|
}
|
|
286
|
+
};
|
|
287
|
+
|
|
288
|
+
const settleReceipt = (
|
|
289
|
+
resultReceiptKey: string,
|
|
290
|
+
settlement: ReceiptSettlement,
|
|
291
|
+
) => {
|
|
292
|
+
if (settledByKey.has(resultReceiptKey)) return;
|
|
293
|
+
const entry = requestedByKey.get(resultReceiptKey);
|
|
294
|
+
if (!entry) return;
|
|
295
|
+
settledByKey.set(resultReceiptKey, settlement);
|
|
296
|
+
// Per receipt, not per wave: the interval a caller opens here is
|
|
297
|
+
// "this row's write settled -> this row resumed", and stamping it
|
|
298
|
+
// when the last receipt of the wave landed charged an earlier row
|
|
299
|
+
// for a later row's flight. Still stamped on failure too, so the
|
|
300
|
+
// interval it opens is closed on both outcomes.
|
|
301
|
+
input.timings?.onCompleteSettled?.({
|
|
302
|
+
memberIds: entry.group.members.map((member) => member.memberId),
|
|
303
|
+
atMs: Date.now(),
|
|
304
|
+
});
|
|
305
|
+
if (!settlement.failed) {
|
|
306
|
+
// A completed-result insert is immutable: another independent
|
|
307
|
+
// cold miss may have published first. That does not retroactively
|
|
308
|
+
// turn this caller into a cache follower. It already executed, so
|
|
309
|
+
// it returns its own result, and the stored row matters only to a
|
|
310
|
+
// later cache read.
|
|
311
|
+
owned.delete(resultReceiptKey);
|
|
312
|
+
resolveGroup(entry.group, entry.result);
|
|
313
|
+
for (const member of entry.group.members) {
|
|
314
|
+
results.set(member.memberId, entry.result);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
settleReadyClaims();
|
|
318
|
+
};
|
|
319
|
+
|
|
320
|
+
try {
|
|
321
|
+
await input.store.complete({
|
|
322
|
+
ownerRunId,
|
|
323
|
+
ownerAttempt: owner.attempt,
|
|
324
|
+
receipts: requested.map(({ group, lease, result }) => ({
|
|
325
|
+
resultReceiptKey: group.resultReceiptKey,
|
|
326
|
+
lease,
|
|
327
|
+
result,
|
|
328
|
+
})),
|
|
329
|
+
onReceiptSettled: (settlement) =>
|
|
330
|
+
settleReceipt(
|
|
331
|
+
settlement.resultReceiptKey,
|
|
332
|
+
settlement.status === 'failed'
|
|
333
|
+
? { failed: true, error: settlement.error }
|
|
334
|
+
: { failed: false, error: undefined },
|
|
335
|
+
),
|
|
336
|
+
});
|
|
337
|
+
for (const entry of requested) {
|
|
338
|
+
settleReceipt(entry.group.resultReceiptKey, {
|
|
339
|
+
failed: false,
|
|
340
|
+
error: undefined,
|
|
341
|
+
});
|
|
342
|
+
}
|
|
212
343
|
} catch (error) {
|
|
213
|
-
for (const
|
|
344
|
+
for (const entry of requested) {
|
|
345
|
+
settleReceipt(entry.group.resultReceiptKey, {
|
|
346
|
+
failed: true,
|
|
347
|
+
error,
|
|
348
|
+
});
|
|
349
|
+
}
|
|
214
350
|
}
|
|
215
351
|
};
|
|
216
352
|
|
|
@@ -353,6 +489,20 @@ type PendingPublication = {
|
|
|
353
489
|
reject(error: unknown): void;
|
|
354
490
|
};
|
|
355
491
|
|
|
492
|
+
type RequestedReceipt = {
|
|
493
|
+
memberId: string;
|
|
494
|
+
result: ToolExecuteResult;
|
|
495
|
+
} & OwnedGroup;
|
|
496
|
+
|
|
497
|
+
/** One queued publication and the receipts it is waiting to become durable. */
|
|
498
|
+
type PublicationClaim = {
|
|
499
|
+
publication: PendingPublication;
|
|
500
|
+
resultReceiptKeys: string[];
|
|
501
|
+
settled: boolean;
|
|
502
|
+
};
|
|
503
|
+
|
|
504
|
+
type ReceiptSettlement = { failed: boolean; error: unknown };
|
|
505
|
+
|
|
356
506
|
function groupMembers(
|
|
357
507
|
members: ToolCallReceiptCohortMember[],
|
|
358
508
|
): Map<string, Group> {
|
package/dist/bundling-sources/shared_libs/play-runtime/tool-call/runtime-step-receipts-adapter.ts
CHANGED
|
@@ -424,23 +424,38 @@ export function createRuntimeStepToolResultReceipts(
|
|
|
424
424
|
if (record.force || !input.store.enabled) {
|
|
425
425
|
return { kind: 'stored', result: candidate.result };
|
|
426
426
|
}
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
427
|
+
// A completion is a write, not a read. The bounded hydration window
|
|
428
|
+
// exists to cap how many stored payloads expand at once; holding one
|
|
429
|
+
// of its slots across this gateway round trip only queued unrelated
|
|
430
|
+
// rows behind this row's write, so the mutation runs outside it.
|
|
431
|
+
const persisted = await input.store.complete(
|
|
432
|
+
record.resultReceiptKey,
|
|
433
|
+
ownerRunId,
|
|
434
|
+
serializeToolExecuteResult(candidate.result),
|
|
435
|
+
COMPLETED_RECEIPT_CACHE_INSERT_LEASE_ID,
|
|
435
436
|
);
|
|
436
|
-
// Completion transports may acknowledge only status/ownership.
|
|
437
|
-
//
|
|
438
|
-
//
|
|
439
|
-
//
|
|
437
|
+
// Completion transports may acknowledge only status/ownership. A
|
|
438
|
+
// compact acknowledgement that says this insert won is complete
|
|
439
|
+
// information: the immutable stored result is the one just written, so
|
|
440
|
+
// the caller's own result is it. Nothing is read back and nothing
|
|
441
|
+
// hydrates.
|
|
442
|
+
if (
|
|
443
|
+
(persisted?.status === 'completed' ||
|
|
444
|
+
persisted?.status === 'skipped') &&
|
|
445
|
+
persisted.output === undefined &&
|
|
446
|
+
persisted.completionDisposition === 'persisted'
|
|
447
|
+
) {
|
|
448
|
+
return { kind: 'stored', result: candidate.result };
|
|
449
|
+
}
|
|
450
|
+
// Anything else — a concurrent winner, or a gateway too old to say who
|
|
451
|
+
// won — really does need the durable row, so it decodes under the gate
|
|
452
|
+
// or is read back below.
|
|
440
453
|
const result =
|
|
441
454
|
persisted?.output === undefined
|
|
442
455
|
? null
|
|
443
|
-
:
|
|
456
|
+
: await hydrate(async () =>
|
|
457
|
+
completedResult(persisted, record.resultReceiptKey),
|
|
458
|
+
);
|
|
444
459
|
if (result) {
|
|
445
460
|
await input.onStored?.({
|
|
446
461
|
result,
|
|
@@ -50,6 +50,45 @@ export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_CODE =
|
|
|
50
50
|
'GATEWAY_INVOCATION_IN_PROGRESS';
|
|
51
51
|
export const TOOL_EXECUTE_GATEWAY_INVOCATION_IN_PROGRESS_MAX_ATTEMPTS = 65;
|
|
52
52
|
|
|
53
|
+
/**
|
|
54
|
+
* Runner-to-gateway capability advertisement. A runner sends the outcomes it
|
|
55
|
+
* can absorb; a gateway may only emit an outcome the caller in front of it
|
|
56
|
+
* understands. This exists because the runner ships inside a Play artifact, so
|
|
57
|
+
* a gateway is always talking to a mixture of runner generations and can never
|
|
58
|
+
* infer the caller's behavior from its own deploy.
|
|
59
|
+
*/
|
|
60
|
+
export const TOOL_EXECUTE_RUNNER_ACCEPTS_HEADER = 'x-deepline-runner-accepts';
|
|
61
|
+
/**
|
|
62
|
+
* "I will pace on a Deepline concurrency shed instead of failing the row."
|
|
63
|
+
* Only a runner whose retry gate admits `concurrency_backpressure` for every
|
|
64
|
+
* tool — not just a transient-HTTP-safe or read-only one — may advertise this.
|
|
65
|
+
*/
|
|
66
|
+
export const TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE =
|
|
67
|
+
'concurrency-backpressure';
|
|
68
|
+
|
|
69
|
+
/** The capability tokens this runner build advertises on provider execute. */
|
|
70
|
+
export const TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE =
|
|
71
|
+
TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE;
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Parse the advertisement as a comma-separated token list so a later runner can
|
|
75
|
+
* add a capability without a second header. An absent or unparsable value means
|
|
76
|
+
* "assume nothing": the gateway falls back to the behavior that is safe for the
|
|
77
|
+
* oldest deployed artifact.
|
|
78
|
+
*/
|
|
79
|
+
export function runnerAcceptsConcurrencyBackpressure(
|
|
80
|
+
headerValue: string | null | undefined,
|
|
81
|
+
): boolean {
|
|
82
|
+
if (typeof headerValue !== 'string') return false;
|
|
83
|
+
return headerValue
|
|
84
|
+
.split(',')
|
|
85
|
+
.some(
|
|
86
|
+
(token) =>
|
|
87
|
+
token.trim().toLowerCase() ===
|
|
88
|
+
TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE,
|
|
89
|
+
);
|
|
90
|
+
}
|
|
91
|
+
|
|
53
92
|
export class ToolExecuteAuthScopeChangedError extends Error {
|
|
54
93
|
readonly code = TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE;
|
|
55
94
|
readonly status = 409;
|
|
@@ -503,6 +542,10 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
503
542
|
retryDecision.reason !== 'gateway_invocation_in_progress' &&
|
|
504
543
|
retryDecision.reason !== 'concurrency_backpressure' &&
|
|
505
544
|
retryDecision.reason !== 'runtime_scheduler_capacity',
|
|
506
|
-
|
|
545
|
+
// Scheduler capacity and the relay's explicit concurrency shed both occur
|
|
546
|
+
// before a provider request is dispatched. Carry that proof in the shared
|
|
547
|
+
// classification so the retry boundary can safely pace either response
|
|
548
|
+
// without asking a non-idempotent provider call to be replayed.
|
|
549
|
+
knownNotDispatched: runtimeSchedulerCapacity || concurrencyBackpressure,
|
|
507
550
|
};
|
|
508
551
|
}
|