deepline 0.1.181 → 0.1.183
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +203 -6
- package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +8 -1
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +633 -107
- package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +3 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +1932 -1838
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +116 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/map-chunk-plan.ts +542 -78
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/output-datasets.ts +124 -12
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +292 -11
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/run-work-dispatcher.ts +519 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +2381 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +18 -5
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/work-budget.ts +130 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/worker-platform-budget.ts +39 -0
- package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +2 -0
- package/dist/bundling-sources/sdk/src/client.ts +48 -5
- package/dist/bundling-sources/sdk/src/http.ts +23 -8
- package/dist/bundling-sources/sdk/src/play.ts +200 -26
- package/dist/bundling-sources/sdk/src/plays/harness-stub.ts +56 -3
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/sdk/src/types.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +132 -10
- package/dist/bundling-sources/shared_libs/play-runtime/auth-scope-resolver.ts +92 -0
- package/dist/bundling-sources/shared_libs/play-runtime/batch-runtime.ts +4 -4
- package/dist/bundling-sources/shared_libs/play-runtime/batching-types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +101 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +1857 -314
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +106 -6
- package/dist/bundling-sources/shared_libs/play-runtime/dedup-backend.ts +0 -0
- package/dist/bundling-sources/shared_libs/play-runtime/default-batch-strategies.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durability-store.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +116 -16
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +186 -20
- package/dist/bundling-sources/shared_libs/play-runtime/dynamic-worker-version.ts +2 -0
- package/dist/bundling-sources/shared_libs/play-runtime/execution-ledger-store.ts +133 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +245 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +5 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +14 -0
- package/dist/bundling-sources/shared_libs/play-runtime/lease-policy.ts +52 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +41 -0
- package/dist/bundling-sources/shared_libs/play-runtime/providers.ts +3 -2
- package/dist/bundling-sources/shared_libs/play-runtime/query-result-dataset.ts +120 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +124 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-status.ts +6 -2
- package/dist/bundling-sources/shared_libs/play-runtime/row-isolation.ts +48 -2
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +158 -15
- package/dist/bundling-sources/shared_libs/play-runtime/run-terminal-source.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-actions.ts +9 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +2037 -262
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-contract.ts +42 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-attempt-state-machine.ts +183 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-sheet-errors.ts +88 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +8 -1
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +19 -1
- package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +343 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +43 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +139 -17
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +437 -0
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipts.ts +83 -6
- package/dist/bundling-sources/shared_libs/security/safe-outbound-fetch.ts +24 -17
- package/dist/cli/index.js +85 -23
- package/dist/cli/index.mjs +85 -23
- package/dist/index.d.mts +4 -1
- package/dist/index.d.ts +4 -1
- package/dist/index.js +274 -35
- package/dist/index.mjs +274 -35
- package/dist/plays/bundle-play-file.mjs +2 -2
- package/package.json +1 -1
|
@@ -0,0 +1,343 @@
|
|
|
1
|
+
export const PLAY_RUNTIME_TEST_FAULT_HEADER = 'x-deepline-test-fault';
|
|
2
|
+
|
|
3
|
+
export type RuntimeTestFaultName =
|
|
4
|
+
| 'receipt_complete_write_fail'
|
|
5
|
+
| 'receipt_fail_write_fail'
|
|
6
|
+
| 'worker_receipt_complete_write_fail';
|
|
7
|
+
|
|
8
|
+
export type RuntimeTestFaultRegistry = {
|
|
9
|
+
consume(name: RuntimeTestFaultName): boolean;
|
|
10
|
+
remaining(): Record<string, number>;
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
type ParsedRuntimeTestFaults =
|
|
14
|
+
| { ok: true; registry: RuntimeTestFaultRegistry | null }
|
|
15
|
+
| { ok: false; status: 400 | 403; error: string };
|
|
16
|
+
|
|
17
|
+
type RuntimeTestSeamContext = {
|
|
18
|
+
internalTokenHeader?: string | null;
|
|
19
|
+
syntheticRunHeader?: string | null;
|
|
20
|
+
};
|
|
21
|
+
|
|
22
|
+
type ParsedRuntimeTestFaultCounts =
|
|
23
|
+
| {
|
|
24
|
+
ok: true;
|
|
25
|
+
counts: Record<string, number> | null;
|
|
26
|
+
headerValue: string | null;
|
|
27
|
+
}
|
|
28
|
+
| { ok: false; status: 400 | 403; error: string };
|
|
29
|
+
|
|
30
|
+
type ValidatedRuntimeTestFaultHeader =
|
|
31
|
+
| { ok: true; headerValue: string | null }
|
|
32
|
+
| { ok: false; status: 400 | 403; error: string };
|
|
33
|
+
|
|
34
|
+
export type RuntimeTestPolicyOverrides = {
|
|
35
|
+
receiptLeaseTtlMs?: number;
|
|
36
|
+
sheetAttemptLeaseMs?: number;
|
|
37
|
+
heartbeatIntervalMs?: number;
|
|
38
|
+
workBudgetYieldLimits?: Partial<
|
|
39
|
+
Record<
|
|
40
|
+
| 'elapsed'
|
|
41
|
+
| 'subrequest'
|
|
42
|
+
| 'provider'
|
|
43
|
+
| 'tool'
|
|
44
|
+
| 'receipt'
|
|
45
|
+
| 'sheet'
|
|
46
|
+
| 'log'
|
|
47
|
+
| 'egress',
|
|
48
|
+
number
|
|
49
|
+
>
|
|
50
|
+
>;
|
|
51
|
+
batchGraceMs?: number;
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
type ValidatedRuntimeTestPolicyOverrides =
|
|
55
|
+
| { ok: true; overrides: RuntimeTestPolicyOverrides | null }
|
|
56
|
+
| { ok: false; status: 400 | 403; error: string };
|
|
57
|
+
|
|
58
|
+
const SUPPORTED_RUNTIME_TEST_FAULTS = new Set<RuntimeTestFaultName>([
|
|
59
|
+
'receipt_complete_write_fail',
|
|
60
|
+
'receipt_fail_write_fail',
|
|
61
|
+
'worker_receipt_complete_write_fail',
|
|
62
|
+
]);
|
|
63
|
+
|
|
64
|
+
const RUNTIME_TEST_POLICY_MS_FIELDS = new Set([
|
|
65
|
+
'receiptLeaseTtlMs',
|
|
66
|
+
'sheetAttemptLeaseMs',
|
|
67
|
+
'heartbeatIntervalMs',
|
|
68
|
+
'batchGraceMs',
|
|
69
|
+
]);
|
|
70
|
+
|
|
71
|
+
const RUNTIME_TEST_POLICY_WORK_BUDGET_KEYS = new Set([
|
|
72
|
+
'elapsed',
|
|
73
|
+
'subrequest',
|
|
74
|
+
'provider',
|
|
75
|
+
'tool',
|
|
76
|
+
'receipt',
|
|
77
|
+
'sheet',
|
|
78
|
+
'log',
|
|
79
|
+
'egress',
|
|
80
|
+
]);
|
|
81
|
+
|
|
82
|
+
const MAX_RUNTIME_TEST_POLICY_MS = 10 * 60_000;
|
|
83
|
+
const MAX_RUNTIME_TEST_POLICY_YIELD_LIMIT = 1_000_000;
|
|
84
|
+
|
|
85
|
+
function isProductionRuntime(): boolean {
|
|
86
|
+
const deeplineEnv = process.env.DEEPLINE_ENV?.trim().toLowerCase();
|
|
87
|
+
if (deeplineEnv === 'production' || deeplineEnv === 'prod') return true;
|
|
88
|
+
const vercelEnv = process.env.VERCEL_ENV?.trim().toLowerCase();
|
|
89
|
+
if (vercelEnv === 'production') return true;
|
|
90
|
+
if (vercelEnv === 'preview' || vercelEnv === 'development') return false;
|
|
91
|
+
return process.env.NODE_ENV === 'production';
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function testRuntimeSeamsEnabled(): boolean {
|
|
95
|
+
return process.env.DEEPLINE_TEST_RUNTIME_SEAMS === '1';
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function internalTokenAllowsRuntimeSeams(
|
|
99
|
+
context: RuntimeTestSeamContext | undefined,
|
|
100
|
+
): boolean {
|
|
101
|
+
const deeplineEnv = process.env.DEEPLINE_ENV?.trim().toLowerCase();
|
|
102
|
+
if (deeplineEnv === 'prod') return false;
|
|
103
|
+
if (deeplineEnv === 'production') return false;
|
|
104
|
+
const expected = process.env.DEEPLINE_INTERNAL_TOKEN?.trim();
|
|
105
|
+
const actual = context?.internalTokenHeader?.trim();
|
|
106
|
+
return Boolean(expected && actual && expected === actual);
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function runtimeTestSeamsAuthorized(
|
|
110
|
+
context: RuntimeTestSeamContext | undefined,
|
|
111
|
+
): boolean {
|
|
112
|
+
return (
|
|
113
|
+
testRuntimeSeamsEnabled() ||
|
|
114
|
+
internalTokenAllowsRuntimeSeams(context) ||
|
|
115
|
+
Boolean(
|
|
116
|
+
context?.syntheticRunHeader?.trim() &&
|
|
117
|
+
context.syntheticRunHeader.trim() !== '0',
|
|
118
|
+
)
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
function parseRuntimeTestFaultHeader(
|
|
123
|
+
rawHeader: string,
|
|
124
|
+
): Map<RuntimeTestFaultName, number> | { error: string } {
|
|
125
|
+
const faults = new Map<RuntimeTestFaultName, number>();
|
|
126
|
+
for (const rawPart of rawHeader.split(',')) {
|
|
127
|
+
const part = rawPart.trim();
|
|
128
|
+
if (!part) continue;
|
|
129
|
+
const [rawName, rawCount] = part.split(':');
|
|
130
|
+
const name = rawName?.trim() as RuntimeTestFaultName | undefined;
|
|
131
|
+
if (!name || !SUPPORTED_RUNTIME_TEST_FAULTS.has(name)) {
|
|
132
|
+
return { error: `Unsupported runtime test fault "${rawName ?? part}".` };
|
|
133
|
+
}
|
|
134
|
+
const count =
|
|
135
|
+
rawCount === undefined || rawCount.trim() === ''
|
|
136
|
+
? 1
|
|
137
|
+
: Number.parseInt(rawCount.trim(), 10);
|
|
138
|
+
if (!Number.isInteger(count) || count < 1) {
|
|
139
|
+
return {
|
|
140
|
+
error: `Runtime test fault "${name}" count must be a positive integer.`,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
faults.set(name, (faults.get(name) ?? 0) + count);
|
|
144
|
+
}
|
|
145
|
+
return faults;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function readPositiveIntegerField(input: {
|
|
149
|
+
value: unknown;
|
|
150
|
+
path: string;
|
|
151
|
+
max: number;
|
|
152
|
+
}): number | { error: string } {
|
|
153
|
+
if (
|
|
154
|
+
typeof input.value !== 'number' ||
|
|
155
|
+
!Number.isFinite(input.value) ||
|
|
156
|
+
!Number.isInteger(input.value) ||
|
|
157
|
+
input.value < 1 ||
|
|
158
|
+
input.value > input.max
|
|
159
|
+
) {
|
|
160
|
+
return {
|
|
161
|
+
error: `${input.path} must be an integer between 1 and ${input.max}.`,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
return input.value;
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function parseRuntimeTestPolicyOverrides(
|
|
168
|
+
value: unknown,
|
|
169
|
+
): RuntimeTestPolicyOverrides | { error: string } | null {
|
|
170
|
+
if (value === undefined || value === null) {
|
|
171
|
+
return null;
|
|
172
|
+
}
|
|
173
|
+
if (typeof value !== 'object' || Array.isArray(value)) {
|
|
174
|
+
return { error: 'testPolicyOverrides must be a JSON object.' };
|
|
175
|
+
}
|
|
176
|
+
const record = value as Record<string, unknown>;
|
|
177
|
+
const unknownKeys = Object.keys(record).filter(
|
|
178
|
+
(key) =>
|
|
179
|
+
!RUNTIME_TEST_POLICY_MS_FIELDS.has(key) &&
|
|
180
|
+
key !== 'workBudgetYieldLimits',
|
|
181
|
+
);
|
|
182
|
+
if (unknownKeys.length > 0) {
|
|
183
|
+
return {
|
|
184
|
+
error: `Unsupported testPolicyOverrides field "${unknownKeys[0]}".`,
|
|
185
|
+
};
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
const overrides: RuntimeTestPolicyOverrides = {};
|
|
189
|
+
for (const field of RUNTIME_TEST_POLICY_MS_FIELDS) {
|
|
190
|
+
if (!(field in record)) continue;
|
|
191
|
+
const parsed = readPositiveIntegerField({
|
|
192
|
+
value: record[field],
|
|
193
|
+
path: `testPolicyOverrides.${field}`,
|
|
194
|
+
max: MAX_RUNTIME_TEST_POLICY_MS,
|
|
195
|
+
});
|
|
196
|
+
if (typeof parsed !== 'number') return parsed;
|
|
197
|
+
(overrides as Record<string, number>)[field] = parsed;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if ('workBudgetYieldLimits' in record) {
|
|
201
|
+
const rawLimits = record.workBudgetYieldLimits;
|
|
202
|
+
if (
|
|
203
|
+
rawLimits === null ||
|
|
204
|
+
typeof rawLimits !== 'object' ||
|
|
205
|
+
Array.isArray(rawLimits)
|
|
206
|
+
) {
|
|
207
|
+
return {
|
|
208
|
+
error: 'testPolicyOverrides.workBudgetYieldLimits must be an object.',
|
|
209
|
+
};
|
|
210
|
+
}
|
|
211
|
+
const limits: NonNullable<
|
|
212
|
+
RuntimeTestPolicyOverrides['workBudgetYieldLimits']
|
|
213
|
+
> = {};
|
|
214
|
+
for (const [key, rawLimit] of Object.entries(
|
|
215
|
+
rawLimits as Record<string, unknown>,
|
|
216
|
+
)) {
|
|
217
|
+
if (!RUNTIME_TEST_POLICY_WORK_BUDGET_KEYS.has(key)) {
|
|
218
|
+
return {
|
|
219
|
+
error: `Unsupported testPolicyOverrides.workBudgetYieldLimits field "${key}".`,
|
|
220
|
+
};
|
|
221
|
+
}
|
|
222
|
+
const parsed = readPositiveIntegerField({
|
|
223
|
+
value: rawLimit,
|
|
224
|
+
path: `testPolicyOverrides.workBudgetYieldLimits.${key}`,
|
|
225
|
+
max: MAX_RUNTIME_TEST_POLICY_YIELD_LIMIT,
|
|
226
|
+
});
|
|
227
|
+
if (typeof parsed !== 'number') return parsed;
|
|
228
|
+
limits[key as keyof typeof limits] = parsed;
|
|
229
|
+
}
|
|
230
|
+
if (Object.keys(limits).length > 0) {
|
|
231
|
+
overrides.workBudgetYieldLimits = limits;
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
return Object.keys(overrides).length > 0 ? overrides : null;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
export function validateRuntimeTestPolicyOverrides(input: {
|
|
239
|
+
value: unknown;
|
|
240
|
+
internalTokenHeader?: string | null;
|
|
241
|
+
syntheticRunHeader?: string | null;
|
|
242
|
+
}): ValidatedRuntimeTestPolicyOverrides {
|
|
243
|
+
if (input.value === undefined || input.value === null) {
|
|
244
|
+
return { ok: true, overrides: null };
|
|
245
|
+
}
|
|
246
|
+
if (!runtimeTestSeamsAuthorized(input) || isProductionRuntime()) {
|
|
247
|
+
return {
|
|
248
|
+
ok: false,
|
|
249
|
+
status: 403,
|
|
250
|
+
error:
|
|
251
|
+
'Runtime test policy overrides are disabled. Set DEEPLINE_TEST_RUNTIME_SEAMS=1 only in dev/preview/CI harnesses.',
|
|
252
|
+
};
|
|
253
|
+
}
|
|
254
|
+
const parsed = parseRuntimeTestPolicyOverrides(input.value);
|
|
255
|
+
if (parsed === null) {
|
|
256
|
+
return { ok: true, overrides: null };
|
|
257
|
+
}
|
|
258
|
+
if ('error' in parsed) {
|
|
259
|
+
return { ok: false, status: 400, error: parsed.error };
|
|
260
|
+
}
|
|
261
|
+
return { ok: true, overrides: parsed };
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
export function readRuntimeTestFaultRegistry(input: {
|
|
265
|
+
headerValue: string | null;
|
|
266
|
+
internalTokenHeader?: string | null;
|
|
267
|
+
syntheticRunHeader?: string | null;
|
|
268
|
+
}): ParsedRuntimeTestFaults {
|
|
269
|
+
const headerValue = input.headerValue?.trim();
|
|
270
|
+
if (!headerValue) {
|
|
271
|
+
return { ok: true, registry: null };
|
|
272
|
+
}
|
|
273
|
+
if (!runtimeTestSeamsAuthorized(input) || isProductionRuntime()) {
|
|
274
|
+
return {
|
|
275
|
+
ok: false,
|
|
276
|
+
status: 403,
|
|
277
|
+
error:
|
|
278
|
+
'Runtime test seams are disabled. Set DEEPLINE_TEST_RUNTIME_SEAMS=1 only in dev/preview/CI harnesses.',
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
const parsed = parseRuntimeTestFaultHeader(headerValue);
|
|
282
|
+
if ('error' in parsed) {
|
|
283
|
+
return { ok: false, status: 400, error: parsed.error };
|
|
284
|
+
}
|
|
285
|
+
return {
|
|
286
|
+
ok: true,
|
|
287
|
+
registry: {
|
|
288
|
+
consume(name) {
|
|
289
|
+
const count = parsed.get(name) ?? 0;
|
|
290
|
+
if (count <= 0) return false;
|
|
291
|
+
if (count === 1) {
|
|
292
|
+
parsed.delete(name);
|
|
293
|
+
} else {
|
|
294
|
+
parsed.set(name, count - 1);
|
|
295
|
+
}
|
|
296
|
+
return true;
|
|
297
|
+
},
|
|
298
|
+
remaining() {
|
|
299
|
+
return Object.fromEntries(parsed.entries());
|
|
300
|
+
},
|
|
301
|
+
},
|
|
302
|
+
};
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
export function validateRuntimeTestFaultHeader(input: {
|
|
306
|
+
headerValue: string | null;
|
|
307
|
+
internalTokenHeader?: string | null;
|
|
308
|
+
syntheticRunHeader?: string | null;
|
|
309
|
+
}): ValidatedRuntimeTestFaultHeader {
|
|
310
|
+
const parsed = parseRuntimeTestFaultCounts(input);
|
|
311
|
+
if (parsed.ok === false) {
|
|
312
|
+
return { ok: false, status: parsed.status, error: parsed.error };
|
|
313
|
+
}
|
|
314
|
+
return { ok: true, headerValue: parsed.headerValue };
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
export function parseRuntimeTestFaultCounts(input: {
|
|
318
|
+
headerValue: string | null;
|
|
319
|
+
internalTokenHeader?: string | null;
|
|
320
|
+
syntheticRunHeader?: string | null;
|
|
321
|
+
}): ParsedRuntimeTestFaultCounts {
|
|
322
|
+
const headerValue = input.headerValue?.trim();
|
|
323
|
+
if (!headerValue) {
|
|
324
|
+
return { ok: true, counts: null, headerValue: null };
|
|
325
|
+
}
|
|
326
|
+
if (!runtimeTestSeamsAuthorized(input) || isProductionRuntime()) {
|
|
327
|
+
return {
|
|
328
|
+
ok: false,
|
|
329
|
+
status: 403,
|
|
330
|
+
error:
|
|
331
|
+
'Runtime test seams are disabled. Set DEEPLINE_TEST_RUNTIME_SEAMS=1 only in dev/preview/CI harnesses.',
|
|
332
|
+
};
|
|
333
|
+
}
|
|
334
|
+
const parsed = parseRuntimeTestFaultHeader(headerValue);
|
|
335
|
+
if ('error' in parsed) {
|
|
336
|
+
return { ok: false, status: 400, error: parsed.error };
|
|
337
|
+
}
|
|
338
|
+
return {
|
|
339
|
+
ok: true,
|
|
340
|
+
counts: Object.fromEntries(parsed.entries()),
|
|
341
|
+
headerValue,
|
|
342
|
+
};
|
|
343
|
+
}
|
|
@@ -12,6 +12,17 @@ export const TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS = 1_000;
|
|
|
12
12
|
export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000;
|
|
13
13
|
export const TOOL_EXECUTE_RETRY_DELAY_MAX_MS = 5_000;
|
|
14
14
|
export const TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS = 60_000;
|
|
15
|
+
export const TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE = 'AUTH_SCOPE_CHANGED';
|
|
16
|
+
|
|
17
|
+
export class ToolExecuteAuthScopeChangedError extends Error {
|
|
18
|
+
readonly code = TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE;
|
|
19
|
+
readonly status = 409;
|
|
20
|
+
|
|
21
|
+
constructor(message = 'Tool execution auth scope changed.') {
|
|
22
|
+
super(message);
|
|
23
|
+
this.name = 'ToolExecuteAuthScopeChangedError';
|
|
24
|
+
}
|
|
25
|
+
}
|
|
15
26
|
|
|
16
27
|
export type ToolExecuteHttpRetryDecision = {
|
|
17
28
|
retryable: boolean;
|
|
@@ -42,6 +53,35 @@ export type ToolExecuteHttpFailureAttemptTracker = {
|
|
|
42
53
|
}): number;
|
|
43
54
|
};
|
|
44
55
|
|
|
56
|
+
function parseJsonObject(text: string): Record<string, unknown> | null {
|
|
57
|
+
if (!text.trim()) return null;
|
|
58
|
+
try {
|
|
59
|
+
const parsed = JSON.parse(text) as unknown;
|
|
60
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
61
|
+
? (parsed as Record<string, unknown>)
|
|
62
|
+
: null;
|
|
63
|
+
} catch {
|
|
64
|
+
return null;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
export function parseToolExecuteAuthScopeChangedError(input: {
|
|
69
|
+
status: number;
|
|
70
|
+
bodyText: string;
|
|
71
|
+
}): ToolExecuteAuthScopeChangedError | null {
|
|
72
|
+
if (input.status !== 409) return null;
|
|
73
|
+
const body = parseJsonObject(input.bodyText);
|
|
74
|
+
const code = typeof body?.code === 'string' ? body.code.trim() : '';
|
|
75
|
+
if (code !== TOOL_EXECUTE_AUTH_SCOPE_CHANGED_CODE) return null;
|
|
76
|
+
const message =
|
|
77
|
+
typeof body?.error === 'string' && body.error.trim()
|
|
78
|
+
? body.error.trim()
|
|
79
|
+
: typeof body?.message === 'string' && body.message.trim()
|
|
80
|
+
? body.message.trim()
|
|
81
|
+
: undefined;
|
|
82
|
+
return new ToolExecuteAuthScopeChangedError(message);
|
|
83
|
+
}
|
|
84
|
+
|
|
45
85
|
function decideToolExecuteHttpRetry(input: {
|
|
46
86
|
status: number;
|
|
47
87
|
hardBillingFailure?: boolean;
|
|
@@ -164,6 +204,9 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
164
204
|
});
|
|
165
205
|
const shouldRetry =
|
|
166
206
|
retryDecision.retryable && input.attempt < retryDecision.attemptCap;
|
|
207
|
+
error.receiptFailureKind = retryDecision.retryable
|
|
208
|
+
? 'repairable'
|
|
209
|
+
: 'terminal';
|
|
167
210
|
const retryAfterMs = parseToolExecuteRetryAfterMs(
|
|
168
211
|
input.retryAfterHeader,
|
|
169
212
|
input.nowMs,
|
|
@@ -1,17 +1,22 @@
|
|
|
1
|
+
import type { WorkReceiptFailureKind } from './work-receipts';
|
|
2
|
+
|
|
1
3
|
export class ToolHttpError extends Error {
|
|
2
4
|
readonly billing: Record<string, unknown> | null;
|
|
3
5
|
/** HTTP status of the failed tool-execute response (e.g. 429, 502). */
|
|
4
6
|
readonly status: number;
|
|
7
|
+
receiptFailureKind: WorkReceiptFailureKind | null;
|
|
5
8
|
|
|
6
9
|
constructor(
|
|
7
10
|
message: string,
|
|
8
11
|
billing: Record<string, unknown> | null,
|
|
9
12
|
status: number,
|
|
13
|
+
receiptFailureKind: WorkReceiptFailureKind | null = null,
|
|
10
14
|
) {
|
|
11
15
|
super(message);
|
|
12
16
|
this.name = 'ToolHttpError';
|
|
13
17
|
this.billing = billing;
|
|
14
18
|
this.status = status;
|
|
19
|
+
this.receiptFailureKind = receiptFailureKind;
|
|
15
20
|
}
|
|
16
21
|
}
|
|
17
22
|
|
|
@@ -247,3 +252,9 @@ export function isHardBillingToolHttpError(error: unknown): boolean {
|
|
|
247
252
|
export function isRateLimitToolHttpError(error: unknown): boolean {
|
|
248
253
|
return error instanceof ToolHttpError && error.status === 429;
|
|
249
254
|
}
|
|
255
|
+
|
|
256
|
+
export function getToolHttpErrorReceiptFailureKind(
|
|
257
|
+
error: unknown,
|
|
258
|
+
): WorkReceiptFailureKind | null {
|
|
259
|
+
return error instanceof ToolHttpError ? error.receiptFailureKind : null;
|
|
260
|
+
}
|
|
@@ -52,7 +52,12 @@ type PathSegment = string | number | '*';
|
|
|
52
52
|
|
|
53
53
|
const SERIALIZED_TOOL_EXECUTE_RESULT_KIND = 'deepline.tool_execute_result.v1';
|
|
54
54
|
const SERIALIZED_TOOL_RESULT_LIST_PREVIEW_LIMIT = 5;
|
|
55
|
+
const LIVE_TOOL_RESULT_LIST_DATASET_REGISTRY_LIMIT = 256;
|
|
55
56
|
const SERIALIZED_TOOL_LIST_ROWS = Symbol('deepline.serialized_tool_list_rows');
|
|
57
|
+
const liveToolResultListDatasets = new Map<
|
|
58
|
+
string,
|
|
59
|
+
PlayDataset<Record<string, unknown>>
|
|
60
|
+
>();
|
|
56
61
|
|
|
57
62
|
const TARGET_FALLBACK_KEYS: Record<string, readonly RegExp[]> = {
|
|
58
63
|
email: [/^email$/i, /^address$/i, /email/i],
|
|
@@ -1169,6 +1174,9 @@ export function readList(
|
|
|
1169
1174
|
if (selector) {
|
|
1170
1175
|
const paths = Array.isArray(selector) ? selector : [selector];
|
|
1171
1176
|
const declaredList = findDeclaredListAccessor(result, paths);
|
|
1177
|
+
if (declaredList && preservedSerializedListRows(result)) {
|
|
1178
|
+
return declaredList.get();
|
|
1179
|
+
}
|
|
1172
1180
|
const found = findFirstTargetByPath(resultRootOf(result), paths)?.value;
|
|
1173
1181
|
const rows = normalizeRows(found);
|
|
1174
1182
|
if (rows) {
|
|
@@ -1186,6 +1194,55 @@ export function readList(
|
|
|
1186
1194
|
return Object.values(result.extractedLists)[0]?.get() ?? [];
|
|
1187
1195
|
}
|
|
1188
1196
|
|
|
1197
|
+
export function attachToolResultListDataset<T extends Record<string, unknown>>(
|
|
1198
|
+
result: ToolExecuteResult,
|
|
1199
|
+
input: {
|
|
1200
|
+
name: string;
|
|
1201
|
+
path: string;
|
|
1202
|
+
dataset: PlayDataset<T>;
|
|
1203
|
+
count: number;
|
|
1204
|
+
keys?: Record<string, string>;
|
|
1205
|
+
},
|
|
1206
|
+
): ToolExecuteResult {
|
|
1207
|
+
const existing = result._metadata.lists[input.name];
|
|
1208
|
+
result._metadata.lists[input.name] = {
|
|
1209
|
+
path: input.path,
|
|
1210
|
+
count: input.count,
|
|
1211
|
+
keys: input.keys ?? existing?.keys ?? {},
|
|
1212
|
+
};
|
|
1213
|
+
const accessor = {
|
|
1214
|
+
path: input.path,
|
|
1215
|
+
count: input.count,
|
|
1216
|
+
keys: result._metadata.lists[input.name].keys,
|
|
1217
|
+
} as ToolResultListAccessor<T>;
|
|
1218
|
+
Object.defineProperty(accessor, 'get', {
|
|
1219
|
+
value() {
|
|
1220
|
+
return input.dataset;
|
|
1221
|
+
},
|
|
1222
|
+
enumerable: false,
|
|
1223
|
+
});
|
|
1224
|
+
result.extractedLists[input.name] = accessor;
|
|
1225
|
+
const serialized = input.dataset.toJSON();
|
|
1226
|
+
let root: unknown = { toolResponse: { raw: result.toolResponse.raw } };
|
|
1227
|
+
const candidates = [...candidateResultPaths(input.path)].filter(
|
|
1228
|
+
(candidate, index, all) => all.indexOf(candidate) === index,
|
|
1229
|
+
);
|
|
1230
|
+
for (const candidate of candidates) {
|
|
1231
|
+
if (!Array.isArray(getAtPath(root, candidate))) continue;
|
|
1232
|
+
root = replaceAtPath(root, candidate, serialized.preview);
|
|
1233
|
+
break;
|
|
1234
|
+
}
|
|
1235
|
+
if (
|
|
1236
|
+
isRecord(root) &&
|
|
1237
|
+
isRecord(root.toolResponse) &&
|
|
1238
|
+
Object.prototype.hasOwnProperty.call(root.toolResponse, 'raw')
|
|
1239
|
+
) {
|
|
1240
|
+
result.toolResponse.raw = root.toolResponse.raw;
|
|
1241
|
+
result.toolOutput.raw = root.toolResponse.raw;
|
|
1242
|
+
}
|
|
1243
|
+
return result;
|
|
1244
|
+
}
|
|
1245
|
+
|
|
1189
1246
|
function normalizeListPathForMatch(path: string): string {
|
|
1190
1247
|
return path.replace(/\[(?:\*|\d+)\]$/g, '');
|
|
1191
1248
|
}
|
|
@@ -1237,10 +1294,12 @@ function serializedListDatasetsFromResult(
|
|
|
1237
1294
|
): Record<string, SerializedPlayDataset<Record<string, unknown>>> | undefined {
|
|
1238
1295
|
const entries = Object.entries(value.extractedLists).flatMap(
|
|
1239
1296
|
([name, accessor]) => {
|
|
1240
|
-
const
|
|
1297
|
+
const dataset = accessor.get();
|
|
1298
|
+
const serialized = dataset.toJSON();
|
|
1241
1299
|
if (!isSerializedPlayDataset<Record<string, unknown>>(serialized)) {
|
|
1242
1300
|
return [];
|
|
1243
1301
|
}
|
|
1302
|
+
rememberLiveToolResultListDataset(serialized.datasetId, dataset);
|
|
1244
1303
|
return [
|
|
1245
1304
|
[
|
|
1246
1305
|
name,
|
|
@@ -1255,6 +1314,32 @@ function serializedListDatasetsFromResult(
|
|
|
1255
1314
|
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
|
|
1256
1315
|
}
|
|
1257
1316
|
|
|
1317
|
+
function rememberLiveToolResultListDataset(
|
|
1318
|
+
datasetId: string,
|
|
1319
|
+
dataset: PlayDataset<Record<string, unknown>>,
|
|
1320
|
+
): void {
|
|
1321
|
+
if (!datasetId) return;
|
|
1322
|
+
liveToolResultListDatasets.delete(datasetId);
|
|
1323
|
+
liveToolResultListDatasets.set(datasetId, dataset);
|
|
1324
|
+
while (
|
|
1325
|
+
liveToolResultListDatasets.size > LIVE_TOOL_RESULT_LIST_DATASET_REGISTRY_LIMIT
|
|
1326
|
+
) {
|
|
1327
|
+
const oldest = liveToolResultListDatasets.keys().next().value;
|
|
1328
|
+
if (typeof oldest !== 'string') break;
|
|
1329
|
+
liveToolResultListDatasets.delete(oldest);
|
|
1330
|
+
}
|
|
1331
|
+
}
|
|
1332
|
+
|
|
1333
|
+
function readLiveToolResultListDataset(
|
|
1334
|
+
serialized: SerializedPlayDataset<Record<string, unknown>>,
|
|
1335
|
+
): PlayDataset<Record<string, unknown>> | null {
|
|
1336
|
+
const dataset = liveToolResultListDatasets.get(serialized.datasetId);
|
|
1337
|
+
if (!dataset) return null;
|
|
1338
|
+
const current = dataset.toJSON();
|
|
1339
|
+
if (current.count !== serialized.count) return null;
|
|
1340
|
+
return dataset;
|
|
1341
|
+
}
|
|
1342
|
+
|
|
1258
1343
|
function serializedListRowsFromRaw(input: {
|
|
1259
1344
|
raw: unknown;
|
|
1260
1345
|
metadata: ToolResultMetadataInput;
|
|
@@ -1273,11 +1358,13 @@ function serializedListRowsFromRaw(input: {
|
|
|
1273
1358
|
function preservedSerializedListRows(
|
|
1274
1359
|
value: ToolExecuteResult,
|
|
1275
1360
|
): Record<string, Array<Record<string, unknown>>> | undefined {
|
|
1276
|
-
const rows = (
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1280
|
-
|
|
1361
|
+
const rows = (
|
|
1362
|
+
value as ToolExecuteResult & {
|
|
1363
|
+
[SERIALIZED_TOOL_LIST_ROWS]?:
|
|
1364
|
+
| Record<string, Array<Record<string, unknown>>>
|
|
1365
|
+
| undefined;
|
|
1366
|
+
}
|
|
1367
|
+
)[SERIALIZED_TOOL_LIST_ROWS];
|
|
1281
1368
|
return rows && Object.keys(rows).length > 0 ? rows : undefined;
|
|
1282
1369
|
}
|
|
1283
1370
|
|
|
@@ -1301,10 +1388,7 @@ function serializedListRowsFromResult(input: {
|
|
|
1301
1388
|
const entries = [...names].flatMap((name) => {
|
|
1302
1389
|
const dataset = input.listDatasets?.[name];
|
|
1303
1390
|
const preservedRows = preserved?.[name];
|
|
1304
|
-
if (
|
|
1305
|
-
preservedRows &&
|
|
1306
|
-
(!dataset || preservedRows.length >= dataset.count)
|
|
1307
|
-
) {
|
|
1391
|
+
if (preservedRows && (!dataset || preservedRows.length >= dataset.count)) {
|
|
1308
1392
|
return [[name, preservedRows]] as const;
|
|
1309
1393
|
}
|
|
1310
1394
|
const rows = rawRows?.[name];
|
|
@@ -1352,6 +1436,41 @@ function serializedRawWithListPreviews(input: {
|
|
|
1352
1436
|
: input.raw;
|
|
1353
1437
|
}
|
|
1354
1438
|
|
|
1439
|
+
function rawWithSerializedListRows(input: {
|
|
1440
|
+
raw: unknown;
|
|
1441
|
+
lists: Record<string, ToolResultListMetadata>;
|
|
1442
|
+
listDatasets:
|
|
1443
|
+
| Record<string, SerializedPlayDataset<Record<string, unknown>>>
|
|
1444
|
+
| undefined;
|
|
1445
|
+
listRows: Record<string, Array<Record<string, unknown>>> | undefined;
|
|
1446
|
+
}): unknown {
|
|
1447
|
+
if (!input.listRows || Object.keys(input.listRows).length === 0) {
|
|
1448
|
+
return input.raw;
|
|
1449
|
+
}
|
|
1450
|
+
|
|
1451
|
+
let root: unknown = { toolResponse: { raw: input.raw } };
|
|
1452
|
+
for (const [name, rows] of Object.entries(input.listRows)) {
|
|
1453
|
+
const sourcePath =
|
|
1454
|
+
input.listDatasets?.[name]?.sourceLabel ??
|
|
1455
|
+
input.lists[name]?.path ??
|
|
1456
|
+
name;
|
|
1457
|
+
const candidates = [...candidateResultPaths(sourcePath)].filter(
|
|
1458
|
+
(candidate, index, all) => all.indexOf(candidate) === index,
|
|
1459
|
+
);
|
|
1460
|
+
for (const candidate of candidates) {
|
|
1461
|
+
if (!Array.isArray(getAtPath(root, candidate))) continue;
|
|
1462
|
+
root = replaceAtPath(root, candidate, rows);
|
|
1463
|
+
break;
|
|
1464
|
+
}
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
return isRecord(root) &&
|
|
1468
|
+
isRecord(root.toolResponse) &&
|
|
1469
|
+
Object.prototype.hasOwnProperty.call(root.toolResponse, 'raw')
|
|
1470
|
+
? root.toolResponse.raw
|
|
1471
|
+
: input.raw;
|
|
1472
|
+
}
|
|
1473
|
+
|
|
1355
1474
|
function createDatasetFromSerializedToolList(
|
|
1356
1475
|
serialized: SerializedPlayDataset<Record<string, unknown>>,
|
|
1357
1476
|
rows?: Array<Record<string, unknown>>,
|
|
@@ -1411,6 +1530,12 @@ function applySerializedListDatasets(
|
|
|
1411
1530
|
): ToolExecuteResult {
|
|
1412
1531
|
if (!listDatasets) return result;
|
|
1413
1532
|
if (listRows && Object.keys(listRows).length > 0) {
|
|
1533
|
+
result.toolResponse.raw = rawWithSerializedListRows({
|
|
1534
|
+
raw: result.toolResponse.raw,
|
|
1535
|
+
lists: result._metadata.lists,
|
|
1536
|
+
listDatasets,
|
|
1537
|
+
listRows,
|
|
1538
|
+
});
|
|
1414
1539
|
Object.defineProperty(result, SERIALIZED_TOOL_LIST_ROWS, {
|
|
1415
1540
|
value: listRows,
|
|
1416
1541
|
enumerable: false,
|
|
@@ -1422,10 +1547,9 @@ function applySerializedListDatasets(
|
|
|
1422
1547
|
if (!isSerializedPlayDataset<Record<string, unknown>>(serialized)) {
|
|
1423
1548
|
continue;
|
|
1424
1549
|
}
|
|
1425
|
-
const dataset =
|
|
1426
|
-
serialized
|
|
1427
|
-
listRows?.[name]
|
|
1428
|
-
);
|
|
1550
|
+
const dataset =
|
|
1551
|
+
readLiveToolResultListDataset(serialized) ??
|
|
1552
|
+
createDatasetFromSerializedToolList(serialized, listRows?.[name]);
|
|
1429
1553
|
const existingMetadata = result._metadata.lists[name];
|
|
1430
1554
|
result._metadata.lists[name] = {
|
|
1431
1555
|
path: serialized.sourceLabel ?? existingMetadata?.path ?? name,
|
|
@@ -1528,9 +1652,7 @@ export function deserializeToolExecuteResult(
|
|
|
1528
1652
|
jobId: value.job_id,
|
|
1529
1653
|
result: {
|
|
1530
1654
|
data: value.toolResponse.raw,
|
|
1531
|
-
...(value.toolResponse.meta
|
|
1532
|
-
? { meta: value.toolResponse.meta }
|
|
1533
|
-
: {}),
|
|
1655
|
+
...(value.toolResponse.meta ? { meta: value.toolResponse.meta } : {}),
|
|
1534
1656
|
},
|
|
1535
1657
|
metadata: value.metadata,
|
|
1536
1658
|
execution: value.execution,
|