deepline 0.3.184 → 0.3.185
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +53 -0
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +751 -193
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +4 -0
- package/dist/bundling-sources/shared_libs/play-runtime/durable-receipt-execution.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +437 -150
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +47 -4
- package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +3 -1
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/map-execution-frame.ts +3 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +22 -2
- package/dist/bundling-sources/shared_libs/play-runtime/resource-governor.ts +6 -1
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +11 -5
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-flow.ts +102 -12
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-operation-contract.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/absurd-runtime-worker.ts +136 -28
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts +80 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +897 -207
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +120 -34
- package/dist/bundling-sources/shared_libs/play-runtime/tool-result.ts +82 -2
- package/dist/bundling-sources/shared_libs/plays/tool-result-types.ts +7 -2
- package/dist/cli/index.js +1 -1
- package/dist/cli/index.mjs +1 -1
- package/dist/index.js +1 -1
- package/dist/index.mjs +1 -1
- package/dist/install-integrity.json +1 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -269,7 +269,10 @@ const POSTGRES_SCHEDULER_SCHEMA_INIT_STATEMENT_TIMEOUT_MS = 60_000;
|
|
|
269
269
|
// deployment retries do not re-enter the old cross-table DDL transaction.
|
|
270
270
|
// v45 adds the bucket-leading expiration index used by rate-reservation
|
|
271
271
|
// cleanup. Shared schemas build it concurrently through the deploy migrator.
|
|
272
|
-
|
|
272
|
+
// v46 adds a bounded provider-scope Retry-After hold with shared adaptive-rate
|
|
273
|
+
// state. The scope owns learning and dispatch across runs and tool rules;
|
|
274
|
+
// existing deadline rows expand additively and initialize on their first seed.
|
|
275
|
+
export const POSTGRES_SCHEDULER_SCHEMA_VERSION = 46;
|
|
273
276
|
const POSTGRES_SCHEDULER_SCHEMA_MIGRATIONS_TABLE =
|
|
274
277
|
'scheduler_schema_migrations';
|
|
275
278
|
export const POSTGRES_SCHEDULER_OPEN_RECOVERY_INDEX =
|
|
@@ -1065,6 +1068,8 @@ export type PostgresSchedulerSchemaShapeRequirements = Readonly<{
|
|
|
1065
1068
|
cronScheduleSchema?: boolean;
|
|
1066
1069
|
/** v43 added per-run fair-share tracking for rate buckets; v42 ignores it. */
|
|
1067
1070
|
rateRequesterFairShare?: boolean;
|
|
1071
|
+
/** v45 added durable Retry-After holds shared across rules and runs. */
|
|
1072
|
+
rateScopeRetryAfter?: boolean;
|
|
1068
1073
|
}>;
|
|
1069
1074
|
|
|
1070
1075
|
export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
@@ -1371,6 +1376,31 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1371
1376
|
)
|
|
1372
1377
|
)
|
|
1373
1378
|
)
|
|
1379
|
+
AND (
|
|
1380
|
+
$36::boolean = false
|
|
1381
|
+
OR (
|
|
1382
|
+
to_regclass($37) IS NOT NULL
|
|
1383
|
+
AND (
|
|
1384
|
+
SELECT count(DISTINCT column_name)
|
|
1385
|
+
FROM information_schema.columns
|
|
1386
|
+
WHERE table_schema = $1
|
|
1387
|
+
AND table_name = 'rate_scope_holds'
|
|
1388
|
+
AND column_name IN (
|
|
1389
|
+
'bucket_id', 'retry_at_ms', 'refill_per_sec', 'capacity', 'max_rps',
|
|
1390
|
+
'tokens', 'refilled_at_ms', 'next_dispatch_at_ms', 'seen_backpressure',
|
|
1391
|
+
'cool_until_ms', 'last_ramp_at_ms', 'rate_observation_started_at_ms',
|
|
1392
|
+
'rate_successes', 'rate_recovery_interval_ms',
|
|
1393
|
+
'rate_throttle_ceiling_rps'
|
|
1394
|
+
)
|
|
1395
|
+
) = 15
|
|
1396
|
+
AND (
|
|
1397
|
+
SELECT count(DISTINCT column_name)
|
|
1398
|
+
FROM information_schema.columns
|
|
1399
|
+
WHERE table_schema = $1 AND table_name = 'rate_bucket'
|
|
1400
|
+
AND column_name IN ('rate_observation_started_at_ms', 'rate_successes', 'rate_recovery_interval_ms')
|
|
1401
|
+
) = 3
|
|
1402
|
+
)
|
|
1403
|
+
)
|
|
1374
1404
|
AND (
|
|
1375
1405
|
$32::boolean = false
|
|
1376
1406
|
OR (
|
|
@@ -1695,6 +1725,8 @@ export async function hasCurrentPostgresSchedulerSchemaShape(
|
|
|
1695
1725
|
`${quotedSchema}.${quoteIdent(POSTGRES_SCHEDULER_CRON_SCHEDULES_TABLE)}`,
|
|
1696
1726
|
requirements.rateRequesterFairShare !== false,
|
|
1697
1727
|
`${quotedSchema}.${quoteIdent('rate_requesters')}`,
|
|
1728
|
+
requirements.rateScopeRetryAfter !== false,
|
|
1729
|
+
`${quotedSchema}.${quoteIdent('rate_scope_holds')}`,
|
|
1698
1730
|
],
|
|
1699
1731
|
);
|
|
1700
1732
|
if (currentShape.rows[0]?.ready !== true) return false;
|
|
@@ -14,6 +14,9 @@ export type PlayExecutionSuspension =
|
|
|
14
14
|
boundaryId: string;
|
|
15
15
|
eventKey: string;
|
|
16
16
|
timeoutMs: number;
|
|
17
|
+
eventExpiresAtMs?: number;
|
|
18
|
+
/** A terminal timeout's deliberate long park, not its original event wait. */
|
|
19
|
+
postTimeoutPark?: boolean;
|
|
17
20
|
messageRef?: {
|
|
18
21
|
channel: string;
|
|
19
22
|
ts: string;
|
|
@@ -21,10 +24,13 @@ export type PlayExecutionSuspension =
|
|
|
21
24
|
}
|
|
22
25
|
| {
|
|
23
26
|
kind: 'integration_event_batch';
|
|
27
|
+
/** Earliest pending provider row; absence preserves older event-only waits. */
|
|
28
|
+
providerWakeAtMs?: number;
|
|
24
29
|
boundaries: Array<{
|
|
25
30
|
boundaryId: string;
|
|
26
31
|
eventKey: string;
|
|
27
32
|
timeoutMs: number;
|
|
33
|
+
eventExpiresAtMs?: number;
|
|
28
34
|
messageRef?: {
|
|
29
35
|
channel: string;
|
|
30
36
|
ts: string;
|
|
@@ -123,6 +129,7 @@ export class PlayRowExecutionSuspendedError extends Error {
|
|
|
123
129
|
boundaryId: string;
|
|
124
130
|
eventKey: string;
|
|
125
131
|
timeoutMs: number;
|
|
132
|
+
eventExpiresAtMs?: number;
|
|
126
133
|
messageRef?: {
|
|
127
134
|
channel: string;
|
|
128
135
|
ts: string;
|
|
@@ -133,6 +140,7 @@ export class PlayRowExecutionSuspendedError extends Error {
|
|
|
133
140
|
boundaryId: string;
|
|
134
141
|
eventKey: string;
|
|
135
142
|
timeoutMs: number;
|
|
143
|
+
eventExpiresAtMs?: number;
|
|
136
144
|
messageRef?: {
|
|
137
145
|
channel: string;
|
|
138
146
|
ts: string;
|
|
@@ -146,6 +154,19 @@ export class PlayRowExecutionSuspendedError extends Error {
|
|
|
146
154
|
}
|
|
147
155
|
}
|
|
148
156
|
|
|
157
|
+
/** A provider wait is a control handoff, never a failed cell or receipt. */
|
|
158
|
+
export class PlayRowProviderBackpressureError extends Error {
|
|
159
|
+
constructor(
|
|
160
|
+
readonly provider: string,
|
|
161
|
+
readonly retryAtMs: number,
|
|
162
|
+
) {
|
|
163
|
+
super(
|
|
164
|
+
`Play row is waiting for ${provider} provider backpressure to clear.`,
|
|
165
|
+
);
|
|
166
|
+
this.name = 'PlayRowProviderBackpressureError';
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
|
|
149
170
|
export function isPlayRowExecutionSuspendedError(
|
|
150
171
|
value: unknown,
|
|
151
172
|
): value is PlayRowExecutionSuspendedError {
|
|
@@ -29,6 +29,8 @@ export const TOOL_EXECUTE_CONCURRENCY_BACKPRESSURE_MAX_ATTEMPTS = 65;
|
|
|
29
29
|
// for an operation that has not opted into ordinary transient-5xx retries.
|
|
30
30
|
// Keep its window aligned with other local control-plane queue signals.
|
|
31
31
|
export const TOOL_EXECUTE_RUNTIME_SCHEDULER_CAPACITY_MAX_ATTEMPTS = 65;
|
|
32
|
+
/** Re-admission after a stale shared provider permit is safely replayable. */
|
|
33
|
+
export const TOOL_EXECUTE_PROVIDER_PACING_WAIT_MAX_ATTEMPTS = 65;
|
|
32
34
|
export const TOOL_EXECUTE_TRANSPORT_MAX_ATTEMPTS = 3;
|
|
33
35
|
export const TOOL_EXECUTE_TRANSPORT_RETRY_DELAY_MS = 1_000;
|
|
34
36
|
export const TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS = 1_000;
|
|
@@ -69,6 +71,15 @@ export const TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE =
|
|
|
69
71
|
/** The capability tokens this runner build advertises on provider execute. */
|
|
70
72
|
export const TOOL_EXECUTE_RUNNER_ACCEPTS_VALUE =
|
|
71
73
|
TOOL_EXECUTE_RUNNER_ACCEPTS_CONCURRENCY_BACKPRESSURE;
|
|
74
|
+
/** Runner proof headers consumed only by the execution gateway. */
|
|
75
|
+
export const TOOL_EXECUTE_PROVIDER_RATE_SCOPE_HEADER =
|
|
76
|
+
'x-deepline-provider-rate-scope';
|
|
77
|
+
export const TOOL_EXECUTE_PROVIDER_RATE_RETRY_SNAPSHOT_HEADER =
|
|
78
|
+
'x-deepline-provider-rate-retry-snapshot';
|
|
79
|
+
/** Server-minted only for an internal executor's provider-owned 429. */
|
|
80
|
+
export const TOOL_EXECUTE_PROVIDER_THROTTLE_HEADER =
|
|
81
|
+
'x-deepline-provider-throttle';
|
|
82
|
+
export const TOOL_EXECUTE_PROVIDER_PACING_WAIT_CODE = 'PROVIDER_PACING_WAIT';
|
|
72
83
|
|
|
73
84
|
/**
|
|
74
85
|
* Parse the advertisement as a comma-separated token list so a later runner can
|
|
@@ -106,6 +117,7 @@ export type ToolExecuteHttpRetryDecision = {
|
|
|
106
117
|
| 'rate_limit'
|
|
107
118
|
| 'concurrency_backpressure'
|
|
108
119
|
| 'runtime_scheduler_capacity'
|
|
120
|
+
| 'provider_pacing_wait'
|
|
109
121
|
| 'idempotency_in_progress'
|
|
110
122
|
| 'gateway_invocation_in_progress'
|
|
111
123
|
| 'customer_db_storage_unavailable'
|
|
@@ -122,6 +134,8 @@ export type ToolExecuteHttpFailureOutcome = ToolExecuteHttpRetryDecision & {
|
|
|
122
134
|
fate: 'retry' | 'settle_row_failure' | 'fail_run';
|
|
123
135
|
retryDelayMs: number;
|
|
124
136
|
backpressureDelayMs: number | null;
|
|
137
|
+
/** Retry-After for known-not-dispatched gateway pacing; not provider feedback. */
|
|
138
|
+
providerPacingWaitMs: number | null;
|
|
125
139
|
chargeRetryBudget: boolean;
|
|
126
140
|
/** The response proves the provider request was never dispatched. */
|
|
127
141
|
knownNotDispatched: boolean;
|
|
@@ -214,6 +228,19 @@ function isRuntimeSchedulerCapacityResponse(input: {
|
|
|
214
228
|
);
|
|
215
229
|
}
|
|
216
230
|
|
|
231
|
+
function isProviderPacingWaitResponse(input: {
|
|
232
|
+
status: number;
|
|
233
|
+
bodyText: string;
|
|
234
|
+
}): boolean {
|
|
235
|
+
const body = parseJsonObject(input.bodyText);
|
|
236
|
+
return (
|
|
237
|
+
input.status === 503 &&
|
|
238
|
+
body?.code === TOOL_EXECUTE_PROVIDER_PACING_WAIT_CODE &&
|
|
239
|
+
body.failure_origin === 'deepline_rate_limit' &&
|
|
240
|
+
body.provider_request_dispatched === false
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
|
|
217
244
|
function idempotencyInProgressRetryDelayMs(attempt: number): number {
|
|
218
245
|
return (
|
|
219
246
|
TOOL_EXECUTE_IDEMPOTENCY_IN_PROGRESS_RETRY_DELAYS_MS[
|
|
@@ -249,6 +276,7 @@ function decideToolExecuteHttpRetry(input: {
|
|
|
249
276
|
customerDbStorageUnavailable?: boolean;
|
|
250
277
|
concurrencyBackpressure?: boolean;
|
|
251
278
|
runtimeSchedulerCapacity?: boolean;
|
|
279
|
+
providerPacingWait?: boolean;
|
|
252
280
|
hardBillingFailure?: boolean;
|
|
253
281
|
hasRetryAfterHeader?: boolean;
|
|
254
282
|
transientHttpRetrySafe?: boolean;
|
|
@@ -289,6 +317,13 @@ function decideToolExecuteHttpRetry(input: {
|
|
|
289
317
|
reason: 'runtime_scheduler_capacity',
|
|
290
318
|
};
|
|
291
319
|
}
|
|
320
|
+
if (input.providerPacingWait) {
|
|
321
|
+
return {
|
|
322
|
+
retryable: true,
|
|
323
|
+
attemptCap: TOOL_EXECUTE_PROVIDER_PACING_WAIT_MAX_ATTEMPTS,
|
|
324
|
+
reason: 'provider_pacing_wait',
|
|
325
|
+
};
|
|
326
|
+
}
|
|
292
327
|
if (input.gatewayInvocationInProgress) {
|
|
293
328
|
return {
|
|
294
329
|
retryable: true,
|
|
@@ -342,6 +377,7 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
|
|
|
342
377
|
rate_limit: 0,
|
|
343
378
|
concurrency_backpressure: 0,
|
|
344
379
|
runtime_scheduler_capacity: 0,
|
|
380
|
+
provider_pacing_wait: 0,
|
|
345
381
|
idempotency_in_progress: 0,
|
|
346
382
|
gateway_invocation_in_progress: 0,
|
|
347
383
|
customer_db_storage_unavailable: 0,
|
|
@@ -376,6 +412,10 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
|
|
|
376
412
|
status: input.status,
|
|
377
413
|
bodyText: input.bodyText ?? '',
|
|
378
414
|
}),
|
|
415
|
+
providerPacingWait: isProviderPacingWaitResponse({
|
|
416
|
+
status: input.status,
|
|
417
|
+
bodyText: input.bodyText ?? '',
|
|
418
|
+
}),
|
|
379
419
|
hasRetryAfterHeader: true,
|
|
380
420
|
transientHttpRetrySafe: input.transientHttpRetrySafe === true,
|
|
381
421
|
});
|
|
@@ -391,12 +431,12 @@ export function parseToolExecuteRetryAfterMs(
|
|
|
391
431
|
): number {
|
|
392
432
|
if (!header) return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
|
|
393
433
|
const seconds = Number(header);
|
|
394
|
-
if (Number.isFinite(seconds) && seconds
|
|
434
|
+
if (Number.isFinite(seconds) && seconds >= 0) {
|
|
395
435
|
return Math.ceil(seconds * 1000);
|
|
396
436
|
}
|
|
397
437
|
const retryAt = Date.parse(header);
|
|
398
438
|
if (Number.isFinite(retryAt)) {
|
|
399
|
-
return Math.max(
|
|
439
|
+
return Math.max(0, retryAt - nowMs);
|
|
400
440
|
}
|
|
401
441
|
return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
|
|
402
442
|
}
|
|
@@ -423,6 +463,7 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
423
463
|
isCustomerDbStorageUnavailableResponse(input);
|
|
424
464
|
const concurrencyBackpressure = isConcurrencyBackpressureResponse(input);
|
|
425
465
|
const runtimeSchedulerCapacity = isRuntimeSchedulerCapacityResponse(input);
|
|
466
|
+
const providerPacingWait = isProviderPacingWaitResponse(input);
|
|
426
467
|
const initialRetryDecision = decideToolExecuteHttpRetry({
|
|
427
468
|
status: input.status,
|
|
428
469
|
idempotencyInProgress,
|
|
@@ -430,13 +471,33 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
430
471
|
customerDbStorageUnavailable,
|
|
431
472
|
concurrencyBackpressure,
|
|
432
473
|
runtimeSchedulerCapacity,
|
|
474
|
+
providerPacingWait,
|
|
433
475
|
hasRetryAfterHeader,
|
|
434
476
|
transientHttpRetrySafe,
|
|
435
477
|
});
|
|
436
|
-
const
|
|
437
|
-
input.
|
|
438
|
-
|
|
439
|
-
|
|
478
|
+
const providerPacingWaitBody = providerPacingWait
|
|
479
|
+
? parseJsonObject(input.bodyText)
|
|
480
|
+
: null;
|
|
481
|
+
const rawProviderPacingWaitMs = providerPacingWaitBody?.retry_after_ms;
|
|
482
|
+
const exactProviderPacingWaitMs =
|
|
483
|
+
typeof rawProviderPacingWaitMs === 'number' &&
|
|
484
|
+
Number.isFinite(rawProviderPacingWaitMs) &&
|
|
485
|
+
rawProviderPacingWaitMs >= 0
|
|
486
|
+
? rawProviderPacingWaitMs
|
|
487
|
+
: null;
|
|
488
|
+
const attestedProviderDeadline =
|
|
489
|
+
providerPacingWait &&
|
|
490
|
+
(providerPacingWaitBody?.reason === 'retry_after' ||
|
|
491
|
+
providerPacingWaitBody?.reason === 'stale_permit') &&
|
|
492
|
+
typeof providerPacingWaitBody?.retry_at_ms === 'number' &&
|
|
493
|
+
Number.isFinite(providerPacingWaitBody.retry_at_ms) &&
|
|
494
|
+
providerPacingWaitBody.retry_at_ms > (input.nowMs ?? Date.now()) &&
|
|
495
|
+
exactProviderPacingWaitMs !== null &&
|
|
496
|
+
exactProviderPacingWaitMs > 0;
|
|
497
|
+
const retryAfterMs =
|
|
498
|
+
providerPacingWait && exactProviderPacingWaitMs !== null
|
|
499
|
+
? exactProviderPacingWaitMs
|
|
500
|
+
: parseToolExecuteRetryAfterMs(input.retryAfterHeader, input.nowMs);
|
|
440
501
|
const error = normalizeToolHttpErrorMessage({
|
|
441
502
|
toolId: input.toolId,
|
|
442
503
|
status: input.status,
|
|
@@ -446,7 +507,9 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
446
507
|
schemaVersion: input.schemaVersion,
|
|
447
508
|
retryable: initialRetryDecision.retryable,
|
|
448
509
|
retryAfterMs:
|
|
449
|
-
(input.status === 429 && hasRetryAfterHeader) ||
|
|
510
|
+
(input.status === 429 && hasRetryAfterHeader) ||
|
|
511
|
+
runtimeSchedulerCapacity ||
|
|
512
|
+
providerPacingWait
|
|
450
513
|
? retryAfterMs
|
|
451
514
|
: null,
|
|
452
515
|
});
|
|
@@ -458,6 +521,11 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
458
521
|
error.message = `${error.message} (provider call ${Math.round(input.providerLatencyMs)}ms)`;
|
|
459
522
|
}
|
|
460
523
|
const hardBillingFailure = isHardBillingToolHttpError(error);
|
|
524
|
+
const providerOwnedRateLimit =
|
|
525
|
+
input.status === 429 &&
|
|
526
|
+
!hardBillingFailure &&
|
|
527
|
+
error instanceof ToolExecutionError &&
|
|
528
|
+
error.origin === 'provider';
|
|
461
529
|
const retryDecision = decideToolExecuteHttpRetry({
|
|
462
530
|
status: input.status,
|
|
463
531
|
idempotencyInProgress,
|
|
@@ -465,12 +533,15 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
465
533
|
customerDbStorageUnavailable,
|
|
466
534
|
concurrencyBackpressure,
|
|
467
535
|
runtimeSchedulerCapacity,
|
|
536
|
+
providerPacingWait,
|
|
468
537
|
hardBillingFailure,
|
|
469
538
|
hasRetryAfterHeader,
|
|
470
539
|
transientHttpRetrySafe,
|
|
471
540
|
});
|
|
472
541
|
const shouldRetry =
|
|
473
|
-
|
|
542
|
+
providerOwnedRateLimit ||
|
|
543
|
+
attestedProviderDeadline ||
|
|
544
|
+
(retryDecision.retryable && input.attempt < retryDecision.attemptCap);
|
|
474
545
|
// In-attempt retry safety and later-run repairability are different
|
|
475
546
|
// decisions. A non-idempotent 5xx must not be repeated automatically inside
|
|
476
547
|
// this execution attempt, but an explicit later `plays run` must be able to
|
|
@@ -496,23 +567,37 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
496
567
|
TOOL_EXECUTE_RETRY_DELAY_MAX_MS,
|
|
497
568
|
Math.max(retryAfterMs, TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS),
|
|
498
569
|
)
|
|
499
|
-
:
|
|
500
|
-
?
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
570
|
+
: retryDecision.reason === 'provider_pacing_wait'
|
|
571
|
+
? retryAfterMs
|
|
572
|
+
: input.status === 429
|
|
573
|
+
? hasRetryAfterHeader
|
|
574
|
+
? providerOwnedRateLimit
|
|
575
|
+
? // The provider's explicit deadline is authoritative. Attempt-
|
|
576
|
+
// scaled fallback backoff is only for a bare 429; extending a
|
|
577
|
+
// valid Retry-After by the attempt number needlessly idles this
|
|
578
|
+
// row after the shared provider pacer has already enforced the
|
|
579
|
+
// deadline for every caller using the credential.
|
|
580
|
+
retryAfterMs
|
|
581
|
+
: Math.max(
|
|
582
|
+
retryAfterMs,
|
|
583
|
+
TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
|
|
584
|
+
)
|
|
585
|
+
: Math.min(
|
|
586
|
+
TOOL_EXECUTE_RATE_LIMIT_RETRY_DELAY_MAX_MS,
|
|
587
|
+
Math.max(
|
|
588
|
+
retryAfterMs,
|
|
589
|
+
TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS * input.attempt,
|
|
590
|
+
),
|
|
591
|
+
)
|
|
592
|
+
: retryDecision.reason === 'idempotency_in_progress' &&
|
|
511
593
|
!hasRetryAfterHeader
|
|
512
594
|
? idempotencyInProgressRetryDelayMs(input.attempt)
|
|
513
|
-
:
|
|
514
|
-
|
|
515
|
-
|
|
595
|
+
: retryDecision.reason === 'gateway_invocation_in_progress' &&
|
|
596
|
+
!hasRetryAfterHeader
|
|
597
|
+
? idempotencyInProgressRetryDelayMs(input.attempt)
|
|
598
|
+
: retryAfterMs > 0
|
|
599
|
+
? Math.min(TOOL_EXECUTE_RETRY_DELAY_MAX_MS, retryAfterMs)
|
|
600
|
+
: TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
|
|
516
601
|
return {
|
|
517
602
|
...retryDecision,
|
|
518
603
|
error,
|
|
@@ -528,24 +613,25 @@ export function classifyToolExecuteHttpFailure(input: {
|
|
|
528
613
|
// caller, and malformed responses are never evidence that a provider is
|
|
529
614
|
// throttling; feeding any of them into the pacer can incorrectly exhaust
|
|
530
615
|
// every customer sharing that provider credential.
|
|
531
|
-
backpressureDelayMs:
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
: TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS
|
|
539
|
-
: null,
|
|
616
|
+
backpressureDelayMs: providerOwnedRateLimit
|
|
617
|
+
? hasRetryAfterHeader
|
|
618
|
+
? retryAfterMs
|
|
619
|
+
: TOOL_EXECUTE_BARE_RATE_LIMIT_BACKPRESSURE_MS
|
|
620
|
+
: null,
|
|
621
|
+
providerPacingWaitMs:
|
|
622
|
+
attestedProviderDeadline && shouldRetry ? retryAfterMs : null,
|
|
540
623
|
chargeRetryBudget:
|
|
541
624
|
shouldRetry &&
|
|
625
|
+
!providerOwnedRateLimit &&
|
|
542
626
|
retryDecision.reason !== 'gateway_invocation_in_progress' &&
|
|
543
627
|
retryDecision.reason !== 'concurrency_backpressure' &&
|
|
544
|
-
retryDecision.reason !== 'runtime_scheduler_capacity'
|
|
628
|
+
retryDecision.reason !== 'runtime_scheduler_capacity' &&
|
|
629
|
+
retryDecision.reason !== 'provider_pacing_wait',
|
|
545
630
|
// Scheduler capacity and the relay's explicit concurrency shed both occur
|
|
546
631
|
// before a provider request is dispatched. Carry that proof in the shared
|
|
547
632
|
// classification so the retry boundary can safely pace either response
|
|
548
633
|
// without asking a non-idempotent provider call to be replayed.
|
|
549
|
-
knownNotDispatched:
|
|
634
|
+
knownNotDispatched:
|
|
635
|
+
runtimeSchedulerCapacity || concurrencyBackpressure || providerPacingWait,
|
|
550
636
|
};
|
|
551
637
|
}
|
|
@@ -1593,6 +1593,22 @@ function serializedListRowsFromRaw(input: {
|
|
|
1593
1593
|
return entries.length > 0 ? Object.fromEntries(entries) : undefined;
|
|
1594
1594
|
}
|
|
1595
1595
|
|
|
1596
|
+
/**
|
|
1597
|
+
* V2 receipts keep the complete provider body in rawV2. Use that body as the
|
|
1598
|
+
* durable source for list handles instead of writing the same rows to the
|
|
1599
|
+
* compatibility-only listRows field.
|
|
1600
|
+
*/
|
|
1601
|
+
function serializedListRowsFromRawV2(input: {
|
|
1602
|
+
rawV2: unknown;
|
|
1603
|
+
view: 'data' | 'rawV2';
|
|
1604
|
+
metadata: ToolResultMetadataInput;
|
|
1605
|
+
}): Record<string, Array<Record<string, unknown>>> | undefined {
|
|
1606
|
+
return serializedListRowsFromRaw({
|
|
1607
|
+
raw: legacyRawFromToolResponseRawV2(input.rawV2, input.view),
|
|
1608
|
+
metadata: input.metadata,
|
|
1609
|
+
});
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1596
1612
|
function preservedSerializedListRows(
|
|
1597
1613
|
value: ToolExecuteResult,
|
|
1598
1614
|
): Record<string, Array<Record<string, unknown>>> | undefined {
|
|
@@ -1864,6 +1880,52 @@ function sameJsonValue(left: unknown, right: unknown): boolean {
|
|
|
1864
1880
|
}
|
|
1865
1881
|
}
|
|
1866
1882
|
|
|
1883
|
+
/**
|
|
1884
|
+
* New V2 receipts must not carry a second full copy of list rows. Keep the
|
|
1885
|
+
* optional field when the rows cannot be proven identical to the canonical
|
|
1886
|
+
* rawV2 body; this makes the compaction conservative for unusual or historical
|
|
1887
|
+
* envelopes while allowing ordinary V2 receipts to store rows exactly once.
|
|
1888
|
+
*/
|
|
1889
|
+
function listRowsAreInCanonicalRawV2(input: {
|
|
1890
|
+
rawV2: unknown;
|
|
1891
|
+
view: 'data' | 'rawV2';
|
|
1892
|
+
metadata: ToolResultMetadataInput;
|
|
1893
|
+
listRows: Record<string, Array<Record<string, unknown>>> | undefined;
|
|
1894
|
+
}): boolean {
|
|
1895
|
+
if (!input.listRows || Object.keys(input.listRows).length === 0) {
|
|
1896
|
+
return false;
|
|
1897
|
+
}
|
|
1898
|
+
const canonicalRows = serializedListRowsFromRawV2({
|
|
1899
|
+
rawV2: input.rawV2,
|
|
1900
|
+
view: input.view,
|
|
1901
|
+
metadata: input.metadata,
|
|
1902
|
+
});
|
|
1903
|
+
if (!canonicalRows) return false;
|
|
1904
|
+
return Object.entries(input.listRows).every(([name, rows]) =>
|
|
1905
|
+
sameJsonValue(canonicalRows[name], rows),
|
|
1906
|
+
);
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
function completeListRowsForDatasets(input: {
|
|
1910
|
+
listRows: Record<string, Array<Record<string, unknown>>> | undefined;
|
|
1911
|
+
listDatasets:
|
|
1912
|
+
| Record<string, SerializedPlayDataset<Record<string, unknown>>>
|
|
1913
|
+
| undefined;
|
|
1914
|
+
}): Record<string, Array<Record<string, unknown>>> | undefined {
|
|
1915
|
+
if (!input.listRows || !input.listDatasets) return undefined;
|
|
1916
|
+
const datasets = Object.entries(input.listDatasets);
|
|
1917
|
+
if (datasets.length === 0) return undefined;
|
|
1918
|
+
if (
|
|
1919
|
+
!datasets.every(([name, dataset]) => {
|
|
1920
|
+
const rows = input.listRows?.[name];
|
|
1921
|
+
return rows !== undefined && rows.length >= dataset.count;
|
|
1922
|
+
})
|
|
1923
|
+
) {
|
|
1924
|
+
return undefined;
|
|
1925
|
+
}
|
|
1926
|
+
return input.listRows;
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1867
1929
|
function responseMetaOutsideRawV2(input: {
|
|
1868
1930
|
rawV2: unknown;
|
|
1869
1931
|
view: 'data' | 'rawV2';
|
|
@@ -1931,6 +1993,14 @@ export function serializeToolExecuteResult(
|
|
|
1931
1993
|
});
|
|
1932
1994
|
if (Object.prototype.hasOwnProperty.call(value.toolResponse, 'rawV2')) {
|
|
1933
1995
|
const view = value.toolResponse.view ?? 'rawV2';
|
|
1996
|
+
const persistedListRows = listRowsAreInCanonicalRawV2({
|
|
1997
|
+
rawV2: value.toolResponse.rawV2,
|
|
1998
|
+
view,
|
|
1999
|
+
metadata,
|
|
2000
|
+
listRows,
|
|
2001
|
+
})
|
|
2002
|
+
? undefined
|
|
2003
|
+
: listRows;
|
|
1934
2004
|
const responseMeta = responseMetaOutsideRawV2({
|
|
1935
2005
|
rawV2: value.toolResponse.rawV2,
|
|
1936
2006
|
view,
|
|
@@ -1960,7 +2030,7 @@ export function serializeToolExecuteResult(
|
|
|
1960
2030
|
...(responseMeta ? { responseMeta } : {}),
|
|
1961
2031
|
},
|
|
1962
2032
|
...(listDatasets ? { listDatasets } : {}),
|
|
1963
|
-
...(
|
|
2033
|
+
...(persistedListRows ? { listRows: persistedListRows } : {}),
|
|
1964
2034
|
...(targetValues ? { targetValues } : {}),
|
|
1965
2035
|
metadata,
|
|
1966
2036
|
execution: value._metadata.execution,
|
|
@@ -2023,6 +2093,16 @@ export function deserializeToolExecuteResult(
|
|
|
2023
2093
|
)
|
|
2024
2094
|
? value.toolResponse.legacyRaw
|
|
2025
2095
|
: raw;
|
|
2096
|
+
const listRows =
|
|
2097
|
+
value.listRows ??
|
|
2098
|
+
completeListRowsForDatasets({
|
|
2099
|
+
listRows: serializedListRowsFromRawV2({
|
|
2100
|
+
rawV2,
|
|
2101
|
+
view,
|
|
2102
|
+
metadata: value.metadata,
|
|
2103
|
+
}),
|
|
2104
|
+
listDatasets: value.listDatasets,
|
|
2105
|
+
});
|
|
2026
2106
|
return applySerializedListDatasets(
|
|
2027
2107
|
applySerializedTargetValues(
|
|
2028
2108
|
createToolExecuteResult({
|
|
@@ -2045,7 +2125,7 @@ export function deserializeToolExecuteResult(
|
|
|
2045
2125
|
value.targetValues,
|
|
2046
2126
|
),
|
|
2047
2127
|
value.listDatasets,
|
|
2048
|
-
|
|
2128
|
+
listRows,
|
|
2049
2129
|
);
|
|
2050
2130
|
}
|
|
2051
2131
|
return applySerializedListDatasets(
|
|
@@ -119,7 +119,7 @@ export type SerializedToolExecuteResultV1 = {
|
|
|
119
119
|
meta?: Record<string, unknown>;
|
|
120
120
|
};
|
|
121
121
|
listDatasets?: Record<string, SerializedPlayDataset<Record<string, unknown>>>;
|
|
122
|
-
/** Full declared list rows
|
|
122
|
+
/** Full declared list rows retained by historical V1 receipts. */
|
|
123
123
|
listRows?: Record<string, Array<Record<string, unknown>>>;
|
|
124
124
|
/** Computed scalar/object getters preserved before raw list previews are clipped. */
|
|
125
125
|
targetValues?: Record<string, ToolResultTargetMetadata>;
|
|
@@ -133,7 +133,7 @@ export type SerializedToolExecuteResultV1 = {
|
|
|
133
133
|
*/
|
|
134
134
|
export type SerializedToolExecuteResultV2 = Omit<
|
|
135
135
|
SerializedToolExecuteResultV1,
|
|
136
|
-
'__kind' | 'toolResponse'
|
|
136
|
+
'__kind' | 'toolResponse' | 'listRows'
|
|
137
137
|
> & {
|
|
138
138
|
__kind: 'deepline.tool_execute_result.v2';
|
|
139
139
|
toolResponse: {
|
|
@@ -144,6 +144,11 @@ export type SerializedToolExecuteResultV2 = Omit<
|
|
|
144
144
|
/** Deepline-owned additions not already represented in `rawV2.meta`. */
|
|
145
145
|
responseMeta?: Record<string, unknown>;
|
|
146
146
|
};
|
|
147
|
+
/**
|
|
148
|
+
* Compatibility fallback for older V2 receipts whose rawV2 body was not
|
|
149
|
+
* complete. New V2 receipts derive list rows from rawV2 and omit this copy.
|
|
150
|
+
*/
|
|
151
|
+
listRows?: Record<string, Array<Record<string, unknown>>>;
|
|
147
152
|
};
|
|
148
153
|
|
|
149
154
|
export type SerializedToolExecuteResult =
|
package/dist/cli/index.js
CHANGED
|
@@ -3051,7 +3051,7 @@ var SDK_RELEASE = {
|
|
|
3051
3051
|
// getters keep their established compatibility behavior.
|
|
3052
3052
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3053
3053
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3054
|
-
version: "0.3.
|
|
3054
|
+
version: "0.3.185",
|
|
3055
3055
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3056
3056
|
packageCapabilities: {
|
|
3057
3057
|
updatePreferences: 1
|
package/dist/cli/index.mjs
CHANGED
|
@@ -3046,7 +3046,7 @@ var SDK_RELEASE = {
|
|
|
3046
3046
|
// getters keep their established compatibility behavior.
|
|
3047
3047
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
3048
3048
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
3049
|
-
version: "0.3.
|
|
3049
|
+
version: "0.3.185",
|
|
3050
3050
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
3051
3051
|
packageCapabilities: {
|
|
3052
3052
|
updatePreferences: 1
|
package/dist/index.js
CHANGED
|
@@ -864,7 +864,7 @@ var SDK_RELEASE = {
|
|
|
864
864
|
// getters keep their established compatibility behavior.
|
|
865
865
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
866
866
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
867
|
-
version: "0.3.
|
|
867
|
+
version: "0.3.185",
|
|
868
868
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
869
869
|
packageCapabilities: {
|
|
870
870
|
updatePreferences: 1
|
package/dist/index.mjs
CHANGED
|
@@ -768,7 +768,7 @@ var SDK_RELEASE = {
|
|
|
768
768
|
// getters keep their established compatibility behavior.
|
|
769
769
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
770
770
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
771
|
-
version: "0.3.
|
|
771
|
+
version: "0.3.185",
|
|
772
772
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
773
773
|
packageCapabilities: {
|
|
774
774
|
updatePreferences: 1
|
|
@@ -388,6 +388,7 @@
|
|
|
388
388
|
"dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-budget-state.ts",
|
|
389
389
|
"dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts",
|
|
390
390
|
"dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-progress.ts",
|
|
391
|
+
"dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-scope.ts",
|
|
391
392
|
"dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts",
|
|
392
393
|
"dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts",
|
|
393
394
|
"dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-completion-coalescer.ts",
|
package/dist/release.d.mts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.185";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.d.ts
CHANGED
|
@@ -149,7 +149,7 @@ type SdkRelease = {
|
|
|
149
149
|
supportPolicy: SdkSupportPolicy;
|
|
150
150
|
};
|
|
151
151
|
declare const SDK_RELEASE: {
|
|
152
|
-
readonly version: "0.3.
|
|
152
|
+
readonly version: "0.3.185";
|
|
153
153
|
readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
|
|
154
154
|
readonly packageCapabilities: {
|
|
155
155
|
readonly updatePreferences: 1;
|
package/dist/release.js
CHANGED
|
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
|
|
|
74
74
|
// getters keep their established compatibility behavior.
|
|
75
75
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
76
76
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
77
|
-
version: "0.3.
|
|
77
|
+
version: "0.3.185",
|
|
78
78
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
79
79
|
packageCapabilities: {
|
|
80
80
|
updatePreferences: 1
|
package/dist/release.mjs
CHANGED
|
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
|
|
|
48
48
|
// getters keep their established compatibility behavior.
|
|
49
49
|
// 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
|
|
50
50
|
// 0.3.90 is the first deliberately versioned SDK release for API v3.
|
|
51
|
-
version: "0.3.
|
|
51
|
+
version: "0.3.185",
|
|
52
52
|
updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
|
|
53
53
|
packageCapabilities: {
|
|
54
54
|
updatePreferences: 1
|