deepline 0.3.201 → 0.3.203
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/sdk/src/client.ts +62 -1
- package/dist/bundling-sources/sdk/src/http.ts +55 -6
- package/dist/bundling-sources/sdk/src/index.ts +1 -0
- package/dist/bundling-sources/sdk/src/release.ts +1 -1
- package/dist/bundling-sources/sdk/src/types.ts +8 -0
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +4 -2
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +460 -96
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +11 -1
- package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +6 -2
- package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +401 -2
- package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +458 -0
- package/dist/bundling-sources/shared_libs/play-runtime/provider-no-progress.ts +40 -0
- package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +7 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +25 -3
- package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +119 -171
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-work-counters.ts +37 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +238 -204
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +16 -7
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +136 -25
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +21 -0
- package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +64 -15
- package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +5 -5
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-call/result-receipt-policy.ts +46 -0
- package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +14 -5
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +5 -1
- package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +3 -9
- package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +1 -1
- package/dist/cli/index.js +362 -55
- package/dist/cli/index.mjs +365 -57
- package/dist/index.d.mts +129 -1
- package/dist/index.d.ts +129 -1
- package/dist/index.js +64 -7
- package/dist/index.mjs +64 -7
- package/dist/install-integrity.json +3 -0
- package/dist/release.d.mts +1 -1
- package/dist/release.d.ts +1 -1
- package/dist/release.js +1 -1
- package/dist/release.mjs +1 -1
- package/package.json +1 -1
|
@@ -84,6 +84,7 @@ import {
|
|
|
84
84
|
type PlayRunLedgerIngressEvent,
|
|
85
85
|
} from '../run-ledger';
|
|
86
86
|
import { stableStringify } from '../../plays/row-identity';
|
|
87
|
+
import { runtimeWorkCountersFromResources } from '../runtime-work-counters';
|
|
87
88
|
import {
|
|
88
89
|
isCtxFetchHttpFailureDiagnostic,
|
|
89
90
|
parseCtxFetchHttpFailureDiagnostic,
|
|
@@ -13045,7 +13046,7 @@ function terminalLogLinesForOutbox(
|
|
|
13045
13046
|
.slice(-LOG_TAIL_LIMIT);
|
|
13046
13047
|
}
|
|
13047
13048
|
|
|
13048
|
-
function attachRunnerWorkProgressMetadata(
|
|
13049
|
+
export function attachRunnerWorkProgressMetadata(
|
|
13049
13050
|
output: unknown,
|
|
13050
13051
|
runnerResult: unknown,
|
|
13051
13052
|
options: {
|
|
@@ -13097,6 +13098,13 @@ function attachRunnerWorkProgressMetadata(
|
|
|
13097
13098
|
const runtimeResources = metadata?.runtimeResources
|
|
13098
13099
|
? null
|
|
13099
13100
|
: boundedRuntimeResourcesMetadata(runnerResult);
|
|
13101
|
+
// Keep the two scalar counts when the detailed runtimeResources record
|
|
13102
|
+
// exceeds its metadata budget. Compact status must not scan a call log to
|
|
13103
|
+
// reconstruct physical work.
|
|
13104
|
+
const runtimeCounters = runtimeWorkCountersFromResources(
|
|
13105
|
+
(runnerResult as { stats?: { runtimeResources?: unknown } }).stats
|
|
13106
|
+
?.runtimeResources,
|
|
13107
|
+
);
|
|
13100
13108
|
const outputWarnings = Array.isArray(
|
|
13101
13109
|
(runnerResult as { outputWarnings?: unknown }).outputWarnings,
|
|
13102
13110
|
)
|
|
@@ -13139,6 +13147,8 @@ function attachRunnerWorkProgressMetadata(
|
|
|
13139
13147
|
(total === null || skipped === null || metadata?.workProgress) &&
|
|
13140
13148
|
!runLogTail &&
|
|
13141
13149
|
!runtimeResources &&
|
|
13150
|
+
!runtimeCounters &&
|
|
13151
|
+
!metadata?.runtimeCounters &&
|
|
13142
13152
|
!rowOutcomes &&
|
|
13143
13153
|
mergedOutputWarnings.length === 0
|
|
13144
13154
|
) {
|
|
@@ -13170,6 +13180,9 @@ function attachRunnerWorkProgressMetadata(
|
|
|
13170
13180
|
...record,
|
|
13171
13181
|
_metadata: {
|
|
13172
13182
|
...(metadata ?? {}),
|
|
13183
|
+
// A previous attempt's scalar evidence cannot describe this terminal
|
|
13184
|
+
// attempt. Undefined removes it from persisted JSON when unavailable.
|
|
13185
|
+
runtimeCounters: runtimeCounters ?? undefined,
|
|
13173
13186
|
...(workProgress ? { workProgress } : {}),
|
|
13174
13187
|
...(rowOutcomes ? { rowOutcomes } : {}),
|
|
13175
13188
|
...(runLogTail ? { runLogTail } : {}),
|
|
@@ -15667,8 +15680,63 @@ export async function readPostgresSchedulerResult(
|
|
|
15667
15680
|
export async function readPostgresSchedulerRunStatus(
|
|
15668
15681
|
client: PostgresSchedulerQueryClient,
|
|
15669
15682
|
runId: string,
|
|
15670
|
-
options?: Pick<PostgresSchedulerOptions, 'schema'
|
|
15683
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'> & {
|
|
15684
|
+
includeDetails?: boolean;
|
|
15685
|
+
},
|
|
15671
15686
|
): Promise<PostgresSchedulerRunStatusSnapshot | null> {
|
|
15687
|
+
if (options?.includeDetails === false) {
|
|
15688
|
+
// An admission projection gap needs lifecycle truth only. Launch, result,
|
|
15689
|
+
// suspension and error JSON may contain row/call-sized detail: never even
|
|
15690
|
+
// select them for status polling. Missing progress remains unconfirmed.
|
|
15691
|
+
const result = await client.query<
|
|
15692
|
+
Pick<
|
|
15693
|
+
RunStatusRow,
|
|
15694
|
+
| 'run_id'
|
|
15695
|
+
| 'org_id'
|
|
15696
|
+
| 'status'
|
|
15697
|
+
| 'play_name'
|
|
15698
|
+
| 'graph_hash'
|
|
15699
|
+
| 'runtime_profile'
|
|
15700
|
+
| 'created_at'
|
|
15701
|
+
| 'updated_at'
|
|
15702
|
+
| 'started_at'
|
|
15703
|
+
| 'finished_at'
|
|
15704
|
+
>
|
|
15705
|
+
>(
|
|
15706
|
+
`
|
|
15707
|
+
SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
|
|
15708
|
+
created_at, updated_at, started_at, finished_at
|
|
15709
|
+
FROM ${tableName('runs', options)}
|
|
15710
|
+
WHERE run_id = $1
|
|
15711
|
+
`,
|
|
15712
|
+
[runId],
|
|
15713
|
+
);
|
|
15714
|
+
const row = result.rows[0];
|
|
15715
|
+
if (!row) return null;
|
|
15716
|
+
return {
|
|
15717
|
+
runId: row.run_id,
|
|
15718
|
+
orgId: row.org_id ?? '',
|
|
15719
|
+
status: row.status,
|
|
15720
|
+
playName: row.play_name,
|
|
15721
|
+
graphHash: row.graph_hash,
|
|
15722
|
+
runtimeBackend: null,
|
|
15723
|
+
schedulerBackend: 'absurd',
|
|
15724
|
+
executionProfile: row.runtime_profile,
|
|
15725
|
+
createdAt: row.created_at?.getTime() ?? null,
|
|
15726
|
+
updatedAt: row.updated_at?.getTime() ?? null,
|
|
15727
|
+
startedAt: row.started_at?.getTime() ?? null,
|
|
15728
|
+
finishedAt: row.finished_at?.getTime() ?? null,
|
|
15729
|
+
durationMs:
|
|
15730
|
+
row.finished_at && row.started_at
|
|
15731
|
+
? Math.max(0, row.finished_at.getTime() - row.started_at.getTime())
|
|
15732
|
+
: null,
|
|
15733
|
+
runtimeTiming: null,
|
|
15734
|
+
result: null,
|
|
15735
|
+
error: null,
|
|
15736
|
+
suspension: null,
|
|
15737
|
+
staticPipeline: null,
|
|
15738
|
+
};
|
|
15739
|
+
}
|
|
15672
15740
|
const result = await client.query<RunStatusRow>(
|
|
15673
15741
|
`
|
|
15674
15742
|
SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
|
|
@@ -15883,6 +15951,10 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
15883
15951
|
runId: string;
|
|
15884
15952
|
idempotencyKey: string;
|
|
15885
15953
|
leaseOwner?: string;
|
|
15954
|
+
projectedEvents?: readonly AcceptedPlayRunLedgerEvent[];
|
|
15955
|
+
normalizeProjectedEvents?: (
|
|
15956
|
+
events: readonly PlayRunLedgerIngressEvent[],
|
|
15957
|
+
) => readonly AcceptedPlayRunLedgerEvent[];
|
|
15886
15958
|
},
|
|
15887
15959
|
options?: Pick<PostgresSchedulerOptions, 'schema'>,
|
|
15888
15960
|
): Promise<void> {
|
|
@@ -15893,6 +15965,8 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
15893
15965
|
lease_live: boolean;
|
|
15894
15966
|
due: boolean;
|
|
15895
15967
|
blocked: boolean;
|
|
15968
|
+
payload_json: unknown;
|
|
15969
|
+
created_at: Date;
|
|
15896
15970
|
}>(
|
|
15897
15971
|
`
|
|
15898
15972
|
SELECT outbox.delivery_key,
|
|
@@ -15900,7 +15974,9 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
15900
15974
|
outbox.lease_owner,
|
|
15901
15975
|
outbox.lease_expires_at > now() AS lease_live,
|
|
15902
15976
|
coalesce(outbox.next_attempt_at, outbox.created_at) <= now() AS due,
|
|
15903
|
-
outbox.failed_at IS NOT NULL AS blocked
|
|
15977
|
+
outbox.failed_at IS NOT NULL AS blocked,
|
|
15978
|
+
outbox.payload_json,
|
|
15979
|
+
outbox.created_at
|
|
15904
15980
|
FROM ${tableName('outbox', options)} AS outbox
|
|
15905
15981
|
JOIN ${tableName('runs', options)} AS runs ON runs.run_id = outbox.run_id
|
|
15906
15982
|
WHERE outbox.run_id = $1 AND runs.org_id = $2
|
|
@@ -15915,12 +15991,9 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
15915
15991
|
throw new RunLedgerProjectionLeaseLostError();
|
|
15916
15992
|
}
|
|
15917
15993
|
|
|
15918
|
-
//
|
|
15919
|
-
//
|
|
15920
|
-
//
|
|
15921
|
-
// one row at a time. Validate the submitted key against a *contiguous due
|
|
15922
|
-
// prefix*, not every row carrying that owner; otherwise a valid cancellation
|
|
15923
|
-
// head is rejected whenever later facts share the cancellation lease.
|
|
15994
|
+
// Cancellation may own several pending facts at once. Validate the exact
|
|
15995
|
+
// contiguous, due prefix named by the receipt instead of all rows sharing
|
|
15996
|
+
// that cancellation lease.
|
|
15924
15997
|
const prefix: typeof rows = [];
|
|
15925
15998
|
for (const row of rows) {
|
|
15926
15999
|
if (
|
|
@@ -15940,6 +16013,9 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
15940
16013
|
candidate.event_type,
|
|
15941
16014
|
),
|
|
15942
16015
|
);
|
|
16016
|
+
const prefixDigest = createHash('sha256')
|
|
16017
|
+
.update(prefix.map((candidate) => candidate.delivery_key).join('\n'))
|
|
16018
|
+
.digest('hex');
|
|
15943
16019
|
let expectedKey: string;
|
|
15944
16020
|
if (terminal) {
|
|
15945
16021
|
expectedKey = buildRunLedgerTerminalIdempotencyKey({
|
|
@@ -15955,29 +16031,58 @@ export async function assertPostgresRunLedgerDelivery(
|
|
|
15955
16031
|
deliveryKey: first.delivery_key,
|
|
15956
16032
|
});
|
|
15957
16033
|
} else {
|
|
15958
|
-
const digest = createHash('sha256')
|
|
15959
|
-
.update(prefix.map((candidate) => candidate.delivery_key).join('\n'))
|
|
15960
|
-
.digest('hex');
|
|
15961
16034
|
expectedKey = buildRunLedgerEventsIdempotencyKey({
|
|
15962
16035
|
runId: input.runId,
|
|
15963
|
-
deliveryDigest:
|
|
16036
|
+
deliveryDigest: prefixDigest,
|
|
15964
16037
|
});
|
|
15965
16038
|
}
|
|
15966
|
-
if (input.idempotencyKey === expectedKey) return;
|
|
15967
16039
|
|
|
15968
|
-
//
|
|
15969
|
-
//
|
|
15970
|
-
|
|
15971
|
-
|
|
16040
|
+
// Older batch dispatchers may use the digest receipt even when the batch
|
|
16041
|
+
// contains a terminal fact. Keep accepting it for this exact prefix.
|
|
16042
|
+
const batchKey = buildRunLedgerEventsIdempotencyKey({
|
|
16043
|
+
runId: input.runId,
|
|
16044
|
+
deliveryDigest: prefixDigest,
|
|
16045
|
+
});
|
|
16046
|
+
|
|
16047
|
+
// Preserve the old singleton key for non-start, nonterminal facts.
|
|
16048
|
+
const legacySingletonKey =
|
|
15972
16049
|
prefix.length === 1 &&
|
|
15973
16050
|
!['run.created', 'run.started'].includes(first.event_type) &&
|
|
15974
16051
|
input.idempotencyKey ===
|
|
15975
16052
|
buildRunLedgerTerminalIdempotencyKey({
|
|
15976
16053
|
runId: input.runId,
|
|
15977
16054
|
deliveryKey: first.delivery_key,
|
|
15978
|
-
})
|
|
16055
|
+
});
|
|
16056
|
+
if (
|
|
16057
|
+
input.idempotencyKey === expectedKey ||
|
|
16058
|
+
input.idempotencyKey === batchKey ||
|
|
16059
|
+
legacySingletonKey
|
|
15979
16060
|
) {
|
|
15980
|
-
return;
|
|
16061
|
+
if (input.projectedEvents === undefined) return;
|
|
16062
|
+
const prefixEvents: PlayRunLedgerIngressEvent[] = [];
|
|
16063
|
+
for (const prefixRow of prefix) {
|
|
16064
|
+
prefixEvents.push(
|
|
16065
|
+
...buildRunLedgerEventsFromPostgresSchedulerOutbox({
|
|
16066
|
+
eventId: prefixRow.delivery_key,
|
|
16067
|
+
runId: input.runId,
|
|
16068
|
+
eventType: prefixRow.event_type,
|
|
16069
|
+
payload: prefixRow.payload_json,
|
|
16070
|
+
createdAt: new Date(prefixRow.created_at),
|
|
16071
|
+
leaseOwner: prefixRow.lease_owner ?? undefined,
|
|
16072
|
+
deliveryKey: prefixRow.delivery_key,
|
|
16073
|
+
}),
|
|
16074
|
+
);
|
|
16075
|
+
}
|
|
16076
|
+
const comparablePrefixEvents = input.normalizeProjectedEvents
|
|
16077
|
+
? input.normalizeProjectedEvents(prefixEvents)
|
|
16078
|
+
: prefixEvents;
|
|
16079
|
+
if (
|
|
16080
|
+
stableStringify(input.projectedEvents) ===
|
|
16081
|
+
stableStringify(comparablePrefixEvents)
|
|
16082
|
+
) {
|
|
16083
|
+
return;
|
|
16084
|
+
}
|
|
16085
|
+
throw new RunLedgerProjectionLeaseLostError();
|
|
15981
16086
|
}
|
|
15982
16087
|
|
|
15983
16088
|
if (
|
|
@@ -16157,7 +16262,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
16157
16262
|
)
|
|
16158
16263
|
)
|
|
16159
16264
|
)
|
|
16160
|
-
ORDER BY outbox
|
|
16265
|
+
ORDER BY ${runLedgerOrderSql('outbox')}
|
|
16161
16266
|
FOR UPDATE OF outbox
|
|
16162
16267
|
LIMIT $1
|
|
16163
16268
|
), terminal_batch AS (
|
|
@@ -16193,7 +16298,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
|
|
|
16193
16298
|
SELECT event_id, run_id, event_type, payload_json, created_at, lease_owner,
|
|
16194
16299
|
delivery_key
|
|
16195
16300
|
FROM claimed
|
|
16196
|
-
ORDER BY
|
|
16301
|
+
ORDER BY ${runLedgerOrderSql('claimed')}
|
|
16197
16302
|
`,
|
|
16198
16303
|
[limit, runId, leaseOwner, leaseSeconds],
|
|
16199
16304
|
);
|
|
@@ -16287,7 +16392,9 @@ export async function readPostgresSchedulerOutboxNextDueAt(
|
|
|
16287
16392
|
export async function readPostgresSchedulerOutboxDeliveryStatus(
|
|
16288
16393
|
client: PostgresSchedulerQueryClient,
|
|
16289
16394
|
input: { runId: string; eventId: string },
|
|
16290
|
-
options?: Pick<PostgresSchedulerOptions, 'schema'
|
|
16395
|
+
options?: Pick<PostgresSchedulerOptions, 'schema'> & {
|
|
16396
|
+
includeBlocked?: boolean;
|
|
16397
|
+
},
|
|
16291
16398
|
): Promise<PostgresSchedulerOutboxDeliveryStatus> {
|
|
16292
16399
|
const runId = input.runId.trim();
|
|
16293
16400
|
const eventId = input.eventId.trim();
|
|
@@ -16298,13 +16405,17 @@ export async function readPostgresSchedulerOutboxDeliveryStatus(
|
|
|
16298
16405
|
}
|
|
16299
16406
|
const result = await client.query<{ delivered: boolean; blocked: boolean }>(
|
|
16300
16407
|
`SELECT target.delivered_at IS NOT NULL AS delivered,
|
|
16301
|
-
|
|
16408
|
+
${
|
|
16409
|
+
options?.includeBlocked === false
|
|
16410
|
+
? 'false'
|
|
16411
|
+
: `target.failed_at IS NOT NULL OR EXISTS (
|
|
16302
16412
|
SELECT 1
|
|
16303
16413
|
FROM ${tableName('outbox', options)} predecessor
|
|
16304
16414
|
WHERE predecessor.run_id = target.run_id
|
|
16305
16415
|
AND predecessor.failed_at IS NOT NULL
|
|
16306
16416
|
AND ${runLedgerPrecedesSql('predecessor', 'target', true)}
|
|
16307
|
-
)
|
|
16417
|
+
)`
|
|
16418
|
+
} AS blocked
|
|
16308
16419
|
FROM ${tableName('outbox', options)} target
|
|
16309
16420
|
WHERE target.run_id = $1 AND target.event_id = $2`,
|
|
16310
16421
|
[runId, eventId],
|
package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts
CHANGED
|
@@ -66,6 +66,7 @@ import {
|
|
|
66
66
|
createRuntimeReceiptValuePersister,
|
|
67
67
|
createRuntimeReceiptValueReader,
|
|
68
68
|
RUNTIME_RECEIPT_VALUE_INLINE_MAX_BYTES,
|
|
69
|
+
RuntimeValueStoreUnavailableError,
|
|
69
70
|
type RuntimeReceiptValueBlobStore,
|
|
70
71
|
type RuntimeReceiptValuePersister,
|
|
71
72
|
type RuntimeReceiptValueReader,
|
|
@@ -250,6 +251,26 @@ export function receiptGatewayFailureResponse(
|
|
|
250
251
|
},
|
|
251
252
|
);
|
|
252
253
|
}
|
|
254
|
+
if (error instanceof RuntimeValueStoreUnavailableError) {
|
|
255
|
+
return receiptGatewayJsonResponse(
|
|
256
|
+
{
|
|
257
|
+
error: `${operationLabel} object store is temporarily unavailable.`,
|
|
258
|
+
code: 'runtime_value_store_unavailable',
|
|
259
|
+
retryable: true,
|
|
260
|
+
retry_after_ms: 1_000,
|
|
261
|
+
detail: error.message,
|
|
262
|
+
...(context.action ? { action: context.action } : {}),
|
|
263
|
+
},
|
|
264
|
+
{
|
|
265
|
+
status: 503,
|
|
266
|
+
headers: {
|
|
267
|
+
'retry-after': '1',
|
|
268
|
+
'x-deepline-error-code': 'runtime_value_store_unavailable',
|
|
269
|
+
'x-deepline-error-detail': error.message,
|
|
270
|
+
},
|
|
271
|
+
},
|
|
272
|
+
);
|
|
273
|
+
}
|
|
253
274
|
if (isRuntimeSchedulerCapacityError(error)) {
|
|
254
275
|
const retryAfterMs = runtimeSchedulerCapacityRetryAfterMs(error);
|
|
255
276
|
return receiptGatewayJsonResponse(
|
|
@@ -25,6 +25,40 @@ export type RuntimeValueBlobStore = {
|
|
|
25
25
|
/** Backward-compatible receipt-specific name for the shared object store. */
|
|
26
26
|
export type RuntimeReceiptValueBlobStore = RuntimeValueBlobStore;
|
|
27
27
|
|
|
28
|
+
export class RuntimeValueStoreUnavailableError extends Error {
|
|
29
|
+
constructor(
|
|
30
|
+
readonly operation: 'put' | 'head',
|
|
31
|
+
readonly status: number,
|
|
32
|
+
cause: unknown,
|
|
33
|
+
) {
|
|
34
|
+
super(`Runtime value object store ${operation} returned HTTP ${status}.`, {
|
|
35
|
+
cause,
|
|
36
|
+
});
|
|
37
|
+
this.name = 'RuntimeValueStoreUnavailableError';
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function transientStoreStatus(error: unknown): number | null {
|
|
42
|
+
if (!error || typeof error !== 'object') return null;
|
|
43
|
+
const metadata = (error as { $metadata?: { httpStatusCode?: unknown } })
|
|
44
|
+
.$metadata;
|
|
45
|
+
const status = metadata?.httpStatusCode;
|
|
46
|
+
return typeof status === 'number' &&
|
|
47
|
+
(status === 408 || status === 429 || status >= 500)
|
|
48
|
+
? status
|
|
49
|
+
: null;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function transientStoreError(
|
|
53
|
+
operation: 'put' | 'head',
|
|
54
|
+
error: unknown,
|
|
55
|
+
): RuntimeValueStoreUnavailableError | null {
|
|
56
|
+
const status = transientStoreStatus(error);
|
|
57
|
+
return status === null
|
|
58
|
+
? null
|
|
59
|
+
: new RuntimeValueStoreUnavailableError(operation, status, error);
|
|
60
|
+
}
|
|
61
|
+
|
|
28
62
|
/**
|
|
29
63
|
* Foundation owns value references and bounded I/O admission; the deployment
|
|
30
64
|
* injects its object-store adapter. This keeps R2 credentials and transport
|
|
@@ -185,7 +219,13 @@ export async function persistRuntimeReceiptValue(
|
|
|
185
219
|
// An immutable deterministic object that is observable after a failed PUT
|
|
186
220
|
// is the same attempted publication. Treat it as a recovered write, never
|
|
187
221
|
// as permission to call the provider again.
|
|
188
|
-
|
|
222
|
+
let exists: boolean;
|
|
223
|
+
try {
|
|
224
|
+
exists = await store.exists(outputRef.key);
|
|
225
|
+
} catch (probeError) {
|
|
226
|
+
throw transientStoreError('head', probeError) ?? probeError;
|
|
227
|
+
}
|
|
228
|
+
if (!exists) throw transientStoreError('put', error) ?? error;
|
|
189
229
|
}
|
|
190
230
|
return { kind: 'reference', outputJson: null, outputRef };
|
|
191
231
|
}
|
|
@@ -209,11 +249,10 @@ export function createRuntimeReceiptValuePersister(
|
|
|
209
249
|
serialized: string;
|
|
210
250
|
inlineFields?: RuntimeReceiptValueInlineFields;
|
|
211
251
|
};
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
>({
|
|
252
|
+
type WriteOutcome =
|
|
253
|
+
| { ok: true; value: PersistedRuntimeReceiptValue }
|
|
254
|
+
| { ok: false; error: unknown };
|
|
255
|
+
const writes = new BoundedBatchFunction<Write, WriteOutcome, 'r2'>({
|
|
217
256
|
keyId: () => 'r2',
|
|
218
257
|
estimateBytes: (value) => Buffer.byteLength(value.serialized, 'utf8'),
|
|
219
258
|
maxBatchItems: 4,
|
|
@@ -224,16 +263,24 @@ export function createRuntimeReceiptValuePersister(
|
|
|
224
263
|
maxRetainedUnits: 32,
|
|
225
264
|
maxInFlightBatches: 2,
|
|
226
265
|
flushMs: 0,
|
|
266
|
+
// R2 has no multi-PUT. One failed object must not reject the other
|
|
267
|
+
// independent receipts admitted alongside it.
|
|
227
268
|
execute: async ({ values }) =>
|
|
228
269
|
await Promise.all(
|
|
229
|
-
values.map(
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
270
|
+
values.map(async (value): Promise<WriteOutcome> => {
|
|
271
|
+
try {
|
|
272
|
+
return {
|
|
273
|
+
ok: true,
|
|
274
|
+
value: await persistRuntimeReceiptValue({
|
|
275
|
+
...value,
|
|
276
|
+
inlineMaxBytes,
|
|
277
|
+
...(input.store ? { store: input.store } : {}),
|
|
278
|
+
}),
|
|
279
|
+
};
|
|
280
|
+
} catch (error) {
|
|
281
|
+
return { ok: false, error };
|
|
282
|
+
}
|
|
283
|
+
}),
|
|
237
284
|
),
|
|
238
285
|
});
|
|
239
286
|
return async (value) => {
|
|
@@ -244,7 +291,9 @@ export function createRuntimeReceiptValuePersister(
|
|
|
244
291
|
...(input.store ? { store: input.store } : {}),
|
|
245
292
|
});
|
|
246
293
|
}
|
|
247
|
-
|
|
294
|
+
const outcome = await writes.submit({ key: 'r2', value });
|
|
295
|
+
if (!outcome.ok) throw outcome.error;
|
|
296
|
+
return outcome.value;
|
|
248
297
|
};
|
|
249
298
|
}
|
|
250
299
|
|
|
@@ -41,11 +41,11 @@ export function releasableRuntimeSheetAttemptFenceSql(
|
|
|
41
41
|
}
|
|
42
42
|
|
|
43
43
|
/**
|
|
44
|
-
* Predicate for
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
*
|
|
44
|
+
* Predicate for callers that need to identify a terminal row from the same
|
|
45
|
+
* attempt owner + epoch. Runtime map persistence now enforces first-terminal
|
|
46
|
+
* idempotency inline because its write-version and row-key context are also
|
|
47
|
+
* required; keep this helper exported for compatibility with external
|
|
48
|
+
* conformance checks and integrations.
|
|
49
49
|
*/
|
|
50
50
|
export function sameOwnerTerminalAttemptEpochSql(
|
|
51
51
|
tableAlias: string,
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { scopeToolReceiptToInvocation } from '../durable-call-cache';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Freshness chooses which completed results a run may reuse. It never turns
|
|
5
|
+
* off durable progress inside that run. A forced invocation gets an immutable
|
|
6
|
+
* result of its own; a scheduler retry reads it like any other completed key.
|
|
7
|
+
*
|
|
8
|
+
* The canonical key already binds payload, authorization, action and response
|
|
9
|
+
* semantics. Keep it in the derivation so a changed input cannot recover an
|
|
10
|
+
* unrelated result. Provider operation identities and public receipt metadata
|
|
11
|
+
* remain canonical and are deliberately outside this storage policy.
|
|
12
|
+
*/
|
|
13
|
+
export function resolveToolCallResultReceiptPolicy(input: {
|
|
14
|
+
orgId?: string | null;
|
|
15
|
+
toolId: string;
|
|
16
|
+
resultReceiptKey: string;
|
|
17
|
+
runId: string;
|
|
18
|
+
logicalCallId: string;
|
|
19
|
+
force: boolean;
|
|
20
|
+
/** Historical local contexts may have no durable result store. */
|
|
21
|
+
canStoreResult?: boolean;
|
|
22
|
+
}): { resultReceiptKey: string; force: boolean } {
|
|
23
|
+
if (!input.resultReceiptKey.trim()) {
|
|
24
|
+
throw new Error('Tool Call result policy requires a result receipt key.');
|
|
25
|
+
}
|
|
26
|
+
if (!input.force) {
|
|
27
|
+
return { resultReceiptKey: input.resultReceiptKey, force: false };
|
|
28
|
+
}
|
|
29
|
+
if (input.canStoreResult === false) {
|
|
30
|
+
return { resultReceiptKey: input.resultReceiptKey, force: true };
|
|
31
|
+
}
|
|
32
|
+
if (!input.runId.trim() || !input.logicalCallId.trim()) {
|
|
33
|
+
throw new Error(
|
|
34
|
+
'Forced Tool Call requires stable run and call identities.',
|
|
35
|
+
);
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
resultReceiptKey: scopeToolReceiptToInvocation({
|
|
39
|
+
orgId: input.orgId,
|
|
40
|
+
toolId: input.toolId,
|
|
41
|
+
receiptKey: input.resultReceiptKey,
|
|
42
|
+
invocationIdentity: `forced-result:${input.runId}:${input.logicalCallId}`,
|
|
43
|
+
}),
|
|
44
|
+
force: false,
|
|
45
|
+
};
|
|
46
|
+
}
|
|
@@ -425,20 +425,29 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
|
|
|
425
425
|
};
|
|
426
426
|
}
|
|
427
427
|
|
|
428
|
-
|
|
428
|
+
// Keep expired absolute deadlines intact: durable response replay must not
|
|
429
|
+
// turn old provider feedback into a new AIMD throttle episode.
|
|
430
|
+
export function parseToolExecuteRetryAtMs(
|
|
429
431
|
header: string | null | undefined,
|
|
430
432
|
nowMs = Date.now(),
|
|
431
433
|
): number {
|
|
432
|
-
if (!header) return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
|
|
434
|
+
if (!header) return nowMs + TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
|
|
433
435
|
const seconds = Number(header);
|
|
434
436
|
if (Number.isFinite(seconds) && seconds >= 0) {
|
|
435
|
-
return Math.ceil(seconds * 1000);
|
|
437
|
+
return nowMs + Math.ceil(seconds * 1000);
|
|
436
438
|
}
|
|
437
439
|
const retryAt = Date.parse(header);
|
|
438
440
|
if (Number.isFinite(retryAt)) {
|
|
439
|
-
return
|
|
441
|
+
return retryAt;
|
|
440
442
|
}
|
|
441
|
-
return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
|
|
443
|
+
return nowMs + TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
export function parseToolExecuteRetryAfterMs(
|
|
447
|
+
header: string | null | undefined,
|
|
448
|
+
nowMs = Date.now(),
|
|
449
|
+
): number {
|
|
450
|
+
return Math.max(0, parseToolExecuteRetryAtMs(header, nowMs) - nowMs);
|
|
442
451
|
}
|
|
443
452
|
|
|
444
453
|
export function classifyToolExecuteHttpFailure(input: {
|
|
@@ -265,7 +265,11 @@ function receiptFailureKindForToolErrorPayload(
|
|
|
265
265
|
// A credential connection can be added or replaced between play runs. Keep
|
|
266
266
|
// the failure terminal within its owning run, but let a later run reclaim
|
|
267
267
|
// the durable receipt instead of replaying stale authorization state.
|
|
268
|
-
|
|
268
|
+
if (code === 'INTEGRATION_CREDENTIALS_MISSING') return 'repairable';
|
|
269
|
+
// Managed AI has billing evidence for completed model steps, but no stored
|
|
270
|
+
// responses from which to resume the later rate-limited step. Its 409 stops
|
|
271
|
+
// automatic replay; an explicit later run may intentionally repeat work.
|
|
272
|
+
return code === 'UPSTREAM_PARTIAL_EXECUTION' ? 'repairable' : 'terminal';
|
|
269
273
|
}
|
|
270
274
|
|
|
271
275
|
function formatHardBillingFailureMessage(input: {
|
|
@@ -246,17 +246,11 @@ export function decideWorkReceiptClaim(input: {
|
|
|
246
246
|
const sameLogicalRun =
|
|
247
247
|
normalize(input.claimantRunId) !== null &&
|
|
248
248
|
normalize(input.claimantRunId) === normalize(input.receipt.runId);
|
|
249
|
-
const sameRunNewerLeaseAttempt =
|
|
250
|
-
state.reusable &&
|
|
251
|
-
state.lease.kind === 'active' &&
|
|
252
|
-
state.lease.ownerRunId === normalize(input.claimantRunId) &&
|
|
253
|
-
state.lease.ownerAttempt < normalizeAttempt(input.claimantRunAttempt);
|
|
254
249
|
// Force refresh means new intent across runs. Within one run it is a replay,
|
|
255
250
|
// so reuse terminal output instead of creating a second terminal payload
|
|
256
|
-
// under the run's existing Runtime Sheet write version. A
|
|
257
|
-
//
|
|
258
|
-
|
|
259
|
-
if (state.reusable && sameLogicalRun && !sameRunNewerLeaseAttempt) {
|
|
251
|
+
// under the run's existing Runtime Sheet write version. A scheduler resume
|
|
252
|
+
// increases the attempt number but does not create another refresh intent.
|
|
253
|
+
if (state.reusable && sameLogicalRun) {
|
|
260
254
|
return { kind: 'reuse_terminal' };
|
|
261
255
|
}
|
|
262
256
|
if (state.reusable && input.forceRefresh !== true) {
|
|
@@ -239,7 +239,7 @@ function defaultThreshold(
|
|
|
239
239
|
}
|
|
240
240
|
|
|
241
241
|
function formatCredits(value: number): string {
|
|
242
|
-
return value.toLocaleString('en-US', { maximumFractionDigits:
|
|
242
|
+
return value.toLocaleString('en-US', { maximumFractionDigits: 3 });
|
|
243
243
|
}
|
|
244
244
|
|
|
245
245
|
export const DEEPLINE_DOCS_URL = 'https://deepline.com/docs';
|