deepline 0.3.201 → 0.3.203

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. package/dist/bundling-sources/sdk/src/client.ts +62 -1
  2. package/dist/bundling-sources/sdk/src/http.ts +55 -6
  3. package/dist/bundling-sources/sdk/src/index.ts +1 -0
  4. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  5. package/dist/bundling-sources/sdk/src/types.ts +8 -0
  6. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +4 -2
  7. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +460 -96
  8. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +11 -1
  9. package/dist/bundling-sources/shared_libs/play-runtime/execution-gateway-invocation-fence.ts +6 -2
  10. package/dist/bundling-sources/shared_libs/play-runtime/fixture-behavior.ts +401 -2
  11. package/dist/bundling-sources/shared_libs/play-runtime/output-size-limits.ts +458 -0
  12. package/dist/bundling-sources/shared_libs/play-runtime/provider-no-progress.ts +40 -0
  13. package/dist/bundling-sources/shared_libs/play-runtime/receipt-sql.ts +7 -0
  14. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +25 -3
  15. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +1 -0
  16. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +119 -171
  17. package/dist/bundling-sources/shared_libs/play-runtime/runtime-work-counters.ts +37 -0
  18. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +238 -204
  19. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-rate-state.ts +16 -7
  20. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +136 -25
  21. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +21 -0
  22. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/runtime-receipt-value-storage.ts +64 -15
  23. package/dist/bundling-sources/shared_libs/play-runtime/sheet-attempt-sql.ts +5 -5
  24. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/index.ts +1 -0
  25. package/dist/bundling-sources/shared_libs/play-runtime/tool-call/result-receipt-policy.ts +46 -0
  26. package/dist/bundling-sources/shared_libs/play-runtime/tool-execute-retry-policy.ts +14 -5
  27. package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +5 -1
  28. package/dist/bundling-sources/shared_libs/play-runtime/work-receipt-state-machine.ts +3 -9
  29. package/dist/bundling-sources/shared_libs/product-notifications/catalog.ts +1 -1
  30. package/dist/cli/index.js +362 -55
  31. package/dist/cli/index.mjs +365 -57
  32. package/dist/index.d.mts +129 -1
  33. package/dist/index.d.ts +129 -1
  34. package/dist/index.js +64 -7
  35. package/dist/index.mjs +64 -7
  36. package/dist/install-integrity.json +3 -0
  37. package/dist/release.d.mts +1 -1
  38. package/dist/release.d.ts +1 -1
  39. package/dist/release.js +1 -1
  40. package/dist/release.mjs +1 -1
  41. package/package.json +1 -1
@@ -84,6 +84,7 @@ import {
84
84
  type PlayRunLedgerIngressEvent,
85
85
  } from '../run-ledger';
86
86
  import { stableStringify } from '../../plays/row-identity';
87
+ import { runtimeWorkCountersFromResources } from '../runtime-work-counters';
87
88
  import {
88
89
  isCtxFetchHttpFailureDiagnostic,
89
90
  parseCtxFetchHttpFailureDiagnostic,
@@ -13045,7 +13046,7 @@ function terminalLogLinesForOutbox(
13045
13046
  .slice(-LOG_TAIL_LIMIT);
13046
13047
  }
13047
13048
 
13048
- function attachRunnerWorkProgressMetadata(
13049
+ export function attachRunnerWorkProgressMetadata(
13049
13050
  output: unknown,
13050
13051
  runnerResult: unknown,
13051
13052
  options: {
@@ -13097,6 +13098,13 @@ function attachRunnerWorkProgressMetadata(
13097
13098
  const runtimeResources = metadata?.runtimeResources
13098
13099
  ? null
13099
13100
  : boundedRuntimeResourcesMetadata(runnerResult);
13101
+ // Keep the two scalar counts when the detailed runtimeResources record
13102
+ // exceeds its metadata budget. Compact status must not scan a call log to
13103
+ // reconstruct physical work.
13104
+ const runtimeCounters = runtimeWorkCountersFromResources(
13105
+ (runnerResult as { stats?: { runtimeResources?: unknown } }).stats
13106
+ ?.runtimeResources,
13107
+ );
13100
13108
  const outputWarnings = Array.isArray(
13101
13109
  (runnerResult as { outputWarnings?: unknown }).outputWarnings,
13102
13110
  )
@@ -13139,6 +13147,8 @@ function attachRunnerWorkProgressMetadata(
13139
13147
  (total === null || skipped === null || metadata?.workProgress) &&
13140
13148
  !runLogTail &&
13141
13149
  !runtimeResources &&
13150
+ !runtimeCounters &&
13151
+ !metadata?.runtimeCounters &&
13142
13152
  !rowOutcomes &&
13143
13153
  mergedOutputWarnings.length === 0
13144
13154
  ) {
@@ -13170,6 +13180,9 @@ function attachRunnerWorkProgressMetadata(
13170
13180
  ...record,
13171
13181
  _metadata: {
13172
13182
  ...(metadata ?? {}),
13183
+ // A previous attempt's scalar evidence cannot describe this terminal
13184
+ // attempt. Undefined removes it from persisted JSON when unavailable.
13185
+ runtimeCounters: runtimeCounters ?? undefined,
13173
13186
  ...(workProgress ? { workProgress } : {}),
13174
13187
  ...(rowOutcomes ? { rowOutcomes } : {}),
13175
13188
  ...(runLogTail ? { runLogTail } : {}),
@@ -15667,8 +15680,63 @@ export async function readPostgresSchedulerResult(
15667
15680
  export async function readPostgresSchedulerRunStatus(
15668
15681
  client: PostgresSchedulerQueryClient,
15669
15682
  runId: string,
15670
- options?: Pick<PostgresSchedulerOptions, 'schema'>,
15683
+ options?: Pick<PostgresSchedulerOptions, 'schema'> & {
15684
+ includeDetails?: boolean;
15685
+ },
15671
15686
  ): Promise<PostgresSchedulerRunStatusSnapshot | null> {
15687
+ if (options?.includeDetails === false) {
15688
+ // An admission projection gap needs lifecycle truth only. Launch, result,
15689
+ // suspension and error JSON may contain row/call-sized detail: never even
15690
+ // select them for status polling. Missing progress remains unconfirmed.
15691
+ const result = await client.query<
15692
+ Pick<
15693
+ RunStatusRow,
15694
+ | 'run_id'
15695
+ | 'org_id'
15696
+ | 'status'
15697
+ | 'play_name'
15698
+ | 'graph_hash'
15699
+ | 'runtime_profile'
15700
+ | 'created_at'
15701
+ | 'updated_at'
15702
+ | 'started_at'
15703
+ | 'finished_at'
15704
+ >
15705
+ >(
15706
+ `
15707
+ SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
15708
+ created_at, updated_at, started_at, finished_at
15709
+ FROM ${tableName('runs', options)}
15710
+ WHERE run_id = $1
15711
+ `,
15712
+ [runId],
15713
+ );
15714
+ const row = result.rows[0];
15715
+ if (!row) return null;
15716
+ return {
15717
+ runId: row.run_id,
15718
+ orgId: row.org_id ?? '',
15719
+ status: row.status,
15720
+ playName: row.play_name,
15721
+ graphHash: row.graph_hash,
15722
+ runtimeBackend: null,
15723
+ schedulerBackend: 'absurd',
15724
+ executionProfile: row.runtime_profile,
15725
+ createdAt: row.created_at?.getTime() ?? null,
15726
+ updatedAt: row.updated_at?.getTime() ?? null,
15727
+ startedAt: row.started_at?.getTime() ?? null,
15728
+ finishedAt: row.finished_at?.getTime() ?? null,
15729
+ durationMs:
15730
+ row.finished_at && row.started_at
15731
+ ? Math.max(0, row.finished_at.getTime() - row.started_at.getTime())
15732
+ : null,
15733
+ runtimeTiming: null,
15734
+ result: null,
15735
+ error: null,
15736
+ suspension: null,
15737
+ staticPipeline: null,
15738
+ };
15739
+ }
15672
15740
  const result = await client.query<RunStatusRow>(
15673
15741
  `
15674
15742
  SELECT run_id, org_id, status, play_name, graph_hash, runtime_profile,
@@ -15883,6 +15951,10 @@ export async function assertPostgresRunLedgerDelivery(
15883
15951
  runId: string;
15884
15952
  idempotencyKey: string;
15885
15953
  leaseOwner?: string;
15954
+ projectedEvents?: readonly AcceptedPlayRunLedgerEvent[];
15955
+ normalizeProjectedEvents?: (
15956
+ events: readonly PlayRunLedgerIngressEvent[],
15957
+ ) => readonly AcceptedPlayRunLedgerEvent[];
15886
15958
  },
15887
15959
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
15888
15960
  ): Promise<void> {
@@ -15893,6 +15965,8 @@ export async function assertPostgresRunLedgerDelivery(
15893
15965
  lease_live: boolean;
15894
15966
  due: boolean;
15895
15967
  blocked: boolean;
15968
+ payload_json: unknown;
15969
+ created_at: Date;
15896
15970
  }>(
15897
15971
  `
15898
15972
  SELECT outbox.delivery_key,
@@ -15900,7 +15974,9 @@ export async function assertPostgresRunLedgerDelivery(
15900
15974
  outbox.lease_owner,
15901
15975
  outbox.lease_expires_at > now() AS lease_live,
15902
15976
  coalesce(outbox.next_attempt_at, outbox.created_at) <= now() AS due,
15903
- outbox.failed_at IS NOT NULL AS blocked
15977
+ outbox.failed_at IS NOT NULL AS blocked,
15978
+ outbox.payload_json,
15979
+ outbox.created_at
15904
15980
  FROM ${tableName('outbox', options)} AS outbox
15905
15981
  JOIN ${tableName('runs', options)} AS runs ON runs.run_id = outbox.run_id
15906
15982
  WHERE outbox.run_id = $1 AND runs.org_id = $2
@@ -15915,12 +15991,9 @@ export async function assertPostgresRunLedgerDelivery(
15915
15991
  throw new RunLedgerProjectionLeaseLostError();
15916
15992
  }
15917
15993
 
15918
- // A normal projector leases only the batch it is about to send. Cancellation
15919
- // intentionally transfers the entire pending causal prefix to one stable
15920
- // owner before synchronously draining it, while run.events are still claimed
15921
- // one row at a time. Validate the submitted key against a *contiguous due
15922
- // prefix*, not every row carrying that owner; otherwise a valid cancellation
15923
- // head is rejected whenever later facts share the cancellation lease.
15994
+ // Cancellation may own several pending facts at once. Validate the exact
15995
+ // contiguous, due prefix named by the receipt instead of all rows sharing
15996
+ // that cancellation lease.
15924
15997
  const prefix: typeof rows = [];
15925
15998
  for (const row of rows) {
15926
15999
  if (
@@ -15940,6 +16013,9 @@ export async function assertPostgresRunLedgerDelivery(
15940
16013
  candidate.event_type,
15941
16014
  ),
15942
16015
  );
16016
+ const prefixDigest = createHash('sha256')
16017
+ .update(prefix.map((candidate) => candidate.delivery_key).join('\n'))
16018
+ .digest('hex');
15943
16019
  let expectedKey: string;
15944
16020
  if (terminal) {
15945
16021
  expectedKey = buildRunLedgerTerminalIdempotencyKey({
@@ -15955,29 +16031,58 @@ export async function assertPostgresRunLedgerDelivery(
15955
16031
  deliveryKey: first.delivery_key,
15956
16032
  });
15957
16033
  } else {
15958
- const digest = createHash('sha256')
15959
- .update(prefix.map((candidate) => candidate.delivery_key).join('\n'))
15960
- .digest('hex');
15961
16034
  expectedKey = buildRunLedgerEventsIdempotencyKey({
15962
16035
  runId: input.runId,
15963
- deliveryDigest: digest,
16036
+ deliveryDigest: prefixDigest,
15964
16037
  });
15965
16038
  }
15966
- if (input.idempotencyKey === expectedKey) return;
15967
16039
 
15968
- // The pre-batch projector used run-terminal:<delivery-key> for a single
15969
- // nonterminal fact. Retain that wire-compatibility key for the current
15970
- // singleton head, but never let it authorize a later row in the prefix.
15971
- if (
16040
+ // Older batch dispatchers may use the digest receipt even when the batch
16041
+ // contains a terminal fact. Keep accepting it for this exact prefix.
16042
+ const batchKey = buildRunLedgerEventsIdempotencyKey({
16043
+ runId: input.runId,
16044
+ deliveryDigest: prefixDigest,
16045
+ });
16046
+
16047
+ // Preserve the old singleton key for non-start, nonterminal facts.
16048
+ const legacySingletonKey =
15972
16049
  prefix.length === 1 &&
15973
16050
  !['run.created', 'run.started'].includes(first.event_type) &&
15974
16051
  input.idempotencyKey ===
15975
16052
  buildRunLedgerTerminalIdempotencyKey({
15976
16053
  runId: input.runId,
15977
16054
  deliveryKey: first.delivery_key,
15978
- })
16055
+ });
16056
+ if (
16057
+ input.idempotencyKey === expectedKey ||
16058
+ input.idempotencyKey === batchKey ||
16059
+ legacySingletonKey
15979
16060
  ) {
15980
- return;
16061
+ if (input.projectedEvents === undefined) return;
16062
+ const prefixEvents: PlayRunLedgerIngressEvent[] = [];
16063
+ for (const prefixRow of prefix) {
16064
+ prefixEvents.push(
16065
+ ...buildRunLedgerEventsFromPostgresSchedulerOutbox({
16066
+ eventId: prefixRow.delivery_key,
16067
+ runId: input.runId,
16068
+ eventType: prefixRow.event_type,
16069
+ payload: prefixRow.payload_json,
16070
+ createdAt: new Date(prefixRow.created_at),
16071
+ leaseOwner: prefixRow.lease_owner ?? undefined,
16072
+ deliveryKey: prefixRow.delivery_key,
16073
+ }),
16074
+ );
16075
+ }
16076
+ const comparablePrefixEvents = input.normalizeProjectedEvents
16077
+ ? input.normalizeProjectedEvents(prefixEvents)
16078
+ : prefixEvents;
16079
+ if (
16080
+ stableStringify(input.projectedEvents) ===
16081
+ stableStringify(comparablePrefixEvents)
16082
+ ) {
16083
+ return;
16084
+ }
16085
+ throw new RunLedgerProjectionLeaseLostError();
15981
16086
  }
15982
16087
 
15983
16088
  if (
@@ -16157,7 +16262,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
16157
16262
  )
16158
16263
  )
16159
16264
  )
16160
- ORDER BY outbox.created_at, outbox.event_id
16265
+ ORDER BY ${runLedgerOrderSql('outbox')}
16161
16266
  FOR UPDATE OF outbox
16162
16267
  LIMIT $1
16163
16268
  ), terminal_batch AS (
@@ -16193,7 +16298,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
16193
16298
  SELECT event_id, run_id, event_type, payload_json, created_at, lease_owner,
16194
16299
  delivery_key
16195
16300
  FROM claimed
16196
- ORDER BY created_at, event_id
16301
+ ORDER BY ${runLedgerOrderSql('claimed')}
16197
16302
  `,
16198
16303
  [limit, runId, leaseOwner, leaseSeconds],
16199
16304
  );
@@ -16287,7 +16392,9 @@ export async function readPostgresSchedulerOutboxNextDueAt(
16287
16392
  export async function readPostgresSchedulerOutboxDeliveryStatus(
16288
16393
  client: PostgresSchedulerQueryClient,
16289
16394
  input: { runId: string; eventId: string },
16290
- options?: Pick<PostgresSchedulerOptions, 'schema'>,
16395
+ options?: Pick<PostgresSchedulerOptions, 'schema'> & {
16396
+ includeBlocked?: boolean;
16397
+ },
16291
16398
  ): Promise<PostgresSchedulerOutboxDeliveryStatus> {
16292
16399
  const runId = input.runId.trim();
16293
16400
  const eventId = input.eventId.trim();
@@ -16298,13 +16405,17 @@ export async function readPostgresSchedulerOutboxDeliveryStatus(
16298
16405
  }
16299
16406
  const result = await client.query<{ delivered: boolean; blocked: boolean }>(
16300
16407
  `SELECT target.delivered_at IS NOT NULL AS delivered,
16301
- target.failed_at IS NOT NULL OR EXISTS (
16408
+ ${
16409
+ options?.includeBlocked === false
16410
+ ? 'false'
16411
+ : `target.failed_at IS NOT NULL OR EXISTS (
16302
16412
  SELECT 1
16303
16413
  FROM ${tableName('outbox', options)} predecessor
16304
16414
  WHERE predecessor.run_id = target.run_id
16305
16415
  AND predecessor.failed_at IS NOT NULL
16306
16416
  AND ${runLedgerPrecedesSql('predecessor', 'target', true)}
16307
- ) AS blocked
16417
+ )`
16418
+ } AS blocked
16308
16419
  FROM ${tableName('outbox', options)} target
16309
16420
  WHERE target.run_id = $1 AND target.event_id = $2`,
16310
16421
  [runId, eventId],
@@ -66,6 +66,7 @@ import {
66
66
  createRuntimeReceiptValuePersister,
67
67
  createRuntimeReceiptValueReader,
68
68
  RUNTIME_RECEIPT_VALUE_INLINE_MAX_BYTES,
69
+ RuntimeValueStoreUnavailableError,
69
70
  type RuntimeReceiptValueBlobStore,
70
71
  type RuntimeReceiptValuePersister,
71
72
  type RuntimeReceiptValueReader,
@@ -250,6 +251,26 @@ export function receiptGatewayFailureResponse(
250
251
  },
251
252
  );
252
253
  }
254
+ if (error instanceof RuntimeValueStoreUnavailableError) {
255
+ return receiptGatewayJsonResponse(
256
+ {
257
+ error: `${operationLabel} object store is temporarily unavailable.`,
258
+ code: 'runtime_value_store_unavailable',
259
+ retryable: true,
260
+ retry_after_ms: 1_000,
261
+ detail: error.message,
262
+ ...(context.action ? { action: context.action } : {}),
263
+ },
264
+ {
265
+ status: 503,
266
+ headers: {
267
+ 'retry-after': '1',
268
+ 'x-deepline-error-code': 'runtime_value_store_unavailable',
269
+ 'x-deepline-error-detail': error.message,
270
+ },
271
+ },
272
+ );
273
+ }
253
274
  if (isRuntimeSchedulerCapacityError(error)) {
254
275
  const retryAfterMs = runtimeSchedulerCapacityRetryAfterMs(error);
255
276
  return receiptGatewayJsonResponse(
@@ -25,6 +25,40 @@ export type RuntimeValueBlobStore = {
25
25
  /** Backward-compatible receipt-specific name for the shared object store. */
26
26
  export type RuntimeReceiptValueBlobStore = RuntimeValueBlobStore;
27
27
 
28
+ export class RuntimeValueStoreUnavailableError extends Error {
29
+ constructor(
30
+ readonly operation: 'put' | 'head',
31
+ readonly status: number,
32
+ cause: unknown,
33
+ ) {
34
+ super(`Runtime value object store ${operation} returned HTTP ${status}.`, {
35
+ cause,
36
+ });
37
+ this.name = 'RuntimeValueStoreUnavailableError';
38
+ }
39
+ }
40
+
41
+ function transientStoreStatus(error: unknown): number | null {
42
+ if (!error || typeof error !== 'object') return null;
43
+ const metadata = (error as { $metadata?: { httpStatusCode?: unknown } })
44
+ .$metadata;
45
+ const status = metadata?.httpStatusCode;
46
+ return typeof status === 'number' &&
47
+ (status === 408 || status === 429 || status >= 500)
48
+ ? status
49
+ : null;
50
+ }
51
+
52
+ function transientStoreError(
53
+ operation: 'put' | 'head',
54
+ error: unknown,
55
+ ): RuntimeValueStoreUnavailableError | null {
56
+ const status = transientStoreStatus(error);
57
+ return status === null
58
+ ? null
59
+ : new RuntimeValueStoreUnavailableError(operation, status, error);
60
+ }
61
+
28
62
  /**
29
63
  * Foundation owns value references and bounded I/O admission; the deployment
30
64
  * injects its object-store adapter. This keeps R2 credentials and transport
@@ -185,7 +219,13 @@ export async function persistRuntimeReceiptValue(
185
219
  // An immutable deterministic object that is observable after a failed PUT
186
220
  // is the same attempted publication. Treat it as a recovered write, never
187
221
  // as permission to call the provider again.
188
- if (!(await store.exists(outputRef.key))) throw error;
222
+ let exists: boolean;
223
+ try {
224
+ exists = await store.exists(outputRef.key);
225
+ } catch (probeError) {
226
+ throw transientStoreError('head', probeError) ?? probeError;
227
+ }
228
+ if (!exists) throw transientStoreError('put', error) ?? error;
189
229
  }
190
230
  return { kind: 'reference', outputJson: null, outputRef };
191
231
  }
@@ -209,11 +249,10 @@ export function createRuntimeReceiptValuePersister(
209
249
  serialized: string;
210
250
  inlineFields?: RuntimeReceiptValueInlineFields;
211
251
  };
212
- const writes = new BoundedBatchFunction<
213
- Write,
214
- PersistedRuntimeReceiptValue,
215
- 'r2'
216
- >({
252
+ type WriteOutcome =
253
+ | { ok: true; value: PersistedRuntimeReceiptValue }
254
+ | { ok: false; error: unknown };
255
+ const writes = new BoundedBatchFunction<Write, WriteOutcome, 'r2'>({
217
256
  keyId: () => 'r2',
218
257
  estimateBytes: (value) => Buffer.byteLength(value.serialized, 'utf8'),
219
258
  maxBatchItems: 4,
@@ -224,16 +263,24 @@ export function createRuntimeReceiptValuePersister(
224
263
  maxRetainedUnits: 32,
225
264
  maxInFlightBatches: 2,
226
265
  flushMs: 0,
266
+ // R2 has no multi-PUT. One failed object must not reject the other
267
+ // independent receipts admitted alongside it.
227
268
  execute: async ({ values }) =>
228
269
  await Promise.all(
229
- values.map(
230
- async (value) =>
231
- await persistRuntimeReceiptValue({
232
- ...value,
233
- inlineMaxBytes,
234
- ...(input.store ? { store: input.store } : {}),
235
- }),
236
- ),
270
+ values.map(async (value): Promise<WriteOutcome> => {
271
+ try {
272
+ return {
273
+ ok: true,
274
+ value: await persistRuntimeReceiptValue({
275
+ ...value,
276
+ inlineMaxBytes,
277
+ ...(input.store ? { store: input.store } : {}),
278
+ }),
279
+ };
280
+ } catch (error) {
281
+ return { ok: false, error };
282
+ }
283
+ }),
237
284
  ),
238
285
  });
239
286
  return async (value) => {
@@ -244,7 +291,9 @@ export function createRuntimeReceiptValuePersister(
244
291
  ...(input.store ? { store: input.store } : {}),
245
292
  });
246
293
  }
247
- return await writes.submit({ key: 'r2', value });
294
+ const outcome = await writes.submit({ key: 'r2', value });
295
+ if (!outcome.ok) throw outcome.error;
296
+ return outcome.value;
248
297
  };
249
298
  }
250
299
 
@@ -41,11 +41,11 @@ export function releasableRuntimeSheetAttemptFenceSql(
41
41
  }
42
42
 
43
43
  /**
44
- * Predicate for durable replay idempotency: matches rows that are already
45
- * terminal (enriched or failed) from the same attempt owner + epoch. Used to
46
- * suppress same-version conflicts when a worker restarts and re-runs map rows
47
- * that it already completed. First terminal state wins; payload/error drift on
48
- * replay is accepted as a no-op rather than rejected as a conflict.
44
+ * Predicate for callers that need to identify a terminal row from the same
45
+ * attempt owner + epoch. Runtime map persistence now enforces first-terminal
46
+ * idempotency inline because its write-version and row-key context are also
47
+ * required; keep this helper exported for compatibility with external
48
+ * conformance checks and integrations.
49
49
  */
50
50
  export function sameOwnerTerminalAttemptEpochSql(
51
51
  tableAlias: string,
@@ -35,6 +35,7 @@ export {
35
35
  type AlwaysFreshToolCall,
36
36
  type AlwaysFreshToolCallInput,
37
37
  } from './always-fresh-adapter';
38
+ export { resolveToolCallResultReceiptPolicy } from './result-receipt-policy';
38
39
 
39
40
  export type {
40
41
  ToolCallCompatibilityIdentity,
@@ -0,0 +1,46 @@
1
+ import { scopeToolReceiptToInvocation } from '../durable-call-cache';
2
+
3
+ /**
4
+ * Freshness chooses which completed results a run may reuse. It never turns
5
+ * off durable progress inside that run. A forced invocation gets an immutable
6
+ * result of its own; a scheduler retry reads it like any other completed key.
7
+ *
8
+ * The canonical key already binds payload, authorization, action and response
9
+ * semantics. Keep it in the derivation so a changed input cannot recover an
10
+ * unrelated result. Provider operation identities and public receipt metadata
11
+ * remain canonical and are deliberately outside this storage policy.
12
+ */
13
+ export function resolveToolCallResultReceiptPolicy(input: {
14
+ orgId?: string | null;
15
+ toolId: string;
16
+ resultReceiptKey: string;
17
+ runId: string;
18
+ logicalCallId: string;
19
+ force: boolean;
20
+ /** Historical local contexts may have no durable result store. */
21
+ canStoreResult?: boolean;
22
+ }): { resultReceiptKey: string; force: boolean } {
23
+ if (!input.resultReceiptKey.trim()) {
24
+ throw new Error('Tool Call result policy requires a result receipt key.');
25
+ }
26
+ if (!input.force) {
27
+ return { resultReceiptKey: input.resultReceiptKey, force: false };
28
+ }
29
+ if (input.canStoreResult === false) {
30
+ return { resultReceiptKey: input.resultReceiptKey, force: true };
31
+ }
32
+ if (!input.runId.trim() || !input.logicalCallId.trim()) {
33
+ throw new Error(
34
+ 'Forced Tool Call requires stable run and call identities.',
35
+ );
36
+ }
37
+ return {
38
+ resultReceiptKey: scopeToolReceiptToInvocation({
39
+ orgId: input.orgId,
40
+ toolId: input.toolId,
41
+ receiptKey: input.resultReceiptKey,
42
+ invocationIdentity: `forced-result:${input.runId}:${input.logicalCallId}`,
43
+ }),
44
+ force: false,
45
+ };
46
+ }
@@ -425,20 +425,29 @@ export function createToolExecuteHttpFailureAttemptTracker(): ToolExecuteHttpFai
425
425
  };
426
426
  }
427
427
 
428
- export function parseToolExecuteRetryAfterMs(
428
+ // Keep expired absolute deadlines intact: durable response replay must not
429
+ // turn old provider feedback into a new AIMD throttle episode.
430
+ export function parseToolExecuteRetryAtMs(
429
431
  header: string | null | undefined,
430
432
  nowMs = Date.now(),
431
433
  ): number {
432
- if (!header) return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
434
+ if (!header) return nowMs + TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
433
435
  const seconds = Number(header);
434
436
  if (Number.isFinite(seconds) && seconds >= 0) {
435
- return Math.ceil(seconds * 1000);
437
+ return nowMs + Math.ceil(seconds * 1000);
436
438
  }
437
439
  const retryAt = Date.parse(header);
438
440
  if (Number.isFinite(retryAt)) {
439
- return Math.max(0, retryAt - nowMs);
441
+ return retryAt;
440
442
  }
441
- return TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
443
+ return nowMs + TOOL_EXECUTE_RETRY_DELAY_FALLBACK_MS;
444
+ }
445
+
446
+ export function parseToolExecuteRetryAfterMs(
447
+ header: string | null | undefined,
448
+ nowMs = Date.now(),
449
+ ): number {
450
+ return Math.max(0, parseToolExecuteRetryAtMs(header, nowMs) - nowMs);
442
451
  }
443
452
 
444
453
  export function classifyToolExecuteHttpFailure(input: {
@@ -265,7 +265,11 @@ function receiptFailureKindForToolErrorPayload(
265
265
  // A credential connection can be added or replaced between play runs. Keep
266
266
  // the failure terminal within its owning run, but let a later run reclaim
267
267
  // the durable receipt instead of replaying stale authorization state.
268
- return code === 'INTEGRATION_CREDENTIALS_MISSING' ? 'repairable' : 'terminal';
268
+ if (code === 'INTEGRATION_CREDENTIALS_MISSING') return 'repairable';
269
+ // Managed AI has billing evidence for completed model steps, but no stored
270
+ // responses from which to resume the later rate-limited step. Its 409 stops
271
+ // automatic replay; an explicit later run may intentionally repeat work.
272
+ return code === 'UPSTREAM_PARTIAL_EXECUTION' ? 'repairable' : 'terminal';
269
273
  }
270
274
 
271
275
  function formatHardBillingFailureMessage(input: {
@@ -246,17 +246,11 @@ export function decideWorkReceiptClaim(input: {
246
246
  const sameLogicalRun =
247
247
  normalize(input.claimantRunId) !== null &&
248
248
  normalize(input.claimantRunId) === normalize(input.receipt.runId);
249
- const sameRunNewerLeaseAttempt =
250
- state.reusable &&
251
- state.lease.kind === 'active' &&
252
- state.lease.ownerRunId === normalize(input.claimantRunId) &&
253
- state.lease.ownerAttempt < normalizeAttempt(input.claimantRunAttempt);
254
249
  // Force refresh means new intent across runs. Within one run it is a replay,
255
250
  // so reuse terminal output instead of creating a second terminal payload
256
- // under the run's existing Runtime Sheet write version. A newer lease
257
- // generation is the exception: it is a new attempt and retains the existing
258
- // stale-owner fence semantics.
259
- if (state.reusable && sameLogicalRun && !sameRunNewerLeaseAttempt) {
251
+ // under the run's existing Runtime Sheet write version. A scheduler resume
252
+ // increases the attempt number but does not create another refresh intent.
253
+ if (state.reusable && sameLogicalRun) {
260
254
  return { kind: 'reuse_terminal' };
261
255
  }
262
256
  if (state.reusable && input.forceRefresh !== true) {
@@ -239,7 +239,7 @@ function defaultThreshold(
239
239
  }
240
240
 
241
241
  function formatCredits(value: number): string {
242
- return value.toLocaleString('en-US', { maximumFractionDigits: 2 });
242
+ return value.toLocaleString('en-US', { maximumFractionDigits: 3 });
243
243
  }
244
244
 
245
245
  export const DEEPLINE_DOCS_URL = 'https://deepline.com/docs';