deepline 0.3.150 → 0.3.152

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
200
200
  // getters keep their established compatibility behavior.
201
201
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
202
202
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
203
- version: '0.3.150',
203
+ version: '0.3.152',
204
204
  updateSummary:
205
205
  'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
206
206
  packageCapabilities: {
@@ -40,7 +40,8 @@ export type DlqPathAction =
40
40
  | 'retry'
41
41
  | 'quarantine'
42
42
  | 'release'
43
- | 'settle';
43
+ | 'settle'
44
+ | 'cancel';
44
45
 
45
46
  export type DlqLookupIndex = {
46
47
  readonly table: string;
@@ -302,6 +303,7 @@ export function normalizeQueueItemAction(
302
303
  quarantine: 'quarantine',
303
304
  release: 'cancel',
304
305
  settle: 'resume',
306
+ cancel: 'cancel',
305
307
  };
306
308
  return action in aliases
307
309
  ? aliases[action as QueueItemLegacyAction]
@@ -343,6 +345,17 @@ export type QueueItemFailedTimeRange = {
343
345
  readonly snapshotAt: string;
344
346
  };
345
347
 
348
+ /** Server attestation for a bounded source-indexed creation-time plan. */
349
+ export type QueueItemCreatedTimeRange = {
350
+ readonly kind: 'created-time-range';
351
+ readonly after: string | null;
352
+ readonly before: string;
353
+ readonly index: string;
354
+ readonly indexed: true;
355
+ /** Stable across every page in the same cursor-bound plan. */
356
+ readonly snapshotAt: string;
357
+ };
358
+
346
359
  export type QueueItemList = {
347
360
  readonly contractVersion: typeof QUEUE_ITEM_CONTROL_CONTRACT_VERSION;
348
361
  readonly queueId: string;
@@ -354,6 +367,8 @@ export type QueueItemList = {
354
367
  readonly cursor: string | null;
355
368
  /** Present only for a successfully measured indexed failure-time range. */
356
369
  readonly failedTimeRange?: QueueItemFailedTimeRange;
370
+ /** Present only for a successfully measured indexed creation-time range. */
371
+ readonly createdTimeRange?: QueueItemCreatedTimeRange;
357
372
  readonly unavailable?: QueueItemUnavailable;
358
373
  };
359
374
 
@@ -470,6 +485,11 @@ export type QueueItemConnector = {
470
485
  context: QueueItemConnectorContext,
471
486
  range: { after?: string; before: string },
472
487
  ) => Promise<QueueItemList>;
488
+ /** Indexed creation-time listing for owner-declared retirement plans. */
489
+ readonly listCreatedInRange?: (
490
+ context: QueueItemConnectorContext,
491
+ range: { after?: string; before: string },
492
+ ) => Promise<QueueItemList>;
473
493
  readonly get: (
474
494
  context: QueueItemConnectorContext,
475
495
  itemId: string,
@@ -810,6 +830,15 @@ const DLQ_PATH_METADATA: Readonly<Record<string, DlqPathMetadata>> = {
810
830
  bounded: true,
811
831
  },
812
832
  transitions: [
833
+ {
834
+ action: 'cancel',
835
+ fromStates: ['ready'],
836
+ toState: 'cancelled',
837
+ enabled: true,
838
+ requiresApproval: true,
839
+ reason:
840
+ 'Cancellation is owner-controlled and revision-fenced; retirement can be recorded only after the job reaches this terminal state.',
841
+ },
813
842
  {
814
843
  action: 'retry',
815
844
  fromStates: ['failed'],
@@ -1,5 +1,5 @@
1
1
  import { createWriteStream } from 'node:fs';
2
- import { randomUUID } from 'node:crypto';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
3
  import { stat } from 'node:fs/promises';
4
4
  import { Readable } from 'node:stream';
5
5
  import { pipeline } from 'node:stream/promises';
@@ -47,6 +47,7 @@ import {
47
47
  LEGACY_RUNTIME_RECEIPT_WIRE_HEADER,
48
48
  } from './/runtime-contract';
49
49
  import { PLAY_RUNTIME_API_COMPAT_PATH } from './/runtime-api-paths';
50
+ import { buildRunLedgerEventsIdempotencyKey } from './run-ledger-projection-contract';
50
51
  import {
51
52
  RUNTIME_API_ACCEPTS_HEADER,
52
53
  gzipRuntimeApiRequestBody,
@@ -157,15 +158,21 @@ type RuntimeApiRequest =
157
158
  action: 'create_db_session';
158
159
  } & CreateDbSessionRequest)
159
160
  | {
160
- action: 'append_run_events';
161
+ action: 'append_run_events' | 'project_run_events';
161
162
  playId: string;
162
163
  events: PlayRunLedgerIngressEvent[];
163
164
  idempotencyKey?: string;
165
+ deliveryIdempotencyKey?: string;
166
+ deliveryLeaseOwner?: string;
164
167
  }
165
168
  | {
166
169
  action: 'start_run';
170
+ deliveryLeaseOwner?: string;
167
171
  idempotencyKey?: string;
168
172
  playName: string;
173
+ playReference?: string | null;
174
+ definitionScope?: 'org' | 'system';
175
+ revisionId?: string | null;
169
176
  runId: string;
170
177
  artifactStorageKey?: string | null;
171
178
  artifactHash?: string | null;
@@ -200,6 +207,7 @@ type RuntimeApiRequest =
200
207
  }
201
208
  | {
202
209
  action: 'project_run_created';
210
+ deliveryLeaseOwner?: string;
203
211
  idempotencyKey: string;
204
212
  playName: string;
205
213
  playReference?: string | null;
@@ -736,7 +744,8 @@ function isRetryableAppRuntimeResponse(input: {
736
744
  // projection. Do not make every 403 retryable: only this exact Vercel HTML
737
745
  // response on a replay-safe action is an intermediary failure.
738
746
  if (
739
- input.action === 'append_run_events' &&
747
+ (input.action === 'append_run_events' ||
748
+ input.action === 'project_run_events') &&
740
749
  isVercelSecurityCheckpointResponse(input)
741
750
  ) {
742
751
  return true;
@@ -964,6 +973,9 @@ function isRetryableAppRuntimeAction(
964
973
  ): boolean {
965
974
  return (
966
975
  action === 'append_run_events' ||
976
+ // Classification remains replay-safe for the durable outbox even though
977
+ // projectRunEventsViaAppRuntime disables in-process transport retries.
978
+ action === 'project_run_events' ||
967
979
  action === 'apply_row_updates' ||
968
980
  action === 'compute_billing_finalize' ||
969
981
  action === 'compute_billing_record_item' ||
@@ -2312,6 +2324,85 @@ async function appendRunEventsViaAppRuntimeRaw(
2312
2324
  });
2313
2325
  }
2314
2326
 
2327
+ /** Leased delivery only. Every transport piece has its own stable receipt. */
2328
+ export async function projectRunEventsViaAppRuntime(
2329
+ context: WorkerRuntimeApiContext,
2330
+ input: {
2331
+ playId: string;
2332
+ events: PlayRunLedgerIngressEvent[];
2333
+ idempotencyKey: string;
2334
+ deliveryLeaseOwner: string;
2335
+ beforeBatch: () => Promise<void>;
2336
+ },
2337
+ ): Promise<void> {
2338
+ const batches = partitionRunEventsForAppRuntime(input.events);
2339
+ const receiptBatch = runLedgerReceiptBatchIndex(batches);
2340
+ for (const [index, events] of batches.entries()) {
2341
+ await input.beforeBatch();
2342
+ // Retain the original receipt for the terminal (or final) piece so an acknowledged
2343
+ // legacy delivery stays acknowledged across the reader rollout.
2344
+ const idempotencyKey =
2345
+ index === receiptBatch
2346
+ ? input.idempotencyKey
2347
+ : buildRunLedgerEventsIdempotencyKey({
2348
+ runId: input.playId,
2349
+ deliveryDigest: createHash('sha256')
2350
+ .update(JSON.stringify([input.idempotencyKey, index, events]))
2351
+ .digest('hex'),
2352
+ });
2353
+ const request = {
2354
+ playId: input.playId,
2355
+ events,
2356
+ idempotencyKey,
2357
+ deliveryIdempotencyKey: input.idempotencyKey,
2358
+ deliveryLeaseOwner: input.deliveryLeaseOwner,
2359
+ };
2360
+ // The durable outbox owns recovery; this helper does not retry OCCs or
2361
+ // other delivery failures inside an already-leased invocation.
2362
+ const deliveryContext = { ...context, retryPolicy: 'none' as const };
2363
+ try {
2364
+ await postAppRuntimeApi(deliveryContext, {
2365
+ action: 'project_run_events',
2366
+ ...request,
2367
+ });
2368
+ } catch (error) {
2369
+ if (
2370
+ !(error instanceof AppRuntimeApiResponseError) ||
2371
+ !isLegacyProjectRunCreatedUnsupportedActionResponse({
2372
+ status: error.status,
2373
+ body: error.detail,
2374
+ })
2375
+ )
2376
+ throw error;
2377
+ // Launch-pinned old apps already implement equivalent keyed projection.
2378
+ // A real auth refusal never selects this compatibility adapter.
2379
+ // The capability probe was a network round trip. Re-check ownership
2380
+ // before the legacy request too; reclamation must stop a stale sender.
2381
+ await input.beforeBatch();
2382
+ await postAppRuntimeApi(deliveryContext, {
2383
+ action: 'append_run_events',
2384
+ ...request,
2385
+ });
2386
+ }
2387
+ }
2388
+ }
2389
+
2390
+ function runLedgerReceiptBatchIndex(
2391
+ batches: PlayRunLedgerIngressEvent[][],
2392
+ ): number {
2393
+ for (let index = batches.length - 1; index >= 0; index -= 1) {
2394
+ if (
2395
+ batches[index]!.some((event) =>
2396
+ ['run.completed', 'run.failed', 'run.cancelled'].includes(
2397
+ event.key ?? event.type ?? '',
2398
+ ),
2399
+ )
2400
+ )
2401
+ return index;
2402
+ }
2403
+ return batches.length - 1;
2404
+ }
2405
+
2315
2406
  /**
2316
2407
  * Append Run Ledger events through the bounded transport used by every runtime
2317
2408
  * producer. Scheduler terminal outbox events can carry a complete log replay,
@@ -2368,7 +2459,11 @@ export async function appendRunEventsViaAppRuntime(
2368
2459
  export async function startRunViaAppRuntime(
2369
2460
  context: WorkerRuntimeApiContext,
2370
2461
  input: {
2462
+ deliveryLeaseOwner?: string;
2371
2463
  playName: string;
2464
+ playReference?: string | null;
2465
+ definitionScope?: 'org' | 'system';
2466
+ revisionId?: string | null;
2372
2467
  runId: string;
2373
2468
  artifactStorageKey?: string | null;
2374
2469
  artifactHash?: string | null;
@@ -215,6 +215,7 @@ import {
215
215
  buildDurableToolPayloadProviderIdempotencyKey,
216
216
  buildProviderOperationIdempotencyKey,
217
217
  buildDurableToolReceiptPrefix,
218
+ scopeToolReceiptToInvocation,
218
219
  resolveDurableCallCachePolicy,
219
220
  } from './durable-call-cache';
220
221
  import {
@@ -12531,6 +12532,27 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
12531
12532
  (await this.resolveToolAuthScopeDigest(toolId))?.trim() ?? null;
12532
12533
  const eventWaitHandler =
12533
12534
  (await this.#options.getIntegrationEventWaitHandler?.(toolId)) ?? null;
12535
+ const invocationScoped =
12536
+ !eventWaitHandler &&
12537
+ (await this.#options.getToolIdempotencyScope?.(toolId)) === 'invocation';
12538
+ const invocationIdentity = `${this.currentRunId}:${logicalCallId}`;
12539
+ const scopeReceipt = (receiptKey: string): string =>
12540
+ invocationScoped
12541
+ ? scopeToolReceiptToInvocation({
12542
+ orgId: this.#options.orgId,
12543
+ toolId,
12544
+ receiptKey,
12545
+ invocationIdentity,
12546
+ })
12547
+ : receiptKey;
12548
+ const scopeOperation = (operationKey: string): string =>
12549
+ invocationScoped
12550
+ ? buildProviderOperationIdempotencyKey({
12551
+ operationKey,
12552
+ force: true,
12553
+ fallbackAttemptId: invocationIdentity,
12554
+ })
12555
+ : operationKey;
12534
12556
  const explicitForce =
12535
12557
  options?.force === true ||
12536
12558
  toolExecutionOverrides.getStore()?.force === true;
@@ -12545,6 +12567,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
12545
12567
  executionAuthScopeDigest,
12546
12568
  staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
12547
12569
  });
12570
+ compatibleDurableCacheKeys = compatibleDurableCacheKeys.map(scopeReceipt);
12548
12571
  let durableCacheKey = compatibleDurableCacheKeys[0];
12549
12572
  if (!durableCacheKey) {
12550
12573
  throw new Error(`Tool ${toolId} is missing a primary durable cache key.`);
@@ -12567,6 +12590,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
12567
12590
  providerActionVersion,
12568
12591
  payload: input,
12569
12592
  });
12593
+ if (legacyDurableCacheKey)
12594
+ legacyDurableCacheKey = scopeReceipt(legacyDurableCacheKey);
12595
+ providerIdempotencyKeyBase = scopeOperation(providerIdempotencyKeyBase);
12570
12596
  let directResultReceiptKey = durableCacheKey;
12571
12597
  // A raw-v2 artifact must never read an older receipt that cannot render
12572
12598
  // raw-v2. If both the primary and its revisioned recovery receipt are
@@ -12589,12 +12615,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
12589
12615
  input,
12590
12616
  eventWaitHandler,
12591
12617
  explicitForce,
12618
+ logicalCallId,
12592
12619
  );
12593
12620
  }
12594
12621
  if (toolId === 'run_javascript') {
12595
12622
  throw new Error(DISALLOWED_RUN_JAVASCRIPT_TOOL_MESSAGE);
12596
12623
  }
12597
- if (cacheableToolResult && !toolCachePolicy.force) {
12624
+ if (cacheableToolResult && !toolCachePolicy.force && !invocationScoped) {
12598
12625
  const sourceRunRecovered = await this.recoverSourceRunToolReceipt({
12599
12626
  toolId,
12600
12627
  logicalCallId,
@@ -13315,6 +13342,8 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13315
13342
  executionAuthScopeDigest,
13316
13343
  staleAfterSeconds: toolCachePolicy.staleAfterSeconds,
13317
13344
  });
13345
+ compatibleDurableCacheKeys =
13346
+ compatibleDurableCacheKeys.map(scopeReceipt);
13318
13347
  durableCacheKey = compatibleDurableCacheKeys[0];
13319
13348
  if (!durableCacheKey) {
13320
13349
  throw new Error(
@@ -13335,6 +13364,9 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13335
13364
  providerActionVersion,
13336
13365
  payload: input,
13337
13366
  });
13367
+ if (legacyDurableCacheKey)
13368
+ legacyDurableCacheKey = scopeReceipt(legacyDurableCacheKey);
13369
+ providerIdempotencyKeyBase = scopeOperation(providerIdempotencyKeyBase);
13338
13370
  directResultReceiptKey = durableCacheKey;
13339
13371
  bypassDirectResultCache = false;
13340
13372
  checkpointCacheKeys.splice(
@@ -13354,6 +13386,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13354
13386
  input: Record<string, unknown>,
13355
13387
  handler: IntegrationEventWaitHandler,
13356
13388
  force = false,
13389
+ toolCallId?: string,
13357
13390
  ): Promise<unknown> {
13358
13391
  this.assertInlineChildContract('suspending_child');
13359
13392
  if (!this.#options.durableBoundaries) {
@@ -13367,6 +13400,7 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
13367
13400
  workflowId: this.#options.workflowId,
13368
13401
  orgId: this.#options.orgId,
13369
13402
  executorToken: this.#options.executorToken,
13403
+ toolCallId,
13370
13404
  slackHitlDefaultTimeoutMs:
13371
13405
  this.currentRuntimeSemantics.slackHitl.defaultTimeoutMs,
13372
13406
  slackHitlTimeoutOutcome:
@@ -670,6 +670,8 @@ export type IntegrationEventWaitRuntimeContext = {
670
670
  workflowId?: string;
671
671
  orgId?: string;
672
672
  executorToken?: string;
673
+ /** Stable runtime identity for one logical tools.execute invocation. */
674
+ toolCallId?: string;
673
675
  slackHitlDefaultTimeoutMs?: number;
674
676
  /** Behavior selected by the admitted Play artifact, never worker version. */
675
677
  slackHitlTimeoutOutcome?: 'park' | 'error';
@@ -981,6 +983,9 @@ export interface ContextOptions {
981
983
  getToolQueueHints?: (toolId: string) => Promise<readonly PlayQueueHint[]>;
982
984
  getToolProvider?: (toolId: string) => Promise<string | null>;
983
985
  getToolOperation?: (toolId: string) => Promise<string | null>;
986
+ /** Identity policy is required before reading receipts; lookup errors are
987
+ * fatal, unlike the legacy best-effort retry advice below. */
988
+ getToolIdempotencyScope?: (toolId: string) => Promise<'invocation' | null>;
984
989
  getToolRetryPolicy?: (
985
990
  toolId: string,
986
991
  input: Record<string, unknown>,
@@ -226,6 +226,24 @@ export function buildDurableToolReceiptPrefix(input: {
226
226
  return `ctx:${orgId}:call:tool:${normalizeTableNamespace(toolId)}:`;
227
227
  }
228
228
 
229
+ /** Partition a completed-result receipt without changing its accepted wire
230
+ * shape. The semantic key still includes payload/auth/action/presentation. */
231
+ export function scopeToolReceiptToInvocation(input: {
232
+ orgId?: string | null;
233
+ toolId: string;
234
+ receiptKey: string;
235
+ invocationIdentity: string;
236
+ }): string {
237
+ if (!input.invocationIdentity.trim())
238
+ throw new Error('Missing tool invocation identity.');
239
+ return `${buildDurableToolReceiptPrefix(input)}${sha256Hex(
240
+ stableStringify({
241
+ receiptKey: input.receiptKey,
242
+ invocationIdentity: input.invocationIdentity,
243
+ }),
244
+ )}`;
245
+ }
246
+
229
247
  export function buildProviderOperationIdempotencyKey(input: {
230
248
  operationKey: string;
231
249
  force?: boolean;
@@ -1,5 +1,6 @@
1
1
  export const EVENT_WAIT_TOOL_IDS: ReadonlySet<string> = new Set([
2
2
  'slack_message_with_hitl',
3
+ 'slack_wait_for_action',
3
4
  'slack_send_message_with_hitl',
4
5
  'test_wait_for_event',
5
6
  ]);
@@ -10,6 +10,13 @@ const UUID_PATTERN =
10
10
  /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
11
11
  const SHA256_HEX_PATTERN = /^[0-9a-f]{64}$/i;
12
12
 
13
+ export class RunLedgerProjectionLeaseLostError extends Error {
14
+ constructor() {
15
+ super('Run Ledger projection requires the current outbox delivery lease.');
16
+ this.name = 'RunLedgerProjectionLeaseLostError';
17
+ }
18
+ }
19
+
13
20
  function requireRunId(runId: string): void {
14
21
  if (!runId.trim()) {
15
22
  throw new Error('Run Ledger projection idempotency key requires a run id.');
@@ -760,6 +760,12 @@ function mergeMonotonicStepProgress(input: {
760
760
  }): PlayRunLedgerStepProgress {
761
761
  const current = input.current ?? {};
762
762
  const next = input.next;
763
+ const supersededRows = maxProgressNumber(
764
+ current.supersededRows,
765
+ next.supersededRows === undefined
766
+ ? undefined
767
+ : Math.max(0, next.supersededRows),
768
+ );
763
769
  return {
764
770
  ...current,
765
771
  ...next,
@@ -786,11 +792,7 @@ function mergeMonotonicStepProgress(input: {
786
792
  ),
787
793
  }
788
794
  : {}),
789
- ...(next.supersededRows !== undefined
790
- ? { supersededRows: Math.max(0, next.supersededRows) }
791
- : current.supersededRows !== undefined
792
- ? { supersededRows: current.supersededRows }
793
- : {}),
795
+ ...(supersededRows !== undefined ? { supersededRows } : {}),
794
796
  };
795
797
  }
796
798
 
@@ -285,7 +285,14 @@ const RUNTIME_RECEIPT_GATEWAY_BATCH_SIZE = 200;
285
285
  const RUNTIME_RECEIPT_GATEWAY_BATCH_MAX_WAIT_MS = 15;
286
286
  const RUNTIME_RECEIPT_GATEWAY_REQUEST_TIMEOUT_MS =
287
287
  RUNTIME_CAPACITY_POLICY.receiptGateway.requestTimeoutMs;
288
- const RETRY_SAFE_EVENT_WAIT_ARM_TOOLS = new Set(['test_wait_for_event']);
288
+ // Slack's native action wait persists its interaction intent before posting or
289
+ // updating a card. Replaying the same boundary either reuses its bound message
290
+ // or refuses an ambiguous unbound post, so transient gateway responses are safe
291
+ // to retry without creating a second card.
292
+ const RETRY_SAFE_EVENT_WAIT_ARM_TOOLS = new Set([
293
+ 'test_wait_for_event',
294
+ 'slack_wait_for_action',
295
+ ]);
289
296
 
290
297
  /**
291
298
  * The Context already bounds receipt completion to sixteen flights. The
@@ -329,6 +336,7 @@ type RuntimeToolMetadata = {
329
336
  actionCacheReadVersions?: string[];
330
337
  queueHints?: unknown;
331
338
  retry?: {
339
+ idempotencyScope?: 'invocation';
332
340
  retrySafeTransientHttp?: boolean;
333
341
  requiresExecutionFence?: boolean;
334
342
  };
@@ -2184,6 +2192,12 @@ async function run(
2184
2192
  ? operation.trim()
2185
2193
  : null;
2186
2194
  },
2195
+ getToolIdempotencyScope: async (toolId) => {
2196
+ const metadata = await getRuntimeToolMetadata(toolId);
2197
+ return metadata.retry?.idempotencyScope === 'invocation'
2198
+ ? 'invocation'
2199
+ : null;
2200
+ },
2187
2201
  getToolRetryPolicy: async (toolId) => {
2188
2202
  const metadata = await getRuntimeToolMetadata(toolId);
2189
2203
  return {
@@ -35,6 +35,7 @@ export const RUNTIME_OPERATION_CAPABILITIES = {
35
35
  create_db_session: 'db.session',
36
36
  repair_runtime_storage_grants: 'sheet.write',
37
37
  append_run_events: 'run.progress',
38
+ project_run_events: 'run.progress',
38
39
  save_results: 'run.progress',
39
40
  start_run: 'run.progress',
40
41
  record_first_claim_latency: 'run.progress',
@@ -56,6 +57,10 @@ export const RUNTIME_OPERATION_CONTRACT: DeployedOperationContract =
56
57
  id: 'runtime-api-actions',
57
58
  actions: RUNTIME_OPERATION_CAPABILITIES,
58
59
  olderReceiver: {
60
+ project_run_events: {
61
+ kind: 'equivalent-fallback',
62
+ action: 'append_run_events',
63
+ },
59
64
  // The runner retains the complete result before attempting this
60
65
  // compact completion. An older app receiver reports the action as
61
66
  // unsupported, and the adapter retries the established full-payload
@@ -250,7 +250,7 @@ export type PlaySchedulerRunHandle = {
250
250
  signal?: AbortSignal;
251
251
  }): AsyncIterable<PlaySchedulerProgressEvent>;
252
252
  /** Cooperatively cancel the run. */
253
- cancel(): Promise<void>;
253
+ cancel(reason?: string): Promise<void>;
254
254
  /** Inject an external event (HITL, webhook). */
255
255
  signal(payload: PlaySchedulerSignalPayload): Promise<void>;
256
256
  /** Block until terminal state and return final envelope. */
@@ -216,14 +216,14 @@ function absurdRunHandle(input: {
216
216
  return error;
217
217
  };
218
218
 
219
- const requestCancellation = async () => {
219
+ const requestCancellation = async (reason?: string) => {
220
220
  await runCancellationSingleFlight(cancellationKey, async () => {
221
221
  // First move the durable engine task into its cancellation lane. If the
222
222
  // scheduler tables are briefly locked, the worker reconciler can finish
223
223
  // the terminal write later without another user request piling up.
224
224
  await cancelAbsurdTask();
225
225
  try {
226
- await delegate.cancel();
226
+ await delegate.cancel(reason);
227
227
  } catch (error) {
228
228
  throw cancellationPendingError(error);
229
229
  }
@@ -77,9 +77,11 @@ import {
77
77
  import type { PlayStaticPipeline } from '../../plays/static-pipeline';
78
78
  import {
79
79
  LOG_TAIL_LIMIT,
80
+ type AcceptedPlayRunLedgerEvent,
80
81
  type PlayRunLedgerEvent,
81
82
  type PlayRunLedgerIngressEvent,
82
83
  } from '../run-ledger';
84
+ import { stableStringify } from '../../plays/row-identity';
83
85
  import {
84
86
  isCtxFetchHttpFailureDiagnostic,
85
87
  parseCtxFetchHttpFailureDiagnostic,
@@ -88,6 +90,7 @@ import {
88
90
  } from '../log-provenance';
89
91
  import {
90
92
  buildRunLedgerEventsIdempotencyKey,
93
+ RunLedgerProjectionLeaseLostError,
91
94
  buildRunLedgerStartIdempotencyKey,
92
95
  buildRunLedgerTerminalIdempotencyKey,
93
96
  } from '../run-ledger-projection-contract';
@@ -101,7 +104,7 @@ import {
101
104
  import {
102
105
  AppRuntimeApiResponseError,
103
106
  AppRuntimeApiTransportError,
104
- appendRunEventsViaAppRuntime,
107
+ projectRunEventsViaAppRuntime,
105
108
  finalizeComputeBillingSessionViaAppRuntime,
106
109
  recordComputeBillingItemViaAppRuntime,
107
110
  upsertComputeBillingSessionViaAppRuntime,
@@ -15092,15 +15095,221 @@ function isMissingOutboxDeliveryBatchMarker(error: unknown): boolean {
15092
15095
  }
15093
15096
 
15094
15097
  /**
15095
- * v34's terminal-batch marker is a delivery-consistency improvement. A v33
15096
- * lane cannot expose that column, so the query has an explicit temporary
15097
- * compatibility Adapter rather than making the whole runtime refuse work.
15098
- *
15099
- * New databases optimistically use the current query. If the database reports
15100
- * the additive column absent, retry the untouched v33 operation once and
15101
- * cache that fact briefly. After an in-flight expansion completes the current
15102
- * query is automatically retried; there is no permanent downgrade or process
15103
- * restart requirement.
15098
+ * Persist a validated event batch without modifying the mutable run snapshot.
15099
+ * Reader-first: deploy the run.events decoder to both projectors before any
15100
+ * ingress adapter calls this function. The existing INSERT trigger wakes them.
15101
+ */
15102
+ export async function appendPostgresRunLedgerEvents(
15103
+ client: PostgresSchedulerQueryClient,
15104
+ input: {
15105
+ orgId: string;
15106
+ runId: string;
15107
+ events: readonly AcceptedPlayRunLedgerEvent[];
15108
+ },
15109
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
15110
+ ): Promise<PostgresSchedulerLifecycleOutboxEvent> {
15111
+ if (
15112
+ input.events.length === 0 ||
15113
+ input.events.some((event) => event.runId !== input.runId)
15114
+ ) {
15115
+ throw new Error(
15116
+ 'Run Ledger ingress requires a nonempty batch for exactly one run.',
15117
+ );
15118
+ }
15119
+ const payload = stableStringify({ events: input.events });
15120
+ // The HTTP adapters own the existing request limits. Postgres intake must
15121
+ // not impose Convex's document limit on a batch: projection partitions its
15122
+ // transport pieces and stores individual events separately.
15123
+ // Stable across a lost response. The complete event identity (including log
15124
+ // channel offsets and original timestamps) participates in the digest.
15125
+ const eventId = `run-events:${input.runId}:${createHash('sha256').update(payload).digest('hex')}`;
15126
+ return appendRunLedgerOutboxFact(
15127
+ client,
15128
+ { ...input, eventId, eventType: 'run.events', payload },
15129
+ options,
15130
+ );
15131
+ }
15132
+
15133
+ export type RunLedgerStartCallback = Omit<
15134
+ Parameters<typeof startRunViaAppRuntime>[1],
15135
+ 'idempotencyKey' | 'deliveryLeaseOwner'
15136
+ >;
15137
+
15138
+ /** Preserve the legacy start/create operation, but deliver it under one lease. */
15139
+ export async function appendPostgresRunLedgerStart(
15140
+ client: PostgresSchedulerQueryClient,
15141
+ input: { orgId: string; runId: string; start: RunLedgerStartCallback },
15142
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
15143
+ ): Promise<PostgresSchedulerLifecycleOutboxEvent> {
15144
+ if (input.start.runId !== input.runId) {
15145
+ throw new Error('Run Ledger start must belong to exactly one run.');
15146
+ }
15147
+ return appendRunLedgerOutboxFact(
15148
+ client,
15149
+ {
15150
+ orgId: input.orgId,
15151
+ runId: input.runId,
15152
+ eventId: `run-start-callback:${input.runId}`,
15153
+ eventType: 'run.started',
15154
+ payload: stableStringify(input.start),
15155
+ },
15156
+ options,
15157
+ );
15158
+ }
15159
+
15160
+ async function appendRunLedgerOutboxFact(
15161
+ client: PostgresSchedulerQueryClient,
15162
+ input: {
15163
+ orgId: string;
15164
+ runId: string;
15165
+ eventId: string;
15166
+ eventType: 'run.events' | 'run.started';
15167
+ payload: string;
15168
+ },
15169
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
15170
+ ): Promise<PostgresSchedulerLifecycleOutboxEvent> {
15171
+ const result = await client.query<{ run_id: string }>(
15172
+ `
15173
+ WITH admitted AS MATERIALIZED (
15174
+ SELECT run_id FROM ${tableName('runs', options)}
15175
+ WHERE run_id = $1 AND org_id = $2
15176
+ ), recorded AS (
15177
+ INSERT INTO ${tableName('outbox', options)}
15178
+ (event_id, run_id, event_type, payload_json, created_at)
15179
+ SELECT $3, run_id, $5, $4::jsonb,
15180
+ ${causalOutboxCreatedAt('admitted.run_id', options)}
15181
+ FROM admitted
15182
+ ON CONFLICT (event_id) DO NOTHING
15183
+ )
15184
+ SELECT run_id FROM admitted
15185
+ `,
15186
+ [input.runId, input.orgId, input.eventId, input.payload, input.eventType],
15187
+ );
15188
+ if (result.rows.length !== 1) {
15189
+ throw new Error(
15190
+ 'Run Ledger ingress run is missing or outside the organization scope.',
15191
+ );
15192
+ }
15193
+ return { runId: input.runId, eventId: input.eventId };
15194
+ }
15195
+
15196
+ export { RunLedgerProjectionLeaseLostError } from '../run-ledger-projection-contract';
15197
+
15198
+ /** Extend only our still-live lease before another bounded transport piece. */
15199
+ export async function renewPostgresRunLedgerDelivery(
15200
+ client: PostgresSchedulerQueryClient,
15201
+ input: { eventIds: string[]; leaseOwner: string; leaseSeconds: number },
15202
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
15203
+ ): Promise<void> {
15204
+ const result = await client.query<{ event_id: string }>(
15205
+ `
15206
+ UPDATE ${tableName('outbox', options)}
15207
+ SET lease_expires_at = now() + make_interval(secs => $3::integer)
15208
+ WHERE event_id = ANY($1::text[]) AND lease_owner = $2
15209
+ AND delivered_at IS NULL AND failed_at IS NULL AND lease_expires_at > now()
15210
+ RETURNING event_id
15211
+ `,
15212
+ [
15213
+ input.eventIds,
15214
+ input.leaseOwner,
15215
+ Math.max(5, Math.min(300, input.leaseSeconds)),
15216
+ ],
15217
+ );
15218
+ if (result.rows.length !== input.eventIds.length)
15219
+ throw new RunLedgerProjectionLeaseLostError();
15220
+ }
15221
+
15222
+ /** Only a currently claimed outbox batch may use the projection write path. */
15223
+ export async function assertPostgresRunLedgerDelivery(
15224
+ client: PostgresSchedulerQueryClient,
15225
+ input: {
15226
+ orgId: string;
15227
+ runId: string;
15228
+ idempotencyKey: string;
15229
+ leaseOwner?: string;
15230
+ },
15231
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
15232
+ ): Promise<void> {
15233
+ const result = await client.query<{
15234
+ delivery_key: string;
15235
+ event_type: string;
15236
+ lease_owner: string;
15237
+ }>(
15238
+ `
15239
+ SELECT outbox.delivery_key, outbox.event_type, outbox.lease_owner
15240
+ FROM ${tableName('outbox', options)} AS outbox
15241
+ JOIN ${tableName('runs', options)} AS runs ON runs.run_id = outbox.run_id
15242
+ WHERE outbox.run_id = $1 AND runs.org_id = $2
15243
+ AND outbox.delivered_at IS NULL AND outbox.failed_at IS NULL
15244
+ AND outbox.lease_owner IS NOT NULL AND outbox.lease_expires_at > now()
15245
+ AND ($3::text IS NULL OR outbox.lease_owner = $3)
15246
+ ORDER BY outbox.created_at, outbox.event_id
15247
+ `,
15248
+ [input.runId, input.orgId, input.leaseOwner ?? null],
15249
+ );
15250
+ const groups = new Map<string, typeof result.rows>();
15251
+ for (const row of result.rows) {
15252
+ const group = groups.get(row.lease_owner) ?? [];
15253
+ group.push(row);
15254
+ groups.set(row.lease_owner, group);
15255
+ }
15256
+ for (const rows of groups.values()) {
15257
+ const digest = createHash('sha256')
15258
+ .update(rows.map((row) => row.delivery_key).join('\n'))
15259
+ .digest('hex');
15260
+ const batchKey = buildRunLedgerEventsIdempotencyKey({
15261
+ runId: input.runId,
15262
+ deliveryDigest: digest,
15263
+ });
15264
+ const terminal = [...rows]
15265
+ .reverse()
15266
+ .find((row) =>
15267
+ ['run.completed', 'run.failed', 'run.cancelled'].includes(
15268
+ row.event_type,
15269
+ ),
15270
+ );
15271
+ // The old one-row dispatcher uses run-terminal for nonterminal facts too.
15272
+ const single = rows.length === 1 ? rows[0] : terminal;
15273
+ const terminalKey = single
15274
+ ? buildRunLedgerTerminalIdempotencyKey({
15275
+ runId: input.runId,
15276
+ deliveryKey: single.delivery_key,
15277
+ })
15278
+ : null;
15279
+ const startKey =
15280
+ rows.length === 1 &&
15281
+ single &&
15282
+ ['run.created', 'run.started'].includes(single.event_type)
15283
+ ? buildRunLedgerStartIdempotencyKey({
15284
+ runId: input.runId,
15285
+ deliveryKey: single.delivery_key,
15286
+ })
15287
+ : null;
15288
+ if (
15289
+ input.idempotencyKey === batchKey ||
15290
+ input.idempotencyKey === terminalKey ||
15291
+ input.idempotencyKey === startKey
15292
+ )
15293
+ return;
15294
+ }
15295
+ throw new RunLedgerProjectionLeaseLostError();
15296
+ }
15297
+
15298
+ /** Snapshot-creating facts precede progress, even when their intake was late. */
15299
+ function runLedgerPrecedesSql(
15300
+ predecessor: string,
15301
+ successor: string,
15302
+ inclusive = false,
15303
+ ): string {
15304
+ const position = (alias: string) =>
15305
+ `(CASE ${alias}.event_type WHEN 'run.created' THEN 0 WHEN 'run.started' THEN 1 ELSE 2 END, ${alias}.created_at, ${alias}.event_id)`;
15306
+ return `${position(predecessor)} ${inclusive ? '<=' : '<'} ${position(successor)}`;
15307
+ }
15308
+
15309
+ /**
15310
+ * Claim a causal prefix, preserving the v33 adapter while the additive batch
15311
+ * marker is absent. An enclosing transaction uses a savepoint so the missing
15312
+ * column does not leave the compatibility claim in an aborted transaction.
15104
15313
  */
15105
15314
  export async function readPendingPostgresSchedulerOutboxEvents(
15106
15315
  client: PostgresSchedulerQueryClient,
@@ -15109,6 +15318,8 @@ export async function readPendingPostgresSchedulerOutboxEvents(
15109
15318
  runId?: string | null;
15110
15319
  leaseOwner?: string | null;
15111
15320
  leaseSeconds?: number;
15321
+ /** The dispatcher owns a short, exclusive claim transaction. */
15322
+ inTransaction?: boolean;
15112
15323
  } = {},
15113
15324
  options?: Pick<PostgresSchedulerOptions, 'schema'>,
15114
15325
  ): Promise<PostgresSchedulerOutboxEvent[]> {
@@ -15137,6 +15348,16 @@ export async function readPendingPostgresSchedulerOutboxEvents(
15137
15348
  AND coalesce(candidate.next_attempt_at, candidate.created_at) <= now()
15138
15349
  AND ($2::text IS NULL OR candidate.run_id = $2)
15139
15350
  AND ($3::text IS NULL OR candidate.lease_expires_at IS NULL OR candidate.lease_expires_at <= now())
15351
+ -- A producer transaction may commit an older timestamp after a
15352
+ -- delivery has started. Ownership covers the run, not just the
15353
+ -- current oldest row: that late predecessor must wait as well.
15354
+ AND NOT EXISTS (
15355
+ SELECT 1 FROM ${tableName('outbox', options)} AS active
15356
+ WHERE active.run_id = candidate.run_id
15357
+ AND active.delivered_at IS NULL
15358
+ AND active.lease_owner IS NOT NULL
15359
+ AND active.lease_expires_at > now()
15360
+ )
15140
15361
  -- Competing projectors acquire one run through its oldest event.
15141
15362
  -- Locking this head prevents another projector from splitting the
15142
15363
  -- run while still letting it skip to unrelated runs.
@@ -15145,13 +15366,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
15145
15366
  FROM ${tableName('outbox', options)} AS predecessor
15146
15367
  WHERE predecessor.run_id = candidate.run_id
15147
15368
  AND predecessor.delivered_at IS NULL
15148
- AND (
15149
- predecessor.created_at < candidate.created_at
15150
- OR (
15151
- predecessor.created_at = candidate.created_at
15152
- AND predecessor.event_id < candidate.event_id
15153
- )
15154
- )
15369
+ AND ${runLedgerPrecedesSql('predecessor', 'candidate')}
15155
15370
  )
15156
15371
  -- A new durable admission has no causal dependency on unrelated
15157
15372
  -- terminal facts. Project it first so a lifecycle backlog cannot make
@@ -15173,9 +15388,12 @@ export async function readPendingPostgresSchedulerOutboxEvents(
15173
15388
  -- Admission is causally first and must be projected alone. A later
15174
15389
  -- start/terminal append may not leapfrog a delayed queued row.
15175
15390
  AND (
15176
- head.event_type NOT IN ('run.created', 'run.started')
15391
+ head.event_type NOT IN ('run.created', 'run.started', 'run.events')
15177
15392
  OR outbox.event_id = head.event_id
15178
15393
  )
15394
+ -- An ingress row already contains a producer's bounded batch. Keep
15395
+ -- its receipt stable independently of the recovery caller's limit.
15396
+ AND (outbox.event_type <> 'run.events' OR outbox.event_id = head.event_id)
15179
15397
  AND coalesce(outbox.next_attempt_at, outbox.created_at) <= now()
15180
15398
  AND ($3::text IS NULL OR outbox.lease_expires_at IS NULL OR outbox.lease_expires_at <= now())
15181
15399
  -- A started terminal batch is deliberately stable across a retry.
@@ -15202,13 +15420,7 @@ export async function readPendingPostgresSchedulerOutboxEvents(
15202
15420
  AND terminal_predecessor.event_type IN (
15203
15421
  'run.completed', 'run.failed', 'run.cancelled'
15204
15422
  )
15205
- AND (
15206
- terminal_predecessor.created_at < outbox.created_at
15207
- OR (
15208
- terminal_predecessor.created_at = outbox.created_at
15209
- AND terminal_predecessor.event_id < outbox.event_id
15210
- )
15211
- )
15423
+ AND ${runLedgerPrecedesSql('terminal_predecessor', 'outbox')}
15212
15424
  )
15213
15425
  -- Claim only the contiguous due prefix. A delayed, leased, or failed
15214
15426
  -- predecessor retains ownership of everything that follows it.
@@ -15217,15 +15429,10 @@ export async function readPendingPostgresSchedulerOutboxEvents(
15217
15429
  FROM ${tableName('outbox', options)} AS predecessor
15218
15430
  WHERE predecessor.run_id = outbox.run_id
15219
15431
  AND predecessor.delivered_at IS NULL
15432
+ AND ${runLedgerPrecedesSql('predecessor', 'outbox')}
15220
15433
  AND (
15221
- predecessor.created_at < outbox.created_at
15222
- OR (
15223
- predecessor.created_at = outbox.created_at
15224
- AND predecessor.event_id < outbox.event_id
15225
- )
15226
- )
15227
- AND (
15228
- predecessor.failed_at IS NOT NULL
15434
+ predecessor.event_type = 'run.events'
15435
+ OR predecessor.failed_at IS NOT NULL
15229
15436
  OR coalesce(predecessor.next_attempt_at, predecessor.created_at) > now()
15230
15437
  OR (
15231
15438
  $3::text IS NOT NULL
@@ -15275,17 +15482,22 @@ export async function readPendingPostgresSchedulerOutboxEvents(
15275
15482
  [limit, runId, leaseOwner, leaseSeconds],
15276
15483
  );
15277
15484
  let result;
15485
+ if (input.inTransaction) await client.query('SAVEPOINT run_ledger_claim');
15278
15486
  if (supportsOutboxDeliveryBatchMarker(options)) {
15279
15487
  try {
15280
15488
  result = await claim(true);
15281
15489
  } catch (error) {
15282
15490
  if (!isMissingOutboxDeliveryBatchMarker(error)) throw error;
15491
+ if (input.inTransaction)
15492
+ await client.query('ROLLBACK TO SAVEPOINT run_ledger_claim');
15283
15493
  markOutboxDeliveryBatchMarkerUnavailable(options);
15284
15494
  result = await claim(false);
15285
15495
  }
15286
15496
  } else {
15287
15497
  result = await claim(false);
15288
15498
  }
15499
+ if (input.inTransaction)
15500
+ await client.query('RELEASE SAVEPOINT run_ledger_claim');
15289
15501
  return result.rows.map((row) => {
15290
15502
  if (!row.delivery_key) {
15291
15503
  throw new Error(
@@ -15317,13 +15529,18 @@ export async function readPostgresSchedulerOutboxNextDueAt(
15317
15529
  ): Promise<Date | null> {
15318
15530
  const result = await client.query<{ due_at: Date | string | null }>(
15319
15531
  `
15320
- SELECT min(
15532
+ SELECT min(greatest(
15321
15533
  CASE
15322
15534
  WHEN lease_expires_at IS NOT NULL AND lease_expires_at > now()
15323
15535
  THEN lease_expires_at
15324
15536
  ELSE coalesce(next_attempt_at, created_at)
15325
- END
15326
- ) AS due_at
15537
+ END,
15538
+ (SELECT max(active.lease_expires_at)
15539
+ FROM ${tableName('outbox', options)} AS active
15540
+ WHERE active.run_id = candidate.run_id
15541
+ AND active.delivered_at IS NULL AND active.lease_owner IS NOT NULL
15542
+ AND active.lease_expires_at > now())
15543
+ )) AS due_at
15327
15544
  FROM ${tableName('outbox', options)} AS candidate
15328
15545
  WHERE candidate.delivered_at IS NULL
15329
15546
  AND candidate.failed_at IS NULL
@@ -15334,13 +15551,7 @@ export async function readPostgresSchedulerOutboxNextDueAt(
15334
15551
  FROM ${tableName('outbox', options)} AS predecessor
15335
15552
  WHERE predecessor.run_id = candidate.run_id
15336
15553
  AND predecessor.delivered_at IS NULL
15337
- AND (
15338
- predecessor.created_at < candidate.created_at
15339
- OR (
15340
- predecessor.created_at = candidate.created_at
15341
- AND predecessor.event_id < candidate.event_id
15342
- )
15343
- )
15554
+ AND ${runLedgerPrecedesSql('predecessor', 'candidate')}
15344
15555
  )
15345
15556
  `,
15346
15557
  );
@@ -15376,13 +15587,7 @@ export async function readPostgresSchedulerOutboxDeliveryStatus(
15376
15587
  FROM ${tableName('outbox', options)} predecessor
15377
15588
  WHERE predecessor.run_id = target.run_id
15378
15589
  AND predecessor.failed_at IS NOT NULL
15379
- AND (
15380
- predecessor.created_at < target.created_at
15381
- OR (
15382
- predecessor.created_at = target.created_at
15383
- AND predecessor.event_id <= target.event_id
15384
- )
15385
- )
15590
+ AND ${runLedgerPrecedesSql('predecessor', 'target', true)}
15386
15591
  ) AS blocked
15387
15592
  FROM ${tableName('outbox', options)} target
15388
15593
  WHERE target.run_id = $1 AND target.event_id = $2`,
@@ -15783,6 +15988,30 @@ export function buildRunLedgerEventsFromPostgresSchedulerOutbox(
15783
15988
  event: PostgresSchedulerOutboxEvent,
15784
15989
  ): PlayRunLedgerIngressEvent[] {
15785
15990
  const occurredAt = event.createdAt.getTime();
15991
+ if (event.eventType === 'run.events') {
15992
+ const events = payloadField(event.payload, 'events');
15993
+ if (
15994
+ !Array.isArray(events) ||
15995
+ events.length === 0 ||
15996
+ events.some(
15997
+ (value) =>
15998
+ !recordPayload(value) ||
15999
+ value.runId !== event.runId ||
16000
+ typeof value.type !== 'string',
16001
+ )
16002
+ ) {
16003
+ throw new AppRuntimeApiResponseError({
16004
+ action: 'project_run_events',
16005
+ status: 422,
16006
+ code: 'run_ledger_ingress_invalid',
16007
+ retryable: false,
16008
+ detail: `Invalid Run Ledger ingress batch ${event.eventId}.`,
16009
+ });
16010
+ }
16011
+ // Preserve event time, source, opaque payloads and positional log identity;
16012
+ // the outbox timestamp orders delivery, not the customer's execution trace.
16013
+ return events as AcceptedPlayRunLedgerEvent[];
16014
+ }
15786
16015
  if (event.eventType === 'run.started') {
15787
16016
  return [
15788
16017
  {
@@ -15964,6 +16193,7 @@ export type PostgresSchedulerOutboxMaterializationPlan = {
15964
16193
  };
15965
16194
 
15966
16195
  const RUN_LEDGER_KNOWN_OUTBOX_EVENT_TYPES = new Set([
16196
+ 'run.events',
15967
16197
  'run.started',
15968
16198
  'run.completed',
15969
16199
  'run.failed',
@@ -16272,18 +16502,58 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
16272
16502
  ): Promise<PostgresSchedulerOutboxProjectionResult> {
16273
16503
  const leaseOwner =
16274
16504
  input.leaseOwner?.trim() || `run-ledger-outbox-dispatch:${randomUUID()}`;
16275
- const events = await withClient((client) =>
16276
- readPendingPostgresSchedulerOutboxEvents(
16277
- client,
16278
- {
16279
- limit: input.limit ?? 1,
16280
- runId: input.runId,
16281
- leaseOwner,
16282
- leaseSeconds: input.leaseSeconds ?? 120,
16283
- },
16284
- options,
16285
- ),
16286
- );
16505
+ const events = await withClient(async (client) => {
16506
+ // The checkout is exclusive. Hold the existing scheduler run's row lock
16507
+ // only while choosing/claiming its outbox prefix, never over HTTP. A
16508
+ // second statement takes a fresh READ COMMITTED snapshot after the lock:
16509
+ // late producer commits cannot split one run between different heads.
16510
+ await client.query('BEGIN');
16511
+ try {
16512
+ const selected = await client.query<{ run_id: string }>(
16513
+ `
16514
+ SELECT runs.run_id
16515
+ FROM ${tableName('outbox', options)} AS head
16516
+ JOIN ${tableName('runs', options)} AS runs ON runs.run_id = head.run_id
16517
+ WHERE ($1::text IS NULL OR runs.run_id = $1)
16518
+ AND head.delivered_at IS NULL AND head.failed_at IS NULL
16519
+ AND coalesce(head.next_attempt_at, head.created_at) <= now()
16520
+ AND NOT EXISTS (
16521
+ SELECT 1 FROM ${tableName('outbox', options)} AS predecessor
16522
+ WHERE predecessor.run_id = head.run_id AND predecessor.delivered_at IS NULL
16523
+ AND ${runLedgerPrecedesSql('predecessor', 'head')}
16524
+ )
16525
+ AND NOT EXISTS (
16526
+ SELECT 1 FROM ${tableName('outbox', options)} AS active
16527
+ WHERE active.run_id = runs.run_id AND active.delivered_at IS NULL
16528
+ AND active.lease_owner IS NOT NULL AND active.lease_expires_at > now()
16529
+ )
16530
+ ORDER BY CASE WHEN head.event_type = 'run.created' THEN 0 ELSE 1 END,
16531
+ head.created_at, runs.run_id
16532
+ FOR UPDATE OF runs SKIP LOCKED LIMIT 1
16533
+ `,
16534
+ [input.runId?.trim() || null],
16535
+ );
16536
+ const runId = selected.rows[0]?.run_id;
16537
+ const claimed = runId
16538
+ ? await readPendingPostgresSchedulerOutboxEvents(
16539
+ client,
16540
+ {
16541
+ limit: input.limit ?? 1,
16542
+ runId,
16543
+ leaseOwner,
16544
+ leaseSeconds: input.leaseSeconds ?? 120,
16545
+ inTransaction: true,
16546
+ },
16547
+ options,
16548
+ )
16549
+ : [];
16550
+ await client.query('COMMIT');
16551
+ return claimed;
16552
+ } catch (error) {
16553
+ await client.query('ROLLBACK');
16554
+ throw error;
16555
+ }
16556
+ });
16287
16557
  if (events.length === 0) {
16288
16558
  return {
16289
16559
  attempted: 0,
@@ -16298,7 +16568,7 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
16298
16568
 
16299
16569
  try {
16300
16570
  const first = events[0]!;
16301
- const runtimeApi =
16571
+ const resolvedRuntimeApi =
16302
16572
  input.runtimeApi ??
16303
16573
  (await withClient((client) =>
16304
16574
  loadPostgresSchedulerRuntimeApiForRun(
@@ -16311,6 +16581,20 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
16311
16581
  options,
16312
16582
  ),
16313
16583
  ));
16584
+ const runtimeApi = { ...resolvedRuntimeApi, retryPolicy: 'none' as const };
16585
+ const renewDelivery = () =>
16586
+ withClient((client) =>
16587
+ renewPostgresRunLedgerDelivery(
16588
+ client,
16589
+ {
16590
+ eventIds: events.map((event) => event.eventId),
16591
+ leaseOwner,
16592
+ leaseSeconds: input.leaseSeconds ?? 120,
16593
+ },
16594
+ options,
16595
+ ),
16596
+ );
16597
+ await renewDelivery();
16314
16598
  // A pending start is always claimed alone because later events require its
16315
16599
  // snapshot. It cannot share the append endpoint, so retain that mutation
16316
16600
  // as its own exact delivery.
@@ -16347,9 +16631,11 @@ export async function dispatchRunLedgerOutboxToLedgerBatchOnceWithClientScope(
16347
16631
  const batchDeliveryKey = createHash('sha256')
16348
16632
  .update(events.map((event) => event.deliveryKey).join('\n'))
16349
16633
  .digest('hex');
16350
- await appendRunEventsViaAppRuntime(runtimeApi, {
16634
+ await projectRunEventsViaAppRuntime(runtimeApi, {
16351
16635
  playId: first.runId,
16352
16636
  events: ledgerEvents,
16637
+ deliveryLeaseOwner: leaseOwner,
16638
+ beforeBatch: renewDelivery,
16353
16639
  idempotencyKey: terminal
16354
16640
  ? buildRunLedgerTerminalIdempotencyKey({
16355
16641
  runId: first.runId,
@@ -16431,6 +16717,7 @@ async function projectPostgresSchedulerCreatedEvent(
16431
16717
  const replayedFromRunId = value('replayedFromRunId');
16432
16718
  const asyncAncestryPlayIds = raw('asyncAncestryPlayIds');
16433
16719
  await projectRunCreatedViaAppRuntime(runtimeApi, {
16720
+ deliveryLeaseOwner: event.leaseOwner ?? undefined,
16434
16721
  idempotencyKey: buildRunLedgerStartIdempotencyKey({
16435
16722
  runId: event.runId,
16436
16723
  deliveryKey: event.deliveryKey,
@@ -16504,11 +16791,15 @@ async function projectPostgresSchedulerStartEvent(
16504
16791
  launch: (PlaySchedulerSubmitInput & { firstClaimLatencyMs?: number }) | null,
16505
16792
  ): Promise<void> {
16506
16793
  await startRunViaAppRuntime(runtimeApi, {
16794
+ deliveryLeaseOwner: event.leaseOwner ?? undefined,
16507
16795
  idempotencyKey: buildRunLedgerStartIdempotencyKey({
16508
16796
  runId: event.runId,
16509
16797
  deliveryKey: event.deliveryKey,
16510
16798
  }),
16511
16799
  playName: stringPayloadField(event.payload, 'playName') ?? event.runId,
16800
+ playReference: stringPayloadField(event.payload, 'playReference'),
16801
+ definitionScope: launch?.definitionScope ?? undefined,
16802
+ revisionId: launch?.revisionId,
16512
16803
  runId: event.runId,
16513
16804
  artifactStorageKey: stringPayloadField(event.payload, 'artifactStorageKey'),
16514
16805
  artifactHash: stringPayloadField(event.payload, 'artifactHash'),
@@ -16517,6 +16808,7 @@ async function projectPostgresSchedulerStartEvent(
16517
16808
  schedulerBackend: stringPayloadField(event.payload, 'schedulerBackend'),
16518
16809
  schedulerSchema: stringPayloadField(event.payload, 'schedulerSchema'),
16519
16810
  executionProfile: stringPayloadField(event.payload, 'executionProfile'),
16811
+ maxCreditsPerRun: numberPayloadField(event.payload, 'maxCreditsPerRun'),
16520
16812
  staticPipeline: payloadField(event.payload, 'staticPipeline'),
16521
16813
  source: sourcePayloadField(event.payload, 'source'),
16522
16814
  inputFileId:
@@ -16534,7 +16826,10 @@ async function projectPostgresSchedulerStartEvent(
16534
16826
  replayedFromRunId:
16535
16827
  launch?.replayedFromRunId ??
16536
16828
  stringPayloadField(event.payload, 'replayedFromRunId'),
16537
- firstClaimLatencyMs: launch?.firstClaimLatencyMs,
16829
+ firstClaimLatencyMs:
16830
+ launch?.firstClaimLatencyMs ??
16831
+ numberPayloadField(event.payload, 'firstClaimLatencyMs') ??
16832
+ undefined,
16538
16833
  });
16539
16834
  }
16540
16835
 
@@ -17063,7 +17358,7 @@ export function postgresSchedulerRunHandle(
17063
17358
  hubLease?.release();
17064
17359
  }
17065
17360
  },
17066
- cancel: async () => {
17361
+ cancel: async (reason) => {
17067
17362
  const poolOptions = resolve();
17068
17363
  await withRuntimeSchedulerTransaction(
17069
17364
  poolOptions,
@@ -17071,7 +17366,8 @@ export function postgresSchedulerRunHandle(
17071
17366
  lockTimeoutMs: RUNTIME_SCHEDULER_CANCEL_LOCK_TIMEOUT_MS,
17072
17367
  statementTimeoutMs: RUNTIME_SCHEDULER_CANCEL_STATEMENT_TIMEOUT_MS,
17073
17368
  },
17074
- (client) => cancelPostgresSchedulerRun(client, { runId }, options),
17369
+ (client) =>
17370
+ cancelPostgresSchedulerRun(client, { runId, reason }, options),
17075
17371
  {
17076
17372
  runId,
17077
17373
  phase: 'cancel_run',
package/dist/cli/index.js CHANGED
@@ -3068,7 +3068,7 @@ var SDK_RELEASE = {
3068
3068
  // getters keep their established compatibility behavior.
3069
3069
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
3070
3070
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
3071
- version: "0.3.150",
3071
+ version: "0.3.152",
3072
3072
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
3073
3073
  packageCapabilities: {
3074
3074
  updatePreferences: 1
@@ -3063,7 +3063,7 @@ var SDK_RELEASE = {
3063
3063
  // getters keep their established compatibility behavior.
3064
3064
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
3065
3065
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
3066
- version: "0.3.150",
3066
+ version: "0.3.152",
3067
3067
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
3068
3068
  packageCapabilities: {
3069
3069
  updatePreferences: 1
package/dist/index.js CHANGED
@@ -864,7 +864,7 @@ var SDK_RELEASE = {
864
864
  // getters keep their established compatibility behavior.
865
865
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
866
866
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
867
- version: "0.3.150",
867
+ version: "0.3.152",
868
868
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
869
869
  packageCapabilities: {
870
870
  updatePreferences: 1
package/dist/index.mjs CHANGED
@@ -768,7 +768,7 @@ var SDK_RELEASE = {
768
768
  // getters keep their established compatibility behavior.
769
769
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
770
770
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
771
- version: "0.3.150",
771
+ version: "0.3.152",
772
772
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
773
773
  packageCapabilities: {
774
774
  updatePreferences: 1
@@ -149,7 +149,7 @@ type SdkRelease = {
149
149
  supportPolicy: SdkSupportPolicy;
150
150
  };
151
151
  declare const SDK_RELEASE: {
152
- readonly version: "0.3.150";
152
+ readonly version: "0.3.152";
153
153
  readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
154
154
  readonly packageCapabilities: {
155
155
  readonly updatePreferences: 1;
package/dist/release.d.ts CHANGED
@@ -149,7 +149,7 @@ type SdkRelease = {
149
149
  supportPolicy: SdkSupportPolicy;
150
150
  };
151
151
  declare const SDK_RELEASE: {
152
- readonly version: "0.3.150";
152
+ readonly version: "0.3.152";
153
153
  readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
154
154
  readonly packageCapabilities: {
155
155
  readonly updatePreferences: 1;
package/dist/release.js CHANGED
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
74
74
  // getters keep their established compatibility behavior.
75
75
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
76
76
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
77
- version: "0.3.150",
77
+ version: "0.3.152",
78
78
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
79
79
  packageCapabilities: {
80
80
  updatePreferences: 1
package/dist/release.mjs CHANGED
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
48
48
  // getters keep their established compatibility behavior.
49
49
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
50
50
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
51
- version: "0.3.150",
51
+ version: "0.3.152",
52
52
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
53
53
  packageCapabilities: {
54
54
  updatePreferences: 1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "deepline",
3
- "version": "0.3.150",
3
+ "version": "0.3.152",
4
4
  "description": "GTM data CLI and TypeScript SDK for coding agents",
5
5
  "license": "MIT",
6
6
  "homepage": "https://code.deepline.com",