deepline 0.3.243 → 0.3.245

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (27) hide show
  1. package/dist/bundling-sources/sdk/src/release.ts +1 -1
  2. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +7 -1
  3. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +4 -2
  4. package/dist/bundling-sources/shared_libs/play-runtime/launch-identity.ts +1 -0
  5. package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +43 -1
  6. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/index.ts +2 -0
  7. package/dist/bundling-sources/shared_libs/play-runtime/runner-app/runtime-terminal-push.ts +15 -8
  8. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +22 -12
  9. package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/local-process.ts +8 -1
  10. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +13 -2
  11. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +2 -3
  12. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres-pool.ts +14 -1
  13. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/postgres.ts +114 -13
  14. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backends/receipt-gateway.ts +4 -1
  15. package/dist/bundling-sources/shared_libs/plays/execution-pipeline.ts +37 -11
  16. package/dist/bundling-sources/shared_libs/plays/retained-pipeline-graph.ts +291 -0
  17. package/dist/bundling-sources/shared_libs/plays/static-pipeline-storage-codec.ts +53 -1
  18. package/dist/cli/index.js +1 -1
  19. package/dist/cli/index.mjs +1 -1
  20. package/dist/index.js +1 -1
  21. package/dist/index.mjs +1 -1
  22. package/dist/install-integrity.json +1 -0
  23. package/dist/release.d.mts +1 -1
  24. package/dist/release.d.ts +1 -1
  25. package/dist/release.js +1 -1
  26. package/dist/release.mjs +1 -1
  27. package/package.json +1 -1
@@ -200,7 +200,7 @@ export const SDK_RELEASE = {
200
200
  // getters keep their established compatibility behavior.
201
201
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
202
202
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
203
- version: '0.3.243',
203
+ version: '0.3.245',
204
204
  updateSummary:
205
205
  'Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.',
206
206
  packageCapabilities: {
@@ -1,3 +1,4 @@
1
+ import { prepareRetainedPlayPipelinesForTransport } from '../plays/execution-pipeline';
1
2
  import { createWriteStream } from 'node:fs';
2
3
  import { createHash, randomUUID } from 'node:crypto';
3
4
  import { stat } from 'node:fs/promises';
@@ -206,6 +207,7 @@ type RuntimeApiRequest =
206
207
  executionProfile?: string | null;
207
208
  maxCreditsPerRun?: number | null;
208
209
  staticPipeline?: unknown;
210
+ retainedPipelineGraph?: unknown;
209
211
  source?: 'published' | 'ad_hoc' | 'draft';
210
212
  inputFileId?: string;
211
213
  inputBytes?: number;
@@ -262,6 +264,7 @@ type RuntimeApiRequest =
262
264
  runtimeHarnessWorkerName?: string | null;
263
265
  maxCreditsPerRun?: number | null;
264
266
  staticPipeline?: unknown;
267
+ retainedPipelineGraph?: unknown;
265
268
  source?: 'published' | 'ad_hoc' | 'draft';
266
269
  triggerSource?: 'webhook' | 'cron' | 'sql_listener' | 'api';
267
270
  inputFileId?: string;
@@ -1448,7 +1451,9 @@ async function postAppRuntimeApi<TResponse>(
1448
1451
  // attempts (that identity is what makes a receipt retry safe), so a second
1449
1452
  // `JSON.stringify` of a megabyte payload would be pure cost.
1450
1453
  const serializationStartedAt = Date.now();
1451
- const serializedBody = JSON.stringify(body);
1454
+ const serializedBody = JSON.stringify(
1455
+ prepareRetainedPlayPipelinesForTransport(body),
1456
+ );
1452
1457
  const requestBodySerializeMs = Math.max(
1453
1458
  0,
1454
1459
  Date.now() - serializationStartedAt,
@@ -2503,6 +2508,7 @@ export async function startRunViaAppRuntime(
2503
2508
  executionProfile?: string | null;
2504
2509
  maxCreditsPerRun?: number | null;
2505
2510
  staticPipeline?: unknown;
2511
+ retainedPipelineGraph?: unknown;
2506
2512
  source?: 'published' | 'ad_hoc' | 'draft';
2507
2513
  inputFileId?: string;
2508
2514
  inputBytes?: number;
@@ -14173,11 +14173,13 @@ export class PlayContextImpl implements ScalarPlayAuthoringRuntimeContext {
14173
14173
  });
14174
14174
  return result as TOutput;
14175
14175
  } catch (childError) {
14176
- this.recordInlineChildFailure(resolvedName, childError);
14176
+ if (!isControlledPlaySuspension(childError)) {
14177
+ this.recordInlineChildFailure(resolvedName, childError);
14178
+ }
14177
14179
  throw childError;
14178
14180
  }
14179
14181
  } catch (error) {
14180
- if (isPlayExecutionSuspendedError(error)) {
14182
+ if (isControlledPlaySuspension(error)) {
14181
14183
  throw error;
14182
14184
  }
14183
14185
  if (rowStore) {
@@ -55,6 +55,7 @@ export const PLAY_LAUNCH_ENVELOPE_FIELD_ROLES: {
55
55
  runtimeArtifact: 'detail',
56
56
  runtimeProtocolVersion: 'detail',
57
57
  staticPipeline: 'detail',
58
+ retainedPipelineGraph: 'detail',
58
59
  inputSummary: 'detail',
59
60
  inputBytes: 'detail',
60
61
  inputFileId: 'detail',
@@ -1,3 +1,9 @@
1
+ import { deployed, readDeployedObject } from '@deepline/deployed-contracts';
2
+ import {
3
+ type RetainedPipelineCarrier,
4
+ hydrateRetainedPlayPipelines,
5
+ prepareRetainedPlayPipelinesForTransport,
6
+ } from '../plays/execution-pipeline';
1
7
  import type { PlayBundleArtifact } from '../plays/artifact-types';
2
8
  import type {
3
9
  PlayCheckpoint,
@@ -177,7 +183,7 @@ export type PlayRunnerBudgetChargeResult = {
177
183
  counters: Record<string, number>;
178
184
  };
179
185
 
180
- export interface PlayRunnerContextConfig {
186
+ export interface PlayRunnerContextConfig extends RetainedPipelineCarrier {
181
187
  executorToken?: string;
182
188
  baseUrl?: string;
183
189
  /**
@@ -447,3 +453,39 @@ export type PlayRunnerRowOutcomeSummary = {
447
453
  /** Rows omitted because a newer Runtime Sheet write took precedence. */
448
454
  supersededRows?: number;
449
455
  };
456
+
457
+ export const RETAINED_PLAY_PIPELINE_CONTRACT = deployed.object({
458
+ id: 'retained-play-pipeline',
459
+ unknown: 'preserve',
460
+ fields: {
461
+ retainedPipelineGraph: deployed.optional(deployed.record(), {
462
+ whenAbsent:
463
+ 'read the historical inline staticPipeline or stored staticPipelineV2',
464
+ }),
465
+ staticPipeline: deployed.optional(deployed.json(), {
466
+ whenAbsent:
467
+ 'read the named retained pipeline root, otherwise no pipeline',
468
+ }),
469
+ },
470
+ });
471
+
472
+ /** Runner envelope ownership stays here; the shared codec owns only contracts. */
473
+ export function preparePlayRunnerConfigForTransport(
474
+ config: PlayRunnerExecutionConfig,
475
+ ): PlayRunnerExecutionConfig {
476
+ return {
477
+ ...config,
478
+ context: prepareRetainedPlayPipelinesForTransport(config.context),
479
+ };
480
+ }
481
+
482
+ export function hydratePlayRunnerConfig(
483
+ config: PlayRunnerExecutionConfig,
484
+ ): PlayRunnerExecutionConfig {
485
+ return {
486
+ ...config,
487
+ context: hydrateRetainedPlayPipelines(
488
+ readDeployedObject(RETAINED_PLAY_PIPELINE_CONTRACT, config.context).value,
489
+ ) as unknown as PlayRunnerContextConfig,
490
+ };
491
+ }
@@ -1,3 +1,4 @@
1
+ import { hydratePlayRunnerConfig } from '../protocol';
1
2
  import { appendFileSync, createReadStream, writeFileSync } from 'node:fs';
2
3
  import { readFile } from 'node:fs/promises';
3
4
  import { dirname, isAbsolute, resolve } from 'node:path';
@@ -1908,6 +1909,7 @@ async function run(
1908
1909
  config: PlayRunnerExecutionConfig,
1909
1910
  options?: ExecutePlayRunnerOptions,
1910
1911
  ): Promise<PlayRunnerResult> {
1912
+ config = hydratePlayRunnerConfig(config);
1911
1913
  const runStartedAt = Date.now();
1912
1914
  const runnerRunAttempt =
1913
1915
  typeof config.context.runAttempt === 'number' &&
@@ -10,9 +10,7 @@ import {
10
10
  jsonByteLength,
11
11
  jsonByteLengthUpTo,
12
12
  } from '../output-size-limits';
13
- import {
14
- PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER,
15
- } from '../runtime-contract';
13
+ import { PLAY_RUNTIME_TRANSPORT_ATTEMPT_HEADER } from '../runtime-contract';
16
14
  import {
17
15
  createRunnerGatewayTransportAttemptId,
18
16
  runnerGatewayTransportErrorCode,
@@ -595,8 +593,14 @@ export async function pushRunnerTerminal(
595
593
  : 'redirect',
596
594
  { status: response.status },
597
595
  );
598
- // 4xx (scope/fence rejections) are deterministic — retrying cannot help.
599
- if (response.status >= 400 && response.status < 500) {
596
+ // Gateway queue admission can shed this first-write-wins terminal with
597
+ // 429 before acquiring a database client. Retry the same terminal fact
598
+ // within the existing attempt budget; scope/fence rejections stay final.
599
+ if (
600
+ response.status >= 400 &&
601
+ response.status < 500 &&
602
+ response.status !== 429
603
+ ) {
600
604
  warn('terminal_push_rejected', {
601
605
  runId: options.runId,
602
606
  status: response.status,
@@ -606,9 +610,12 @@ export async function pushRunnerTerminal(
606
610
  }
607
611
  } catch (error) {
608
612
  lastError = error instanceof Error ? error.message : String(error);
609
- reportTransport('network_error', {
610
- errorCode: runnerGatewayTransportErrorCode(error),
611
- });
613
+ reportTransport(
614
+ requestController.signal.aborted ? 'timeout' : 'network_error',
615
+ {
616
+ errorCode: runnerGatewayTransportErrorCode(error),
617
+ },
618
+ );
612
619
  } finally {
613
620
  clearTimeout(requestTimeout);
614
621
  }
@@ -1,3 +1,4 @@
1
+ import { preparePlayRunnerConfigForTransport } from '../../protocol';
1
2
  import { randomUUID } from 'node:crypto';
2
3
  import { readFile } from 'node:fs/promises';
3
4
  import { gzipSync } from 'node:zlib';
@@ -487,10 +488,15 @@ async function main() {
487
488
  checkpointBytes: 0,
488
489
  tableNamespace: null,
489
490
  });
491
+ const requestBody = JSON.stringify({
492
+ action: 'runner_terminal',
493
+ runId: push.runId,
494
+ result,
495
+ });
490
496
  for (let attempt = 1; attempt <= 3; attempt += 1) {
497
+ const controller = new AbortController();
498
+ const timer = setTimeout(() => controller.abort(), 10000);
491
499
  try {
492
- const controller = new AbortController();
493
- const timer = setTimeout(() => controller.abort(), 10000);
494
500
  const response = await fetch(gateway + '/api/v2/plays/runtime', {
495
501
  method: 'POST',
496
502
  headers: {
@@ -500,28 +506,32 @@ async function main() {
500
506
  ? { 'fly-force-instance-id': preferredGatewayMachineId }
501
507
  : {}),
502
508
  },
503
- body: JSON.stringify({
504
- action: 'runner_terminal',
505
- runId: push.runId,
506
- result,
507
- }),
509
+ body: requestBody,
508
510
  signal: controller.signal,
509
511
  redirect: 'manual',
510
512
  });
511
- clearTimeout(timer);
513
+ // Keep the HTTP deadline through body consumption so retries do not
514
+ // leave an unread socket behind. The status already acknowledges the
515
+ // durable terminal even if its diagnostic response body is interrupted.
516
+ await response.text().catch(() => '');
512
517
  if (response.ok) {
513
518
  writeDiagnostic('[crash-terminal] pushed status=' + result.status);
514
519
  return;
515
520
  }
516
521
  writeDiagnostic('[crash-terminal] rejected http=' + response.status);
517
- // Deterministic scope/fence rejections cannot succeed on retry.
518
- if (response.status >= 400 && response.status < 500) return;
522
+ // A 429 is gateway admission pressure, not a scope/fence rejection.
523
+ // Replaying this terminal fact never replays customer execution.
524
+ if (response.status >= 400 && response.status < 500 && response.status !== 429) return;
519
525
  } catch (error) {
520
526
  writeDiagnostic(
521
527
  '[crash-terminal] transport ' + (error && error.message ? error.message : String(error)),
522
528
  );
529
+ } finally {
530
+ clearTimeout(timer);
531
+ }
532
+ if (attempt < 3) {
533
+ await new Promise((resolveDelay) => setTimeout(resolveDelay, attempt * 1000));
523
534
  }
524
- await new Promise((resolveDelay) => setTimeout(resolveDelay, attempt * 1000));
525
535
  }
526
536
  writeDiagnostic('[crash-terminal] exhausted retries');
527
537
  }
@@ -695,7 +705,7 @@ export async function stageRunnerPayload(input: {
695
705
  runnerCode: bundle,
696
706
  artifactBundledCode: compactedArtifact.bundledCode,
697
707
  artifactSourceMap: input.config.artifact.sourceMap,
698
- config: remoteConfig,
708
+ config: preparePlayRunnerConfigForTransport(remoteConfig),
699
709
  crashPusherCode:
700
710
  buildDaytonaCrashTerminalPusherSource(runtimeLimitSeconds),
701
711
  }),
@@ -1,3 +1,4 @@
1
+ import { preparePlayRunnerConfigForTransport } from '../../protocol';
1
2
  import { randomUUID } from 'node:crypto';
2
3
  import { spawn } from 'node:child_process';
3
4
  import { cp, mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
@@ -407,7 +408,13 @@ export const localProcessPlayRunnerBackend: PlayRunnerBackend = {
407
408
  },
408
409
  async () => {
409
410
  await writeFile(runnerPath, bundle, 'utf-8');
410
- await writeFile(configPath, JSON.stringify(localConfig), 'utf-8');
411
+ await writeFile(
412
+ configPath,
413
+ JSON.stringify(
414
+ preparePlayRunnerConfigForTransport(localConfig),
415
+ ),
416
+ 'utf-8',
417
+ );
411
418
  },
412
419
  );
413
420
  logLocalRunnerPerf(
@@ -1,3 +1,7 @@
1
+ import {
2
+ prepareRetainedPlayPipelinesForTransport,
3
+ hydrateRetainedPlayPipelines,
4
+ } from '../plays/execution-pipeline';
1
5
  import { createHash, randomUUID } from 'node:crypto';
2
6
  import {
3
7
  createRuntimePool,
@@ -514,6 +518,7 @@ type RuntimeApiActionRequest =
514
518
  | {
515
519
  action: 'resolve_play';
516
520
  playRef: string;
521
+ acceptRetainedPipelineGraph?: boolean;
517
522
  }
518
523
  | {
519
524
  action: 'ensure_sheet';
@@ -784,7 +789,7 @@ async function postRuntimeApi<TResponse>(
784
789
  response = await fetch(url, {
785
790
  method: 'POST',
786
791
  headers: resolveRuntimeApiHeaders(context, vercelHeaders),
787
- body: JSON.stringify(body),
792
+ body: JSON.stringify(prepareRetainedPlayPipelinesForTransport(body)),
788
793
  signal: context.abortSignal
789
794
  ? AbortSignal.any([context.abortSignal, abortController.signal])
790
795
  : abortController.signal,
@@ -3579,13 +3584,19 @@ export async function resolveRuntimeReferencedPlay(
3579
3584
  context: RuntimeApiContext,
3580
3585
  playRef: string,
3581
3586
  ): Promise<ResolvedRuntimePlay | null> {
3582
- const response = await postRuntimeApi<{
3587
+ const wire = await postRuntimeApi<{
3583
3588
  play: ResolvedRuntimePlay | null;
3584
3589
  manifest?: { staticPipeline?: PlayStaticPipeline | null } | null;
3585
3590
  }>(context, {
3586
3591
  action: 'resolve_play',
3587
3592
  playRef,
3593
+ acceptRetainedPipelineGraph: true,
3588
3594
  });
3595
+ const response = {
3596
+ ...wire,
3597
+ play: hydrateRetainedPlayPipelines(wire.play),
3598
+ manifest: hydrateRetainedPlayPipelines(wire.manifest),
3599
+ };
3589
3600
  if (!response.play) {
3590
3601
  return null;
3591
3602
  }
@@ -1,3 +1,4 @@
1
+ import type { RetainedPipelineCarrier } from '../plays/execution-pipeline';
1
2
  /**
2
3
  * Scheduler-backend interface — owns play workflow lifecycle.
3
4
  *
@@ -39,7 +40,7 @@ export type PlaySchedulerBackendId =
39
40
 
40
41
  export type { PlayRunInputPayload };
41
42
 
42
- export type PlaySchedulerSubmitInput = {
43
+ export type PlaySchedulerSubmitInput = RetainedPipelineCarrier & {
43
44
  runId: string;
44
45
  /**
45
46
  * Admit an exact replay only. This is an ephemeral admission instruction:
@@ -92,8 +93,6 @@ export type PlaySchedulerSubmitInput = {
92
93
  /** Immutable scheduler/worker/runner ABI snapshot (not a Git SHA). */
93
94
  runtimeProtocolVersion?: PlayRuntimeProtocolVersion | null;
94
95
  input: PlayRunInputPayload;
95
- /** Bounded launch metadata used only by the asynchronous ledger projector. */
96
- staticPipeline?: unknown;
97
96
  inputSummary?: Record<string, unknown>;
98
97
  /** Convex metadata for the exact input saved before scheduler submission. */
99
98
  inputFileId?: string;
@@ -937,6 +937,7 @@ export async function withRuntimeSchedulerClient<T>(
937
937
  throw error;
938
938
  }
939
939
  const admissionWaitMs = performance.now() - admissionStartedAt;
940
+ let unsampledAdmissionWaitMs = admissionWaitMs;
940
941
  let lastError: unknown = null;
941
942
  try {
942
943
  throwIfSchedulerAborted(signal);
@@ -996,7 +997,14 @@ export async function withRuntimeSchedulerClient<T>(
996
997
  entry.stats.peakTotalClients,
997
998
  entry.pool.totalCount,
998
999
  );
999
- pushSample(entry.stats.acquireWait, acquireWaitMs);
1000
+ // The admission gate owns the local queue before pg.Pool sees this
1001
+ // checkout. Include that wait once, on the operation's first acquired
1002
+ // client; retries retain the permit and only add their own connect wait.
1003
+ pushSample(
1004
+ entry.stats.acquireWait,
1005
+ unsampledAdmissionWaitMs + acquireWaitMs,
1006
+ );
1007
+ unsampledAdmissionWaitMs = 0;
1000
1008
 
1001
1009
  // A checked-out pg client emits 'error' (not via the pool) when its TCP
1002
1010
  // connection dies while idle — e.g. a worker holding a client across a
@@ -1255,6 +1263,7 @@ export type RuntimeSchedulerPoolStats = {
1255
1263
  idleCount: number;
1256
1264
  waitingCount: number;
1257
1265
  acquires: number;
1266
+ /** Successful checkout latency, including local admission once per operation. */
1258
1267
  acquireWaitP50Ms: number | null;
1259
1268
  acquireWaitP95Ms: number | null;
1260
1269
  saturationRejections: number;
@@ -1388,6 +1397,8 @@ export async function probeRuntimeSchedulerDirectSession(input: {
1388
1397
  connectionString: deriveDirectDatabaseUrl(input.url),
1389
1398
  application_name:
1390
1399
  input.applicationName ?? 'deepline-runtime-direct-session-preflight',
1400
+ connectionTimeoutMillis: RUNTIME_SCHEDULER_POOL_DEFAULTS.connectTimeoutMs,
1401
+ query_timeout: RUNTIME_SCHEDULER_POOL_DEFAULTS.queryTimeoutMs,
1391
1402
  });
1392
1403
  try {
1393
1404
  await client.connect();
@@ -1648,6 +1659,8 @@ export function createPgNotifyHub(input: {
1648
1659
  const next = (input.createClient ?? ((config) => new Client(config)))({
1649
1660
  connectionString: directUrl,
1650
1661
  application_name: input.applicationName ?? 'deepline-runtime-notify-hub',
1662
+ connectionTimeoutMillis: RUNTIME_SCHEDULER_POOL_DEFAULTS.connectTimeoutMs,
1663
+ query_timeout: RUNTIME_SCHEDULER_POOL_DEFAULTS.queryTimeoutMs,
1651
1664
  });
1652
1665
  try {
1653
1666
  await next.connect();
@@ -1,3 +1,8 @@
1
+ import {
2
+ retainedPlayPipelineFields,
3
+ hydrateRetainedPlayPipelines,
4
+ prepareRetainedPlayPipelinesForTransport,
5
+ } from '../../plays/execution-pipeline';
1
6
  import { mkdir, rm, writeFile } from 'node:fs/promises';
2
7
  import { createHash, randomUUID } from 'node:crypto';
3
8
  import { dirname, isAbsolute, join, relative, sep } from 'node:path';
@@ -4963,7 +4968,9 @@ async function submitPostgresSchedulerRunTransaction(
4963
4968
  'replayedFromRunId', launch_payload.launch_json ->> 'replayedFromRunId',
4964
4969
  'secretRefs', launch_payload.launch_json -> 'secretRefs',
4965
4970
  'inputSummary', launch_payload.launch_json -> 'inputSummary'
4966
- )),
4971
+ )) || CASE WHEN launch_payload.launch_json ? 'retainedPipelineGraph'
4972
+ THEN jsonb_build_object('retainedPipelineGraph', launch_payload.launch_json -> 'retainedPipelineGraph')
4973
+ ELSE '{}'::jsonb END,
4967
4974
  ${causalOutboxCreatedAt('run_insert.run_id', options)}
4968
4975
  FROM run_insert
4969
4976
  CROSS JOIN launch_payload
@@ -5016,7 +5023,7 @@ async function submitPostgresSchedulerRunTransaction(
5016
5023
  input.graphHash,
5017
5024
  input.runtimeBackend,
5018
5025
  input.artifactStorageKey,
5019
- stringifyPostgresJson(input),
5026
+ stringifyPostgresJson(prepareRetainedPlayPipelinesForTransport(input)),
5020
5027
  postgresSchedulerNotifyChannel('queue', options),
5021
5028
  JSON.stringify({ runId: input.runId, playName: input.playName }),
5022
5029
  releaseAdmissionKey,
@@ -5418,6 +5425,7 @@ export type PostgresSchedulerCronSourcePin = {
5418
5425
  artifactHash: string | null;
5419
5426
  graphHash: string | null;
5420
5427
  staticPipeline: unknown | null;
5428
+ retainedPipelineGraph?: unknown;
5421
5429
  input: PlaySchedulerSubmitInput['input'];
5422
5430
  source: 'published' | 'draft';
5423
5431
  schedulerSchema: string | null;
@@ -5442,7 +5450,7 @@ export function postgresSchedulerCronSourcePinFromLaunch(
5442
5450
  artifactKind,
5443
5451
  artifactHash: launch.artifactHash || null,
5444
5452
  graphHash: launch.graphHash || null,
5445
- staticPipeline: launch.staticPipeline ?? null,
5453
+ ...retainedPlayPipelineFields(launch.staticPipeline ?? null, launch),
5446
5454
  input: launch.input,
5447
5455
  source: launch.source === 'draft' ? 'draft' : 'published',
5448
5456
  schedulerSchema: launch.schedulerSchema ?? null,
@@ -5621,7 +5629,9 @@ export async function seedPostgresSchedulerCronSchedule(
5621
5629
  input.dueAtMs,
5622
5630
  input.cron.trim(),
5623
5631
  input.timezone?.trim() || null,
5624
- stringifyPostgresJson(sourcePin),
5632
+ stringifyPostgresJson(
5633
+ prepareRetainedPlayPipelinesForTransport(sourcePin),
5634
+ ),
5625
5635
  input.queueName,
5626
5636
  task.taskName,
5627
5637
  stringifyPostgresJson(task.params),
@@ -6038,7 +6048,9 @@ export async function firePostgresSchedulerCronSchedule(
6038
6048
  'asyncAncestryPlayIds', launch_payload.launch_json -> 'asyncAncestryPlayIds',
6039
6049
  'secretRefs', launch_payload.launch_json -> 'secretRefs',
6040
6050
  'inputSummary', launch_payload.launch_json -> 'inputSummary'
6041
- )),
6051
+ )) || CASE WHEN launch_payload.launch_json ? 'retainedPipelineGraph'
6052
+ THEN jsonb_build_object('retainedPipelineGraph', launch_payload.launch_json -> 'retainedPipelineGraph')
6053
+ ELSE '{}'::jsonb END,
6042
6054
  ${causalOutboxCreatedAt('run_insert.run_id', options)}
6043
6055
  FROM run_insert
6044
6056
  CROSS JOIN launch_payload
@@ -6078,7 +6090,7 @@ export async function firePostgresSchedulerCronSchedule(
6078
6090
  schedule.dueAtMs,
6079
6091
  successorDueAtMs,
6080
6092
  launch.runId,
6081
- stringifyPostgresJson(launch),
6093
+ stringifyPostgresJson(prepareRetainedPlayPipelinesForTransport(launch)),
6082
6094
  input.queueName,
6083
6095
  playTask.taskName,
6084
6096
  stringifyPostgresJson(playTask.params),
@@ -7463,7 +7475,7 @@ export async function claimPostgresSchedulerOverdueDetachedRunnerCeilings(
7463
7475
  runId: row.run_id,
7464
7476
  attempt: row.attempt,
7465
7477
  leaseToken: row.lease_token,
7466
- launch: row.launch_json,
7478
+ launch: parseLaunchJson(row.launch_json),
7467
7479
  suspension: row.suspension_json,
7468
7480
  }));
7469
7481
  }
@@ -8441,6 +8453,80 @@ export async function readPostgresSchedulerLaunchForAttemptLease(
8441
8453
  return row ? parseLaunchJson(row.launch_json) : null;
8442
8454
  }
8443
8455
 
8456
+ /** Identity, signing authority, and origins needed to issue an attempt token. */
8457
+ const POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS = [
8458
+ 'runId',
8459
+ 'orgId',
8460
+ 'playName',
8461
+ 'userId',
8462
+ 'userEmail',
8463
+ 'baseUrl',
8464
+ 'runtimeApiBaseUrl',
8465
+ 'runtimeTokenOrigin',
8466
+ 'runtimeBackend',
8467
+ 'runtimeAuthority',
8468
+ ] as const;
8469
+
8470
+ export type PostgresSchedulerAttemptAuthority = Pick<
8471
+ PlaySchedulerSubmitInput,
8472
+ (typeof POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS)[number]
8473
+ >;
8474
+
8475
+ /**
8476
+ * Prove the same attempt lease as the full execution reader, returning only
8477
+ * the token issuer's authority. Input, graph, and artifact bytes never cross
8478
+ * this SQL seam or occupy a Vercel checkout during JSON decoding. The authority
8479
+ * descriptor is retained intact for the existing validator and deploy skew;
8480
+ * its admitted secrets and fixture configuration are not execution payloads.
8481
+ */
8482
+ export async function readPostgresSchedulerAttemptAuthorityForLease(
8483
+ client: PostgresSchedulerQueryClient,
8484
+ input: { runId: string; attempt: number; leaseToken: string },
8485
+ options?: Pick<PostgresSchedulerOptions, 'schema'>,
8486
+ ): Promise<PostgresSchedulerAttemptAuthority | null> {
8487
+ const fields = POSTGRES_SCHEDULER_ATTEMPT_AUTHORITY_FIELDS;
8488
+ // Keep JSON types intact. A text record column would coerce malformed launch
8489
+ // scalars into strings, bypassing the full reader's required-field checks.
8490
+ const columns = [...fields, 'playId', 'executorToken']
8491
+ .map((field) => `${quoteIdent(field)} jsonb`)
8492
+ .join(', ');
8493
+ const result = await client.query<
8494
+ PostgresSchedulerAttemptAuthority & {
8495
+ launch_required_fields_valid: boolean;
8496
+ }
8497
+ >(
8498
+ `
8499
+ SELECT ${fields.map((field) => `launch.${quoteIdent(field)}`).join(', ')},
8500
+ ${['runId', 'playId', 'playName', 'executorToken', 'baseUrl']
8501
+ .map(
8502
+ (field) =>
8503
+ `jsonb_typeof(launch.${quoteIdent(field)}) = 'string'`,
8504
+ )
8505
+ .join(' AND ')} AS launch_required_fields_valid
8506
+ FROM ${tableName('runs', options)} AS runs
8507
+ JOIN ${tableName('attempts', options)} AS attempts
8508
+ ON attempts.run_id = runs.run_id
8509
+ CROSS JOIN LATERAL jsonb_to_record(runs.launch_json) AS launch(${columns})
8510
+ WHERE runs.run_id = $1
8511
+ AND runs.status IN ('queued', 'running', 'waiting')
8512
+ AND attempts.attempt = $2
8513
+ AND attempts.lease_token = $3
8514
+ AND attempts.status IN ('claimed', 'running', 'waiting')
8515
+ LIMIT 1
8516
+ `,
8517
+ [input.runId, input.attempt, input.leaseToken],
8518
+ );
8519
+ const row = result.rows[0];
8520
+ if (!row) return null;
8521
+ const { launch_required_fields_valid, ...authority } = row;
8522
+ if (launch_required_fields_valid !== true) {
8523
+ throw new Error(
8524
+ 'Postgres scheduler launch_json is missing required fields.',
8525
+ );
8526
+ }
8527
+ return authority;
8528
+ }
8529
+
8444
8530
  /**
8445
8531
  * Absurd scheduler backend: persist the vendored engine's task id on the run
8446
8532
  * row after `spawn`. Idempotent (last write wins); a re-submit under the same
@@ -13924,7 +14010,10 @@ export function buildPostgresSchedulerRunnerContext(
13924
14010
  toolResponseReceiptRevision: toolResponseReceiptRevisionFromLaunch(
13925
14011
  launch.contractSnapshot,
13926
14012
  ),
13927
- staticPipeline: staticPipelineFromContractSnapshot(launch.contractSnapshot),
14013
+ ...retainedPlayPipelineFields(
14014
+ staticPipelineFromContractSnapshot(launch.contractSnapshot),
14015
+ launch,
14016
+ ),
13928
14017
  sandboxRuntimeLimits: sandboxRuntimeLimitsFromContractSnapshot(
13929
14018
  launch.contractSnapshot,
13930
14019
  ),
@@ -16221,7 +16310,9 @@ export async function appendPostgresRunLedgerStart(
16221
16310
  runId: input.runId,
16222
16311
  eventId: `run-start-callback:${input.runId}`,
16223
16312
  eventType: 'run.started',
16224
- payload: stableStringify(input.start),
16313
+ payload: stableStringify(
16314
+ prepareRetainedPlayPipelinesForTransport(input.start),
16315
+ ),
16225
16316
  },
16226
16317
  options,
16227
16318
  );
@@ -18093,6 +18184,7 @@ async function projectPostgresSchedulerCreatedEvent(
18093
18184
  runtimeApi: WorkerRuntimeApiContext,
18094
18185
  event: PostgresSchedulerOutboxEvent,
18095
18186
  ): Promise<void> {
18187
+ event = { ...event, payload: hydrateRetainedPlayPipelines(event.payload) };
18096
18188
  const value = (key: string) => stringPayloadField(event.payload, key);
18097
18189
  const number = (key: string) => numberPayloadField(event.payload, key);
18098
18190
  const raw = (key: string) => payloadField(event.payload, key);
@@ -18153,7 +18245,10 @@ async function projectPostgresSchedulerCreatedEvent(
18153
18245
  runtimeWorkflowName: value('runtimeWorkflowName'),
18154
18246
  runtimeHarnessWorkerName: value('runtimeHarnessWorkerName'),
18155
18247
  maxCreditsPerRun: number('maxCreditsPerRun'),
18156
- staticPipeline: raw('staticPipeline'),
18248
+ ...retainedPlayPipelineFields(
18249
+ raw('staticPipeline'),
18250
+ event.payload as { retainedPipelineGraph?: unknown },
18251
+ ),
18157
18252
  source: sourcePayloadField(event.payload, 'source'),
18158
18253
  ...(triggerSource === 'api' ||
18159
18254
  triggerSource === 'webhook' ||
@@ -18179,6 +18274,9 @@ async function projectPostgresSchedulerStartEvent(
18179
18274
  event: PostgresSchedulerOutboxEvent,
18180
18275
  launch: (PlaySchedulerSubmitInput & { firstClaimLatencyMs?: number }) | null,
18181
18276
  ): Promise<void> {
18277
+ const payload = hydrateRetainedPlayPipelines(
18278
+ event.payload as Record<string, unknown>,
18279
+ );
18182
18280
  await startRunViaAppRuntime(runtimeApi, {
18183
18281
  deliveryLeaseOwner: event.leaseOwner ?? undefined,
18184
18282
  idempotencyKey: buildRunLedgerStartIdempotencyKey({
@@ -18198,7 +18296,7 @@ async function projectPostgresSchedulerStartEvent(
18198
18296
  schedulerSchema: stringPayloadField(event.payload, 'schedulerSchema'),
18199
18297
  executionProfile: stringPayloadField(event.payload, 'executionProfile'),
18200
18298
  maxCreditsPerRun: numberPayloadField(event.payload, 'maxCreditsPerRun'),
18201
- staticPipeline: payloadField(event.payload, 'staticPipeline'),
18299
+ ...retainedPlayPipelineFields(payload.staticPipeline, payload),
18202
18300
  source: sourcePayloadField(event.payload, 'source'),
18203
18301
  inputFileId:
18204
18302
  launch?.inputFileId ??
@@ -18619,7 +18717,7 @@ function parseLaunchJson(value: unknown): PlaySchedulerSubmitInput {
18619
18717
  'Postgres scheduler launch_json is missing required fields.',
18620
18718
  );
18621
18719
  }
18622
- return launch as PlaySchedulerSubmitInput;
18720
+ return hydrateRetainedPlayPipelines(launch as PlaySchedulerSubmitInput);
18623
18721
  }
18624
18722
 
18625
18723
  function parsePostgresSchedulerCronSourcePin(
@@ -18628,7 +18726,9 @@ function parsePostgresSchedulerCronSourcePin(
18628
18726
  if (!value || typeof value !== 'object' || Array.isArray(value)) {
18629
18727
  throw new Error('Cron source pin is not an object.');
18630
18728
  }
18631
- const pin = value as Partial<PostgresSchedulerCronSourcePin>;
18729
+ const pin = hydrateRetainedPlayPipelines(
18730
+ value as Partial<PostgresSchedulerCronSourcePin>,
18731
+ );
18632
18732
  if (
18633
18733
  typeof pin.orgId !== 'string' ||
18634
18734
  typeof pin.playName !== 'string' ||
@@ -18639,6 +18739,7 @@ function parsePostgresSchedulerCronSourcePin(
18639
18739
  throw new Error('Cron source pin is missing required immutable fields.');
18640
18740
  }
18641
18741
  return {
18742
+ ...pin,
18642
18743
  orgId: pin.orgId,
18643
18744
  userId: typeof pin.userId === 'string' ? pin.userId : null,
18644
18745
  playName: pin.playName,
@@ -1,3 +1,4 @@
1
+ import { hydrateRetainedPlayPipelines } from '../../plays/execution-pipeline';
1
2
  /**
2
3
  * Runtime receipt gateway dispatch.
3
4
  *
@@ -866,7 +867,9 @@ export function createReceiptGatewayDispatcher(input: {
866
867
  { status: 400 },
867
868
  );
868
869
  }
869
- const body = rawBody as ReceiptGatewayActionBody;
870
+ const body = hydrateRetainedPlayPipelines(
871
+ rawBody as ReceiptGatewayActionBody,
872
+ );
870
873
  const scopeError = requireReceiptScope(scope, body);
871
874
  if (scopeError) return scopeError;
872
875
  const runId = String(body.runId || scope.runId);
@@ -1,24 +1,41 @@
1
+ import type { RetainedPipelineCarrier } from './retained-pipeline-graph';
1
2
  import type { PlayStaticPipeline } from './static-pipeline';
2
3
  import { staticPipelineStorageCodec } from './static-pipeline-storage-codec';
3
4
 
5
+ export {
6
+ retainedPlayPipelineFields,
7
+ type RetainedPlayContract,
8
+ type RetainedPipelineCarrier,
9
+ encodeRetainedPlayPipelines,
10
+ hydrateRetainedPlayPipelines,
11
+ prepareRetainedPlayPipelinesForTransport,
12
+ type RetainedPipelineGraph,
13
+ } from './retained-pipeline-graph';
14
+
4
15
  /**
5
16
  * An in-process execution snapshot, deliberately distinct from display graphs
6
- * and Convex's bounded V1 mirror. Never serialize this wrapper: durable/wire
7
- * contracts continue to carry the original pipeline fields.
17
+ * and Convex's bounded V1 mirror. Boundary adapters use the shared retained
18
+ * contract codec above; JSON serialization of the hydrated pipeline expands
19
+ * shared child definitions and is not a valid transport.
8
20
  */
9
21
  export class PlayExecutionPipeline {
10
22
  readonly #pipeline: PlayStaticPipeline | null;
11
23
 
12
- private constructor(pipeline: PlayStaticPipeline | null) {
24
+ private constructor(
25
+ pipeline: PlayStaticPipeline | null,
26
+ readonly retainedPipelineGraph?: unknown,
27
+ ) {
13
28
  this.#pipeline = pipeline;
14
29
  }
15
30
 
16
31
  /** Read the entire storage carrier, so an existing V2 can never be dropped. */
17
- static fromStored(record: {
18
- staticPipeline?: unknown;
19
- staticPipelineV2?: unknown;
20
- }): PlayExecutionPipeline {
21
- return new PlayExecutionPipeline(staticPipelineStorageCodec.read(record));
32
+ static fromStored(
33
+ record: RetainedPipelineCarrier & { staticPipelineV2?: unknown },
34
+ ): PlayExecutionPipeline {
35
+ return new PlayExecutionPipeline(
36
+ staticPipelineStorageCodec.read(record),
37
+ record.retainedPipelineGraph,
38
+ );
22
39
  }
23
40
 
24
41
  /** Only for complete compiler/admitted-artifact graphs, never a stored field. */
@@ -58,13 +75,22 @@ export function readPlayExecutionPipeline(
58
75
 
59
76
  /** Execution readers cannot see (and accidentally forward) the stored mirror. */
60
77
  export function readStoredPlayExecutionSnapshot<
61
- T extends { staticPipeline?: unknown; staticPipelineV2?: unknown },
78
+ T extends {
79
+ staticPipeline?: unknown;
80
+ staticPipelineV2?: unknown;
81
+ retainedPipelineGraph?: unknown;
82
+ },
62
83
  >(
63
84
  record: T,
64
- ): Omit<T, 'staticPipeline' | 'staticPipelineV2'> & {
85
+ ): Omit<T, 'staticPipeline' | 'staticPipelineV2' | 'retainedPipelineGraph'> & {
65
86
  executionPipeline: PlayExecutionPipeline;
66
87
  } {
67
- const { staticPipeline: _v1, staticPipelineV2: _v2, ...metadata } = record;
88
+ const {
89
+ staticPipeline: _v1,
90
+ staticPipelineV2: _v2,
91
+ retainedPipelineGraph: _graph,
92
+ ...metadata
93
+ } = record;
68
94
  return {
69
95
  ...metadata,
70
96
  executionPipeline: PlayExecutionPipeline.fromStored(record),
@@ -0,0 +1,291 @@
1
+ /**
2
+ * Lossless transport for retained execution graphs. A call site remains a
3
+ * distinct node; equal immutable definition content is stored once. References
4
+ * are scoped to this envelope, never resolved against a mutable Play registry.
5
+ *
6
+ * This is an additive named representation. Readers must be deployed before
7
+ * WRITE_RETAINED_PIPELINE_GRAPHS is enabled in a separate release.
8
+ */
9
+ export const WRITE_RETAINED_PIPELINE_GRAPHS = false;
10
+
11
+ type Scalar = null | boolean | number | string;
12
+ type GraphValue = Scalar | { ref: number };
13
+ type GraphNode = { array: GraphValue[] } | { object: [string, GraphValue][] };
14
+
15
+ export type RetainedPipelineGraph = {
16
+ nodes: GraphNode[];
17
+ roots: Record<string, GraphValue>;
18
+ };
19
+
20
+ /** The owned execution contract; caller envelopes choose their own adapter. */
21
+ export type RetainedPlayContract = {
22
+ retainedPipelineGraph?: unknown;
23
+ staticPipeline?: unknown;
24
+ authoringContract?: { staticPipeline?: unknown } | null;
25
+ contractSnapshot?: {
26
+ staticPipeline?: unknown;
27
+ authoringContract?: { staticPipeline?: unknown } | null;
28
+ } | null;
29
+ };
30
+
31
+ export type RetainedPipelineCarrier = Pick<
32
+ RetainedPlayContract,
33
+ 'staticPipeline' | 'retainedPipelineGraph'
34
+ >;
35
+
36
+ // Named roots have fixed meaning. This is not a recursive JSON serializer:
37
+ // customer inputs, checkpoints, runner configs and API responses are not
38
+ // searched for magic property names. Their owners explicitly pass a contract.
39
+ const PIPELINE_ROOT_PATHS = {
40
+ pipeline: [],
41
+ authoring: ['authoringContract'],
42
+ admitted: ['contractSnapshot'],
43
+ admittedAuthoring: ['contractSnapshot', 'authoringContract'],
44
+ } as const;
45
+
46
+ function contractAt(input: Record<string, unknown>, path: readonly string[]) {
47
+ let owner = input;
48
+ for (const key of path) {
49
+ if (!record(owner[key])) return undefined;
50
+ owner = owner[key];
51
+ }
52
+ return owner;
53
+ }
54
+
55
+ function updateContract(
56
+ input: Record<string, unknown>,
57
+ path: readonly string[],
58
+ pipeline: unknown,
59
+ remove: boolean,
60
+ ): Record<string, unknown> {
61
+ const result = { ...input };
62
+ let owner = result;
63
+ for (const key of path) {
64
+ if (Object.prototype.hasOwnProperty.call(owner, key) && !record(owner[key]))
65
+ invalid('invalid contract container');
66
+ const child = record(owner[key]) ? { ...owner[key] } : {};
67
+ owner[key] = child;
68
+ owner = child;
69
+ }
70
+ if (remove) delete owner.staticPipeline;
71
+ else owner.staticPipeline = pipeline;
72
+ return result;
73
+ }
74
+
75
+ function record(value: unknown): value is Record<string, unknown> {
76
+ return value !== null && typeof value === 'object' && !Array.isArray(value);
77
+ }
78
+
79
+ function invalid(detail: string): never {
80
+ throw new Error(`Invalid retained Play pipeline graph: ${detail}`);
81
+ }
82
+
83
+ /** Explicit encoder, also used by fixtures before production writer enable. */
84
+ export function encodeRetainedPlayPipelines<T>(input: T): T {
85
+ if (!record(input)) return input;
86
+ // Repeated serialization must not expand a previously encoded envelope.
87
+ const prior =
88
+ input.retainedPipelineGraph === undefined
89
+ ? null
90
+ : readGraph(input.retainedPipelineGraph);
91
+ const source = prior ? hydrateContractRoots(input, prior.roots) : input;
92
+ // Retain existing node IDs and all additive graph/node metadata. A reader
93
+ // that forwards a newer envelope must not rebuild away fields it ignores.
94
+ const nodes: GraphNode[] = prior
95
+ ? [...(input.retainedPipelineGraph as RetainedPipelineGraph).nodes]
96
+ : [];
97
+ const interned = new Map<string, number>();
98
+ const objects = new WeakMap<object, number>();
99
+ const visiting = new WeakSet<object>();
100
+ const roots: RetainedPipelineGraph['roots'] = prior
101
+ ? { ...(input.retainedPipelineGraph as RetainedPipelineGraph).roots }
102
+ : {};
103
+ for (let id = 0; id < nodes.length; id += 1) {
104
+ interned.set(JSON.stringify(nodes[id]), id);
105
+ const value = prior!.values[id];
106
+ if (value !== null && typeof value === 'object') objects.set(value, id);
107
+ }
108
+
109
+ function encode(value: unknown): GraphValue {
110
+ if (value === null) return null;
111
+ if (typeof value === 'string' || typeof value === 'boolean') return value;
112
+ if (typeof value === 'number') return Number.isFinite(value) ? value : null;
113
+ if (typeof value !== 'object') invalid('non-JSON pipeline value');
114
+ const cached = objects.get(value);
115
+ if (cached !== undefined) return { ref: cached };
116
+ if (visiting.has(value)) invalid('cyclic pipeline');
117
+ visiting.add(value);
118
+ const prototype = Object.getPrototypeOf(value);
119
+ if (
120
+ !Array.isArray(value) &&
121
+ prototype !== Object.prototype &&
122
+ prototype !== null
123
+ ) {
124
+ invalid('pipeline must contain plain JSON objects');
125
+ }
126
+ const node: GraphNode = Array.isArray(value)
127
+ ? { array: Array.from(value, (entry) => encode(entry ?? null)) }
128
+ : {
129
+ object: Object.entries(value)
130
+ .filter(([, entry]) => entry !== undefined)
131
+ .map(([key, entry]) => [key, encode(entry)]),
132
+ };
133
+ // Node signatures contain references, never recursively expanded children.
134
+ const signature = JSON.stringify(node);
135
+ let id = interned.get(signature);
136
+ if (id === undefined) {
137
+ id = nodes.length;
138
+ nodes.push(node);
139
+ interned.set(signature, id);
140
+ }
141
+ visiting.delete(value);
142
+ objects.set(value, id);
143
+ return { ref: id };
144
+ }
145
+
146
+ let result: Record<string, unknown> = { ...source };
147
+ for (const [name, path] of Object.entries(PIPELINE_ROOT_PATHS)) {
148
+ const owner = contractAt(source, path);
149
+ if (
150
+ owner &&
151
+ Object.prototype.hasOwnProperty.call(owner, 'staticPipeline') &&
152
+ owner.staticPipeline !== undefined
153
+ ) {
154
+ roots[name] = encode(owner.staticPipeline);
155
+ result = updateContract(result, path, undefined, true);
156
+ }
157
+ }
158
+ if (!Object.keys(roots).length) return input;
159
+ return {
160
+ ...result,
161
+ retainedPipelineGraph: {
162
+ ...(record(input.retainedPipelineGraph)
163
+ ? input.retainedPipelineGraph
164
+ : {}),
165
+ nodes,
166
+ roots,
167
+ },
168
+ } as T;
169
+ }
170
+
171
+ function readGraph(graph: unknown): {
172
+ roots: Record<string, unknown>;
173
+ values: unknown[];
174
+ } {
175
+ if (!record(graph) || !Array.isArray(graph.nodes) || !record(graph.roots)) {
176
+ invalid('expected nodes and roots');
177
+ }
178
+ const values: unknown[] = [];
179
+ function decode(value: unknown): unknown {
180
+ if (
181
+ value === null ||
182
+ typeof value === 'string' ||
183
+ typeof value === 'boolean'
184
+ )
185
+ return value;
186
+ if (typeof value === 'number' && Number.isFinite(value)) return value;
187
+ if (
188
+ !record(value) ||
189
+ Object.keys(value).length !== 1 ||
190
+ !Number.isSafeInteger(value.ref) ||
191
+ (value.ref as number) < 0 ||
192
+ (value.ref as number) >= values.length
193
+ ) {
194
+ invalid('reference must name a preceding node');
195
+ }
196
+ return values[value.ref as number];
197
+ }
198
+ for (const node of graph.nodes) {
199
+ if (!record(node)) invalid('expected node object');
200
+ if (Array.isArray(node.array) && node.object === undefined) {
201
+ values.push(node.array.map(decode));
202
+ } else if (Array.isArray(node.object) && node.array === undefined) {
203
+ const entries: [string, unknown][] = [];
204
+ const keys = new Set<string>();
205
+ for (const entry of node.object) {
206
+ if (
207
+ !Array.isArray(entry) ||
208
+ entry.length !== 2 ||
209
+ typeof entry[0] !== 'string' ||
210
+ keys.has(entry[0])
211
+ ) {
212
+ invalid('invalid or duplicate object entry');
213
+ }
214
+ keys.add(entry[0]);
215
+ entries.push([entry[0], decode(entry[1])]);
216
+ }
217
+ values.push(Object.fromEntries(entries));
218
+ } else invalid('expected array or object node');
219
+ }
220
+ return {
221
+ roots: Object.fromEntries(
222
+ Object.entries(graph.roots).map(([name, value]) => [name, decode(value)]),
223
+ ),
224
+ values,
225
+ };
226
+ }
227
+
228
+ /** Old envelopes pass through unchanged; malformed explicit graphs fail loudly. */
229
+ export function hydrateRetainedPlayPipelines<T>(input: T): T {
230
+ if (!record(input) || input.retainedPipelineGraph === undefined) return input;
231
+ return hydrateContractRoots(
232
+ input,
233
+ readGraph(input.retainedPipelineGraph).roots,
234
+ ) as T;
235
+ }
236
+
237
+ function hydrateContractRoots(
238
+ input: Record<string, unknown>,
239
+ roots: Record<string, unknown>,
240
+ ): Record<string, unknown> {
241
+ let result: Record<string, unknown> = { ...input };
242
+ for (const [name, path] of Object.entries(PIPELINE_ROOT_PATHS)) {
243
+ if (!Object.prototype.hasOwnProperty.call(roots, name)) continue;
244
+ const pipeline = roots[name];
245
+ if (
246
+ pipeline !== null &&
247
+ (!record(pipeline) ||
248
+ !Array.isArray(pipeline.fields) ||
249
+ !Array.isArray(pipeline.substeps))
250
+ ) {
251
+ invalid(`root ${name} must be a complete pipeline or null`);
252
+ }
253
+ result = updateContract(result, path, pipeline, false);
254
+ }
255
+ return result;
256
+ }
257
+
258
+ /** Production writers remain inert until the separate reader rollout completes. */
259
+ export function prepareRetainedPlayPipelinesForTransport<T>(input: T): T {
260
+ // Reader-only builds must preserve new representations when forwarding
261
+ // them. The rollout switch controls creation from legacy envelopes only.
262
+ return WRITE_RETAINED_PIPELINE_GRAPHS ||
263
+ (record(input) && input.retainedPipelineGraph !== undefined)
264
+ ? encodeRetainedPlayPipelines(input)
265
+ : input;
266
+ }
267
+
268
+ /** Project one retained pipeline into a new envelope without losing its format. */
269
+ export function retainedPlayPipelineFields<T>(
270
+ pipeline: T,
271
+ source: RetainedPipelineCarrier,
272
+ ): { staticPipeline: T; retainedPipelineGraph?: RetainedPipelineGraph } {
273
+ if (source.retainedPipelineGraph === undefined)
274
+ return { staticPipeline: pipeline };
275
+ readGraph(source.retainedPipelineGraph);
276
+ const graph = source.retainedPipelineGraph as RetainedPipelineGraph;
277
+ // This is an explicit projection to one pipeline, not a reinterpretation of
278
+ // the old root. Keep unknown roles and metadata through the projection.
279
+ const roots = Object.fromEntries(
280
+ Object.entries(graph.roots).filter(
281
+ ([name]) =>
282
+ !Object.prototype.hasOwnProperty.call(PIPELINE_ROOT_PATHS, name),
283
+ ),
284
+ );
285
+ return hydrateRetainedPlayPipelines(
286
+ encodeRetainedPlayPipelines({
287
+ staticPipeline: pipeline,
288
+ retainedPipelineGraph: { ...graph, roots },
289
+ }),
290
+ );
291
+ }
@@ -3,7 +3,58 @@ import {
3
3
  createStoredStaticPipelineV2,
4
4
  hydrateStoredStaticPipelineV2,
5
5
  readStoredStaticPipeline,
6
+ createStaticPipelineV1CompatibilityProjection,
7
+ type PlayStaticPipeline,
6
8
  } from './static-pipeline';
9
+ import {
10
+ retainedPlayPipelineFields,
11
+ encodeRetainedPlayPipelines,
12
+ hydrateRetainedPlayPipelines,
13
+ WRITE_RETAINED_PIPELINE_GRAPHS,
14
+ type RetainedPipelineGraph,
15
+ type RetainedPipelineCarrier,
16
+ } from './retained-pipeline-graph';
17
+
18
+ function readExecutionPipeline(
19
+ record: RetainedPipelineCarrier & { staticPipelineV2?: unknown },
20
+ ): PlayStaticPipeline | null {
21
+ if (record.retainedPipelineGraph !== undefined) {
22
+ const hydrated = hydrateRetainedPlayPipelines({
23
+ retainedPipelineGraph: record.retainedPipelineGraph,
24
+ }) as { staticPipeline?: PlayStaticPipeline | null };
25
+ if (hydrated.staticPipeline === undefined)
26
+ throw new Error('Retained execution graph is missing its pipeline root.');
27
+ return hydrated.staticPipeline;
28
+ }
29
+ return readStoredStaticPipeline(record);
30
+ }
31
+
32
+ /** Runs use the shared graph after all storage readers have been deployed. */
33
+ function executionStorageFields(
34
+ pipeline: PlayStaticPipeline | null | undefined,
35
+ carrier?: RetainedPipelineCarrier,
36
+ ): {
37
+ staticPipeline?: PlayStaticPipeline | null;
38
+ staticPipelineV2?: ReturnType<typeof createStoredStaticPipelineV2>;
39
+ retainedPipelineGraph?: RetainedPipelineGraph;
40
+ } {
41
+ if (
42
+ (!WRITE_RETAINED_PIPELINE_GRAPHS &&
43
+ carrier?.retainedPipelineGraph === undefined) ||
44
+ pipeline === undefined
45
+ ) {
46
+ return createStaticPipelineStorageFields(pipeline);
47
+ }
48
+ const compact = encodeRetainedPlayPipelines(
49
+ retainedPlayPipelineFields(pipeline, carrier ?? {}),
50
+ ) as unknown as {
51
+ retainedPipelineGraph: RetainedPipelineGraph;
52
+ };
53
+ return {
54
+ staticPipeline: createStaticPipelineV1CompatibilityProjection(pipeline),
55
+ retainedPipelineGraph: compact.retainedPipelineGraph,
56
+ };
57
+ }
7
58
 
8
59
  export {
9
60
  isStoredStaticPipelineV2IntegrityError,
@@ -23,5 +74,6 @@ export const staticPipelineStorageCodec = Object.freeze({
23
74
  flatten: createStoredStaticPipelineV2,
24
75
  hydrate: hydrateStoredStaticPipelineV2,
25
76
  storageFields: createStaticPipelineStorageFields,
26
- read: readStoredStaticPipeline,
77
+ read: readExecutionPipeline,
78
+ executionStorageFields,
27
79
  });
package/dist/cli/index.js CHANGED
@@ -3473,7 +3473,7 @@ var SDK_RELEASE = {
3473
3473
  // getters keep their established compatibility behavior.
3474
3474
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
3475
3475
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
3476
- version: "0.3.243",
3476
+ version: "0.3.245",
3477
3477
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
3478
3478
  packageCapabilities: {
3479
3479
  updatePreferences: 1
@@ -3468,7 +3468,7 @@ var SDK_RELEASE = {
3468
3468
  // getters keep their established compatibility behavior.
3469
3469
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
3470
3470
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
3471
- version: "0.3.243",
3471
+ version: "0.3.245",
3472
3472
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
3473
3473
  packageCapabilities: {
3474
3474
  updatePreferences: 1
package/dist/index.js CHANGED
@@ -1286,7 +1286,7 @@ var SDK_RELEASE = {
1286
1286
  // getters keep their established compatibility behavior.
1287
1287
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
1288
1288
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
1289
- version: "0.3.243",
1289
+ version: "0.3.245",
1290
1290
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
1291
1291
  packageCapabilities: {
1292
1292
  updatePreferences: 1
package/dist/index.mjs CHANGED
@@ -1190,7 +1190,7 @@ var SDK_RELEASE = {
1190
1190
  // getters keep their established compatibility behavior.
1191
1191
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
1192
1192
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
1193
- version: "0.3.243",
1193
+ version: "0.3.245",
1194
1194
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
1195
1195
  packageCapabilities: {
1196
1196
  updatePreferences: 1
@@ -506,6 +506,7 @@
506
506
  "dist/bundling-sources/shared_libs/plays/play-grep-search.ts",
507
507
  "dist/bundling-sources/shared_libs/plays/play-reference.ts",
508
508
  "dist/bundling-sources/shared_libs/plays/resolve-static-pipeline.ts",
509
+ "dist/bundling-sources/shared_libs/plays/retained-pipeline-graph.ts",
509
510
  "dist/bundling-sources/shared_libs/plays/row-identity.ts",
510
511
  "dist/bundling-sources/shared_libs/plays/run-input-summary.ts",
511
512
  "dist/bundling-sources/shared_libs/plays/run-summary-types.ts",
@@ -149,7 +149,7 @@ type SdkRelease = {
149
149
  supportPolicy: SdkSupportPolicy;
150
150
  };
151
151
  declare const SDK_RELEASE: {
152
- readonly version: "0.3.243";
152
+ readonly version: "0.3.245";
153
153
  readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
154
154
  readonly packageCapabilities: {
155
155
  readonly updatePreferences: 1;
package/dist/release.d.ts CHANGED
@@ -149,7 +149,7 @@ type SdkRelease = {
149
149
  supportPolicy: SdkSupportPolicy;
150
150
  };
151
151
  declare const SDK_RELEASE: {
152
- readonly version: "0.3.243";
152
+ readonly version: "0.3.245";
153
153
  readonly updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.";
154
154
  readonly packageCapabilities: {
155
155
  readonly updatePreferences: 1;
package/dist/release.js CHANGED
@@ -74,7 +74,7 @@ var SDK_RELEASE = {
74
74
  // getters keep their established compatibility behavior.
75
75
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
76
76
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
77
- version: "0.3.243",
77
+ version: "0.3.245",
78
78
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
79
79
  packageCapabilities: {
80
80
  updatePreferences: 1
package/dist/release.mjs CHANGED
@@ -48,7 +48,7 @@ var SDK_RELEASE = {
48
48
  // getters keep their established compatibility behavior.
49
49
  // 0.3.1 deprecated the legacy `deepline enrich` command in favor of Plays.
50
50
  // 0.3.90 is the first deliberately versioned SDK release for API v3.
51
- version: "0.3.243",
51
+ version: "0.3.245",
52
52
  updateSummary: "Play grep now returns concise results with the matched fields and terms. Automatic CLI updates are enabled by default; use `deepline settings autoupdate off` to opt out, `deepline settings autoupdate on` to re-enable updates, or `deepline settings autoupdate pin <version>` to hold an exact release.",
53
53
  packageCapabilities: {
54
54
  updatePreferences: 1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "deepline",
3
- "version": "0.3.243",
3
+ "version": "0.3.245",
4
4
  "description": "GTM data CLI and TypeScript SDK for coding agents",
5
5
  "license": "MIT",
6
6
  "homepage": "https://code.deepline.com",