deepline 0.1.227 → 0.1.229

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +76 -1842
  2. package/dist/bundling-sources/apps/play-runner-workers/src/dedup-do.ts +0 -113
  3. package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +144 -1000
  4. package/dist/bundling-sources/apps/play-runner-workers/src/runtime/ledger-event-batches.ts +81 -0
  5. package/dist/bundling-sources/apps/play-runner-workers/src/workflow-retry-state.ts +1 -50
  6. package/dist/bundling-sources/sdk/src/client.ts +24 -2
  7. package/dist/bundling-sources/sdk/src/play.ts +2 -2
  8. package/dist/bundling-sources/sdk/src/release.ts +2 -2
  9. package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +382 -31
  10. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +84 -79
  11. package/dist/bundling-sources/shared_libs/play-runtime/context.ts +201 -705
  12. package/dist/bundling-sources/shared_libs/play-runtime/coordinator-headers.ts +3 -10
  13. package/dist/bundling-sources/shared_libs/play-runtime/ctx-contract.ts +2 -2
  14. package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +12 -47
  15. package/dist/bundling-sources/shared_libs/play-runtime/governor/block-reserving-budget-state-backend.ts +0 -2
  16. package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +12 -200
  17. package/dist/bundling-sources/shared_libs/play-runtime/governor/policy.ts +2 -17
  18. package/dist/bundling-sources/shared_libs/play-runtime/live-state-contract.ts +3 -9
  19. package/dist/bundling-sources/shared_libs/play-runtime/play-call-execution.ts +6 -1
  20. package/dist/bundling-sources/shared_libs/play-runtime/play-latency-trace.ts +28 -0
  21. package/dist/bundling-sources/shared_libs/play-runtime/run-execution-scope.ts +16 -64
  22. package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +26 -0
  23. package/dist/bundling-sources/shared_libs/play-runtime/run-lifecycle-policy.ts +5 -2
  24. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api-paths.ts +0 -4
  25. package/dist/bundling-sources/shared_libs/play-runtime/runtime-api.ts +81 -18
  26. package/dist/bundling-sources/shared_libs/play-runtime/scheduler-backend.ts +0 -36
  27. package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +3 -21
  28. package/dist/bundling-sources/shared_libs/play-runtime/test-runtime-seams.ts +9 -1
  29. package/dist/bundling-sources/shared_libs/play-runtime/transport-error-diagnostics.ts +63 -0
  30. package/dist/bundling-sources/shared_libs/plays/static-pipeline.ts +2 -0
  31. package/dist/cli/index.js +78 -11
  32. package/dist/cli/index.mjs +78 -11
  33. package/dist/{compiler-manifest-BhgZ23A4.d.ts → compiler-manifest-CYcwzSOJ.d.mts} +2 -0
  34. package/dist/{compiler-manifest-BhgZ23A4.d.mts → compiler-manifest-CYcwzSOJ.d.ts} +2 -0
  35. package/dist/index.d.mts +3 -3
  36. package/dist/index.d.ts +3 -3
  37. package/dist/index.js +14 -5
  38. package/dist/index.mjs +14 -5
  39. package/dist/plays/bundle-play-file.d.mts +2 -2
  40. package/dist/plays/bundle-play-file.d.ts +2 -2
  41. package/package.json +1 -1
  42. package/dist/bundling-sources/apps/play-runner-workers/src/child-manifest-resolver.ts +0 -108
  43. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-await.ts +0 -374
  44. package/dist/bundling-sources/apps/play-runner-workers/src/child-play-submit.ts +0 -203
  45. package/dist/bundling-sources/shared_libs/play-runtime/child-execution-placement.ts +0 -14
  46. package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +0 -121
@@ -62,13 +62,6 @@ import {
62
62
  pacingPolicyFromUnknownQueueHints,
63
63
  type ResolvedPacingPolicy,
64
64
  } from '../../../shared_libs/play-runtime/pacing';
65
- import {
66
- awaitChildTerminal,
67
- type ChildPlayTerminalWaitResult,
68
- type WorkflowStepLike,
69
- } from './child-play-await';
70
- import { submitChildPlayThroughCoordinator } from './child-play-submit';
71
- import { createRunScopedChildManifestResolver } from './child-manifest-resolver';
72
65
  import { isRetryableWorkerRuntimeApiError } from './runtime-api-retry';
73
66
  import { formatCustomerConsoleValue } from './runtime/customer-console';
74
67
  import {
@@ -91,12 +84,8 @@ import {
91
84
  createToolExecuteHttpFailureAttemptTracker,
92
85
  parseToolExecuteAuthScopeChangedError,
93
86
  } from '../../../shared_libs/play-runtime/tool-execute-retry-policy';
94
- import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
95
87
  import type { PreloadedRuntimeDbSession } from '../../../shared_libs/play-runtime/db-session';
96
- import type {
97
- PlayRuntimeManifest,
98
- PlayRuntimeManifestMap,
99
- } from '../../../shared_libs/plays/compiler-manifest';
88
+ import type { PlayRuntimeManifest } from '../../../shared_libs/plays/compiler-manifest';
100
89
  import {
101
90
  deriveToolRequestIdentity,
102
91
  derivePlayRowIdentity,
@@ -105,10 +94,7 @@ import {
105
94
  stableStringify,
106
95
  } from '../../../shared_libs/plays/row-identity';
107
96
  import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
108
- import {
109
- buildDurableCtxCallCacheKey,
110
- buildDurableRunPlayInvocationScope,
111
- } from '../../../shared_libs/play-runtime/durable-call-cache';
97
+ import { buildDurableCtxCallCacheKey } from '../../../shared_libs/play-runtime/durable-call-cache';
112
98
  import {
113
99
  resolveRuntimeToolReceiptWaitMaxAttempts,
114
100
  resolveRuntimeToolReceiptWaitTimeoutMs,
@@ -140,6 +126,11 @@ import type {
140
126
  PlayRunLedgerStepProgress,
141
127
  PlayRunLedgerStepStatus,
142
128
  } from '../../../shared_libs/play-runtime/run-ledger';
129
+ import {
130
+ MAX_LEDGER_LOG_LINE_LENGTH,
131
+ MAX_LEDGER_LOG_LINES_PER_EVENT,
132
+ } from '../../../shared_libs/play-runtime/ledger-safe-payload';
133
+ import { partitionWorkerLedgerEvents } from './runtime/ledger-event-batches';
143
134
  import type { PlayHarnessRpc } from '../../play-harness-worker/src/rpc-types';
144
135
  import {
145
136
  createCsvDatasetHandle,
@@ -205,7 +196,6 @@ import {
205
196
  runtimeMapJsonByteLength,
206
197
  } from '../../../shared_libs/play-runtime/map-memory-limits';
207
198
  import { createSerialTaskQueue } from '../../../shared_libs/play-runtime/serial-task-queue';
208
- import { resolveChildExecutionPlacement as resolveChildExecutionStrategy } from '../../../shared_libs/play-runtime/child-execution-placement';
209
199
  // The harness stub forwards leaf calls (validation, runtime-api HTTP) into
210
200
  // the long-lived Play Harness Worker via env.HARNESS. We import the
211
201
  // `setHarnessBinding` setter eagerly so it's available the moment
@@ -418,10 +408,6 @@ type RunRequest = {
418
408
  contractSnapshot?: unknown;
419
409
  /** Deepline-owned compact execution plan. Public SDK/users never see this. */
420
410
  executionPlan?: ExecutionPlan | null;
421
- /** Trusted child play launch manifests carried by the top-level Vercel submit. */
422
- childPlayManifests?: PlayRuntimeManifestMap | null;
423
- /** Internal ctx.runPlay lineage. Public SDK/users never see this. */
424
- playCallGovernance?: PlayCallGovernanceSnapshot | null;
425
411
  preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
426
412
  /** Cloudflare coordinator URL for direct Workflow control-plane signals. */
427
413
  coordinatorUrl?: string | null;
@@ -634,41 +620,8 @@ type WorkerEnv = {
634
620
  body: string;
635
621
  }>;
636
622
  };
637
- /**
638
- * Loopback RPC binding into the coordinator Worker. Used for CF-to-CF
639
- * child orchestration so nested plays do not bounce through a public
640
- * coordinator URL when both caller and callee are already inside the same
641
- * Cloudflare Worker graph.
642
- */
623
+ /** Loopback RPC binding for runtime observability and rate coordination. */
643
624
  COORDINATOR?: {
644
- submitChild(
645
- parentRunId: string,
646
- body: Record<string, unknown>,
647
- ): Promise<{
648
- workflowId?: string;
649
- runId?: string;
650
- status?: string;
651
- mode?: string;
652
- output?: unknown;
653
- result?: unknown;
654
- error?: unknown;
655
- logs?: string[];
656
- timings?: Array<{ phase: string; ms: number }>;
657
- }>;
658
- submitWorkflowChild?(
659
- parentRunId: string,
660
- body: Record<string, unknown>,
661
- ): Promise<{
662
- workflowId?: string;
663
- runId?: string;
664
- status?: string;
665
- mode?: string;
666
- output?: unknown;
667
- result?: unknown;
668
- error?: unknown;
669
- logs?: string[];
670
- timings?: Array<{ phase: string; ms: number }>;
671
- }>;
672
625
  signal(
673
626
  runId: string,
674
627
  body: Record<string, unknown>,
@@ -682,11 +635,6 @@ type WorkerEnv = {
682
635
  event: Record<string, unknown>,
683
636
  ): Promise<void>;
684
637
  readTerminalState?(runId: string): Promise<Record<string, unknown> | null>;
685
- readChildTerminalState?(
686
- parentRunId: string,
687
- eventKey: string,
688
- timeoutMs?: number,
689
- ): Promise<Record<string, unknown> | null>;
690
638
  /**
691
639
  * Distributed Rate State Backend RPC. Routes to the per-(org,provider)
692
640
  * rate-bucket Durable Object so the request window is global across
@@ -1225,29 +1173,11 @@ async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
1225
1173
 
1226
1174
  let workerRequestSequence = 0;
1227
1175
 
1228
- const workerRunPlayRowContext = new AsyncLocalStorage<{
1229
- tableNamespace: string;
1230
- rowKey: string;
1231
- fieldName: string;
1232
- }>();
1233
-
1234
1176
  function makeRequestId(): string {
1235
1177
  workerRequestSequence = (workerRequestSequence + 1) % Number.MAX_SAFE_INTEGER;
1236
1178
  return `worker-request-${workerRequestSequence}`;
1237
1179
  }
1238
1180
 
1239
- async function executeFreshChildInvocation<T>(
1240
- input: {
1241
- parentRunId: string;
1242
- key: string;
1243
- invocationScope: string;
1244
- },
1245
- execute: (invocationId: string) => Promise<T>,
1246
- ): Promise<T> {
1247
- const invocationId = `${input.parentRunId}:runPlay:${input.key}:${input.invocationScope}`;
1248
- return await execute(invocationId);
1249
- }
1250
-
1251
1181
  function makeRuntimeSheetAttempt(input: {
1252
1182
  runId: string;
1253
1183
  runAttempt?: number | null;
@@ -1478,65 +1408,14 @@ async function postRuntimeApiBestEffort(
1478
1408
  }
1479
1409
  }
1480
1410
 
1481
- async function readDurableChildRunTerminalState(input: {
1482
- baseUrl: string;
1483
- executorToken: string;
1484
- parentRunId: string;
1485
- childRunId: string;
1486
- childPlayName: string;
1487
- timeoutMs?: number;
1488
- }): Promise<{ data?: unknown } | null> {
1489
- const response = await postRuntimeApi<{
1490
- state?: {
1491
- data?: unknown;
1492
- reason?: string;
1493
- status?: string;
1494
- runId?: string;
1495
- playName?: string;
1496
- parentRunId?: string | null;
1497
- } | null;
1498
- }>(
1499
- input.baseUrl,
1500
- input.executorToken,
1501
- {
1502
- action: 'read_child_run_terminal_snapshot',
1503
- parentRunId: input.parentRunId,
1504
- childRunId: input.childRunId,
1505
- childPlayName: input.childPlayName,
1506
- },
1507
- {
1508
- timeoutMsOverride: input.timeoutMs,
1509
- timeoutErrorMessage: `[play-harness] child run terminal snapshot read timed out after ${Math.max(
1510
- 1,
1511
- Math.ceil(input.timeoutMs ?? RUNTIME_API_TIMEOUT_MS),
1512
- )}ms. childRunId=${input.childRunId}`,
1513
- },
1514
- );
1515
- return response.state ?? null;
1516
- }
1517
-
1518
- function workflowEventType(name: string): string {
1519
- const normalized = name
1520
- .trim()
1521
- .replace(/[^A-Za-z0-9_-]+/g, '_')
1522
- .replace(/^_+|_+$/g, '')
1523
- .slice(0, 100);
1524
- return normalized || 'deepline_event';
1525
- }
1526
-
1527
- function normalizeContextKey(key: unknown, operation: string): string {
1528
- if (typeof key !== 'string' || !key.trim()) {
1529
- throw new Error(`ctx.${operation}(...) requires a non-empty key.`);
1530
- }
1531
- return key
1532
- .trim()
1533
- .replace(/[^A-Za-z0-9_.:-]+/g, '_')
1534
- .slice(0, 120);
1535
- }
1536
-
1537
1411
  function resolvePlayRefName(playRef: unknown): string {
1538
- if (typeof playRef === 'string') return playRef.trim();
1539
- if (typeof playRef === 'function') {
1412
+ if (typeof playRef === 'string' && playRef.trim()) {
1413
+ return playRef.trim();
1414
+ }
1415
+ if (
1416
+ playRef &&
1417
+ (typeof playRef === 'object' || typeof playRef === 'function')
1418
+ ) {
1540
1419
  const record = playRef as unknown as Record<string | symbol, unknown>;
1541
1420
  const playName = record.playName;
1542
1421
  if (typeof playName === 'string' && playName.trim()) return playName.trim();
@@ -1559,189 +1438,11 @@ function resolvePlayRefName(playRef: unknown): string {
1559
1438
  return '';
1560
1439
  }
1561
1440
 
1562
- function extractChildPlayOutput(status: Record<string, unknown>): unknown {
1563
- const result = status.result;
1564
- if (isRecord(result) && 'output' in result) {
1565
- return result.output;
1566
- }
1567
- return result ?? null;
1568
- }
1569
-
1570
- function hydrateChildPlayOutputDatasets(input: {
1571
- req: RunRequest;
1572
- childRunId: string;
1573
- childPlayName: string;
1574
- value: unknown;
1575
- }): unknown {
1576
- return hydrateSerializedResultDatasets(input.value, (dataset) => {
1577
- const tableNamespace = dataset.tableNamespace?.trim();
1578
- if (!tableNamespace) {
1579
- return dataset;
1580
- }
1581
- const count = Math.max(0, Math.floor(dataset.count));
1582
- const previewRows = dataset.preview.filter(isRecord);
1583
- return createPersistedDatasetHandle({
1584
- playName: input.childPlayName,
1585
- name: tableNamespace,
1586
- count,
1587
- datasetKind: dataset.datasetKind,
1588
- previewRows,
1589
- cachedRows: count <= previewRows.length ? previewRows : null,
1590
- sourceLabel: dataset.sourceLabel ?? null,
1591
- readRows: async ({ limit, offset }) => {
1592
- const result = await harnessReadSheetDatasetRows({
1593
- baseUrl: input.req.baseUrl,
1594
- executorToken: input.req.executorToken,
1595
- orgId: input.req.orgId,
1596
- playName: input.childPlayName,
1597
- tableNamespace,
1598
- runId: input.childRunId,
1599
- limit,
1600
- offset,
1601
- userEmail: input.req.userEmail,
1602
- preloadedDbSessions: input.req.preloadedDbSessions ?? null,
1603
- });
1604
- return result.rows as Array<Record<string, unknown>>;
1605
- },
1606
- trace: (phase, ms, extra) =>
1607
- recordRunnerPerfTrace({
1608
- req: input.req,
1609
- phase,
1610
- ms,
1611
- extra: {
1612
- ...(extra ?? {}),
1613
- childRunId: input.childRunId,
1614
- childPlayName: input.childPlayName,
1615
- },
1616
- }),
1617
- nowMs,
1618
- workProgress: dataset._metadata?.workProgress,
1619
- });
1620
- });
1621
- }
1622
-
1623
- async function hashChildPlayEventKey(input: unknown): Promise<string> {
1624
- return (await hashJson(input)).slice(0, 32);
1625
- }
1626
-
1627
- type RuntimeResolvedPlayResponse = {
1628
- manifest?: PlayRuntimeManifest | null;
1629
- };
1630
-
1631
- async function resolveAuthorizedWorkerChildManifest(input: {
1632
- req: RunRequest;
1633
- playRef: string;
1634
- }): Promise<PlayRuntimeManifest | null> {
1635
- const response = await postRuntimeApi<RuntimeResolvedPlayResponse>(
1636
- input.req.baseUrl,
1637
- input.req.executorToken,
1638
- {
1639
- action: 'resolve_play',
1640
- playRef: input.playRef,
1641
- artifactKind: 'esm_workers',
1642
- },
1643
- {
1644
- runtimeTestFaultHeader: input.req.runtimeTestFaultHeader,
1645
- timeoutMsOverride: 30_000,
1646
- },
1647
- );
1648
- return response.manifest ?? null;
1649
- }
1650
-
1651
- async function childPlayEventKey(input: {
1652
- key: string;
1653
- workflowId: string;
1654
- }): Promise<string> {
1655
- const readableKey = workflowEventType(input.key).slice(0, 40);
1656
- const digest = await hashChildPlayEventKey({
1657
- key: input.key,
1658
- workflowId: input.workflowId,
1659
- });
1660
- return `child_play_${digest}_${readableKey}`;
1661
- }
1662
-
1663
1441
  function workflowTimeoutFromMs(timeoutMs: number): string {
1664
1442
  const seconds = Math.max(1, Math.ceil(timeoutMs / 1000));
1665
1443
  return `${seconds} second${seconds === 1 ? '' : 's'}`;
1666
1444
  }
1667
1445
 
1668
- async function signalParentPlayTerminal(input: {
1669
- req: RunRequest;
1670
- status: 'completed' | 'failed' | 'cancelled';
1671
- result?: unknown;
1672
- error?: string | null;
1673
- }): Promise<void> {
1674
- const governance = input.req.playCallGovernance;
1675
- if (!governance?.parentRunId || !governance.key) return;
1676
- const eventKey = await childPlayEventKey({
1677
- key: governance.key,
1678
- workflowId: input.req.runId,
1679
- });
1680
- const data = {
1681
- type: 'play.child.terminal',
1682
- parentRunId: governance.parentRunId,
1683
- childRunId: input.req.runId,
1684
- playName: input.req.playName,
1685
- key: governance.key,
1686
- status: input.status,
1687
- ...(input.result !== undefined ? { result: input.result } : {}),
1688
- ...(input.error
1689
- ? {
1690
- error: {
1691
- code:
1692
- input.status === 'cancelled' ? 'CHILD_CANCELLED' : 'CHILD_FAILED',
1693
- message: input.error,
1694
- phase: 'child_terminal',
1695
- runId: input.req.runId,
1696
- parentRunId: governance.parentRunId,
1697
- childRunId: input.req.runId,
1698
- },
1699
- }
1700
- : {}),
1701
- emittedAt: nowMs(),
1702
- };
1703
- const signalBody = {
1704
- signal: 'integration_event',
1705
- eventKey,
1706
- data,
1707
- };
1708
- if (cachedCoordinatorBinding) {
1709
- await cachedCoordinatorBinding.signal(governance.parentRunId, signalBody);
1710
- return;
1711
- }
1712
- const coordinatorUrl = input.req.coordinatorUrl?.trim();
1713
- if (coordinatorUrl) {
1714
- const res = await fetch(
1715
- `${coordinatorUrl.replace(/\/$/, '')}/workflow/${encodeURIComponent(
1716
- governance.parentRunId,
1717
- )}/signal`,
1718
- {
1719
- method: 'POST',
1720
- headers: {
1721
- 'x-deepline-request-id': makeRequestId(),
1722
- ...coordinatorRequestHeaders({
1723
- runId: governance.parentRunId,
1724
- contentType: 'application/json',
1725
- internalToken: input.req.coordinatorInternalToken,
1726
- }),
1727
- },
1728
- body: JSON.stringify(signalBody),
1729
- },
1730
- );
1731
- if (res.ok) {
1732
- return;
1733
- }
1734
- const text = await res.text().catch(() => '');
1735
- throw new Error(
1736
- text.slice(0, 800) ||
1737
- `Coordinator parent signal failed with ${res.status}.`,
1738
- );
1739
- }
1740
- throw new Error(
1741
- 'Child play terminal signaling requires a coordinator binding in the cf-workflows runtime.',
1742
- );
1743
- }
1744
-
1745
1446
  async function executeTool(
1746
1447
  req: RunRequest,
1747
1448
  args: {
@@ -1875,6 +1576,25 @@ function normalizeToolExecuteArgs(request: unknown): {
1875
1576
  };
1876
1577
  }
1877
1578
 
1579
+ function workflowEventType(name: string): string {
1580
+ const normalized = name
1581
+ .trim()
1582
+ .replace(/[^A-Za-z0-9_-]+/g, '_')
1583
+ .replace(/^_+|_+$/g, '')
1584
+ .slice(0, 100);
1585
+ return normalized || 'deepline_event';
1586
+ }
1587
+
1588
+ function normalizeContextKey(key: unknown, operation: string): string {
1589
+ if (typeof key !== 'string' || !key.trim()) {
1590
+ throw new Error(`ctx.${operation}(...) requires a non-empty key.`);
1591
+ }
1592
+ return key
1593
+ .trim()
1594
+ .replace(/[^A-Za-z0-9_.:-]+/g, '_')
1595
+ .slice(0, 120);
1596
+ }
1597
+
1878
1598
  function integrationEventType(eventKey: string): string {
1879
1599
  return workflowEventType(`integration_event_${eventKey}`);
1880
1600
  }
@@ -3933,34 +3653,6 @@ function formatWorkerRowFailureMessage(error: unknown): string {
3933
3653
  return message.length > 1_000 ? `${message.slice(0, 1_000)}…` : message;
3934
3654
  }
3935
3655
 
3936
- function childPipelineUsesCtxDataset(
3937
- pipeline: PlayStaticPipeline | null | undefined,
3938
- ): boolean {
3939
- if (!pipeline) return false;
3940
- if (typeof pipeline.tableNamespace === 'string' && pipeline.tableNamespace) {
3941
- return true;
3942
- }
3943
- if (pipeline.sheetContract) {
3944
- return true;
3945
- }
3946
- return flattenStaticPipeline(pipeline).some((substep) => {
3947
- if (substep.type === 'dataset') return true;
3948
- if (!isRecord(substep)) return false;
3949
- return (
3950
- ('tableNamespace' in substep &&
3951
- typeof substep.tableNamespace === 'string' &&
3952
- substep.tableNamespace.length > 0) ||
3953
- ('sheetContract' in substep && Boolean(substep.sheetContract))
3954
- );
3955
- });
3956
- }
3957
-
3958
- /**
3959
- * Build the per-(org,provider) rate port the distributed Rate State Backend
3960
- * RPCs through. When the coordinator binding (or its rate RPCs) is absent we
3961
- * fail OPEN — grant immediately — matching customer-rate-limiter semantics so a
3962
- * miswired binding degrades pacing without stalling the run.
3963
- */
3964
3656
  function createCoordinatorRatePort(req: RunRequest): CoordinatorRatePort {
3965
3657
  return {
3966
3658
  async rateAcquire(input) {
@@ -4174,46 +3866,20 @@ function createWorkerPacingResolver(req: RunRequest): WorkerPacingResolver {
4174
3866
  }
4175
3867
 
4176
3868
  /**
4177
- * Build the Governor's lineage snapshot for this worker, seeded from the
4178
- * inherited PlayCallGovernanceSnapshot (threaded via internalRunPlay) so play-
4179
- * call budgets accumulate down the dispatch tree across isolates. The current
4180
- * play id is always `req.playName` so the per-parent child-call counter keys off
4181
- * the executing play (matching the prior worker behavior). Per-run counters the
4182
- * worker lineage does not carry (tool/retry/descendant/waterfall) seed at 0 —
4183
- * these are runaway guards, not exact cross-isolate ledgers, so a per-worker
4184
- * reset is acceptable and matches the Governor contract.
3869
+ * The retired workers-edge runtime has no child-run scheduling. Its Governor
3870
+ * therefore starts from this top-level run only.
4185
3871
  */
4186
3872
  function resumeGovernanceFromRequest(req: RunRequest): GovernanceSnapshot {
4187
- const inherited = req.playCallGovernance;
4188
- const rootRunId = inherited?.rootRunId || req.runId;
4189
- const ancestryPlayIds = inherited?.ancestryPlayIds?.length
4190
- ? // Per the lineage validator the inherited tail equals the parent; ensure
4191
- // the chain ends with the currently-executing play for the cycle guard.
4192
- inherited.ancestryPlayIds[inherited.ancestryPlayIds.length - 1] ===
4193
- req.playName
4194
- ? [...inherited.ancestryPlayIds]
4195
- : [...inherited.ancestryPlayIds, req.playName]
4196
- : [req.playName];
4197
- const ancestryRunIds =
4198
- rootRunId === req.runId ? [req.runId] : [rootRunId, req.runId];
4199
3873
  return {
4200
- rootRunId,
3874
+ rootRunId: req.runId,
4201
3875
  currentRunId: req.runId,
4202
3876
  currentPlayId: req.playName,
4203
- ancestryPlayIds,
4204
- ancestryRunIds,
4205
- callDepth: inherited?.callDepth ?? 0,
4206
- // Seed every lineage-global budget counter from the inherited snapshot so
4207
- // descendant/tool/retry/waterfall budgets accumulate across isolates exactly
4208
- // as they do across the cjs forkChild lineage. Without this they would reset
4209
- // to 0 in each isolate and become per-worker — contradicting the Governor's
4210
- // lineage-global budget contract. Fail-safe to 0 for older callers.
4211
- playCallCount: inherited?.playCallCount ?? 0,
4212
- toolCallCount: inherited?.toolCallCount ?? 0,
4213
- retryCount: inherited?.retryCount ?? 0,
4214
- descendantCount: inherited?.descendantCount ?? 0,
4215
- waterfallStepExecutions: inherited?.waterfallStepExecutions ?? 0,
4216
- parentChildCalls: {},
3877
+ ancestryPlayIds: [req.playName],
3878
+ ancestryRunIds: [req.runId],
3879
+ callDepth: 0,
3880
+ toolCallCount: 0,
3881
+ retryCount: 0,
3882
+ waterfallStepExecutions: 0,
4217
3883
  };
4218
3884
  }
4219
3885
 
@@ -4226,7 +3892,7 @@ function createGovernorForRun(req: RunRequest): {
4226
3892
  adapter: 'esm_workers',
4227
3893
  scope: {
4228
3894
  orgId: req.orgId,
4229
- rootRunId: req.playCallGovernance?.rootRunId ?? req.runId,
3895
+ rootRunId: req.runId,
4230
3896
  },
4231
3897
  rateState: new CoordinatorRateStateBackend(createCoordinatorRatePort(req)),
4232
3898
  budgetState: new BlockReservingBudgetStateBackend(
@@ -4254,7 +3920,7 @@ function createGovernorForRun(req: RunRequest): {
4254
3920
  * - ctx.csv(filename | inline rows) (calls runtime API for file resolve)
4255
3921
  * - ctx.dataset(name, rows).withColumn(name, resolver).run(opts)
4256
3922
  * - ctx.tools.execute({ id, tool, input, ... })
4257
- * - ctx.runPlay(key, playRef, input, opts)
3923
+ * - ctx.runPlay(...), which rejects with CTX_RUN_PLAY_INLINE_ONLY
4258
3924
  *
4259
3925
  * Not supported (will throw):
4260
3926
  * - ctx.fetch, checkpoints, etc.
@@ -4284,16 +3950,7 @@ function createMinimalWorkerCtx(
4284
3950
  createWorkerToolActionCacheVersionResolver(req);
4285
3951
  const resolveToolAuthScopeDigest =
4286
3952
  createWorkerToolAuthScopeDigestResolver(req);
4287
- // Play-call depth/count/per-parent budgets, child-play concurrency, and the
4288
- // lineage snapshot are owned by the Governor (createGovernorForRun above).
4289
- // The worker keeps only substrate mechanism here.
4290
3953
  const stepCallCounts: Record<string, number> = {};
4291
- const resolveChildManifest = createRunScopedChildManifestResolver({
4292
- scopeKey: `${req.orgId}:${req.runId}:${req.runAttempt}`,
4293
- preloaded: req.childPlayManifests,
4294
- resolveManifest: (playRef) =>
4295
- resolveAuthorizedWorkerChildManifest({ req, playRef }),
4296
- });
4297
3954
  const secretRedactor = createSecretRedactionContext();
4298
3955
 
4299
3956
  const resolveSecretAuth = async (auth?: SecretAuth) => {
@@ -4332,8 +3989,6 @@ function createMinimalWorkerCtx(
4332
3989
  : { [auth.header.toLowerCase()]: value };
4333
3990
  };
4334
3991
 
4335
- const rootGovernance = req.playCallGovernance;
4336
- const rootRunId = rootGovernance?.rootRunId ?? req.runId;
4337
3992
  const receiptStore = createHarnessWorkerReceiptStore({
4338
3993
  executorToken: req.executorToken,
4339
3994
  orgId: req.orgId,
@@ -4510,23 +4165,6 @@ function createMinimalWorkerCtx(
4510
4165
  persistenceLatch: rootPersistenceLatch,
4511
4166
  receiptSalvage,
4512
4167
  });
4513
- // Local ancestry chain that always ENDS with the currently-executing play
4514
- // (req.playName). The /api/v2/plays/run lineage validator requires the
4515
- // submitted ancestry's tail to equal the executor token's play name (i.e.
4516
- // the caller). Normalizing here keeps both the cycle-detection check below
4517
- // and the runPlay submission honest: at root we start with [req.playName],
4518
- // and for nested calls we append req.playName onto the parent-supplied
4519
- // chain (which per contract ends with the parent that submitted us).
4520
- const ancestryPlayIds = ((): string[] => {
4521
- if (!rootGovernance?.ancestryPlayIds?.length) {
4522
- return [req.playName];
4523
- }
4524
- const inherited = rootGovernance.ancestryPlayIds;
4525
- return inherited[inherited.length - 1] === req.playName
4526
- ? inherited
4527
- : [...inherited, req.playName];
4528
- })();
4529
- const callDepth = rootGovernance?.callDepth ?? 0;
4530
4168
  const runMap = async <T extends Record<string, unknown>>(
4531
4169
  name: string,
4532
4170
  rows: WorkerDatasetInput<T>,
@@ -5590,49 +5228,37 @@ function createMinimalWorkerCtx(
5590
5228
  })
5591
5229
  : null,
5592
5230
  });
5593
- const resolved = await workerRunPlayRowContext.run(
5594
- {
5595
- tableNamespace: name,
5596
- rowKey: entry.rowKey,
5597
- fieldName: key,
5598
- },
5599
- () =>
5600
- executeWorkerStepResolver(
5601
- value,
5602
- enriched,
5603
- rowCtx,
5604
- absoluteIndex,
5605
- previousCell,
5606
- isWorkerStepProgram(value)
5607
- ? {
5608
- parentField: key,
5609
- path: [],
5610
- outputs: stepProgramOutputs,
5611
- onOutput: async (stepOutput) => {
5612
- generatedOutputFields.add(
5613
- stepOutput.columnName,
5614
- );
5615
- const status =
5616
- stepOutput.status ?? 'completed';
5617
- if (status === 'skipped')
5618
- stepCellsSkipped += 1;
5619
- else stepCellsCompleted += 1;
5620
- await enqueueLiveRowUpdate({
5621
- key: entry.rowKey,
5622
- dataPatch: {
5623
- [stepOutput.columnName]: stepOutput.value,
5624
- },
5625
- cellMetaPatch: {
5626
- [stepOutput.columnName]: {
5627
- status,
5628
- stage: stepOutput.stepId,
5629
- },
5630
- },
5631
- });
5231
+ const resolved = await executeWorkerStepResolver(
5232
+ value,
5233
+ enriched,
5234
+ rowCtx,
5235
+ absoluteIndex,
5236
+ previousCell,
5237
+ isWorkerStepProgram(value)
5238
+ ? {
5239
+ parentField: key,
5240
+ path: [],
5241
+ outputs: stepProgramOutputs,
5242
+ onOutput: async (stepOutput) => {
5243
+ generatedOutputFields.add(stepOutput.columnName);
5244
+ const status = stepOutput.status ?? 'completed';
5245
+ if (status === 'skipped') stepCellsSkipped += 1;
5246
+ else stepCellsCompleted += 1;
5247
+ await enqueueLiveRowUpdate({
5248
+ key: entry.rowKey,
5249
+ dataPatch: {
5250
+ [stepOutput.columnName]: stepOutput.value,
5251
+ },
5252
+ cellMetaPatch: {
5253
+ [stepOutput.columnName]: {
5254
+ status,
5255
+ stage: stepOutput.stepId,
5256
+ },
5632
5257
  },
5633
- }
5634
- : undefined,
5635
- ),
5258
+ });
5259
+ },
5260
+ }
5261
+ : undefined,
5636
5262
  );
5637
5263
  enriched[key] = resolved.value;
5638
5264
  fieldOutputs[key] = resolved.value;
@@ -6966,438 +6592,20 @@ function createMinimalWorkerCtx(
6966
6592
  key: string,
6967
6593
  playRef: string | { playName?: string; name?: string },
6968
6594
  input: Record<string, unknown>,
6969
- options?: {
6595
+ _options?: {
6970
6596
  description?: string;
6971
6597
  timeoutMs?: number;
6972
6598
  execution?: 'inline' | 'child-workflow';
6973
6599
  },
6974
6600
  ): Promise<unknown> {
6975
- const normalizedKey = normalizeContextKey(key, 'runPlay');
6976
- const resolvedName = resolvePlayRefName(playRef);
6601
+ normalizeContextKey(key, 'runPlay');
6977
6602
  assertNoSecretTaint(input, 'ctx.runPlay input');
6603
+ const resolvedName = resolvePlayRefName(playRef);
6978
6604
  if (!resolvedName) {
6979
6605
  throw new Error('ctx.runPlay(...) requires a resolvable play name.');
6980
6606
  }
6981
- const childManifest = await resolveChildManifest(resolvedName);
6982
- const rowScope = workerRunPlayRowContext.getStore();
6983
- const invocationScope = buildDurableRunPlayInvocationScope({
6984
- childPlayName: resolvedName,
6985
- input,
6986
- rowScope: rowScope
6987
- ? {
6988
- fieldName: rowScope.fieldName,
6989
- rowKey: rowScope.rowKey,
6990
- tableNamespace: rowScope.tableNamespace,
6991
- }
6992
- : null,
6993
- });
6994
- return await executeFreshChildInvocation(
6995
- {
6996
- parentRunId: req.runId,
6997
- key: normalizedKey,
6998
- invocationScope,
6999
- },
7000
- async (childInvocationId) => {
7001
- // The Governor owns the play-call lineage: forkChild does the cycle
7002
- // guard, depth/per-parent/playCall/descendant budget charges, and
7003
- // returns the snapshot to thread into the child so budgets accumulate
7004
- // across isolates. Every ctx.runPlay call is a fresh child invocation.
7005
- const childRunId = `${req.runId}:child:${childInvocationId}`;
7006
- // Checkpoint governance admission separately from the child result.
7007
- // Workflow replay reuses this admission and the coordinator's stable
7008
- // child launch identity, but ctx.runPlay never caches child output.
7009
- const forkChild = () =>
7010
- governor.forkChild({
7011
- childPlayName: resolvedName,
7012
- childRunId,
7013
- });
7014
- const childGovernance = workflowStep
7015
- ? await (
7016
- workflowStep.do as unknown as <T>(
7017
- name: string,
7018
- callback: () => Promise<T>,
7019
- ) => Promise<T>
7020
- )(
7021
- `runPlay-governance:${sha256Hex(childInvocationId).slice(0, 24)}`,
7022
- forkChild,
7023
- )
7024
- : await forkChild();
7025
- const nextDepth = childGovernance.callDepth;
7026
- const nextParentCalls =
7027
- governor.snapshot().parentChildCalls[req.playName] ?? 0;
7028
-
7029
- emitEvent({
7030
- type: 'log',
7031
- level: 'info',
7032
- message: `Starting child play ${resolvedName} (${normalizedKey})`,
7033
- ts: nowMs(),
7034
- });
7035
- if (!childManifest) {
7036
- throw new Error(
7037
- `ctx.runPlay(${normalizedKey}) cannot start ${resolvedName}: not callable.`,
7038
- );
7039
- }
7040
- const childIsDatasetBacked = childPipelineUsesCtxDataset(
7041
- childManifest.staticPipeline,
7042
- );
7043
- const childWaitsForEvent = (
7044
- childManifest.staticPipeline
7045
- ? flattenStaticPipeline(childManifest.staticPipeline)
7046
- : []
7047
- ).some(
7048
- (step) =>
7049
- step.type === 'tool' &&
7050
- (step.isEventWait || step.toolId === 'test_wait_for_event'),
7051
- );
7052
- const authoredWorkflow =
7053
- options?.timeoutMs != null ||
7054
- childIsDatasetBacked ||
7055
- childWaitsForEvent;
7056
- if (options?.execution === 'inline' && authoredWorkflow) {
7057
- throw new Error(
7058
- `ctx.runPlay("${resolvedName}") requested inline execution for a child that requires a workflow.`,
7059
- );
7060
- }
7061
- const childExecutionDecision = resolveChildExecutionStrategy({
7062
- requiresWorkflow: authoredWorkflow,
7063
- execution: options?.execution,
7064
- });
7065
- const childNeedsWorkflowScheduler =
7066
- childExecutionDecision.strategy === 'scheduled';
7067
- if (rowScope && childNeedsWorkflowScheduler) {
7068
- throw new Error(
7069
- `ctx.runPlay("${resolvedName}") cannot start a child workflow per dataset row.`,
7070
- );
7071
- }
7072
- recordRunnerOperationalSpan(req, {
7073
- event: 'play.runtime.span',
7074
- phase: 'child_route',
7075
- runId: req.runId,
7076
- parentRunId: req.runId,
7077
- playName: resolvedName,
7078
- graphHash: req.graphHash ?? null,
7079
- depth: nextDepth,
7080
- fanoutIndex: nextParentCalls - 1,
7081
- childIsDatasetBacked,
7082
- childNeedsWorkflowScheduler,
7083
- hasStaticPipeline: Boolean(childManifest.staticPipeline),
7084
- childTableNamespace:
7085
- typeof childManifest.staticPipeline?.tableNamespace === 'string'
7086
- ? childManifest.staticPipeline.tableNamespace
7087
- : null,
7088
- childStageCount: Array.isArray(childManifest.staticPipeline?.stages)
7089
- ? childManifest.staticPipeline.stages.length
7090
- : null,
7091
- childSubstepCount: Array.isArray(
7092
- childManifest.staticPipeline?.substeps,
7093
- )
7094
- ? childManifest.staticPipeline.substeps.length
7095
- : null,
7096
- });
7097
- let childPlaySlot: { release(): void } | null = null;
7098
- try {
7099
- childPlaySlot = await governor.acquireChildSubmitSlot({
7100
- signal: abortSignal,
7101
- });
7102
- const childSubmitStartedAt = nowMs();
7103
- let started: {
7104
- workflowId?: string;
7105
- runId?: string;
7106
- status?: string;
7107
- mode?: string;
7108
- output?: unknown;
7109
- result?: unknown;
7110
- error?: unknown;
7111
- logs?: string[];
7112
- timings?: Array<{ phase: string; ms: number }>;
7113
- };
7114
- try {
7115
- started = await submitChildPlayThroughCoordinator({
7116
- req,
7117
- coordinatorBinding: cachedCoordinatorBinding,
7118
- makeRequestId,
7119
- coordinatorRequestHeaders,
7120
- allowInline: childExecutionDecision.strategy === 'inline',
7121
- body: {
7122
- name: resolvedName,
7123
- childIdempotencyKey: childInvocationId,
7124
- input: isRecord(input) ? input : {},
7125
- orgId: req.orgId,
7126
- callbackUrl: req.callbackUrl,
7127
- baseUrl: req.baseUrl,
7128
- integrationMode: req.integrationMode ?? null,
7129
- parentExecutorToken: req.executorToken,
7130
- userEmail: req.userEmail ?? '',
7131
- profile: 'workers_edge',
7132
- manifest: childManifest,
7133
- childPlayManifests: req.childPlayManifests ?? null,
7134
- internalRunPlay: {
7135
- rootRunId,
7136
- parentRunId: req.runId,
7137
- parentPlayName: req.playName,
7138
- key: normalizedKey,
7139
- // Per the lineage validator: ancestry tail must equal the
7140
- // executor token's play name (the parent making this call).
7141
- ancestryPlayIds,
7142
- callDepth: nextDepth,
7143
- // Cumulative lineage-global budget counters (incl. this
7144
- // launch's play/descendant charges) so the child seeds its
7145
- // budgets from the lineage total instead of resetting to 0 in
7146
- // its isolate. Threading descendantCount in particular keeps
7147
- // fan-out descendant accounting lineage-global, matching cjs.
7148
- playCallCount: childGovernance.playCallCount,
7149
- toolCallCount: childGovernance.toolCallCount,
7150
- retryCount: childGovernance.retryCount,
7151
- descendantCount: childGovernance.descendantCount,
7152
- waterfallStepExecutions:
7153
- childGovernance.waterfallStepExecutions,
7154
- description:
7155
- typeof options?.description === 'string'
7156
- ? options.description
7157
- : null,
7158
- },
7159
- },
7160
- });
7161
- } catch (error) {
7162
- recordRunnerOperationalSpan(req, {
7163
- event: 'play.runtime.span',
7164
- phase: 'child_submit',
7165
- runId: req.runId,
7166
- parentRunId: req.runId,
7167
- playName: resolvedName,
7168
- graphHash: req.graphHash ?? null,
7169
- depth: nextDepth,
7170
- fanoutIndex: nextParentCalls - 1,
7171
- ms: nowMs() - childSubmitStartedAt,
7172
- status: 'failed',
7173
- errorCode: 'CHILD_SUBMIT_FAILED',
7174
- });
7175
- recordRunnerPerfTrace({
7176
- req,
7177
- phase: 'ctx_run_play.child_submit',
7178
- ms: nowMs() - childSubmitStartedAt,
7179
- extra: {
7180
- status: 'failed',
7181
- errorCode: 'CHILD_SUBMIT_FAILED',
7182
- playName: resolvedName,
7183
- key: normalizedKey,
7184
- depth: nextDepth,
7185
- fanoutIndex: nextParentCalls - 1,
7186
- childIsDatasetBacked,
7187
- childNeedsWorkflowScheduler,
7188
- },
7189
- });
7190
- throw error;
7191
- }
7192
- const workflowId = started.workflowId ?? started.runId;
7193
- if (!workflowId) {
7194
- const startedError = isRecord(started.error)
7195
- ? started.error
7196
- : { message: started.error };
7197
- const startedErrorMessage =
7198
- typeof startedError.message === 'string' &&
7199
- startedError.message.trim()
7200
- ? startedError.message.trim()
7201
- : null;
7202
- throw new Error(
7203
- startedErrorMessage ??
7204
- `ctx.runPlay(${normalizedKey}) did not receive a child workflow id.`,
7205
- );
7206
- }
7207
- recordRunnerOperationalSpan(req, {
7208
- event: 'play.runtime.span',
7209
- phase: 'child_submit',
7210
- runId: req.runId,
7211
- parentRunId: req.runId,
7212
- childRunId: workflowId,
7213
- playName: resolvedName,
7214
- graphHash: req.graphHash ?? null,
7215
- depth: nextDepth,
7216
- fanoutIndex: nextParentCalls - 1,
7217
- ms: nowMs() - childSubmitStartedAt,
7218
- status: 'ok',
7219
- });
7220
- recordRunnerPerfTrace({
7221
- req,
7222
- phase: 'ctx_run_play.child_submit',
7223
- ms: nowMs() - childSubmitStartedAt,
7224
- extra: {
7225
- status: 'ok',
7226
- childRunId: workflowId,
7227
- startedStatus: started.status ?? null,
7228
- mode: started.mode ?? null,
7229
- coordinatorTimings: Array.isArray(started.timings)
7230
- ? started.timings
7231
- : null,
7232
- playName: resolvedName,
7233
- key: normalizedKey,
7234
- depth: nextDepth,
7235
- fanoutIndex: nextParentCalls - 1,
7236
- childIsDatasetBacked,
7237
- childNeedsWorkflowScheduler,
7238
- },
7239
- });
7240
- childPlaySlot?.release();
7241
- childPlaySlot = null;
7242
- const startedStatus = String(started.status ?? '').toLowerCase();
7243
- if (startedStatus === 'completed') {
7244
- emitEvent({
7245
- type: 'log',
7246
- level: 'info',
7247
- message: `Completed child play ${resolvedName} (${normalizedKey})`,
7248
- ts: nowMs(),
7249
- });
7250
- return hydrateChildPlayOutputDatasets({
7251
- req,
7252
- childRunId: workflowId,
7253
- childPlayName: resolvedName,
7254
- value: started.output ?? extractChildPlayOutput(started),
7255
- });
7256
- }
7257
- if (startedStatus === 'failed') {
7258
- const startedError = isRecord(started.error)
7259
- ? started.error
7260
- : { message: started.error };
7261
- const startedErrorMessage =
7262
- typeof startedError.message === 'string' &&
7263
- startedError.message.trim()
7264
- ? startedError.message.trim()
7265
- : `Child play ${resolvedName} (${workflowId}) failed.`;
7266
- throw new Error(startedErrorMessage);
7267
- }
7268
- const childWaitStartedAt = nowMs();
7269
- let waitResult: ChildPlayTerminalWaitResult;
7270
- const readCachedChildTerminalState =
7271
- cachedCoordinatorBinding?.readChildTerminalState?.bind(
7272
- cachedCoordinatorBinding,
7273
- );
7274
- try {
7275
- waitResult = await awaitChildTerminal({
7276
- parentRunId: req.runId,
7277
- // CF's WorkflowStep.waitForEvent generic signature is wider than
7278
- // the small structural shape ChildPlayAwait needs; bridge it the
7279
- // same way the inline implementation did.
7280
- workflowStep: workflowStep as unknown as
7281
- | WorkflowStepLike
7282
- | undefined,
7283
- workflowId,
7284
- playName: resolvedName,
7285
- key: normalizedKey,
7286
- timeoutMs: Math.max(
7287
- 1_000,
7288
- Math.min(options?.timeoutMs ?? 5 * 60_000, 30 * 60_000),
7289
- ),
7290
- coordinator: {
7291
- readRunTerminalState: (runId, timeoutMs) =>
7292
- readDurableChildRunTerminalState({
7293
- baseUrl: req.baseUrl,
7294
- executorToken: req.executorToken,
7295
- parentRunId: req.runId,
7296
- childRunId: runId,
7297
- childPlayName: resolvedName,
7298
- timeoutMs,
7299
- }),
7300
- ...(readCachedChildTerminalState
7301
- ? {
7302
- readChildTerminalState: (
7303
- parentRunId: string,
7304
- eventKey: string,
7305
- timeoutMs?: number,
7306
- ) =>
7307
- readCachedChildTerminalState(
7308
- parentRunId,
7309
- eventKey,
7310
- timeoutMs,
7311
- ),
7312
- }
7313
- : {}),
7314
- },
7315
- now: nowMs,
7316
- hashJson,
7317
- });
7318
- } catch (error) {
7319
- recordRunnerOperationalSpan(req, {
7320
- event: 'play.runtime.span',
7321
- phase: 'child_wait',
7322
- runId: req.runId,
7323
- parentRunId: req.runId,
7324
- childRunId: workflowId,
7325
- playName: resolvedName,
7326
- graphHash: req.graphHash ?? null,
7327
- depth: nextDepth,
7328
- fanoutIndex: nextParentCalls - 1,
7329
- ms: nowMs() - childWaitStartedAt,
7330
- status: 'failed',
7331
- errorCode: 'CHILD_WAIT_FAILED',
7332
- });
7333
- recordRunnerPerfTrace({
7334
- req,
7335
- phase: 'ctx_run_play.child_wait',
7336
- ms: nowMs() - childWaitStartedAt,
7337
- extra: {
7338
- status: 'failed',
7339
- errorCode: 'CHILD_WAIT_FAILED',
7340
- childRunId: workflowId,
7341
- playName: resolvedName,
7342
- key: normalizedKey,
7343
- depth: nextDepth,
7344
- fanoutIndex: nextParentCalls - 1,
7345
- childIsDatasetBacked,
7346
- childNeedsWorkflowScheduler,
7347
- },
7348
- });
7349
- throw error;
7350
- }
7351
- recordRunnerOperationalSpan(req, {
7352
- event: 'play.runtime.span',
7353
- phase: 'child_wait',
7354
- runId: req.runId,
7355
- parentRunId: req.runId,
7356
- childRunId: workflowId,
7357
- playName: resolvedName,
7358
- graphHash: req.graphHash ?? null,
7359
- depth: nextDepth,
7360
- fanoutIndex: nextParentCalls - 1,
7361
- ms: nowMs() - childWaitStartedAt,
7362
- status: 'ok',
7363
- waitSource: waitResult.source,
7364
- waitAttempts: waitResult.attempts ?? null,
7365
- reportedWaitMs: waitResult.waitMs,
7366
- });
7367
- recordRunnerPerfTrace({
7368
- req,
7369
- phase: 'ctx_run_play.child_wait',
7370
- ms: nowMs() - childWaitStartedAt,
7371
- extra: {
7372
- status: 'ok',
7373
- childRunId: workflowId,
7374
- playName: resolvedName,
7375
- key: normalizedKey,
7376
- depth: nextDepth,
7377
- fanoutIndex: nextParentCalls - 1,
7378
- childIsDatasetBacked,
7379
- childNeedsWorkflowScheduler,
7380
- waitSource: waitResult.source,
7381
- waitAttempts: waitResult.attempts ?? null,
7382
- reportedWaitMs: waitResult.waitMs,
7383
- },
7384
- });
7385
- emitEvent({
7386
- type: 'log',
7387
- level: 'info',
7388
- message: `Completed child play ${resolvedName} (${normalizedKey})`,
7389
- ts: nowMs(),
7390
- });
7391
- return hydrateChildPlayOutputDatasets({
7392
- req,
7393
- childRunId: workflowId,
7394
- childPlayName: resolvedName,
7395
- value: waitResult.output,
7396
- });
7397
- } finally {
7398
- childPlaySlot?.release();
7399
- }
7400
- },
6607
+ throw new Error(
6608
+ `CTX_RUN_PLAY_INLINE_ONLY: ctx.runPlay("${resolvedName}") is unavailable on the retired workers_edge runtime. Run this play on the current runtime, where resolved scalar children compose inline.`,
7401
6609
  );
7402
6610
  },
7403
6611
  async fetch(
@@ -7778,7 +6986,10 @@ async function executeRunRequest(
7778
6986
  let ledgerFlushQueueDepth = 0;
7779
6987
 
7780
6988
  const appendRunLogLine = (line: string) => {
7781
- const trimmed = redactSecretsFromLogString(line.trim());
6989
+ const trimmed = redactSecretsFromLogString(line.trim()).slice(
6990
+ 0,
6991
+ MAX_LEDGER_LOG_LINE_LENGTH,
6992
+ );
7782
6993
  if (!trimmed) return;
7783
6994
  workBudgetMeter.count('log');
7784
6995
  totalEmittedLogLines += 1;
@@ -7861,17 +7072,27 @@ async function executeRunRequest(
7861
7072
  pendingLedgerEvents = [];
7862
7073
 
7863
7074
  if (pendingRunLogLines.length > 0) {
7864
- events.push({
7865
- type: 'log.appended',
7866
- runId: req.runId,
7867
- source: 'worker',
7868
- occurredAt,
7869
- lines: pendingRunLogLines,
7870
- // Positional cursor: pendingRunLogLines always holds the LAST
7871
- // pending lines emitted on this channel, so the offset of its first
7872
- // line is total-emitted minus pending length.
7873
- channelOffset: totalEmittedLogLines - pendingRunLogLines.length,
7874
- });
7075
+ const channelOffset = totalEmittedLogLines - pendingRunLogLines.length;
7076
+ for (
7077
+ let start = 0;
7078
+ start < pendingRunLogLines.length;
7079
+ start += MAX_LEDGER_LOG_LINES_PER_EVENT
7080
+ ) {
7081
+ events.push({
7082
+ type: 'log.appended',
7083
+ runId: req.runId,
7084
+ source: 'worker',
7085
+ occurredAt,
7086
+ lines: pendingRunLogLines.slice(
7087
+ start,
7088
+ start + MAX_LEDGER_LOG_LINES_PER_EVENT,
7089
+ ),
7090
+ // Positional cursor: pendingRunLogLines always holds the LAST
7091
+ // pending lines emitted on this channel, so the offset of its first
7092
+ // line is total-emitted minus pending length.
7093
+ channelOffset: channelOffset + start,
7094
+ });
7095
+ }
7875
7096
  pendingRunLogLines = [];
7876
7097
  }
7877
7098
 
@@ -7956,19 +7177,26 @@ async function executeRunRequest(
7956
7177
  ledgerFlushInFlight = ledgerFlushInFlight
7957
7178
  .catch(() => undefined)
7958
7179
  .then(async () => {
7959
- try {
7960
- await postRuntimeApi(req.baseUrl, req.executorToken, {
7961
- action: 'append_run_events',
7962
- playId: req.runId,
7963
- events,
7964
- });
7965
- } catch {
7966
- pendingLedgerEvents = [...events, ...pendingLedgerEvents];
7967
- throw new Error('runtime run-ledger append failed');
7968
- } finally {
7969
- ledgerFlushQueueDepth = Math.max(0, ledgerFlushQueueDepth - 1);
7180
+ const batches = partitionWorkerLedgerEvents(events);
7181
+ for (let index = 0; index < batches.length; index += 1) {
7182
+ try {
7183
+ await postRuntimeApi(req.baseUrl, req.executorToken, {
7184
+ action: 'append_run_events',
7185
+ playId: req.runId,
7186
+ events: batches[index]!,
7187
+ });
7188
+ } catch {
7189
+ pendingLedgerEvents = [
7190
+ ...batches.slice(index).flat(),
7191
+ ...pendingLedgerEvents,
7192
+ ];
7193
+ throw new Error('runtime run-ledger append failed');
7194
+ }
7970
7195
  }
7971
7196
  })
7197
+ .finally(() => {
7198
+ ledgerFlushQueueDepth = Math.max(0, ledgerFlushQueueDepth - 1);
7199
+ })
7972
7200
  .catch(() => undefined);
7973
7201
  return force ? ledgerFlushInFlight : Promise.resolve();
7974
7202
  };
@@ -7976,9 +7204,7 @@ async function executeRunRequest(
7976
7204
  const flushTerminalLedgerEvents = async (
7977
7205
  terminalEvent: PlayRunLedgerEvent,
7978
7206
  ): Promise<void> => {
7979
- const shouldPersistTerminalLedger =
7980
- options?.persistResultDatasets === true ||
7981
- Boolean(req.playCallGovernance);
7207
+ const shouldPersistTerminalLedger = options?.persistResultDatasets === true;
7982
7208
  if (!shouldPersistTerminalLedger) return;
7983
7209
  await ledgerFlushInFlight;
7984
7210
  const now = nowMs();
@@ -7988,15 +7214,21 @@ async function executeRunRequest(
7988
7214
  ]);
7989
7215
  const events = [...drainPendingLedgerEvents(now), terminalEvent];
7990
7216
  if (events.length === 0) return;
7991
- try {
7992
- await postRuntimeApi(req.baseUrl, req.executorToken, {
7993
- action: 'append_run_events',
7994
- playId: req.runId,
7995
- events,
7996
- });
7997
- } catch (error) {
7998
- pendingLedgerEvents = [...events, ...pendingLedgerEvents];
7999
- throw error;
7217
+ const batches = partitionWorkerLedgerEvents(events);
7218
+ for (let index = 0; index < batches.length; index += 1) {
7219
+ try {
7220
+ await postRuntimeApi(req.baseUrl, req.executorToken, {
7221
+ action: 'append_run_events',
7222
+ playId: req.runId,
7223
+ events: batches[index]!,
7224
+ });
7225
+ } catch (error) {
7226
+ pendingLedgerEvents = [
7227
+ ...batches.slice(index).flat(),
7228
+ ...pendingLedgerEvents,
7229
+ ];
7230
+ throw error;
7231
+ }
8000
7232
  }
8001
7233
  };
8002
7234
 
@@ -8177,46 +7409,6 @@ async function executeRunRequest(
8177
7409
  ms: nowMs() - resultDatasetStartedAt,
8178
7410
  });
8179
7411
  };
8180
- let parentSignalPromise: Promise<void> | null = null;
8181
- const startParentTerminalSignal = (): Promise<void> => {
8182
- if (!parentSignalPromise) {
8183
- const parentSignalStartedAt = nowMs();
8184
- parentSignalPromise = signalParentPlayTerminal({
8185
- req,
8186
- status: 'completed',
8187
- result: req.playCallGovernance ? serializedResult : terminalResult,
8188
- })
8189
- .catch((error) => {
8190
- void runnerTelemetry
8191
- .child({
8192
- context: {
8193
- runId: req.runId,
8194
- orgId: req.orgId,
8195
- playName: req.playName,
8196
- },
8197
- })
8198
- .error(
8199
- 'runner.parent_completion_signal.failed',
8200
- error,
8201
- undefined,
8202
- {
8203
- tag: '[play-harness] non-fatal parent completion signal failed',
8204
- },
8205
- );
8206
- })
8207
- .finally(() => {
8208
- recordRunnerPerfTrace({
8209
- req,
8210
- phase: 'runner.parent_terminal_signal',
8211
- ms: nowMs() - parentSignalStartedAt,
8212
- });
8213
- });
8214
- }
8215
- return parentSignalPromise;
8216
- };
8217
- if (req.playCallGovernance && !options?.persistResultDatasets) {
8218
- await persistProjectedResultDatasets();
8219
- }
8220
7412
  await releaseRuntimeLeasesOnSettlement({
8221
7413
  req,
8222
7414
  leasedSheetNamespaces,
@@ -8259,7 +7451,6 @@ async function executeRunRequest(
8259
7451
  ms: nowMs() - terminalUpdateStartedAt,
8260
7452
  });
8261
7453
 
8262
- const parentSignal = startParentTerminalSignal();
8263
7454
  if (!capped) {
8264
7455
  const billingStartedAt = nowMs();
8265
7456
  const billingPromise = finalizeWorkerComputeBilling({
@@ -8292,23 +7483,7 @@ async function executeRunRequest(
8292
7483
  await nonBlockingBillingPromise;
8293
7484
  }
8294
7485
  }
8295
- await parentSignal;
8296
- } else if (req.playCallGovernance) {
8297
- const childTerminalStartedAt = nowMs();
8298
- await flushTerminalLedgerEvents({
8299
- type: 'run.completed',
8300
- runId: req.runId,
8301
- source: 'worker',
8302
- occurredAt: nowMs(),
8303
- result: terminalResult,
8304
- });
8305
- recordRunnerPerfTrace({
8306
- req,
8307
- phase: 'runner.child_terminal_ledger_append',
8308
- ms: nowMs() - childTerminalStartedAt,
8309
- });
8310
7486
  }
8311
- await startParentTerminalSignal();
8312
7487
  recordRunnerPerfTrace({
8313
7488
  req,
8314
7489
  phase: 'runner.execute_total',
@@ -8364,7 +7539,7 @@ async function executeRunRequest(
8364
7539
  ? `${message} ${receiptSalvageFailureSuffix(salvage)}`
8365
7540
  : message;
8366
7541
  const terminalLedgerMessage = redactSecretsFromLogString(terminalMessage);
8367
- if (options?.persistResultDatasets || req.playCallGovernance) {
7542
+ if (options?.persistResultDatasets) {
8368
7543
  appendRunLogLine(
8369
7544
  `${aborted ? '[cancelled]' : '[error]'} ${terminalLedgerMessage}`,
8370
7545
  );
@@ -8398,13 +7573,9 @@ async function executeRunRequest(
8398
7573
  });
8399
7574
  recordRunnerPerfTrace({
8400
7575
  req,
8401
- phase: req.playCallGovernance
8402
- ? aborted
8403
- ? 'runner.child_terminal_ledger_append_cancelled'
8404
- : 'runner.child_terminal_ledger_append_failed'
8405
- : aborted
8406
- ? 'runner.terminal_ledger_append_cancelled'
8407
- : 'runner.terminal_ledger_append_failed',
7576
+ phase: aborted
7577
+ ? 'runner.terminal_ledger_append_cancelled'
7578
+ : 'runner.terminal_ledger_append_failed',
8408
7579
  ms: nowMs() - terminalUpdateStartedAt,
8409
7580
  extra: {
8410
7581
  errorCode: failure.code,
@@ -8445,11 +7616,6 @@ async function executeRunRequest(
8445
7616
  });
8446
7617
  }
8447
7618
  }
8448
- await signalParentPlayTerminal({
8449
- req,
8450
- status: aborted ? 'cancelled' : 'failed',
8451
- error: terminalMessage,
8452
- }).catch(() => null);
8453
7619
  recordRunnerPerfTrace({
8454
7620
  req,
8455
7621
  phase: aborted ? 'runner.execute_cancelled' : 'runner.execute_failed',
@@ -8600,12 +7766,6 @@ function runRequestFromWorkflowParams(
8600
7766
  executionPlan: isExecutionPlan(params.executionPlan)
8601
7767
  ? (params.executionPlan as ExecutionPlan)
8602
7768
  : null,
8603
- childPlayManifests: isRecord(params.childPlayManifests)
8604
- ? (params.childPlayManifests as PlayRuntimeManifestMap)
8605
- : null,
8606
- playCallGovernance: isPlayCallGovernanceSnapshot(params.playCallGovernance)
8607
- ? params.playCallGovernance
8608
- : null,
8609
7769
  preloadedDbSessions: Array.isArray(params.preloadedDbSessions)
8610
7770
  ? (params.preloadedDbSessions as PreloadedRuntimeDbSession[])
8611
7771
  : null,
@@ -8642,22 +7802,6 @@ function isExecutionPlan(value: unknown): value is ExecutionPlan {
8642
7802
  );
8643
7803
  }
8644
7804
 
8645
- function isPlayCallGovernanceSnapshot(
8646
- value: unknown,
8647
- ): value is PlayCallGovernanceSnapshot {
8648
- return (
8649
- isRecord(value) &&
8650
- typeof value.rootRunId === 'string' &&
8651
- typeof value.parentRunId === 'string' &&
8652
- typeof value.parentPlayName === 'string' &&
8653
- typeof value.key === 'string' &&
8654
- Array.isArray(value.ancestryPlayIds) &&
8655
- value.ancestryPlayIds.every((entry) => typeof entry === 'string') &&
8656
- typeof value.callDepth === 'number' &&
8657
- Number.isFinite(value.callDepth)
8658
- );
8659
- }
8660
-
8661
7805
  async function persistResultDatasets(
8662
7806
  req: RunRequest,
8663
7807
  resultDatasets: ProjectedResultDatasetHandle[],
@@ -8983,7 +8127,7 @@ export class TenantWorkflow extends WorkflowEntrypoint<
8983
8127
  },
8984
8128
  step,
8985
8129
  {
8986
- persistResultDatasets: !req.playCallGovernance,
8130
+ persistResultDatasets: true,
8987
8131
  abortController,
8988
8132
  waitUntil: (promise) => this.ctx.waitUntil(promise),
8989
8133
  },