deepline 0.1.208 → 0.1.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +0 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +102 -138
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +2 -12
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +12 -4
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +25 -92
- package/dist/bundling-sources/sdk/src/client.ts +10 -0
- package/dist/bundling-sources/sdk/src/play.ts +2 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +16 -31
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +4 -5
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +316 -232
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +16 -2
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts +0 -5
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +2 -4
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +429 -102
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +27 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +31 -12
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +105 -10
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +21 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +32 -86
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +141 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +180 -447
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-kind-guard.ts +112 -0
- package/dist/cli/index.js +33 -8
- package/dist/cli/index.mjs +33 -8
- package/dist/index.d.mts +11 -2
- package/dist/index.d.ts +11 -2
- package/dist/index.js +26 -4
- package/dist/index.mjs +26 -4
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-lifecycle.ts +0 -70
|
@@ -51,11 +51,6 @@ import {
|
|
|
51
51
|
PLAY_RUNTIME_API_COMPAT_PATH,
|
|
52
52
|
PLAY_RUNTIME_CHILD_EXECUTOR_TOKEN_COMPAT_PATH,
|
|
53
53
|
} from './runtime-api-paths';
|
|
54
|
-
import {
|
|
55
|
-
settleInlineChildRunViaAppRuntime,
|
|
56
|
-
startRunViaAppRuntime,
|
|
57
|
-
} from './app-runtime-api';
|
|
58
|
-
import { executeChildRunLifecycle } from './child-run-lifecycle';
|
|
59
54
|
import {
|
|
60
55
|
createRootRunExecutionScope,
|
|
61
56
|
deriveChildRunExecutionScope,
|
|
@@ -79,6 +74,7 @@ import {
|
|
|
79
74
|
type RuntimeResourceGovernor,
|
|
80
75
|
} from './resource-governor';
|
|
81
76
|
import { CTX_FETCH_EGRESS_TOOL_ID } from './builtin-pacing';
|
|
77
|
+
import { ProviderExhaustedError } from './run-failure';
|
|
82
78
|
import { InMemoryRateStateBackend } from './governor/in-memory-rate-state-backend';
|
|
83
79
|
import { pacingPolicyForTool } from './pacing';
|
|
84
80
|
import {
|
|
@@ -111,7 +107,7 @@ import {
|
|
|
111
107
|
} from './tool-execute-retry-policy';
|
|
112
108
|
import {
|
|
113
109
|
buildDurableCtxCallCacheKey,
|
|
114
|
-
|
|
110
|
+
buildDurableRunPlayInvocationScope,
|
|
115
111
|
buildDurableToolAggregateProviderIdempotencyKey,
|
|
116
112
|
buildDurableToolAggregateReceiptKey,
|
|
117
113
|
buildDurableToolCallAuthScopeDigest,
|
|
@@ -136,7 +132,7 @@ import {
|
|
|
136
132
|
isQueryResultDatasetReadRequest,
|
|
137
133
|
isQueryResultDatasetTool,
|
|
138
134
|
} from './query-result-dataset';
|
|
139
|
-
import {
|
|
135
|
+
import { isRowIsolationExemptError } from './row-isolation';
|
|
140
136
|
import {
|
|
141
137
|
createRuntimePersistenceLatch,
|
|
142
138
|
RuntimePersistenceCircuitOpenError,
|
|
@@ -241,7 +237,10 @@ import {
|
|
|
241
237
|
type StepProgramDatasetOptions,
|
|
242
238
|
} from './step-program-dataset-builder';
|
|
243
239
|
import { readRuntimeSheetDatasetRows } from './runtime-api';
|
|
244
|
-
import {
|
|
240
|
+
import {
|
|
241
|
+
resolveChildExecutionStrategy,
|
|
242
|
+
inlineChildScheduledInRowMessage,
|
|
243
|
+
} from './child-execution-strategy';
|
|
245
244
|
|
|
246
245
|
type ResolvedPlayExecutor = (
|
|
247
246
|
ctx: PlayContextImpl,
|
|
@@ -292,6 +291,14 @@ const CHILD_PLAY_LAUNCH_ADMISSION_RETRY_DELAYS_MS = [
|
|
|
292
291
|
] as const;
|
|
293
292
|
const NODE_RUNTIME_MAP_VISIBILITY_MAX_ATTEMPTS = 100;
|
|
294
293
|
const NODE_RUNTIME_MAP_VISIBILITY_RETRY_MS = 25;
|
|
294
|
+
// Per-row sanity cap on distinct inline child invocations. `maxPlayCallDepth`
|
|
295
|
+
// (governor) already bounds nesting; this bounds fan-WIDTH from a single row so
|
|
296
|
+
// a resolver that loops `ctx.runPlay` unbounded fails loudly instead of leaking
|
|
297
|
+
// composition namespaces. Well above any legitimate authored per-row fan-out.
|
|
298
|
+
const MAX_INLINE_CHILD_INVOCATIONS_PER_ROW = 512;
|
|
299
|
+
// Bound the retained failure detail carried in parent aggregates so a fully
|
|
300
|
+
// failing large map cannot grow the progress event without limit.
|
|
301
|
+
const MAX_INLINE_CHILD_FAILURE_DETAIL = 25;
|
|
295
302
|
// A single batched runtime-step-receipt request (get/claim/complete/fail/seed)
|
|
296
303
|
// carries at most this many keys. At map scale a run can hold 5k-10k receipt
|
|
297
304
|
// keys; sending them all in one request made the server claim/complete them
|
|
@@ -812,7 +819,7 @@ function stableDigest(value: string): string {
|
|
|
812
819
|
return sha256Hex(value);
|
|
813
820
|
}
|
|
814
821
|
|
|
815
|
-
type DurableCtxOperation = 'step' | 'tool' | 'fetch'
|
|
822
|
+
type DurableCtxOperation = 'step' | 'tool' | 'fetch';
|
|
816
823
|
|
|
817
824
|
function durableCtxKey(input: {
|
|
818
825
|
orgId?: string | null;
|
|
@@ -1088,6 +1095,30 @@ export class PlayContextImpl {
|
|
|
1088
1095
|
createSecretRedactionContext();
|
|
1089
1096
|
private mapInvocationIndex = 0;
|
|
1090
1097
|
private readonly stepCallIndexByKey = new Map<string, number>();
|
|
1098
|
+
/**
|
|
1099
|
+
* Registry of distinct inline child invocation namespaces seen by this
|
|
1100
|
+
* context. Inline child composition is a function call, not a run; each
|
|
1101
|
+
* invocation's namespace (`child:<playName>#<callKey>[@<rowScope>]`) scopes
|
|
1102
|
+
* its receipts under the parent run. See ADR 0013.
|
|
1103
|
+
*/
|
|
1104
|
+
private readonly inlineChildInvocationOrdinals = new Map<string, number>();
|
|
1105
|
+
/**
|
|
1106
|
+
* Distinct inline child invocations per row scope (top-level counts as one
|
|
1107
|
+
* scope). Backs the per-row fan-width sanity cap so a resolver looping
|
|
1108
|
+
* `ctx.runPlay` unbounded fails loudly instead of leaking namespaces.
|
|
1109
|
+
*/
|
|
1110
|
+
private readonly inlineChildInvocationsPerRow = new Map<string, number>();
|
|
1111
|
+
/**
|
|
1112
|
+
* Parent-level inline-child aggregates. Maintained only by the single-writer
|
|
1113
|
+
* progress path (never per child) so concurrent fan-out cannot contend. See
|
|
1114
|
+
* `flushInlineChildAggregates` and ADR 0013.
|
|
1115
|
+
*/
|
|
1116
|
+
private inlineChildAggregates: {
|
|
1117
|
+
total: number;
|
|
1118
|
+
ok: number;
|
|
1119
|
+
failed: number;
|
|
1120
|
+
failures: Array<{ childPlayName: string; error: string }>;
|
|
1121
|
+
} = { total: 0, ok: 0, failed: 0, failures: [] };
|
|
1091
1122
|
/**
|
|
1092
1123
|
* Runtime persistence-failure circuit breaker (postgres_fast parity with the
|
|
1093
1124
|
* Workers runtime). The first persistence failure — a receipt-completion
|
|
@@ -4906,6 +4937,9 @@ export class PlayContextImpl {
|
|
|
4906
4937
|
completedRowKeys.size,
|
|
4907
4938
|
failedRows,
|
|
4908
4939
|
totalRows,
|
|
4940
|
+
// Inline child aggregates ride the single-writer progress event so
|
|
4941
|
+
// fan-out never contends on a per-child mutation. See ADR 0013.
|
|
4942
|
+
...this.inlineChildAggregateEventFields(),
|
|
4909
4943
|
at: Date.now(),
|
|
4910
4944
|
} as PlayExecutionEvent);
|
|
4911
4945
|
}
|
|
@@ -5448,34 +5482,66 @@ export class PlayContextImpl {
|
|
|
5448
5482
|
const produced: Record<string, unknown> = {};
|
|
5449
5483
|
for (const step of program.steps) {
|
|
5450
5484
|
const stepPath = [...path, step.name];
|
|
5451
|
-
const
|
|
5452
|
-
|
|
5453
|
-
|
|
5454
|
-
|
|
5455
|
-
|
|
5456
|
-
|
|
5457
|
-
|
|
5458
|
-
|
|
5459
|
-
stepPath,
|
|
5460
|
-
undefined,
|
|
5461
|
-
),
|
|
5462
|
-
{
|
|
5463
|
-
semanticKey: stableDigest(
|
|
5464
|
-
stableStringify({
|
|
5485
|
+
const runStep = async () =>
|
|
5486
|
+
options?.checkpointSteps
|
|
5487
|
+
? await this.step(
|
|
5488
|
+
stepPath.join('.'),
|
|
5489
|
+
async () =>
|
|
5490
|
+
await this.executeStepProgramStep(
|
|
5491
|
+
step,
|
|
5492
|
+
currentRow,
|
|
5465
5493
|
index,
|
|
5466
|
-
input: currentRow,
|
|
5467
5494
|
stepPath,
|
|
5468
|
-
|
|
5469
|
-
|
|
5470
|
-
|
|
5471
|
-
|
|
5472
|
-
|
|
5473
|
-
|
|
5474
|
-
|
|
5475
|
-
|
|
5476
|
-
|
|
5477
|
-
|
|
5478
|
-
|
|
5495
|
+
undefined,
|
|
5496
|
+
),
|
|
5497
|
+
{
|
|
5498
|
+
semanticKey: stableDigest(
|
|
5499
|
+
stableStringify({
|
|
5500
|
+
index,
|
|
5501
|
+
input: currentRow,
|
|
5502
|
+
stepPath,
|
|
5503
|
+
}),
|
|
5504
|
+
),
|
|
5505
|
+
},
|
|
5506
|
+
)
|
|
5507
|
+
: await this.executeStepProgramStep(
|
|
5508
|
+
step,
|
|
5509
|
+
currentRow,
|
|
5510
|
+
index,
|
|
5511
|
+
stepPath,
|
|
5512
|
+
undefined,
|
|
5513
|
+
);
|
|
5514
|
+
let value: unknown;
|
|
5515
|
+
try {
|
|
5516
|
+
value = await runStep();
|
|
5517
|
+
} catch (error) {
|
|
5518
|
+
// PROVIDER_EXHAUSTED is a step MISS, not a program abort. The pacer
|
|
5519
|
+
// skipped this provider (no spend, no dispatch); resolve the step as an
|
|
5520
|
+
// empty result so the row falls through to the next step's `runIf` chain
|
|
5521
|
+
// exactly as it would for a provider miss. Any OTHER error keeps its
|
|
5522
|
+
// existing propagation (row-isolation / abort semantics unchanged).
|
|
5523
|
+
if (!(error instanceof ProviderExhaustedError)) throw error;
|
|
5524
|
+
value = null;
|
|
5525
|
+
const rowStore = rowContext.getStore();
|
|
5526
|
+
const fieldName = stepPath.join('.');
|
|
5527
|
+
if (rowStore) {
|
|
5528
|
+
this.emitScopedFieldMetaUpdate({
|
|
5529
|
+
rowId: rowStore.rowId,
|
|
5530
|
+
key: rowStore.rowKey ?? null,
|
|
5531
|
+
tableNamespace: rowStore.tableNamespace ?? null,
|
|
5532
|
+
fieldName,
|
|
5533
|
+
status: 'failed',
|
|
5534
|
+
rowStatus: 'running',
|
|
5535
|
+
stage: 'failed',
|
|
5536
|
+
provider: null,
|
|
5537
|
+
error: this.formatRuntimeError(error),
|
|
5538
|
+
dataPatch: {},
|
|
5539
|
+
});
|
|
5540
|
+
}
|
|
5541
|
+
produced[step.name] = value;
|
|
5542
|
+
currentRow = cloneCsvAliasedRow(currentRow, { [step.name]: value });
|
|
5543
|
+
continue;
|
|
5544
|
+
}
|
|
5479
5545
|
produced[step.name] = value;
|
|
5480
5546
|
const rowStore = rowContext.getStore();
|
|
5481
5547
|
const fieldName = stepPath.join('.');
|
|
@@ -6304,8 +6370,6 @@ export class PlayContextImpl {
|
|
|
6304
6370
|
`Unable to resolve play "${resolvedName}" for ctx.runPlay(...).`,
|
|
6305
6371
|
);
|
|
6306
6372
|
}
|
|
6307
|
-
const childRevisionFingerprint =
|
|
6308
|
-
resolvedPlayRevisionFingerprint(resolvedPlay);
|
|
6309
6373
|
const childExecutionDecision = resolveChildExecutionStrategy({
|
|
6310
6374
|
pipeline: resolvedPlay.staticPipeline,
|
|
6311
6375
|
});
|
|
@@ -6319,14 +6383,40 @@ export class PlayContextImpl {
|
|
|
6319
6383
|
`ctx.runPlay(${normalizedKey}): ${childExecutionDecision.strategy} (${childExecutionDecision.reason})`,
|
|
6320
6384
|
);
|
|
6321
6385
|
const runPlayRowScope = rowContext.getStore();
|
|
6322
|
-
|
|
6386
|
+
// Runtime enforcement of the row-scoped-scheduled-child rule for children
|
|
6387
|
+
// whose name is only known at runtime (static preflight cannot see them).
|
|
6388
|
+
// A scheduling-required child fanned out per row is the 100,000-run-doc
|
|
6389
|
+
// failure mode the inline-child cutover removes. Fail on the FIRST row,
|
|
6390
|
+
// loud, with the SAME message the static preflight emits. See ADR 0013.
|
|
6391
|
+
//
|
|
6392
|
+
// Mirrors the static check's proof standard: reject when the child's
|
|
6393
|
+
// resolved contract PROVES it needs scheduling (dataset/csv, event-wait,
|
|
6394
|
+
// explicit timeout), or when this call would actually launch a durable
|
|
6395
|
+
// scheduled child run per row. A missing static contract alone is not
|
|
6396
|
+
// proof — in-process fallback runtimes legitimately run contract-less
|
|
6397
|
+
// scalar children inline.
|
|
6398
|
+
const provablySchedulingRequired =
|
|
6399
|
+
childExecutionDecision.strategy === 'scheduled' &&
|
|
6400
|
+
childExecutionDecision.reason !== 'missing_static_contract';
|
|
6401
|
+
if (
|
|
6402
|
+
runPlayRowScope &&
|
|
6403
|
+
(provablySchedulingRequired || launchThroughScheduler)
|
|
6404
|
+
) {
|
|
6405
|
+
throw new Error(
|
|
6406
|
+
inlineChildScheduledInRowMessage(
|
|
6407
|
+
resolvedName,
|
|
6408
|
+
runPlayRowScope.tableNamespace ??
|
|
6409
|
+
runPlayRowScope.fieldName ??
|
|
6410
|
+
'dataset',
|
|
6411
|
+
),
|
|
6412
|
+
);
|
|
6413
|
+
}
|
|
6414
|
+
const runPlayInvocationScope = buildDurableRunPlayInvocationScope({
|
|
6323
6415
|
childPlayName: resolvedName,
|
|
6324
|
-
childRevisionFingerprint,
|
|
6325
6416
|
input,
|
|
6326
6417
|
rowScope: runPlayRowScope
|
|
6327
6418
|
? {
|
|
6328
6419
|
fieldName: runPlayRowScope.fieldName,
|
|
6329
|
-
rowId: runPlayRowScope.rowId,
|
|
6330
6420
|
rowKey: runPlayRowScope.rowKey ?? null,
|
|
6331
6421
|
tableNamespace: runPlayRowScope.tableNamespace ?? null,
|
|
6332
6422
|
}
|
|
@@ -6339,33 +6429,35 @@ export class PlayContextImpl {
|
|
|
6339
6429
|
// cycle guard, depth cap, per-parent cap, and play/descendant charges, so
|
|
6340
6430
|
// a recovered receipt (which skips this body) never consumes budget.
|
|
6341
6431
|
//
|
|
6342
|
-
// The
|
|
6343
|
-
// in-process
|
|
6344
|
-
// FORMAT
|
|
6345
|
-
//
|
|
6346
|
-
//
|
|
6347
|
-
//
|
|
6348
|
-
//
|
|
6349
|
-
//
|
|
6350
|
-
const childRunId = buildChildRunId({
|
|
6351
|
-
childPlayName: resolvedName,
|
|
6352
|
-
parentRunId: this.currentRunId,
|
|
6353
|
-
parentPlayName: this.#options.playName,
|
|
6354
|
-
key: normalizedKey,
|
|
6355
|
-
runPlaySemanticKey,
|
|
6356
|
-
input,
|
|
6357
|
-
graphHash: null,
|
|
6358
|
-
});
|
|
6432
|
+
// The SCHEDULED substrate still allocates a durable child run id (through
|
|
6433
|
+
// the shared canonical builder so in-process and the workers_edge
|
|
6434
|
+
// coordinator produce the same FORMAT `play/<slug>/run/child-<digest>`).
|
|
6435
|
+
// INLINE scalar children are pure function composition (ADR 0013): they
|
|
6436
|
+
// carry NO durable run identity — no child run id, no playRuns doc, no
|
|
6437
|
+
// settlement. Their only identity is a replay-stable, call-site scoped
|
|
6438
|
+
// composition namespace (`child:<playName>#<ordinal>`) used to scope tool
|
|
6439
|
+
// receipts under the PARENT run. See child-composition-namespace below.
|
|
6359
6440
|
const inlineChildGovernor = launchThroughScheduler
|
|
6360
6441
|
? null
|
|
6361
6442
|
: await this.governor.forkInlineChild({
|
|
6362
6443
|
childPlayName: resolvedName,
|
|
6363
|
-
childRunId
|
|
6444
|
+
childRunId: this.inlineChildCompositionNamespace(
|
|
6445
|
+
resolvedName,
|
|
6446
|
+
normalizedKey,
|
|
6447
|
+
),
|
|
6364
6448
|
});
|
|
6365
6449
|
const childGovernance = launchThroughScheduler
|
|
6366
6450
|
? await this.governor.forkChild({
|
|
6367
6451
|
childPlayName: resolvedName,
|
|
6368
|
-
childRunId
|
|
6452
|
+
childRunId: buildChildRunId({
|
|
6453
|
+
childPlayName: resolvedName,
|
|
6454
|
+
parentRunId: this.currentRunId,
|
|
6455
|
+
parentPlayName: this.#options.playName,
|
|
6456
|
+
key: normalizedKey,
|
|
6457
|
+
runPlaySemanticKey: runPlayInvocationScope,
|
|
6458
|
+
input,
|
|
6459
|
+
graphHash: null,
|
|
6460
|
+
}),
|
|
6369
6461
|
})
|
|
6370
6462
|
: inlineChildGovernor!.snapshot();
|
|
6371
6463
|
const childPlaySlot = launchThroughScheduler
|
|
@@ -6377,7 +6469,9 @@ export class PlayContextImpl {
|
|
|
6377
6469
|
id: normalizedKey,
|
|
6378
6470
|
playId: resolvedName,
|
|
6379
6471
|
displayName: displayNameFromProducerId(resolvedName),
|
|
6380
|
-
|
|
6472
|
+
// For scheduled children this is the durable child run id; for inline
|
|
6473
|
+
// children it is the composition namespace (`child:<playName>#<ord>`).
|
|
6474
|
+
runId: childGovernance.currentRunId,
|
|
6381
6475
|
};
|
|
6382
6476
|
try {
|
|
6383
6477
|
if (rowStore) {
|
|
@@ -6490,112 +6584,93 @@ export class PlayContextImpl {
|
|
|
6490
6584
|
this.consumePendingChildPlaySuspension() ?? suspension,
|
|
6491
6585
|
);
|
|
6492
6586
|
}
|
|
6587
|
+
// Inline child = pure function composition (ADR 0013). No durable
|
|
6588
|
+
// child run: no playRuns doc, no start/settle round-trip, no
|
|
6589
|
+
// OptimisticConcurrencyControlFailure fan-out. The child's tool
|
|
6590
|
+
// receipts still land durably under the PARENT run (the inline
|
|
6591
|
+
// execution scope keeps the parent's receipt.ownerRunId) scoped by the
|
|
6592
|
+
// replay-stable composition namespace (which also enforces the
|
|
6593
|
+
// per-row inline-invocation sanity cap at creation).
|
|
6594
|
+
const compositionNamespace = childGovernance.currentRunId;
|
|
6493
6595
|
const childExecutionScope = deriveChildRunExecutionScope(
|
|
6494
6596
|
this.executionScope,
|
|
6495
6597
|
{
|
|
6496
6598
|
placement: 'inline',
|
|
6497
|
-
runId:
|
|
6599
|
+
runId: compositionNamespace,
|
|
6498
6600
|
playId: resolvedName,
|
|
6499
|
-
receiptNamespace:
|
|
6601
|
+
receiptNamespace: compositionNamespace,
|
|
6500
6602
|
},
|
|
6501
6603
|
);
|
|
6502
|
-
|
|
6503
|
-
|
|
6504
|
-
|
|
6505
|
-
|
|
6506
|
-
|
|
6507
|
-
|
|
6508
|
-
|
|
6509
|
-
|
|
6510
|
-
|
|
6511
|
-
|
|
6512
|
-
|
|
6513
|
-
|
|
6514
|
-
|
|
6515
|
-
|
|
6516
|
-
|
|
6517
|
-
|
|
6518
|
-
|
|
6519
|
-
|
|
6520
|
-
|
|
6521
|
-
|
|
6522
|
-
|
|
6523
|
-
|
|
6524
|
-
}
|
|
6525
|
-
|
|
6526
|
-
|
|
6527
|
-
|
|
6528
|
-
|
|
6529
|
-
|
|
6530
|
-
|
|
6531
|
-
|
|
6532
|
-
|
|
6533
|
-
|
|
6534
|
-
|
|
6535
|
-
|
|
6536
|
-
|
|
6537
|
-
...this.#options,
|
|
6538
|
-
executionScope: childExecutionScope,
|
|
6539
|
-
playId: resolvedName,
|
|
6540
|
-
playName: resolvedName,
|
|
6541
|
-
runId: childGovernance.currentRunId,
|
|
6542
|
-
executorToken: inlineScalarChild
|
|
6543
|
-
? this.#options.executorToken
|
|
6544
|
-
: await this.mintChildExecutorToken({
|
|
6545
|
-
parentRunId: this.currentRunId,
|
|
6546
|
-
parentPlayName: this.#options.playName,
|
|
6547
|
-
childRunId: childGovernance.currentRunId,
|
|
6548
|
-
childPlayName: resolvedName,
|
|
6549
|
-
}),
|
|
6550
|
-
staticPipeline: resolvedPlay.staticPipeline ?? null,
|
|
6551
|
-
checkpoint: this.checkpoint,
|
|
6552
|
-
governance: undefined,
|
|
6553
|
-
executionGovernor: inlineChildGovernor!,
|
|
6554
|
-
...(inlineScalarChild
|
|
6555
|
-
? {}
|
|
6556
|
-
: {
|
|
6557
|
-
getRuntimeStepReceipt: undefined,
|
|
6558
|
-
getRuntimeStepReceipts: undefined,
|
|
6559
|
-
claimRuntimeStepReceipt: undefined,
|
|
6560
|
-
claimRuntimeStepReceipts: undefined,
|
|
6561
|
-
completeRuntimeStepReceipt: undefined,
|
|
6562
|
-
completeRuntimeStepReceipts: undefined,
|
|
6563
|
-
failRuntimeStepReceipt: undefined,
|
|
6564
|
-
failRuntimeStepReceipts: undefined,
|
|
6565
|
-
heartbeatRuntimeStepReceipts: undefined,
|
|
6566
|
-
skipRuntimeStepReceipt: undefined,
|
|
6567
|
-
}),
|
|
6568
|
-
});
|
|
6569
|
-
const childExecution = this.executeResolvedPlay(
|
|
6570
|
-
resolvedPlay,
|
|
6571
|
-
childContext,
|
|
6572
|
-
input,
|
|
6573
|
-
);
|
|
6574
|
-
await childContext.drainQueuedWork([childExecution]);
|
|
6575
|
-
const result = await childExecution;
|
|
6576
|
-
if (rowStore) {
|
|
6577
|
-
this.emitScopedFieldMetaUpdate({
|
|
6578
|
-
rowId: rowStore.rowId,
|
|
6579
|
-
key: rowStore.rowKey ?? null,
|
|
6580
|
-
tableNamespace: rowStore.tableNamespace ?? null,
|
|
6581
|
-
fieldName: rowStore.fieldName,
|
|
6582
|
-
status: 'completed',
|
|
6583
|
-
rowStatus: 'running',
|
|
6584
|
-
stage: resolvedName,
|
|
6585
|
-
provider: 'deepline_native',
|
|
6586
|
-
error: null,
|
|
6587
|
-
producer,
|
|
6588
|
-
dataPatch: {},
|
|
6589
|
-
});
|
|
6590
|
-
}
|
|
6591
|
-
this.recordPlayCallStep({
|
|
6592
|
-
playId: resolvedName,
|
|
6593
|
-
description: options?.description,
|
|
6594
|
-
nestedSteps: childContext.getSteps(),
|
|
6595
|
-
});
|
|
6596
|
-
return result as TOutput;
|
|
6597
|
-
},
|
|
6604
|
+
this.inlineChildAggregates.total += 1;
|
|
6605
|
+
const inlineScalarChild =
|
|
6606
|
+
childExecutionDecision.reason === 'scalar_child';
|
|
6607
|
+
const childContext = createPlayContext({
|
|
6608
|
+
...this.#options,
|
|
6609
|
+
executionScope: childExecutionScope,
|
|
6610
|
+
playId: resolvedName,
|
|
6611
|
+
playName: resolvedName,
|
|
6612
|
+
runId: compositionNamespace,
|
|
6613
|
+
executorToken: inlineScalarChild
|
|
6614
|
+
? this.#options.executorToken
|
|
6615
|
+
: await this.mintChildExecutorToken({
|
|
6616
|
+
parentRunId: this.currentRunId,
|
|
6617
|
+
parentPlayName: this.#options.playName,
|
|
6618
|
+
childRunId: compositionNamespace,
|
|
6619
|
+
childPlayName: resolvedName,
|
|
6620
|
+
}),
|
|
6621
|
+
staticPipeline: resolvedPlay.staticPipeline ?? null,
|
|
6622
|
+
checkpoint: this.checkpoint,
|
|
6623
|
+
governance: undefined,
|
|
6624
|
+
executionGovernor: inlineChildGovernor!,
|
|
6625
|
+
...(inlineScalarChild
|
|
6626
|
+
? {}
|
|
6627
|
+
: {
|
|
6628
|
+
getRuntimeStepReceipt: undefined,
|
|
6629
|
+
getRuntimeStepReceipts: undefined,
|
|
6630
|
+
claimRuntimeStepReceipt: undefined,
|
|
6631
|
+
claimRuntimeStepReceipts: undefined,
|
|
6632
|
+
completeRuntimeStepReceipt: undefined,
|
|
6633
|
+
completeRuntimeStepReceipts: undefined,
|
|
6634
|
+
failRuntimeStepReceipt: undefined,
|
|
6635
|
+
failRuntimeStepReceipts: undefined,
|
|
6636
|
+
heartbeatRuntimeStepReceipts: undefined,
|
|
6637
|
+
skipRuntimeStepReceipt: undefined,
|
|
6638
|
+
}),
|
|
6598
6639
|
});
|
|
6640
|
+
try {
|
|
6641
|
+
const childExecution = this.executeResolvedPlay(
|
|
6642
|
+
resolvedPlay,
|
|
6643
|
+
childContext,
|
|
6644
|
+
input,
|
|
6645
|
+
);
|
|
6646
|
+
await childContext.drainQueuedWork([childExecution]);
|
|
6647
|
+
const result = await childExecution;
|
|
6648
|
+
this.inlineChildAggregates.ok += 1;
|
|
6649
|
+
if (rowStore) {
|
|
6650
|
+
this.emitScopedFieldMetaUpdate({
|
|
6651
|
+
rowId: rowStore.rowId,
|
|
6652
|
+
key: rowStore.rowKey ?? null,
|
|
6653
|
+
tableNamespace: rowStore.tableNamespace ?? null,
|
|
6654
|
+
fieldName: rowStore.fieldName,
|
|
6655
|
+
status: 'completed',
|
|
6656
|
+
rowStatus: 'running',
|
|
6657
|
+
stage: resolvedName,
|
|
6658
|
+
provider: 'deepline_native',
|
|
6659
|
+
error: null,
|
|
6660
|
+
producer,
|
|
6661
|
+
dataPatch: {},
|
|
6662
|
+
});
|
|
6663
|
+
}
|
|
6664
|
+
this.recordPlayCallStep({
|
|
6665
|
+
playId: resolvedName,
|
|
6666
|
+
description: options?.description,
|
|
6667
|
+
nestedSteps: childContext.getSteps(),
|
|
6668
|
+
});
|
|
6669
|
+
return result as TOutput;
|
|
6670
|
+
} catch (childError) {
|
|
6671
|
+
this.recordInlineChildFailure(resolvedName, childError);
|
|
6672
|
+
throw childError;
|
|
6673
|
+
}
|
|
6599
6674
|
} catch (error) {
|
|
6600
6675
|
if (isPlayExecutionSuspendedError(error)) {
|
|
6601
6676
|
throw error;
|
|
@@ -6621,24 +6696,7 @@ export class PlayContextImpl {
|
|
|
6621
6696
|
}
|
|
6622
6697
|
};
|
|
6623
6698
|
|
|
6624
|
-
return await
|
|
6625
|
-
'runPlay',
|
|
6626
|
-
normalizedKey,
|
|
6627
|
-
this.currentRunId,
|
|
6628
|
-
{
|
|
6629
|
-
semanticKey: runPlaySemanticKey,
|
|
6630
|
-
staleAfterSeconds: options?.staleAfterSeconds,
|
|
6631
|
-
markSkipped: () => {
|
|
6632
|
-
this.log(
|
|
6633
|
-
`ctx.runPlay(${normalizedKey}): no-op due completed receipt`,
|
|
6634
|
-
);
|
|
6635
|
-
},
|
|
6636
|
-
repairRunningReceiptForSameRunAfterWaitTimeout: true,
|
|
6637
|
-
runningReceiptWaitMaxAttempts:
|
|
6638
|
-
resolveRuntimeToolReceiptWaitMaxAttempts(input),
|
|
6639
|
-
execute: executePlayCall,
|
|
6640
|
-
},
|
|
6641
|
-
);
|
|
6699
|
+
return await executePlayCall();
|
|
6642
6700
|
} finally {
|
|
6643
6701
|
scheduledChildPlayCall?.release();
|
|
6644
6702
|
}
|
|
@@ -6695,65 +6753,91 @@ export class PlayContextImpl {
|
|
|
6695
6753
|
return body.executorToken.trim();
|
|
6696
6754
|
}
|
|
6697
6755
|
|
|
6698
|
-
|
|
6699
|
-
|
|
6700
|
-
|
|
6701
|
-
|
|
6702
|
-
|
|
6703
|
-
|
|
6704
|
-
|
|
6705
|
-
|
|
6706
|
-
|
|
6707
|
-
|
|
6708
|
-
|
|
6709
|
-
|
|
6710
|
-
|
|
6711
|
-
|
|
6712
|
-
|
|
6713
|
-
|
|
6714
|
-
|
|
6715
|
-
|
|
6716
|
-
|
|
6717
|
-
|
|
6718
|
-
|
|
6719
|
-
|
|
6720
|
-
|
|
6721
|
-
|
|
6722
|
-
|
|
6723
|
-
|
|
6724
|
-
|
|
6725
|
-
|
|
6726
|
-
}
|
|
6727
|
-
|
|
6756
|
+
/**
|
|
6757
|
+
* Replay-stable composition namespace for one inline `ctx.runPlay`
|
|
6758
|
+
* invocation. Format `child:<playName>#<callKey>[@<rowScope>]`:
|
|
6759
|
+
*
|
|
6760
|
+
* - `callKey` is the author's normalized `ctx.runPlay(key, ...)` call key,
|
|
6761
|
+
* so distinct call sites never collide.
|
|
6762
|
+
* - `rowScope` (table namespace + row key) is appended when the invocation
|
|
6763
|
+
* runs inside a map/dataset row resolver, so the SAME call site invoked
|
|
6764
|
+
* across rows scopes each row's child step receipts separately.
|
|
6765
|
+
*
|
|
6766
|
+
* The namespace is derived purely from authored, content-stable inputs —
|
|
6767
|
+
* never from arrival order — so a retried run attempt reproduces the same
|
|
6768
|
+
* namespace under any row-completion interleaving and the child's tool
|
|
6769
|
+
* receipts (owned by the PARENT run) recover instead of re-billing. Inline
|
|
6770
|
+
* children carry NO durable run id — this namespace is their only identity.
|
|
6771
|
+
* See ADR 0013.
|
|
6772
|
+
*/
|
|
6773
|
+
private inlineChildCompositionNamespace(
|
|
6774
|
+
childPlayName: string,
|
|
6775
|
+
normalizedKey: string,
|
|
6776
|
+
): string {
|
|
6777
|
+
const rowScope = rowContext.getStore();
|
|
6778
|
+
const rowScopeKey = rowScope
|
|
6779
|
+
? `${rowScope.tableNamespace ?? rowScope.fieldName ?? 'rows'}:${
|
|
6780
|
+
rowScope.rowKey ?? String(rowScope.rowId)
|
|
6781
|
+
}`
|
|
6782
|
+
: '';
|
|
6783
|
+
const namespace = `child:${childPlayName}#${normalizedKey}${
|
|
6784
|
+
rowScopeKey ? `@${rowScopeKey}` : ''
|
|
6785
|
+
}`;
|
|
6786
|
+
if (!this.inlineChildInvocationOrdinals.has(namespace)) {
|
|
6787
|
+
// Per-row fan-width guard. `maxPlayCallDepth` (governor) bounds nesting;
|
|
6788
|
+
// this bounds distinct inline invocations from ONE row scope so a
|
|
6789
|
+
// resolver looping runPlay unbounded fails loudly on that row instead of
|
|
6790
|
+
// leaking namespaces.
|
|
6791
|
+
const rowInvocationCount =
|
|
6792
|
+
(this.inlineChildInvocationsPerRow.get(rowScopeKey) ?? 0) + 1;
|
|
6793
|
+
if (rowInvocationCount > MAX_INLINE_CHILD_INVOCATIONS_PER_ROW) {
|
|
6794
|
+
throw new Error(
|
|
6795
|
+
`ctx.runPlay("${childPlayName}") exceeded the inline child invocation cap ` +
|
|
6796
|
+
`(${MAX_INLINE_CHILD_INVOCATIONS_PER_ROW} per row). A resolver is calling ` +
|
|
6797
|
+
'runPlay in an unbounded loop; give the batch ONE dataset child at play level instead.',
|
|
6798
|
+
);
|
|
6799
|
+
}
|
|
6800
|
+
this.inlineChildInvocationsPerRow.set(rowScopeKey, rowInvocationCount);
|
|
6801
|
+
this.inlineChildInvocationOrdinals.set(
|
|
6802
|
+
namespace,
|
|
6803
|
+
this.inlineChildInvocationOrdinals.size,
|
|
6804
|
+
);
|
|
6805
|
+
}
|
|
6806
|
+
return namespace;
|
|
6728
6807
|
}
|
|
6729
6808
|
|
|
6730
|
-
private
|
|
6731
|
-
|
|
6732
|
-
|
|
6733
|
-
|
|
6734
|
-
|
|
6735
|
-
|
|
6736
|
-
|
|
6737
|
-
|
|
6738
|
-
|
|
6809
|
+
private recordInlineChildFailure(
|
|
6810
|
+
childPlayName: string,
|
|
6811
|
+
error: unknown,
|
|
6812
|
+
): void {
|
|
6813
|
+
this.inlineChildAggregates.failed += 1;
|
|
6814
|
+
if (
|
|
6815
|
+
this.inlineChildAggregates.failures.length <
|
|
6816
|
+
MAX_INLINE_CHILD_FAILURE_DETAIL
|
|
6817
|
+
) {
|
|
6818
|
+
this.inlineChildAggregates.failures.push({
|
|
6819
|
+
childPlayName,
|
|
6820
|
+
error: this.formatRuntimeError(error),
|
|
6821
|
+
});
|
|
6822
|
+
}
|
|
6823
|
+
}
|
|
6739
6824
|
|
|
6740
|
-
|
|
6741
|
-
|
|
6742
|
-
|
|
6743
|
-
|
|
6744
|
-
|
|
6745
|
-
|
|
6746
|
-
|
|
6747
|
-
|
|
6748
|
-
|
|
6749
|
-
|
|
6750
|
-
|
|
6751
|
-
|
|
6752
|
-
|
|
6753
|
-
|
|
6754
|
-
|
|
6755
|
-
|
|
6756
|
-
);
|
|
6825
|
+
/**
|
|
6826
|
+
* Snapshot of inline child aggregates for the single-writer progress event.
|
|
6827
|
+
* Read-only; never mutates. Omitted entirely when no inline child has run so
|
|
6828
|
+
* plays without child composition keep byte-identical progress events.
|
|
6829
|
+
*/
|
|
6830
|
+
private inlineChildAggregateEventFields(): {
|
|
6831
|
+
childrenTotal?: number;
|
|
6832
|
+
childrenOk?: number;
|
|
6833
|
+
childrenFailed?: number;
|
|
6834
|
+
} {
|
|
6835
|
+
if (this.inlineChildAggregates.total === 0) return {};
|
|
6836
|
+
return {
|
|
6837
|
+
childrenTotal: this.inlineChildAggregates.total,
|
|
6838
|
+
childrenOk: this.inlineChildAggregates.ok,
|
|
6839
|
+
childrenFailed: this.inlineChildAggregates.failed,
|
|
6840
|
+
};
|
|
6757
6841
|
}
|
|
6758
6842
|
|
|
6759
6843
|
/**
|