deepline 0.1.208 → 0.1.210
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundling-sources/apps/play-runner-workers/src/coordinator-entry.ts +0 -2
- package/dist/bundling-sources/apps/play-runner-workers/src/entry.ts +102 -138
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/receipts.ts +2 -12
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-dispatch.ts +12 -4
- package/dist/bundling-sources/apps/play-runner-workers/src/runtime/tool-receipts.ts +25 -92
- package/dist/bundling-sources/sdk/src/client.ts +10 -0
- package/dist/bundling-sources/sdk/src/play.ts +2 -2
- package/dist/bundling-sources/sdk/src/release.ts +2 -2
- package/dist/bundling-sources/shared_libs/play-runtime/app-runtime-api.ts +16 -31
- package/dist/bundling-sources/shared_libs/play-runtime/child-execution-strategy.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-id.ts +4 -5
- package/dist/bundling-sources/shared_libs/play-runtime/context.ts +316 -232
- package/dist/bundling-sources/shared_libs/play-runtime/ctx-types.ts +16 -2
- package/dist/bundling-sources/shared_libs/play-runtime/daytona-runtime-config.ts +0 -5
- package/dist/bundling-sources/shared_libs/play-runtime/durable-call-cache.ts +2 -4
- package/dist/bundling-sources/shared_libs/play-runtime/execution-capabilities.ts +1 -0
- package/dist/bundling-sources/shared_libs/play-runtime/governor/app-runtime-rate-state-backend.ts +429 -102
- package/dist/bundling-sources/shared_libs/play-runtime/governor/governor.ts +27 -5
- package/dist/bundling-sources/shared_libs/play-runtime/governor/rate-state-backend.ts +18 -0
- package/dist/bundling-sources/shared_libs/play-runtime/protocol.ts +31 -12
- package/dist/bundling-sources/shared_libs/play-runtime/run-failure.ts +103 -0
- package/dist/bundling-sources/shared_libs/play-runtime/run-ledger.ts +105 -10
- package/dist/bundling-sources/shared_libs/play-runtime/run-snapshot-stream.ts +21 -1
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-lifecycle.ts +32 -86
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-payload-transport.ts +141 -2
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona-session-execution.ts +45 -0
- package/dist/bundling-sources/shared_libs/play-runtime/runner-backends/backends/daytona.ts +180 -447
- package/dist/bundling-sources/shared_libs/play-runtime/runtime-constants.ts +12 -0
- package/dist/bundling-sources/shared_libs/play-runtime/suspension.ts +33 -1
- package/dist/bundling-sources/shared_libs/play-runtime/tool-http-errors.ts +11 -0
- package/dist/bundling-sources/shared_libs/plays/artifact-kind-guard.ts +112 -0
- package/dist/cli/index.js +33 -8
- package/dist/cli/index.mjs +33 -8
- package/dist/index.d.mts +11 -2
- package/dist/index.d.ts +11 -2
- package/dist/index.js +26 -4
- package/dist/index.mjs +26 -4
- package/package.json +1 -1
- package/dist/bundling-sources/shared_libs/play-runtime/child-run-lifecycle.ts +0 -70
|
@@ -2445,8 +2445,6 @@ function runRequestFromPlayWorkflowParams(
|
|
|
2445
2445
|
preloadedDbSessions: params.preloadedDbSessions ?? null,
|
|
2446
2446
|
runtimeTestFaultHeader: params.runtimeTestFaultHeader ?? null,
|
|
2447
2447
|
testPolicyOverrides: params.testPolicyOverrides ?? null,
|
|
2448
|
-
inlineChildRunRegistered:
|
|
2449
|
-
params.runtimeBackend === 'cf_workflows_dynamic_worker_inline_child',
|
|
2450
2448
|
coordinatorUrl: params.coordinatorUrl ?? null,
|
|
2451
2449
|
coordinatorInternalToken: params.coordinatorInternalToken ?? null,
|
|
2452
2450
|
totalRows: params.totalRows,
|
|
@@ -32,6 +32,7 @@ import {
|
|
|
32
32
|
type WorkflowEvent,
|
|
33
33
|
type WorkflowStep,
|
|
34
34
|
} from 'cloudflare:workers';
|
|
35
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
35
36
|
import {
|
|
36
37
|
deterministicMapChunkStepName,
|
|
37
38
|
type ExecutionPlan,
|
|
@@ -40,7 +41,6 @@ import {
|
|
|
40
41
|
STANDARD_PLAY_RUNTIME_LIMIT_LABEL,
|
|
41
42
|
STANDARD_PLAY_RUNTIME_LIMIT_SECONDS,
|
|
42
43
|
} from '../../../shared_libs/play-runtime/runtime-constants';
|
|
43
|
-
import { childSubmitIdempotencyKey } from '../../../shared_libs/play-runtime/child-run-id';
|
|
44
44
|
import {
|
|
45
45
|
createPlayExecutionGovernor,
|
|
46
46
|
type GovernanceSnapshot,
|
|
@@ -104,7 +104,7 @@ import {
|
|
|
104
104
|
import { createDeferredPlayDataset } from '../../../shared_libs/plays/dataset';
|
|
105
105
|
import {
|
|
106
106
|
buildDurableCtxCallCacheKey,
|
|
107
|
-
|
|
107
|
+
buildDurableRunPlayInvocationScope,
|
|
108
108
|
} from '../../../shared_libs/play-runtime/durable-call-cache';
|
|
109
109
|
import {
|
|
110
110
|
resolveRuntimeToolReceiptWaitMaxAttempts,
|
|
@@ -368,8 +368,6 @@ type RunRequest = {
|
|
|
368
368
|
/** Internal ctx.runPlay lineage. Public SDK/users never see this. */
|
|
369
369
|
playCallGovernance?: PlayCallGovernanceSnapshot | null;
|
|
370
370
|
preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
|
|
371
|
-
/** Coordinator already created the child run row before invoking /run-inline. */
|
|
372
|
-
inlineChildRunRegistered?: boolean | null;
|
|
373
371
|
/** Cloudflare coordinator URL for direct Workflow control-plane signals. */
|
|
374
372
|
coordinatorUrl?: string | null;
|
|
375
373
|
/** Request-scoped coordinator auth token for preview/dev direct control calls. */
|
|
@@ -1148,11 +1146,29 @@ async function drainRunnerPerfTraces(req: RunRequest): Promise<void> {
|
|
|
1148
1146
|
|
|
1149
1147
|
let workerRequestSequence = 0;
|
|
1150
1148
|
|
|
1149
|
+
const workerRunPlayRowContext = new AsyncLocalStorage<{
|
|
1150
|
+
tableNamespace: string;
|
|
1151
|
+
rowKey: string;
|
|
1152
|
+
fieldName: string;
|
|
1153
|
+
}>();
|
|
1154
|
+
|
|
1151
1155
|
function makeRequestId(): string {
|
|
1152
1156
|
workerRequestSequence = (workerRequestSequence + 1) % Number.MAX_SAFE_INTEGER;
|
|
1153
1157
|
return `worker-request-${workerRequestSequence}`;
|
|
1154
1158
|
}
|
|
1155
1159
|
|
|
1160
|
+
async function executeFreshChildInvocation<T>(
|
|
1161
|
+
input: {
|
|
1162
|
+
parentRunId: string;
|
|
1163
|
+
key: string;
|
|
1164
|
+
invocationScope: string;
|
|
1165
|
+
},
|
|
1166
|
+
execute: (invocationId: string) => Promise<T>,
|
|
1167
|
+
): Promise<T> {
|
|
1168
|
+
const invocationId = `${input.parentRunId}:runPlay:${input.key}:${input.invocationScope}`;
|
|
1169
|
+
return await execute(invocationId);
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1156
1172
|
function makeRuntimeSheetAttempt(input: {
|
|
1157
1173
|
runId: string;
|
|
1158
1174
|
runAttempt?: number | null;
|
|
@@ -1665,27 +1681,6 @@ async function hashChildPlayEventKey(input: unknown): Promise<string> {
|
|
|
1665
1681
|
return (await hashJson(input)).slice(0, 32);
|
|
1666
1682
|
}
|
|
1667
1683
|
|
|
1668
|
-
function workerChildRevisionFingerprint(input: {
|
|
1669
|
-
playName: string;
|
|
1670
|
-
manifest: PlayRuntimeManifest | null | undefined;
|
|
1671
|
-
}): string | null {
|
|
1672
|
-
const manifest = input.manifest;
|
|
1673
|
-
if (!manifest) return null;
|
|
1674
|
-
return sha256Hex(
|
|
1675
|
-
stableStringify({
|
|
1676
|
-
playId: manifest.playName?.trim() || input.playName,
|
|
1677
|
-
codeFormat: 'cjs_module',
|
|
1678
|
-
artifactHash: manifest.artifactHash ?? null,
|
|
1679
|
-
graphHash: manifest.graphHash ?? null,
|
|
1680
|
-
sourceHash: null,
|
|
1681
|
-
sourceCodeHash:
|
|
1682
|
-
typeof manifest.sourceCode === 'string'
|
|
1683
|
-
? sha256Hex(manifest.sourceCode)
|
|
1684
|
-
: null,
|
|
1685
|
-
}),
|
|
1686
|
-
);
|
|
1687
|
-
}
|
|
1688
|
-
|
|
1689
1684
|
type RuntimeResolvedPlayResponse = {
|
|
1690
1685
|
manifest?: PlayRuntimeManifest | null;
|
|
1691
1686
|
};
|
|
@@ -5580,37 +5575,49 @@ function createMinimalWorkerCtx(
|
|
|
5580
5575
|
})
|
|
5581
5576
|
: null,
|
|
5582
5577
|
});
|
|
5583
|
-
const resolved = await
|
|
5584
|
-
|
|
5585
|
-
|
|
5586
|
-
|
|
5587
|
-
|
|
5588
|
-
|
|
5589
|
-
|
|
5590
|
-
|
|
5591
|
-
|
|
5592
|
-
|
|
5593
|
-
|
|
5594
|
-
|
|
5595
|
-
|
|
5596
|
-
|
|
5597
|
-
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5606
|
-
status
|
|
5607
|
-
|
|
5608
|
-
|
|
5578
|
+
const resolved = await workerRunPlayRowContext.run(
|
|
5579
|
+
{
|
|
5580
|
+
tableNamespace: name,
|
|
5581
|
+
rowKey: entry.rowKey,
|
|
5582
|
+
fieldName: key,
|
|
5583
|
+
},
|
|
5584
|
+
() =>
|
|
5585
|
+
executeWorkerStepResolver(
|
|
5586
|
+
value,
|
|
5587
|
+
enriched,
|
|
5588
|
+
rowCtx,
|
|
5589
|
+
absoluteIndex,
|
|
5590
|
+
previousCell,
|
|
5591
|
+
isWorkerStepProgram(value)
|
|
5592
|
+
? {
|
|
5593
|
+
parentField: key,
|
|
5594
|
+
path: [],
|
|
5595
|
+
outputs: stepProgramOutputs,
|
|
5596
|
+
onOutput: async (stepOutput) => {
|
|
5597
|
+
generatedOutputFields.add(
|
|
5598
|
+
stepOutput.columnName,
|
|
5599
|
+
);
|
|
5600
|
+
const status =
|
|
5601
|
+
stepOutput.status ?? 'completed';
|
|
5602
|
+
if (status === 'skipped')
|
|
5603
|
+
stepCellsSkipped += 1;
|
|
5604
|
+
else stepCellsCompleted += 1;
|
|
5605
|
+
await enqueueLiveRowUpdate({
|
|
5606
|
+
key: entry.rowKey,
|
|
5607
|
+
dataPatch: {
|
|
5608
|
+
[stepOutput.columnName]: stepOutput.value,
|
|
5609
|
+
},
|
|
5610
|
+
cellMetaPatch: {
|
|
5611
|
+
[stepOutput.columnName]: {
|
|
5612
|
+
status,
|
|
5613
|
+
stage: stepOutput.stepId,
|
|
5614
|
+
},
|
|
5615
|
+
},
|
|
5616
|
+
});
|
|
5609
5617
|
},
|
|
5610
|
-
}
|
|
5611
|
-
|
|
5612
|
-
|
|
5613
|
-
: undefined,
|
|
5618
|
+
}
|
|
5619
|
+
: undefined,
|
|
5620
|
+
),
|
|
5614
5621
|
);
|
|
5615
5622
|
enriched[key] = resolved.value;
|
|
5616
5623
|
fieldOutputs[key] = resolved.value;
|
|
@@ -6933,7 +6940,6 @@ function createMinimalWorkerCtx(
|
|
|
6933
6940
|
options?: {
|
|
6934
6941
|
description?: string;
|
|
6935
6942
|
timeoutMs?: number;
|
|
6936
|
-
staleAfterSeconds?: number;
|
|
6937
6943
|
},
|
|
6938
6944
|
): Promise<unknown> {
|
|
6939
6945
|
const normalizedKey = normalizeContextKey(key, 'runPlay');
|
|
@@ -6947,34 +6953,49 @@ function createMinimalWorkerCtx(
|
|
|
6947
6953
|
playRef: resolvedName,
|
|
6948
6954
|
cache: childManifestCache,
|
|
6949
6955
|
});
|
|
6950
|
-
const
|
|
6951
|
-
|
|
6952
|
-
|
|
6953
|
-
|
|
6954
|
-
|
|
6955
|
-
|
|
6956
|
-
|
|
6957
|
-
|
|
6958
|
-
|
|
6959
|
-
|
|
6960
|
-
|
|
6961
|
-
input,
|
|
6962
|
-
}),
|
|
6963
|
-
staleAfterSeconds: options?.staleAfterSeconds,
|
|
6956
|
+
const rowScope = workerRunPlayRowContext.getStore();
|
|
6957
|
+
const invocationScope = buildDurableRunPlayInvocationScope({
|
|
6958
|
+
childPlayName: resolvedName,
|
|
6959
|
+
input,
|
|
6960
|
+
rowScope: rowScope
|
|
6961
|
+
? {
|
|
6962
|
+
fieldName: rowScope.fieldName,
|
|
6963
|
+
rowKey: rowScope.rowKey,
|
|
6964
|
+
tableNamespace: rowScope.tableNamespace,
|
|
6965
|
+
}
|
|
6966
|
+
: null,
|
|
6964
6967
|
});
|
|
6965
|
-
return await
|
|
6966
|
-
|
|
6967
|
-
|
|
6968
|
+
return await executeFreshChildInvocation(
|
|
6969
|
+
{
|
|
6970
|
+
parentRunId: req.runId,
|
|
6971
|
+
key: normalizedKey,
|
|
6972
|
+
invocationScope,
|
|
6973
|
+
},
|
|
6974
|
+
async (childInvocationId) => {
|
|
6968
6975
|
// The Governor owns the play-call lineage: forkChild does the cycle
|
|
6969
6976
|
// guard, depth/per-parent/playCall/descendant budget charges, and
|
|
6970
6977
|
// returns the snapshot to thread into the child so budgets accumulate
|
|
6971
|
-
// across isolates.
|
|
6972
|
-
|
|
6973
|
-
|
|
6974
|
-
|
|
6975
|
-
|
|
6976
|
-
|
|
6977
|
-
|
|
6978
|
+
// across isolates. Every ctx.runPlay call is a fresh child invocation.
|
|
6979
|
+
const childRunId = `${req.runId}:child:${childInvocationId}`;
|
|
6980
|
+
// Checkpoint governance admission separately from the child result.
|
|
6981
|
+
// Workflow replay reuses this admission and the coordinator's stable
|
|
6982
|
+
// child launch identity, but ctx.runPlay never caches child output.
|
|
6983
|
+
const forkChild = () =>
|
|
6984
|
+
governor.forkChild({
|
|
6985
|
+
childPlayName: resolvedName,
|
|
6986
|
+
childRunId,
|
|
6987
|
+
});
|
|
6988
|
+
const childGovernance = workflowStep
|
|
6989
|
+
? await (
|
|
6990
|
+
workflowStep.do as unknown as <T>(
|
|
6991
|
+
name: string,
|
|
6992
|
+
callback: () => Promise<T>,
|
|
6993
|
+
) => Promise<T>
|
|
6994
|
+
)(
|
|
6995
|
+
`runPlay-governance:${sha256Hex(childInvocationId).slice(0, 24)}`,
|
|
6996
|
+
forkChild,
|
|
6997
|
+
)
|
|
6998
|
+
: await forkChild();
|
|
6978
6999
|
const nextDepth = childGovernance.callDepth;
|
|
6979
7000
|
const nextParentCalls =
|
|
6980
7001
|
governor.snapshot().parentChildCalls[req.playName] ?? 0;
|
|
@@ -7047,11 +7068,7 @@ function createMinimalWorkerCtx(
|
|
|
7047
7068
|
options?.timeoutMs == null && !childNeedsWorkflowScheduler,
|
|
7048
7069
|
body: {
|
|
7049
7070
|
name: resolvedName,
|
|
7050
|
-
childIdempotencyKey:
|
|
7051
|
-
receiptKey,
|
|
7052
|
-
wasFailed,
|
|
7053
|
-
leaseId: receiptContext.leaseId,
|
|
7054
|
-
}),
|
|
7071
|
+
childIdempotencyKey: childInvocationId,
|
|
7055
7072
|
input: isRecord(input) ? input : {},
|
|
7056
7073
|
orgId: req.orgId,
|
|
7057
7074
|
callbackUrl: req.callbackUrl,
|
|
@@ -7329,12 +7346,6 @@ function createMinimalWorkerCtx(
|
|
|
7329
7346
|
childPlaySlot?.release();
|
|
7330
7347
|
}
|
|
7331
7348
|
},
|
|
7332
|
-
{
|
|
7333
|
-
forceFailedRefresh: true,
|
|
7334
|
-
repairRunningReceiptForSameRunAfterWaitTimeout: true,
|
|
7335
|
-
runningReceiptWaitMaxAttempts:
|
|
7336
|
-
resolveRuntimeToolReceiptWaitMaxAttempts(input),
|
|
7337
|
-
},
|
|
7338
7349
|
);
|
|
7339
7350
|
},
|
|
7340
7351
|
async fetch(
|
|
@@ -7615,13 +7626,6 @@ async function handleRunInline(
|
|
|
7615
7626
|
const probeStartedAt = nowMs();
|
|
7616
7627
|
await probeHarnessOnce(env, runPrefix);
|
|
7617
7628
|
traceInline('inline.probe_harness', probeStartedAt);
|
|
7618
|
-
if (!req.inlineChildRunRegistered) {
|
|
7619
|
-
const registerStartedAt = nowMs();
|
|
7620
|
-
await registerInlineChildRun(req);
|
|
7621
|
-
traceInline('inline.register_child_run', registerStartedAt);
|
|
7622
|
-
} else {
|
|
7623
|
-
traceInline('inline.register_child_run', nowMs(), { skipped: true });
|
|
7624
|
-
}
|
|
7625
7629
|
const executeStartedAt = nowMs();
|
|
7626
7630
|
const output = await executeRunRequest(
|
|
7627
7631
|
req,
|
|
@@ -7667,46 +7671,6 @@ async function handleRunInline(
|
|
|
7667
7671
|
}
|
|
7668
7672
|
}
|
|
7669
7673
|
|
|
7670
|
-
async function registerInlineChildRun(req: RunRequest): Promise<void> {
|
|
7671
|
-
const snapshot = isRecord(req.contractSnapshot) ? req.contractSnapshot : {};
|
|
7672
|
-
const artifactMetadata = isRecord(snapshot.artifactMetadata)
|
|
7673
|
-
? snapshot.artifactMetadata
|
|
7674
|
-
: {};
|
|
7675
|
-
const governance = req.playCallGovernance;
|
|
7676
|
-
await postRuntimeApi(req.baseUrl, req.executorToken, {
|
|
7677
|
-
action: 'start_inline_child_run',
|
|
7678
|
-
playName: req.playName,
|
|
7679
|
-
runId: req.runId,
|
|
7680
|
-
parentRunId: governance?.parentRunId,
|
|
7681
|
-
rootRunId: governance?.rootRunId,
|
|
7682
|
-
workflowFamilyKey:
|
|
7683
|
-
governance?.rootRunId ?? governance?.parentRunId ?? req.runId,
|
|
7684
|
-
artifactStorageKey:
|
|
7685
|
-
typeof artifactMetadata.storageKey === 'string'
|
|
7686
|
-
? artifactMetadata.storageKey
|
|
7687
|
-
: undefined,
|
|
7688
|
-
artifactHash:
|
|
7689
|
-
typeof artifactMetadata.artifactHash === 'string'
|
|
7690
|
-
? artifactMetadata.artifactHash
|
|
7691
|
-
: undefined,
|
|
7692
|
-
graphHash:
|
|
7693
|
-
typeof artifactMetadata.graphHash === 'string'
|
|
7694
|
-
? artifactMetadata.graphHash
|
|
7695
|
-
: undefined,
|
|
7696
|
-
runtimeBackend: 'workers_edge',
|
|
7697
|
-
schedulerBackend: 'inline_child',
|
|
7698
|
-
executionProfile: 'workers_edge',
|
|
7699
|
-
maxCreditsPerRun: extractMaxCreditsPerRun(req.contractSnapshot),
|
|
7700
|
-
staticPipeline: snapshot.staticPipeline ?? null,
|
|
7701
|
-
source:
|
|
7702
|
-
snapshot.source === 'published' ||
|
|
7703
|
-
snapshot.source === 'ad_hoc' ||
|
|
7704
|
-
snapshot.source === 'draft'
|
|
7705
|
-
? snapshot.source
|
|
7706
|
-
: 'published',
|
|
7707
|
-
});
|
|
7708
|
-
}
|
|
7709
|
-
|
|
7710
7674
|
/** Cap on run log lines retained in the terminal output compatibility shape. */
|
|
7711
7675
|
const RUN_LOG_BUFFER_LIMIT = 500;
|
|
7712
7676
|
/** Min wall-clock interval between live run-ledger flushes during a run. */
|
|
@@ -4,7 +4,6 @@ import {
|
|
|
4
4
|
type WorkReceiptBatchMarkRunningCommand,
|
|
5
5
|
type WorkReceiptClaim,
|
|
6
6
|
type WorkReceiptCommand,
|
|
7
|
-
type WorkReceiptFailureKind,
|
|
8
7
|
type WorkReceiptGetCommand,
|
|
9
8
|
type WorkReceiptStatus,
|
|
10
9
|
} from '../../../../shared_libs/play-runtime/work-receipts';
|
|
@@ -17,6 +16,7 @@ import {
|
|
|
17
16
|
isPlayExecutionSuspendedError,
|
|
18
17
|
isPlayRowExecutionSuspendedError,
|
|
19
18
|
} from '../../../../shared_libs/play-runtime/suspension';
|
|
19
|
+
import { runtimeReceiptFailureKindForError } from '../../../../shared_libs/play-runtime/durable-receipt-execution';
|
|
20
20
|
import type { WorkerWorkBudgetMeter } from './work-budget';
|
|
21
21
|
|
|
22
22
|
export type RuntimeReceiptStatus = WorkReceiptStatus;
|
|
@@ -132,15 +132,6 @@ function errorMessage(error: unknown): string {
|
|
|
132
132
|
return error instanceof Error ? error.message : String(error);
|
|
133
133
|
}
|
|
134
134
|
|
|
135
|
-
function runtimeReceiptFailureKindForError(
|
|
136
|
-
error: unknown,
|
|
137
|
-
): WorkReceiptFailureKind {
|
|
138
|
-
return isPlayExecutionSuspendedError(error) ||
|
|
139
|
-
isPlayRowExecutionSuspendedError(error)
|
|
140
|
-
? 'repairable'
|
|
141
|
-
: 'terminal';
|
|
142
|
-
}
|
|
143
|
-
|
|
144
135
|
async function waitForReusableReceipt<T>(input: {
|
|
145
136
|
key: string;
|
|
146
137
|
playName: string;
|
|
@@ -363,8 +354,7 @@ async function executeAndPersistReceipt<T>(input: {
|
|
|
363
354
|
budgetMeter: input.budgetMeter,
|
|
364
355
|
heartbeatIntervalMs: input.heartbeatIntervalMs,
|
|
365
356
|
targets: [{ key: input.key, leaseId }],
|
|
366
|
-
execute: () =>
|
|
367
|
-
input.execute({ leaseId }, input.wasFailed === true),
|
|
357
|
+
execute: () => input.execute({ leaseId }, input.wasFailed === true),
|
|
368
358
|
});
|
|
369
359
|
} catch (error) {
|
|
370
360
|
if (error instanceof RuntimeLeaseLostError) {
|
|
@@ -54,6 +54,7 @@ import {
|
|
|
54
54
|
markWorkerToolReceiptResultCached,
|
|
55
55
|
markWorkerToolReceiptResultExecution,
|
|
56
56
|
planWorkerToolReceiptGroups,
|
|
57
|
+
resolveRunningReceiptWaitMaxAttempts,
|
|
57
58
|
resolveWorkerToolReceiptGroupWaitMaxAttempts,
|
|
58
59
|
resolveWorkerToolRuntimeTimeoutMs,
|
|
59
60
|
} from './tool-receipts';
|
|
@@ -1232,10 +1233,17 @@ export class WorkerToolBatchScheduler {
|
|
|
1232
1233
|
try {
|
|
1233
1234
|
const result = await this.waitForDurableToolReceipt({
|
|
1234
1235
|
receiptKey,
|
|
1235
|
-
maxAttempts:
|
|
1236
|
-
|
|
1237
|
-
|
|
1238
|
-
|
|
1236
|
+
maxAttempts: resolveRunningReceiptWaitMaxAttempts({
|
|
1237
|
+
ownerRunId: claim.receipt.runId,
|
|
1238
|
+
currentRunId: this.options.req.runId,
|
|
1239
|
+
updatedAt: claim.receipt.updatedAt,
|
|
1240
|
+
requestedMaxAttempts:
|
|
1241
|
+
resolveWorkerToolReceiptGroupWaitMaxAttempts(
|
|
1242
|
+
group,
|
|
1243
|
+
(request) => request.receiptWaitMaxAttempts,
|
|
1244
|
+
),
|
|
1245
|
+
nowMs: this.options.nowMs(),
|
|
1246
|
+
}),
|
|
1239
1247
|
});
|
|
1240
1248
|
if (groupState.forceDurableRefresh) {
|
|
1241
1249
|
return await this.claimForcedDurableToolReceiptGroupAfterWait(
|
|
@@ -39,106 +39,39 @@ export function canReclaimTimedOutWorkerToolReceipt(input: {
|
|
|
39
39
|
* conservative constant, not a heartbeat multiple.
|
|
40
40
|
*/
|
|
41
41
|
export const RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS = 60_000;
|
|
42
|
+
export const RUNNING_RECEIPT_POLL_INTERVAL_MS = 250;
|
|
42
43
|
|
|
43
|
-
/**
|
|
44
|
-
|
|
45
|
-
* the completion poll.
|
|
46
|
-
*
|
|
47
|
-
* The poll (`waitForCompletedRuntimeReceipt`) issues one service-binding
|
|
48
|
-
* subrequest per 250ms tick, up to the receipt's wait budget (~1320 ticks for a
|
|
49
|
-
* 5.5-minute default). On resume, a map can carry many receipts left `running`
|
|
50
|
-
* by a DEAD prior run; polling each one burns its full tick budget waiting on a
|
|
51
|
-
* receipt no live worker will complete, which blows Cloudflare's ~1000
|
|
52
|
-
* subrequests-per-invocation budget and kills the resume with
|
|
53
|
-
* "Too many subrequests by single Worker invocation" (prod resume deaths
|
|
54
|
-
* 20260703t230209 / 20260703t232104).
|
|
55
|
-
*
|
|
56
|
-
* A receipt is a dead-owner orphan when it is owned by a DIFFERENT run (a
|
|
57
|
-
* same-run `running` is this run replaying — its owner may still be live) and
|
|
58
|
-
* its last write is stale (or missing entirely — a live owner would carry a
|
|
59
|
-
* recent claim timestamp). Those reclaim immediately, matching the dispatch
|
|
60
|
-
* code's own contract ("on resume it reclaims and re-executes"). A live or
|
|
61
|
-
* freshly-claimed owner keeps the poll.
|
|
62
|
-
*/
|
|
63
|
-
export function shouldReclaimRunningReceiptWithoutPolling(input: {
|
|
44
|
+
/** Bound a follower poll by when a different-run owner becomes reclaimable. */
|
|
45
|
+
export function resolveRunningReceiptWaitMaxAttempts(input: {
|
|
64
46
|
ownerRunId?: string | null;
|
|
65
47
|
currentRunId: string;
|
|
66
48
|
updatedAt?: string | null;
|
|
49
|
+
requestedMaxAttempts: number;
|
|
67
50
|
nowMs?: number;
|
|
68
51
|
stalenessFloorMs?: number;
|
|
69
|
-
|
|
70
|
-
|
|
52
|
+
pollIntervalMs?: number;
|
|
53
|
+
}): number {
|
|
54
|
+
const requested = Math.max(1, Math.floor(input.requestedMaxAttempts));
|
|
71
55
|
const ownerRunId = input.ownerRunId?.trim() ?? '';
|
|
72
|
-
|
|
73
|
-
if (!
|
|
74
|
-
|
|
75
|
-
input.stalenessFloorMs ?? RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS;
|
|
76
|
-
// No freshness signal at all: treat as stale. A live owner would carry a
|
|
77
|
-
// recent claim timestamp; an orphan from a dead run may carry none.
|
|
78
|
-
if (!input.updatedAt) return true;
|
|
79
|
-
const updatedAtMs = Date.parse(input.updatedAt);
|
|
80
|
-
if (!Number.isFinite(updatedAtMs)) return true;
|
|
81
|
-
const now = input.nowMs ?? Date.now();
|
|
82
|
-
return now - updatedAtMs >= floorMs;
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
/** Metadata a `running` receipt carries about its owner and last write. */
|
|
86
|
-
export interface RunningReceiptOwnershipInput {
|
|
87
|
-
ownerRunId?: string | null;
|
|
88
|
-
currentRunId: string;
|
|
89
|
-
updatedAt?: string | null;
|
|
90
|
-
nowMs?: number;
|
|
91
|
-
stalenessFloorMs?: number;
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
/** Outcome of polling a `running` receipt for completion. */
|
|
95
|
-
export type RunningReceiptPollOutcome =
|
|
96
|
-
/** The poll saw the receipt complete and already resolved the group. */
|
|
97
|
-
| { kind: 'completed' }
|
|
98
|
-
/** The poll timed out; the receipt is still `running`. */
|
|
99
|
-
| { kind: 'timed_out'; error: unknown }
|
|
100
|
-
/** The poll failed for a non-timeout reason (e.g. the receipt failed). */
|
|
101
|
-
| { kind: 'errored'; error: unknown };
|
|
102
|
-
|
|
103
|
-
/**
|
|
104
|
-
* Orchestrate the resume-time handling of a `running` receipt: reclaim it
|
|
105
|
-
* immediately when it is a dead-owner orphan, otherwise poll for completion and
|
|
106
|
-
* only reclaim after the poll times out. Kept as a pure, dependency-injected
|
|
107
|
-
* function (the poll and reclaim side effects are passed in) so the
|
|
108
|
-
* "skip the poll for a dead owner" decision is unit-testable with a poll counter
|
|
109
|
-
* — the Worker scheduler that owns the real effects imports `cloudflare:workers`
|
|
110
|
-
* and cannot be loaded under vitest.
|
|
111
|
-
*
|
|
112
|
-
* Behavior (must match the scheduler's prior inline logic exactly):
|
|
113
|
-
* - dead-owner + reclaim-eligible → `reclaim` now, never polling;
|
|
114
|
-
* - otherwise `poll`; on `completed` the group is already resolved (return []);
|
|
115
|
-
* on a non-timeout error `reject` and stop; on timeout, `reclaim` if still
|
|
116
|
-
* eligible else `reject` with the timeout error.
|
|
117
|
-
*/
|
|
118
|
-
export async function reclaimRunningReceiptGroupAfterWait<TClaimed>(input: {
|
|
119
|
-
ownership: RunningReceiptOwnershipInput;
|
|
120
|
-
timeoutError: unknown;
|
|
121
|
-
poll: () => Promise<RunningReceiptPollOutcome>;
|
|
122
|
-
reclaim: (rejectionOnFallthrough: unknown) => Promise<TClaimed[]>;
|
|
123
|
-
reject: (error: unknown) => void;
|
|
124
|
-
}): Promise<TClaimed[]> {
|
|
125
|
-
if (
|
|
126
|
-
shouldReclaimRunningReceiptWithoutPolling(input.ownership) &&
|
|
127
|
-
canReclaimTimedOutWorkerToolReceipt(input.ownership)
|
|
128
|
-
) {
|
|
129
|
-
return await input.reclaim(input.timeoutError);
|
|
130
|
-
}
|
|
131
|
-
const outcome = await input.poll();
|
|
132
|
-
if (outcome.kind === 'completed') return [];
|
|
133
|
-
if (outcome.kind === 'errored') {
|
|
134
|
-
input.reject(outcome.error);
|
|
135
|
-
return [];
|
|
136
|
-
}
|
|
137
|
-
if (!canReclaimTimedOutWorkerToolReceipt(input.ownership)) {
|
|
138
|
-
input.reject(outcome.error);
|
|
139
|
-
return [];
|
|
56
|
+
const currentRunId = input.currentRunId.trim();
|
|
57
|
+
if (!ownerRunId || !currentRunId || ownerRunId === currentRunId) {
|
|
58
|
+
return requested;
|
|
140
59
|
}
|
|
141
|
-
|
|
60
|
+
const updatedAtMs = input.updatedAt ? Date.parse(input.updatedAt) : NaN;
|
|
61
|
+
if (!Number.isFinite(updatedAtMs)) return 1;
|
|
62
|
+
const remainingMs = Math.max(
|
|
63
|
+
0,
|
|
64
|
+
updatedAtMs +
|
|
65
|
+
(input.stalenessFloorMs ?? RUNNING_RECEIPT_DEAD_OWNER_STALENESS_MS) -
|
|
66
|
+
(input.nowMs ?? Date.now()),
|
|
67
|
+
);
|
|
68
|
+
const attemptsUntilReclaim = Math.max(
|
|
69
|
+
1,
|
|
70
|
+
Math.floor(
|
|
71
|
+
remainingMs / (input.pollIntervalMs ?? RUNNING_RECEIPT_POLL_INTERVAL_MS),
|
|
72
|
+
) + 1,
|
|
73
|
+
);
|
|
74
|
+
return Math.min(requested, attemptsUntilReclaim);
|
|
142
75
|
}
|
|
143
76
|
|
|
144
77
|
export function canReclaimFailedWorkerToolReceipt(input: {
|
|
@@ -1807,6 +1807,16 @@ export class DeeplineClient {
|
|
|
1807
1807
|
description?: string;
|
|
1808
1808
|
artifact: Record<string, unknown>;
|
|
1809
1809
|
integrationMode?: 'live' | 'eval_stub' | 'fixture';
|
|
1810
|
+
/**
|
|
1811
|
+
* Sibling plays from the same local bundle graph. Lets the server splice
|
|
1812
|
+
* unpublished local children into the checked plan and enforce the
|
|
1813
|
+
* row-scoped batch rule (ADR 0013).
|
|
1814
|
+
*/
|
|
1815
|
+
importedPlays?: Array<{
|
|
1816
|
+
playName?: string | null;
|
|
1817
|
+
sourceCode: string;
|
|
1818
|
+
sourcePath?: string | null;
|
|
1819
|
+
}>;
|
|
1810
1820
|
}): Promise<PlayCheckResult> {
|
|
1811
1821
|
const integrationMode = normalizePlayRunIntegrationMode(
|
|
1812
1822
|
input.integrationMode ?? process.env.DEEPLINE_EVAL_INTEGRATION_MODE,
|
|
@@ -400,7 +400,7 @@ export type StepOptions<Row, Value = unknown> = {
|
|
|
400
400
|
* Legacy dataset-column recompute flag accepted for older authored plays.
|
|
401
401
|
*
|
|
402
402
|
* Prefer putting freshness on the actual reusable call
|
|
403
|
-
* (`ctx.tools.execute`, `ctx.step`,
|
|
403
|
+
* (`ctx.tools.execute`, `ctx.step`, or `ctx.fetch`).
|
|
404
404
|
*/
|
|
405
405
|
readonly recompute?: boolean;
|
|
406
406
|
/** Legacy error-recompute flag accepted for older authored plays. */
|
|
@@ -889,7 +889,7 @@ export interface DeeplinePlayRuntimeContext {
|
|
|
889
889
|
key: string,
|
|
890
890
|
playRef: string | PlayReferenceLike,
|
|
891
891
|
input: Record<string, unknown>,
|
|
892
|
-
options: { description: string
|
|
892
|
+
options: { description: string },
|
|
893
893
|
): Promise<TOutput>;
|
|
894
894
|
|
|
895
895
|
/**
|
|
@@ -106,10 +106,10 @@ export const SDK_RELEASE = {
|
|
|
106
106
|
// 0.1.154 removes the short-lived generated enrich StepOptions recompute
|
|
107
107
|
// fields shipped in 0.1.153.
|
|
108
108
|
// 0.1.175 ships loud `deepline enrich` empty-waterfall reporting.
|
|
109
|
-
version: '0.1.
|
|
109
|
+
version: '0.1.210',
|
|
110
110
|
apiContract: '2026-06-dataset-handle-results-hard-cutover',
|
|
111
111
|
supportPolicy: {
|
|
112
|
-
latest: '0.1.
|
|
112
|
+
latest: '0.1.210',
|
|
113
113
|
minimumSupported: '0.1.53',
|
|
114
114
|
deprecatedBelow: '0.1.53',
|
|
115
115
|
commandMinimumSupported: [
|