deepline 0.1.64 → 0.1.66
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +236 -199
- package/dist/cli/index.mjs +120 -83
- package/dist/index.js +48 -11
- package/dist/index.mjs +44 -7
- package/dist/repo/apps/play-runner-workers/src/coordinator-entry.ts +475 -69
- package/dist/repo/apps/play-runner-workers/src/dedup-do.ts +134 -0
- package/dist/repo/apps/play-runner-workers/src/entry.ts +37 -49
- package/dist/repo/apps/play-runner-workers/src/runtime/harness-receipt-store.ts +13 -0
- package/dist/repo/apps/play-runner-workers/src/runtime/receipts.ts +10 -136
- package/dist/repo/sdk/src/http.ts +44 -0
- package/dist/repo/sdk/src/plays/harness-stub.ts +4 -0
- package/dist/repo/sdk/src/release.ts +2 -2
- package/dist/repo/shared_libs/play-runtime/db-session-crypto.ts +312 -0
- package/dist/repo/shared_libs/play-runtime/db-session-plan.ts +69 -0
- package/dist/repo/shared_libs/play-runtime/db-session.ts +439 -0
- package/dist/repo/shared_libs/play-runtime/work-receipts.ts +92 -0
- package/package.json +1 -1
|
@@ -23,7 +23,23 @@ import {
|
|
|
23
23
|
} from '@cloudflare/dynamic-workflows';
|
|
24
24
|
import type { ExecutionPlan } from '../../../shared_libs/play-runtime/execution-plan';
|
|
25
25
|
import type { PlayCallGovernanceSnapshot } from '../../../shared_libs/play-runtime/scheduler-backend';
|
|
26
|
-
import
|
|
26
|
+
import {
|
|
27
|
+
DB_SESSION_DEFAULT_TTL_SECONDS,
|
|
28
|
+
createDbSessionResponseSchema,
|
|
29
|
+
type CreateDbSessionResponse,
|
|
30
|
+
type PreloadedRuntimeDbSession,
|
|
31
|
+
} from '../../../shared_libs/play-runtime/db-session';
|
|
32
|
+
import {
|
|
33
|
+
dbSessionPostgresUrlAad,
|
|
34
|
+
decryptDbSessionPostgresUrlWithPrivateKey,
|
|
35
|
+
encryptDbSessionPostgresUrl,
|
|
36
|
+
generateDbSessionPostgresUrlDecryptionKey,
|
|
37
|
+
type PostgresUrlDecryptionKey,
|
|
38
|
+
} from '../../../shared_libs/play-runtime/db-session-crypto';
|
|
39
|
+
import {
|
|
40
|
+
planRuntimeDbSessionRequirements,
|
|
41
|
+
type RuntimeDbSessionRequirement,
|
|
42
|
+
} from '../../../shared_libs/play-runtime/db-session-plan';
|
|
27
43
|
import type {
|
|
28
44
|
PlayRuntimeManifest,
|
|
29
45
|
PlayRuntimeManifestMap,
|
|
@@ -72,6 +88,11 @@ export type PlayWorkflowParams = {
|
|
|
72
88
|
childPlayManifests?: PlayRuntimeManifestMap | null;
|
|
73
89
|
playCallGovernance?: PlayCallGovernanceSnapshot | null;
|
|
74
90
|
preloadedDbSessions?: PreloadedRuntimeDbSession[] | null;
|
|
91
|
+
preloadedDbSessionRef?: {
|
|
92
|
+
runId: string;
|
|
93
|
+
sessionCount: number;
|
|
94
|
+
expiresAt: number;
|
|
95
|
+
} | null;
|
|
75
96
|
dynamicWorkerCode?: string | null;
|
|
76
97
|
executorToken: string;
|
|
77
98
|
baseUrl: string;
|
|
@@ -239,15 +260,12 @@ interface CoordinatorEnv {
|
|
|
239
260
|
* helpers (runtime-API HTTP forwarder, Neon dataset IO, …) that we
|
|
240
261
|
* deliberately keep OUT of every per-graphHash play bundle.
|
|
241
262
|
*
|
|
242
|
-
*
|
|
243
|
-
*
|
|
244
|
-
* coordinator falls back to passing a `null` HARNESS into per-play
|
|
245
|
-
* Workers and the per-play stub throws a clear error if anyone tries
|
|
246
|
-
* to use it. Loud failures > silent fallbacks.
|
|
263
|
+
* Required in every Cloudflare play environment. The coordinator fails
|
|
264
|
+
* submit/load before handing a per-play Worker an unwired harness binding.
|
|
247
265
|
*
|
|
248
266
|
* Wired in wrangler.toml as `[[services]] binding = "HARNESS"`.
|
|
249
267
|
*/
|
|
250
|
-
HARNESS
|
|
268
|
+
HARNESS: import('../../play-harness-worker/src/rpc-types').PlayHarnessRpc;
|
|
251
269
|
}
|
|
252
270
|
|
|
253
271
|
const WORKFLOW_READ_ONLY_ACTIONS = new Set(['', 'result', 'status', 'tail']);
|
|
@@ -605,7 +623,7 @@ function readWorkflowTraceContext(event: unknown): {
|
|
|
605
623
|
} {
|
|
606
624
|
const record = isRecord(event) ? event : {};
|
|
607
625
|
const payload = isRecord(record.payload) ? record.payload : {};
|
|
608
|
-
const params = isRecord(payload.params) ? payload.params :
|
|
626
|
+
const params = isRecord(payload.params) ? payload.params : payload;
|
|
609
627
|
const metadata = isRecord(payload.__dispatcherMetadata)
|
|
610
628
|
? payload.__dispatcherMetadata
|
|
611
629
|
: null;
|
|
@@ -706,6 +724,13 @@ function workflowPoolDurableObject(env: CoordinatorEnv): DurableObjectStub {
|
|
|
706
724
|
);
|
|
707
725
|
}
|
|
708
726
|
|
|
727
|
+
function runScopedDurableObject(
|
|
728
|
+
env: CoordinatorEnv,
|
|
729
|
+
runId: string,
|
|
730
|
+
): DurableObjectStub {
|
|
731
|
+
return env.PLAY_DEDUP.get(env.PLAY_DEDUP.idFromName(runId));
|
|
732
|
+
}
|
|
733
|
+
|
|
709
734
|
async function callWorkflowPool<T>(
|
|
710
735
|
env: CoordinatorEnv,
|
|
711
736
|
path: string,
|
|
@@ -762,6 +787,188 @@ async function callWorkflowPool<T>(
|
|
|
762
787
|
}
|
|
763
788
|
}
|
|
764
789
|
|
|
790
|
+
function assertEncryptedPreloadedDbSessions(
|
|
791
|
+
sessions: PreloadedRuntimeDbSession[],
|
|
792
|
+
): void {
|
|
793
|
+
for (const session of sessions) {
|
|
794
|
+
if (session.session.postgresUrl) {
|
|
795
|
+
throw new Error(
|
|
796
|
+
'Preloaded Runtime DB sessions must not carry raw Postgres URLs.',
|
|
797
|
+
);
|
|
798
|
+
}
|
|
799
|
+
if (!session.session.encryptedPostgresUrl) {
|
|
800
|
+
throw new Error(
|
|
801
|
+
'Preloaded Runtime DB sessions must carry encrypted Postgres URLs.',
|
|
802
|
+
);
|
|
803
|
+
}
|
|
804
|
+
}
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
async function persistWorkflowDbSessions(input: {
|
|
808
|
+
env: CoordinatorEnv;
|
|
809
|
+
runId: string;
|
|
810
|
+
sessions: PreloadedRuntimeDbSession[];
|
|
811
|
+
}): Promise<NonNullable<PlayWorkflowParams['preloadedDbSessionRef']>> {
|
|
812
|
+
assertEncryptedPreloadedDbSessions(input.sessions);
|
|
813
|
+
const response = await runScopedDurableObject(input.env, input.runId).fetch(
|
|
814
|
+
'https://deepline.dedup.internal/db-sessions-put',
|
|
815
|
+
{
|
|
816
|
+
method: 'POST',
|
|
817
|
+
headers: { 'content-type': 'application/json' },
|
|
818
|
+
body: JSON.stringify({
|
|
819
|
+
runId: input.runId,
|
|
820
|
+
sessions: input.sessions,
|
|
821
|
+
ttlMs: DB_SESSION_DEFAULT_TTL_SECONDS * 1000,
|
|
822
|
+
}),
|
|
823
|
+
},
|
|
824
|
+
);
|
|
825
|
+
if (!response.ok) {
|
|
826
|
+
throw new Error(
|
|
827
|
+
`workflow db session storage failed ${response.status}: ${(
|
|
828
|
+
await response.text().catch(() => '')
|
|
829
|
+
).slice(0, 400)}`,
|
|
830
|
+
);
|
|
831
|
+
}
|
|
832
|
+
const body = (await response.json().catch(() => ({}))) as {
|
|
833
|
+
sessionCount?: unknown;
|
|
834
|
+
expiresAt?: unknown;
|
|
835
|
+
};
|
|
836
|
+
return {
|
|
837
|
+
runId: input.runId,
|
|
838
|
+
sessionCount:
|
|
839
|
+
typeof body.sessionCount === 'number' &&
|
|
840
|
+
Number.isFinite(body.sessionCount)
|
|
841
|
+
? body.sessionCount
|
|
842
|
+
: input.sessions.length,
|
|
843
|
+
expiresAt:
|
|
844
|
+
typeof body.expiresAt === 'number' && Number.isFinite(body.expiresAt)
|
|
845
|
+
? body.expiresAt
|
|
846
|
+
: Date.now() + DB_SESSION_DEFAULT_TTL_SECONDS * 1000,
|
|
847
|
+
};
|
|
848
|
+
}
|
|
849
|
+
|
|
850
|
+
async function readWorkflowDbSessions(input: {
|
|
851
|
+
env: CoordinatorEnv;
|
|
852
|
+
ref: NonNullable<PlayWorkflowParams['preloadedDbSessionRef']>;
|
|
853
|
+
}): Promise<PreloadedRuntimeDbSession[]> {
|
|
854
|
+
const response = await runScopedDurableObject(
|
|
855
|
+
input.env,
|
|
856
|
+
input.ref.runId,
|
|
857
|
+
).fetch(
|
|
858
|
+
`https://deepline.dedup.internal/db-sessions-get?runId=${encodeURIComponent(
|
|
859
|
+
input.ref.runId,
|
|
860
|
+
)}`,
|
|
861
|
+
);
|
|
862
|
+
if (!response.ok) {
|
|
863
|
+
throw new Error(
|
|
864
|
+
`workflow db session lookup failed ${response.status}: ${(
|
|
865
|
+
await response.text().catch(() => '')
|
|
866
|
+
).slice(0, 400)}`,
|
|
867
|
+
);
|
|
868
|
+
}
|
|
869
|
+
const body = (await response.json().catch(() => ({}))) as {
|
|
870
|
+
sessions?: unknown;
|
|
871
|
+
};
|
|
872
|
+
if (!Array.isArray(body.sessions)) {
|
|
873
|
+
throw new Error('workflow db session lookup returned no sessions.');
|
|
874
|
+
}
|
|
875
|
+
const sessions = body.sessions as PreloadedRuntimeDbSession[];
|
|
876
|
+
assertEncryptedPreloadedDbSessions(sessions);
|
|
877
|
+
return sessions;
|
|
878
|
+
}
|
|
879
|
+
|
|
880
|
+
async function externalizeWorkflowDbSessions(input: {
|
|
881
|
+
env: CoordinatorEnv;
|
|
882
|
+
params: PlayWorkflowParams;
|
|
883
|
+
recordSubmitTiming?: (timing: CoordinatorTiming) => void;
|
|
884
|
+
}): Promise<PlayWorkflowParams> {
|
|
885
|
+
const sessions = Array.isArray(input.params.preloadedDbSessions)
|
|
886
|
+
? input.params.preloadedDbSessions
|
|
887
|
+
: [];
|
|
888
|
+
if (sessions.length === 0) return input.params;
|
|
889
|
+
const startedAt = Date.now();
|
|
890
|
+
const ref = await persistWorkflowDbSessions({
|
|
891
|
+
env: input.env,
|
|
892
|
+
runId: input.params.runId,
|
|
893
|
+
sessions,
|
|
894
|
+
});
|
|
895
|
+
input.recordSubmitTiming?.({
|
|
896
|
+
phase: 'coordinator.workflow_db_sessions_externalized',
|
|
897
|
+
ms: Date.now() - startedAt,
|
|
898
|
+
graphHash: input.params.graphHash ?? null,
|
|
899
|
+
extra: {
|
|
900
|
+
sessions: sessions.length,
|
|
901
|
+
expiresAt: ref.expiresAt,
|
|
902
|
+
},
|
|
903
|
+
});
|
|
904
|
+
return {
|
|
905
|
+
...input.params,
|
|
906
|
+
preloadedDbSessions: null,
|
|
907
|
+
preloadedDbSessionRef: ref,
|
|
908
|
+
};
|
|
909
|
+
}
|
|
910
|
+
|
|
911
|
+
function readPreloadedDbSessionRef(
|
|
912
|
+
value: unknown,
|
|
913
|
+
): NonNullable<PlayWorkflowParams['preloadedDbSessionRef']> | null {
|
|
914
|
+
if (!isRecord(value)) return null;
|
|
915
|
+
const runId = typeof value.runId === 'string' ? value.runId : '';
|
|
916
|
+
const sessionCount =
|
|
917
|
+
typeof value.sessionCount === 'number' &&
|
|
918
|
+
Number.isFinite(value.sessionCount)
|
|
919
|
+
? value.sessionCount
|
|
920
|
+
: 0;
|
|
921
|
+
const expiresAt =
|
|
922
|
+
typeof value.expiresAt === 'number' && Number.isFinite(value.expiresAt)
|
|
923
|
+
? value.expiresAt
|
|
924
|
+
: 0;
|
|
925
|
+
if (!runId || sessionCount <= 0 || expiresAt <= 0) return null;
|
|
926
|
+
return { runId, sessionCount, expiresAt };
|
|
927
|
+
}
|
|
928
|
+
|
|
929
|
+
async function hydrateWorkflowDbSessions(input: {
|
|
930
|
+
env: CoordinatorEnv;
|
|
931
|
+
event: unknown;
|
|
932
|
+
trace: CoordinatorPerfTraceSink;
|
|
933
|
+
}): Promise<unknown> {
|
|
934
|
+
if (!isRecord(input.event)) return input.event;
|
|
935
|
+
const payload = isRecord(input.event.payload) ? input.event.payload : null;
|
|
936
|
+
const envelopeParams = isRecord(payload?.params) ? payload.params : null;
|
|
937
|
+
const params = envelopeParams ?? payload;
|
|
938
|
+
if (!payload || !params || Array.isArray(params.preloadedDbSessions)) {
|
|
939
|
+
return input.event;
|
|
940
|
+
}
|
|
941
|
+
const ref = readPreloadedDbSessionRef(params.preloadedDbSessionRef);
|
|
942
|
+
if (!ref) return input.event;
|
|
943
|
+
const startedAt = Date.now();
|
|
944
|
+
const sessions = await readWorkflowDbSessions({ env: input.env, ref });
|
|
945
|
+
input.trace({
|
|
946
|
+
runId: ref.runId,
|
|
947
|
+
phase: 'coordinator.workflow_db_sessions_hydrated',
|
|
948
|
+
ms: Date.now() - startedAt,
|
|
949
|
+
graphHash: typeof params.graphHash === 'string' ? params.graphHash : null,
|
|
950
|
+
extra: {
|
|
951
|
+
sessions: sessions.length,
|
|
952
|
+
refSessionCount: ref.sessionCount,
|
|
953
|
+
},
|
|
954
|
+
});
|
|
955
|
+
return {
|
|
956
|
+
...input.event,
|
|
957
|
+
payload: envelopeParams
|
|
958
|
+
? {
|
|
959
|
+
...payload,
|
|
960
|
+
params: {
|
|
961
|
+
...params,
|
|
962
|
+
preloadedDbSessions: sessions,
|
|
963
|
+
},
|
|
964
|
+
}
|
|
965
|
+
: {
|
|
966
|
+
...payload,
|
|
967
|
+
preloadedDbSessions: sessions,
|
|
968
|
+
},
|
|
969
|
+
};
|
|
970
|
+
}
|
|
971
|
+
|
|
765
972
|
type WorkflowPoolCounts = {
|
|
766
973
|
available: number;
|
|
767
974
|
warming: number;
|
|
@@ -1689,7 +1896,8 @@ async function submitViaPooledWorkflow(input: {
|
|
|
1689
1896
|
function readWorkflowPayload(event: unknown): Record<string, unknown> | null {
|
|
1690
1897
|
if (!isRecord(event)) return null;
|
|
1691
1898
|
const payload = event.payload;
|
|
1692
|
-
|
|
1899
|
+
if (!isRecord(payload)) return null;
|
|
1900
|
+
return isRecord(payload.params) ? payload.params : payload;
|
|
1693
1901
|
}
|
|
1694
1902
|
|
|
1695
1903
|
async function markWorkflowRuntimeFailure(input: {
|
|
@@ -1833,6 +2041,185 @@ async function mintChildWorkflowExecutorToken(input: {
|
|
|
1833
2041
|
return executorToken;
|
|
1834
2042
|
}
|
|
1835
2043
|
|
|
2044
|
+
function stripDbSessionUrl(
|
|
2045
|
+
session: CreateDbSessionResponse,
|
|
2046
|
+
): Omit<CreateDbSessionResponse, 'postgresUrl' | 'encryptedPostgresUrl'> {
|
|
2047
|
+
const {
|
|
2048
|
+
postgresUrl: _postgresUrl,
|
|
2049
|
+
encryptedPostgresUrl: _encryptedPostgresUrl,
|
|
2050
|
+
...sessionWithoutUrl
|
|
2051
|
+
} = session;
|
|
2052
|
+
void _postgresUrl;
|
|
2053
|
+
void _encryptedPostgresUrl;
|
|
2054
|
+
return sessionWithoutUrl;
|
|
2055
|
+
}
|
|
2056
|
+
|
|
2057
|
+
function assertChildDbSessionScope(input: {
|
|
2058
|
+
session: CreateDbSessionResponse;
|
|
2059
|
+
requirement: RuntimeDbSessionRequirement;
|
|
2060
|
+
orgId: string;
|
|
2061
|
+
childPlayName: string;
|
|
2062
|
+
}): void {
|
|
2063
|
+
const { session, requirement, orgId, childPlayName } = input;
|
|
2064
|
+
if (
|
|
2065
|
+
session.playName !== childPlayName ||
|
|
2066
|
+
session.target.orgId !== orgId ||
|
|
2067
|
+
session.target.tableNamespace !== requirement.tableNamespace ||
|
|
2068
|
+
session.target.logicalTable !== requirement.logicalTable
|
|
2069
|
+
) {
|
|
2070
|
+
throw new Error(
|
|
2071
|
+
'Runtime API returned a DB session outside the child play scope.',
|
|
2072
|
+
);
|
|
2073
|
+
}
|
|
2074
|
+
const expectedOperations = [...requirement.operations].sort().join(',');
|
|
2075
|
+
const actualOperations = [...session.operations].sort().join(',');
|
|
2076
|
+
if (actualOperations !== expectedOperations) {
|
|
2077
|
+
throw new Error(
|
|
2078
|
+
'Runtime API returned a DB session with unexpected operations.',
|
|
2079
|
+
);
|
|
2080
|
+
}
|
|
2081
|
+
}
|
|
2082
|
+
|
|
2083
|
+
async function reencryptChildDbSessionForExecutor(input: {
|
|
2084
|
+
session: CreateDbSessionResponse;
|
|
2085
|
+
decryptionKey: PostgresUrlDecryptionKey;
|
|
2086
|
+
childExecutorToken: string;
|
|
2087
|
+
}): Promise<CreateDbSessionResponse> {
|
|
2088
|
+
const sessionWithoutUrl = stripDbSessionUrl(input.session);
|
|
2089
|
+
if (!input.session.encryptedPostgresUrl) {
|
|
2090
|
+
throw new Error('Runtime API returned an unencrypted DB session URL.');
|
|
2091
|
+
}
|
|
2092
|
+
const postgresUrl = await decryptDbSessionPostgresUrlWithPrivateKey({
|
|
2093
|
+
encrypted: input.session.encryptedPostgresUrl,
|
|
2094
|
+
privateKey: input.decryptionKey.privateKey,
|
|
2095
|
+
aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
|
|
2096
|
+
});
|
|
2097
|
+
return {
|
|
2098
|
+
...sessionWithoutUrl,
|
|
2099
|
+
encryptedPostgresUrl: await encryptDbSessionPostgresUrl({
|
|
2100
|
+
postgresUrl,
|
|
2101
|
+
secret: input.childExecutorToken,
|
|
2102
|
+
aad: dbSessionPostgresUrlAad(sessionWithoutUrl),
|
|
2103
|
+
}),
|
|
2104
|
+
};
|
|
2105
|
+
}
|
|
2106
|
+
|
|
2107
|
+
async function createChildRuntimeDbSession(input: {
|
|
2108
|
+
env: CoordinatorEnv;
|
|
2109
|
+
baseUrl: string;
|
|
2110
|
+
childExecutorToken: string;
|
|
2111
|
+
childPlayName: string;
|
|
2112
|
+
requirement: RuntimeDbSessionRequirement;
|
|
2113
|
+
userEmail: string;
|
|
2114
|
+
orgId: string;
|
|
2115
|
+
}): Promise<CreateDbSessionResponse> {
|
|
2116
|
+
const decryptionKey = await generateDbSessionPostgresUrlDecryptionKey();
|
|
2117
|
+
const url = `${input.baseUrl.replace(/\/$/, '')}/api/v2/plays/internal/runtime`;
|
|
2118
|
+
const headers = new Headers({
|
|
2119
|
+
authorization: `Bearer ${input.childExecutorToken}`,
|
|
2120
|
+
'content-type': 'application/json',
|
|
2121
|
+
'x-deepline-request-id': crypto.randomUUID(),
|
|
2122
|
+
});
|
|
2123
|
+
if (input.env.VERCEL_PROTECTION_BYPASS_TOKEN?.trim()) {
|
|
2124
|
+
headers.set(
|
|
2125
|
+
'x-vercel-protection-bypass',
|
|
2126
|
+
input.env.VERCEL_PROTECTION_BYPASS_TOKEN.trim(),
|
|
2127
|
+
);
|
|
2128
|
+
}
|
|
2129
|
+
const response = await fetch(url, {
|
|
2130
|
+
method: 'POST',
|
|
2131
|
+
headers,
|
|
2132
|
+
body: JSON.stringify({
|
|
2133
|
+
action: 'create_db_session',
|
|
2134
|
+
playName: input.childPlayName,
|
|
2135
|
+
target: {
|
|
2136
|
+
tableNamespace: input.requirement.tableNamespace,
|
|
2137
|
+
logicalTable: input.requirement.logicalTable,
|
|
2138
|
+
},
|
|
2139
|
+
operations: input.requirement.operations,
|
|
2140
|
+
limits: input.requirement.limits,
|
|
2141
|
+
sheetContract: input.requirement.sheetContract ?? null,
|
|
2142
|
+
ttlSeconds: DB_SESSION_DEFAULT_TTL_SECONDS,
|
|
2143
|
+
userEmail: input.userEmail,
|
|
2144
|
+
postgresUrlEncryption: decryptionKey.request,
|
|
2145
|
+
}),
|
|
2146
|
+
});
|
|
2147
|
+
const text = await response.text().catch(() => '');
|
|
2148
|
+
let parsed: unknown = {};
|
|
2149
|
+
try {
|
|
2150
|
+
parsed = text ? JSON.parse(text) : {};
|
|
2151
|
+
} catch {
|
|
2152
|
+
parsed = {};
|
|
2153
|
+
}
|
|
2154
|
+
if (!response.ok) {
|
|
2155
|
+
const error =
|
|
2156
|
+
isRecord(parsed) && isRecord(parsed.error) ? parsed.error : {};
|
|
2157
|
+
const message =
|
|
2158
|
+
(typeof error.message === 'string' && error.message.trim()) ||
|
|
2159
|
+
(isRecord(parsed) &&
|
|
2160
|
+
typeof parsed.error === 'string' &&
|
|
2161
|
+
parsed.error.trim()
|
|
2162
|
+
? parsed.error.trim()
|
|
2163
|
+
: '') ||
|
|
2164
|
+
text.slice(0, 800) ||
|
|
2165
|
+
`Origin DB session mint failed with ${response.status}.`;
|
|
2166
|
+
throw new Error(message);
|
|
2167
|
+
}
|
|
2168
|
+
const session = createDbSessionResponseSchema.parse(parsed);
|
|
2169
|
+
assertChildDbSessionScope({
|
|
2170
|
+
session,
|
|
2171
|
+
requirement: input.requirement,
|
|
2172
|
+
orgId: input.orgId,
|
|
2173
|
+
childPlayName: input.childPlayName,
|
|
2174
|
+
});
|
|
2175
|
+
return await reencryptChildDbSessionForExecutor({
|
|
2176
|
+
session,
|
|
2177
|
+
decryptionKey,
|
|
2178
|
+
childExecutorToken: input.childExecutorToken,
|
|
2179
|
+
});
|
|
2180
|
+
}
|
|
2181
|
+
|
|
2182
|
+
async function preloadChildRuntimeDbSessions(input: {
|
|
2183
|
+
env: CoordinatorEnv;
|
|
2184
|
+
baseUrl: string;
|
|
2185
|
+
childExecutorToken: string;
|
|
2186
|
+
childRunId: string;
|
|
2187
|
+
childPlayName: string;
|
|
2188
|
+
manifest: PlayRuntimeManifest;
|
|
2189
|
+
orgId: string;
|
|
2190
|
+
userEmail: string;
|
|
2191
|
+
}): Promise<PreloadedRuntimeDbSession[]> {
|
|
2192
|
+
const startedAt = Date.now();
|
|
2193
|
+
const requirements = planRuntimeDbSessionRequirements(
|
|
2194
|
+
input.manifest.staticPipeline ?? null,
|
|
2195
|
+
);
|
|
2196
|
+
const sessions = await Promise.all(
|
|
2197
|
+
requirements.map(async (requirement) => ({
|
|
2198
|
+
tableNamespace: requirement.tableNamespace,
|
|
2199
|
+
logicalTable: requirement.logicalTable,
|
|
2200
|
+
operations: requirement.operations,
|
|
2201
|
+
...(requirement.limits ? { limits: requirement.limits } : {}),
|
|
2202
|
+
session: await createChildRuntimeDbSession({
|
|
2203
|
+
env: input.env,
|
|
2204
|
+
baseUrl: input.baseUrl,
|
|
2205
|
+
childExecutorToken: input.childExecutorToken,
|
|
2206
|
+
childPlayName: input.childPlayName,
|
|
2207
|
+
requirement,
|
|
2208
|
+
userEmail: input.userEmail,
|
|
2209
|
+
orgId: input.orgId,
|
|
2210
|
+
}),
|
|
2211
|
+
})),
|
|
2212
|
+
);
|
|
2213
|
+
recordCoordinatorPerfTrace({
|
|
2214
|
+
runId: input.childRunId,
|
|
2215
|
+
phase: 'coordinator.child_db_session_preload',
|
|
2216
|
+
ms: Date.now() - startedAt,
|
|
2217
|
+
graphHash: input.manifest.graphHash,
|
|
2218
|
+
extra: { sessions: sessions.length },
|
|
2219
|
+
});
|
|
2220
|
+
return sessions;
|
|
2221
|
+
}
|
|
2222
|
+
|
|
1836
2223
|
function buildChildRunId(playName: string): string {
|
|
1837
2224
|
const slug =
|
|
1838
2225
|
playName
|
|
@@ -1973,6 +2360,9 @@ function buildChildWorkflowParams(input: {
|
|
|
1973
2360
|
childToken: string;
|
|
1974
2361
|
orgId: string;
|
|
1975
2362
|
coordinatorUrl: string | null;
|
|
2363
|
+
runtimeBackend: string;
|
|
2364
|
+
dynamicWorkerCode: string | null;
|
|
2365
|
+
preloadedDbSessions: PreloadedRuntimeDbSession[] | null;
|
|
1976
2366
|
}): PlayWorkflowParams {
|
|
1977
2367
|
const {
|
|
1978
2368
|
env,
|
|
@@ -1984,6 +2374,9 @@ function buildChildWorkflowParams(input: {
|
|
|
1984
2374
|
childToken,
|
|
1985
2375
|
orgId,
|
|
1986
2376
|
coordinatorUrl,
|
|
2377
|
+
runtimeBackend,
|
|
2378
|
+
dynamicWorkerCode,
|
|
2379
|
+
preloadedDbSessions,
|
|
1987
2380
|
} = input;
|
|
1988
2381
|
const baseUrl = resolveRuntimeBaseUrl(env, body);
|
|
1989
2382
|
return {
|
|
@@ -2019,13 +2412,14 @@ function buildChildWorkflowParams(input: {
|
|
|
2019
2412
|
? (body.childPlayManifests as PlayRuntimeManifestMap)
|
|
2020
2413
|
: null,
|
|
2021
2414
|
playCallGovernance: governance,
|
|
2022
|
-
|
|
2415
|
+
preloadedDbSessions,
|
|
2416
|
+
dynamicWorkerCode,
|
|
2023
2417
|
executorToken: childToken,
|
|
2024
2418
|
baseUrl,
|
|
2025
2419
|
orgId,
|
|
2026
2420
|
userEmail: typeof body.userEmail === 'string' ? body.userEmail : '',
|
|
2027
2421
|
userId: typeof body.userId === 'string' ? body.userId : null,
|
|
2028
|
-
runtimeBackend
|
|
2422
|
+
runtimeBackend,
|
|
2029
2423
|
dedupBackend: 'in_memory',
|
|
2030
2424
|
coordinatorUrl,
|
|
2031
2425
|
coordinatorInternalToken: env.DEEPLINE_INTERNAL_TOKEN?.trim() || null,
|
|
@@ -2213,9 +2607,10 @@ async function executeChildInline(input: {
|
|
|
2213
2607
|
});
|
|
2214
2608
|
|
|
2215
2609
|
const tokenStartedAt = Date.now();
|
|
2610
|
+
const baseUrl = resolveRuntimeBaseUrl(input.env, input.body);
|
|
2216
2611
|
const childToken = await mintChildWorkflowExecutorToken({
|
|
2217
2612
|
env: input.env,
|
|
2218
|
-
baseUrl
|
|
2613
|
+
baseUrl,
|
|
2219
2614
|
parentExecutorToken,
|
|
2220
2615
|
parentRunId: input.parentRunId,
|
|
2221
2616
|
parentPlayName:
|
|
@@ -2229,6 +2624,22 @@ async function executeChildInline(input: {
|
|
|
2229
2624
|
});
|
|
2230
2625
|
trace('coordinator.inline_child_token', tokenStartedAt);
|
|
2231
2626
|
|
|
2627
|
+
const dbSessionStartedAt = Date.now();
|
|
2628
|
+
const preloadedDbSessions = await preloadChildRuntimeDbSessions({
|
|
2629
|
+
env: input.env,
|
|
2630
|
+
baseUrl,
|
|
2631
|
+
childExecutorToken: childToken,
|
|
2632
|
+
childRunId,
|
|
2633
|
+
childPlayName,
|
|
2634
|
+
manifest,
|
|
2635
|
+
orgId,
|
|
2636
|
+
userEmail:
|
|
2637
|
+
typeof input.body.userEmail === 'string' ? input.body.userEmail : '',
|
|
2638
|
+
});
|
|
2639
|
+
trace('coordinator.inline_child_db_session_preload', dbSessionStartedAt, {
|
|
2640
|
+
sessions: preloadedDbSessions.length,
|
|
2641
|
+
});
|
|
2642
|
+
|
|
2232
2643
|
const params = buildChildWorkflowParams({
|
|
2233
2644
|
env: input.env,
|
|
2234
2645
|
body: input.body,
|
|
@@ -2239,6 +2650,10 @@ async function executeChildInline(input: {
|
|
|
2239
2650
|
childToken,
|
|
2240
2651
|
orgId,
|
|
2241
2652
|
coordinatorUrl: null,
|
|
2653
|
+
runtimeBackend: 'cf_workflows_dynamic_worker_inline_child',
|
|
2654
|
+
dynamicWorkerCode: null,
|
|
2655
|
+
preloadedDbSessions:
|
|
2656
|
+
preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
|
|
2242
2657
|
});
|
|
2243
2658
|
const loaderStartedAt = Date.now();
|
|
2244
2659
|
const stub = loadDynamicPlayWorker(input.env, {
|
|
@@ -2630,6 +3045,11 @@ export class DynamicWorkflow extends WorkflowEntrypoint<
|
|
|
2630
3045
|
},
|
|
2631
3046
|
});
|
|
2632
3047
|
}
|
|
3048
|
+
dispatchedEvent = await hydrateWorkflowDbSessions({
|
|
3049
|
+
env: this.env,
|
|
3050
|
+
event: dispatchedEvent,
|
|
3051
|
+
trace,
|
|
3052
|
+
});
|
|
2633
3053
|
const dispatchTrace = readWorkflowTraceContext(dispatchedEvent);
|
|
2634
3054
|
trace({
|
|
2635
3055
|
runId: dispatchTrace.runId,
|
|
@@ -3152,6 +3572,8 @@ async function handleWorkflowRoute(input: {
|
|
|
3152
3572
|
});
|
|
3153
3573
|
const prewarmPromise = env.HARNESS.prewarmPostgresSessions({
|
|
3154
3574
|
executorToken: params.executorToken,
|
|
3575
|
+
orgId: params.orgId,
|
|
3576
|
+
playName: params.playName,
|
|
3155
3577
|
sessions: preloadedDbSessions,
|
|
3156
3578
|
})
|
|
3157
3579
|
.then((result) => {
|
|
@@ -3179,10 +3601,15 @@ async function handleWorkflowRoute(input: {
|
|
|
3179
3601
|
});
|
|
3180
3602
|
input.ctx?.waitUntil(prewarmPromise);
|
|
3181
3603
|
}
|
|
3604
|
+
const workflowParams = await externalizeWorkflowDbSessions({
|
|
3605
|
+
env,
|
|
3606
|
+
params,
|
|
3607
|
+
recordSubmitTiming,
|
|
3608
|
+
});
|
|
3182
3609
|
await persistWorkflowRetryState({
|
|
3183
3610
|
env,
|
|
3184
3611
|
runId: submittedRunId,
|
|
3185
|
-
params,
|
|
3612
|
+
params: workflowParams,
|
|
3186
3613
|
});
|
|
3187
3614
|
let instance: WorkflowInstance | null = null;
|
|
3188
3615
|
try {
|
|
@@ -3202,7 +3629,7 @@ async function handleWorkflowRoute(input: {
|
|
|
3202
3629
|
const poolAttemptStartedAt = Date.now();
|
|
3203
3630
|
instance = await submitViaPooledWorkflow({
|
|
3204
3631
|
env,
|
|
3205
|
-
params,
|
|
3632
|
+
params: workflowParams,
|
|
3206
3633
|
recordSubmitTiming,
|
|
3207
3634
|
});
|
|
3208
3635
|
recordSubmitTiming({
|
|
@@ -3219,7 +3646,7 @@ async function handleWorkflowRoute(input: {
|
|
|
3219
3646
|
instance = await createDynamicWorkflowInstance({
|
|
3220
3647
|
env,
|
|
3221
3648
|
id: defaultInstanceId,
|
|
3222
|
-
params,
|
|
3649
|
+
params: workflowParams,
|
|
3223
3650
|
});
|
|
3224
3651
|
recordSubmitTiming({
|
|
3225
3652
|
phase: 'coordinator.workflow_create',
|
|
@@ -3368,9 +3795,10 @@ async function handleWorkflowRoute(input: {
|
|
|
3368
3795
|
{ status: 400 },
|
|
3369
3796
|
);
|
|
3370
3797
|
}
|
|
3798
|
+
const baseUrl = resolveRuntimeBaseUrl(env, body);
|
|
3371
3799
|
const childToken = await mintChildWorkflowExecutorToken({
|
|
3372
3800
|
env,
|
|
3373
|
-
baseUrl
|
|
3801
|
+
baseUrl,
|
|
3374
3802
|
parentExecutorToken,
|
|
3375
3803
|
parentRunId: runId,
|
|
3376
3804
|
parentPlayName:
|
|
@@ -3381,6 +3809,34 @@ async function handleWorkflowRoute(input: {
|
|
|
3381
3809
|
childPlayName,
|
|
3382
3810
|
maxCreditsPerRun: manifest.maxCreditsPerRun ?? null,
|
|
3383
3811
|
});
|
|
3812
|
+
const preloadedDbSessions = await preloadChildRuntimeDbSessions({
|
|
3813
|
+
env,
|
|
3814
|
+
baseUrl,
|
|
3815
|
+
childExecutorToken: childToken,
|
|
3816
|
+
childRunId,
|
|
3817
|
+
childPlayName,
|
|
3818
|
+
manifest,
|
|
3819
|
+
orgId,
|
|
3820
|
+
userEmail: typeof body.userEmail === 'string' ? body.userEmail : '',
|
|
3821
|
+
});
|
|
3822
|
+
const params = buildChildWorkflowParams({
|
|
3823
|
+
env,
|
|
3824
|
+
body,
|
|
3825
|
+
manifest,
|
|
3826
|
+
governance,
|
|
3827
|
+
childRunId,
|
|
3828
|
+
childPlayName,
|
|
3829
|
+
childToken,
|
|
3830
|
+
orgId,
|
|
3831
|
+
coordinatorUrl: new URL(request.url).origin,
|
|
3832
|
+
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3833
|
+
dynamicWorkerCode:
|
|
3834
|
+
typeof manifest.bundledCode === 'string'
|
|
3835
|
+
? manifest.bundledCode
|
|
3836
|
+
: null,
|
|
3837
|
+
preloadedDbSessions:
|
|
3838
|
+
preloadedDbSessions.length > 0 ? preloadedDbSessions : null,
|
|
3839
|
+
});
|
|
3384
3840
|
const submitResponse = await handleWorkflowRoute({
|
|
3385
3841
|
runId: childRunId,
|
|
3386
3842
|
action: 'submit',
|
|
@@ -3391,55 +3847,7 @@ async function handleWorkflowRoute(input: {
|
|
|
3391
3847
|
{
|
|
3392
3848
|
method: 'POST',
|
|
3393
3849
|
headers: { 'content-type': 'application/json' },
|
|
3394
|
-
body: JSON.stringify(
|
|
3395
|
-
runId: childRunId,
|
|
3396
|
-
playId: childRunId,
|
|
3397
|
-
playName: childPlayName,
|
|
3398
|
-
artifactStorageKey: manifest.artifactStorageKey,
|
|
3399
|
-
artifactHash: manifest.artifactHash,
|
|
3400
|
-
graphHash: manifest.graphHash,
|
|
3401
|
-
input: isRecord(body.input) ? body.input : {},
|
|
3402
|
-
contractSnapshot: {
|
|
3403
|
-
source: 'published',
|
|
3404
|
-
revisionVersion: null,
|
|
3405
|
-
staticPipeline: manifest.staticPipeline,
|
|
3406
|
-
billingLimit:
|
|
3407
|
-
typeof manifest.maxCreditsPerRun === 'number'
|
|
3408
|
-
? { maxCreditsPerRun: manifest.maxCreditsPerRun }
|
|
3409
|
-
: null,
|
|
3410
|
-
sourceCode: manifest.sourceCode ?? '',
|
|
3411
|
-
artifactMetadata: {
|
|
3412
|
-
storageKey: manifest.artifactStorageKey,
|
|
3413
|
-
artifactHash: manifest.artifactHash,
|
|
3414
|
-
graphHash: manifest.graphHash,
|
|
3415
|
-
},
|
|
3416
|
-
codeFormat: 'cjs_module',
|
|
3417
|
-
compatibility: {
|
|
3418
|
-
apiVersion: 2,
|
|
3419
|
-
runtimeBackend: 'workers_edge',
|
|
3420
|
-
},
|
|
3421
|
-
},
|
|
3422
|
-
executionPlan: null,
|
|
3423
|
-
childPlayManifests: isRecord(body.childPlayManifests)
|
|
3424
|
-
? (body.childPlayManifests as PlayRuntimeManifestMap)
|
|
3425
|
-
: null,
|
|
3426
|
-
playCallGovernance: governance,
|
|
3427
|
-
dynamicWorkerCode:
|
|
3428
|
-
typeof manifest.bundledCode === 'string'
|
|
3429
|
-
? manifest.bundledCode
|
|
3430
|
-
: null,
|
|
3431
|
-
executorToken: childToken,
|
|
3432
|
-
baseUrl: resolveRuntimeBaseUrl(env, body),
|
|
3433
|
-
orgId,
|
|
3434
|
-
userEmail:
|
|
3435
|
-
typeof body.userEmail === 'string' ? body.userEmail : '',
|
|
3436
|
-
userId: typeof body.userId === 'string' ? body.userId : null,
|
|
3437
|
-
runtimeBackend: 'cf_workflows_dynamic_worker',
|
|
3438
|
-
dedupBackend: 'in_memory',
|
|
3439
|
-
coordinatorUrl: new URL(request.url).origin,
|
|
3440
|
-
coordinatorInternalToken:
|
|
3441
|
-
env.DEEPLINE_INTERNAL_TOKEN?.trim() || null,
|
|
3442
|
-
} satisfies PlayWorkflowParams),
|
|
3850
|
+
body: JSON.stringify(params),
|
|
3443
3851
|
},
|
|
3444
3852
|
),
|
|
3445
3853
|
env,
|
|
@@ -3911,10 +4319,8 @@ function loadDynamicPlayWorkerSync(
|
|
|
3911
4319
|
env: {
|
|
3912
4320
|
// Service binding to the long-lived Play Harness Worker.
|
|
3913
4321
|
// Per-play code reaches this via `env.HARNESS.<method>(...)` —
|
|
3914
|
-
// see sdk/src/plays/harness-stub.ts.
|
|
3915
|
-
// environments
|
|
3916
|
-
// which case the per-play stub throws a loud error on first use
|
|
3917
|
-
// (no silent fallbacks — see harness-stub.ts → requireBinding).
|
|
4322
|
+
// see sdk/src/plays/harness-stub.ts. This binding is required;
|
|
4323
|
+
// miswired environments fail before user code starts.
|
|
3918
4324
|
HARNESS: env.HARNESS,
|
|
3919
4325
|
VERCEL_PROTECTION_BYPASS_TOKEN: env.VERCEL_PROTECTION_BYPASS_TOKEN,
|
|
3920
4326
|
// In-process runtime API bridge used by the play harness for status,
|