@cascade-flow/runner 0.2.5 → 0.2.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/checkpoint-ipc.d.ts +82 -0
- package/dist/checkpoint-ipc.d.ts.map +1 -0
- package/dist/index.d.ts +15 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +262 -17
- package/dist/index.js.map +6 -5
- package/dist/step-executor.d.ts +9 -12
- package/dist/step-executor.d.ts.map +1 -1
- package/dist/subprocess-executor.d.ts +6 -1
- package/dist/subprocess-executor.d.ts.map +1 -1
- package/package.json +4 -4
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checkpoint IPC Types and Utilities
|
|
3
|
+
*
|
|
4
|
+
* Handles communication between the subprocess (step execution) and parent process
|
|
5
|
+
* (worker) for checkpoint operations. Uses file-based IPC for simplicity.
|
|
6
|
+
*
|
|
7
|
+
* Flow:
|
|
8
|
+
* 1. Subprocess writes request to req-{id}.json
|
|
9
|
+
* 2. Parent reads request, checks cache
|
|
10
|
+
* 3. Parent writes response to res-{id}.json (hit/miss)
|
|
11
|
+
* 4. If miss: subprocess executes fn, writes result to result-{id}.json
|
|
12
|
+
* 5. Parent persists checkpoint to backend
|
|
13
|
+
*/
|
|
14
|
+
/**
|
|
15
|
+
* Request from subprocess to parent for checkpoint lookup/save
|
|
16
|
+
*/
|
|
17
|
+
export type CheckpointRequest = {
|
|
18
|
+
requestId: string;
|
|
19
|
+
name: string;
|
|
20
|
+
sequenceNumber: number;
|
|
21
|
+
};
|
|
22
|
+
/**
|
|
23
|
+
* Response from parent to subprocess
|
|
24
|
+
*/
|
|
25
|
+
export type CheckpointResponse = {
|
|
26
|
+
requestId: string;
|
|
27
|
+
hit: boolean;
|
|
28
|
+
data?: string;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Result from subprocess after executing checkpoint function (on cache miss)
|
|
32
|
+
*/
|
|
33
|
+
export type CheckpointResult = {
|
|
34
|
+
requestId: string;
|
|
35
|
+
data: string;
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* Data passed to onCheckpoint callback
|
|
39
|
+
*/
|
|
40
|
+
export type CheckpointData = {
|
|
41
|
+
name: string;
|
|
42
|
+
sequenceNumber: number;
|
|
43
|
+
data: string;
|
|
44
|
+
};
|
|
45
|
+
/**
|
|
46
|
+
* Failure from subprocess when checkpoint function throws (on cache miss)
|
|
47
|
+
*/
|
|
48
|
+
export type CheckpointFailure = {
|
|
49
|
+
requestId: string;
|
|
50
|
+
name: string;
|
|
51
|
+
sequenceNumber: number;
|
|
52
|
+
error: string;
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Data passed to onCheckpointFailed callback
|
|
56
|
+
*/
|
|
57
|
+
export type CheckpointFailedData = {
|
|
58
|
+
name: string;
|
|
59
|
+
sequenceNumber: number;
|
|
60
|
+
error: string;
|
|
61
|
+
};
|
|
62
|
+
/**
|
|
63
|
+
* Get path for checkpoint request file
|
|
64
|
+
*/
|
|
65
|
+
export declare function getRequestPath(dir: string, requestId: string): string;
|
|
66
|
+
/**
|
|
67
|
+
* Get path for checkpoint response file
|
|
68
|
+
*/
|
|
69
|
+
export declare function getResponsePath(dir: string, requestId: string): string;
|
|
70
|
+
/**
|
|
71
|
+
* Get path for checkpoint result file
|
|
72
|
+
*/
|
|
73
|
+
export declare function getResultPath(dir: string, requestId: string): string;
|
|
74
|
+
/**
|
|
75
|
+
* Get path for checkpoint failure file
|
|
76
|
+
*/
|
|
77
|
+
export declare function getFailurePath(dir: string, requestId: string): string;
|
|
78
|
+
/**
|
|
79
|
+
* Generate unique request ID for checkpoint IPC
|
|
80
|
+
*/
|
|
81
|
+
export declare function generateRequestId(): string;
|
|
82
|
+
//# sourceMappingURL=checkpoint-ipc.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint-ipc.d.ts","sourceRoot":"","sources":["../src/checkpoint-ipc.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAIH;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;CACxB,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,kBAAkB,GAAG;IAC/B,SAAS,EAAE,MAAM,CAAC;IAClB,GAAG,EAAE,OAAO,CAAC;IACb,IAAI,CAAC,EAAE,MAAM,CAAC;CACf,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC7B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,iBAAiB,GAAG;IAC9B,SAAS,EAAE,MAAM,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF;;GAEG;AACH,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF;;GAEG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAErE;AAED;;GAEG;AACH,wBAAgB,eAAe,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAEtE;AAED;;GAEG;AACH,wBAAgB,aAAa,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAEpE;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,GAAG,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,MAAM,CAErE;AAED;;GAEG;AACH,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C"}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,13 +1,27 @@
|
|
|
1
1
|
import type { Backend, LogEntry } from "@cascade-flow/backend-interface";
|
|
2
2
|
import type { StepOutput, RunnerContext } from "@cascade-flow/workflow";
|
|
3
3
|
import type { LoadedStep } from "./types";
|
|
4
|
+
import type { CheckpointData, CheckpointFailedData } from "./checkpoint-ipc";
|
|
4
5
|
export type { LoadedStep };
|
|
6
|
+
export type { CheckpointData, CheckpointFailedData } from "./checkpoint-ipc";
|
|
5
7
|
/**
|
|
6
8
|
* Execute a step in an isolated child process
|
|
7
9
|
*
|
|
8
10
|
* Wrapper around executeStepInSubprocess that generates the output path.
|
|
11
|
+
*
|
|
12
|
+
* @param stepFile - Absolute path to the step.ts file
|
|
13
|
+
* @param stepId - Unique identifier of the step
|
|
14
|
+
* @param dependencies - Resolved dependency outputs
|
|
15
|
+
* @param ctx - Runner context passed to step
|
|
16
|
+
* @param attemptNumber - Current attempt number (for retries)
|
|
17
|
+
* @param backend - Backend for generating output path
|
|
18
|
+
* @param onLog - Optional callback for real-time log emission
|
|
19
|
+
* @param onCheckpoint - Optional callback for persisting checkpoints
|
|
20
|
+
* @param onCheckpointFailed - Optional callback for persisting checkpoint failures
|
|
21
|
+
* @param existingCheckpoints - Existing checkpoints to replay (name -> data[] by sequence)
|
|
22
|
+
* @param options - Additional options (signal for abort)
|
|
9
23
|
*/
|
|
10
|
-
export declare function executeStepInProcess(stepFile: string, stepId: string, dependencies: Record<string, unknown>, ctx: RunnerContext, attemptNumber: number, backend: Backend, onLog?: (log: LogEntry) => void | Promise<void>, options?: {
|
|
24
|
+
export declare function executeStepInProcess(stepFile: string, stepId: string, dependencies: Record<string, unknown>, ctx: RunnerContext, attemptNumber: number, backend: Backend, onLog?: (log: LogEntry) => void | Promise<void>, onCheckpoint?: (checkpoint: CheckpointData) => Promise<void>, onCheckpointFailed?: (checkpoint: CheckpointFailedData) => Promise<void>, existingCheckpoints?: Map<string, string[]>, options?: {
|
|
11
25
|
signal?: AbortSignal;
|
|
12
26
|
}): Promise<{
|
|
13
27
|
result: StepOutput;
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAEzE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAEzE,OAAO,KAAK,EAAE,UAAU,EAAE,aAAa,EAAE,MAAM,wBAAwB,CAAC;AACxE,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,SAAS,CAAC;AAC1C,OAAO,KAAK,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAG7E,YAAY,EAAE,UAAU,EAAE,CAAC;AAG3B,YAAY,EAAE,cAAc,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAE7E;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAsB,oBAAoB,CACxC,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EACrC,GAAG,EAAE,aAAa,EAClB,aAAa,EAAE,MAAM,EACrB,OAAO,EAAE,OAAO,EAChB,KAAK,CAAC,EAAE,CAAC,GAAG,EAAE,QAAQ,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,EAC/C,YAAY,CAAC,EAAE,CAAC,UAAU,EAAE,cAAc,KAAK,OAAO,CAAC,IAAI,CAAC,EAC5D,kBAAkB,CAAC,EAAE,CAAC,UAAU,EAAE,oBAAoB,KAAK,OAAO,CAAC,IAAI,CAAC,EACxE,mBAAmB,CAAC,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAC3C,OAAO,CAAC,EAAE;IAAE,MAAM,CAAC,EAAE,WAAW,CAAA;CAAE,GACjC,OAAO,CAAC;IAAE,MAAM,EAAE,UAAU,CAAC;IAAC,IAAI,EAAE,QAAQ,EAAE,CAAA;CAAE,CAAC,CAuBnD;AAGD,OAAO,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AAG/D,OAAO,EAAE,qBAAqB,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAGjE,OAAO,EAAE,gBAAgB,EAAE,uBAAuB,EAAE,MAAM,cAAc,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -11,10 +11,132 @@ var __export = (target, all) => {
|
|
|
11
11
|
|
|
12
12
|
// src/subprocess-executor.ts
|
|
13
13
|
import { spawn } from "node:child_process";
|
|
14
|
-
import { resolve, dirname } from "node:path";
|
|
14
|
+
import { resolve, dirname, join as join2 } from "node:path";
|
|
15
15
|
import { fileURLToPath } from "node:url";
|
|
16
|
-
import { mkdir, readFile, unlink } from "node:fs/promises";
|
|
16
|
+
import { mkdir, readFile, unlink, access, writeFile, readdir, rm } from "node:fs/promises";
|
|
17
|
+
import { tmpdir } from "node:os";
|
|
17
18
|
import { getMicrosecondTimestamp, ensureErrorMessage } from "@cascade-flow/backend-interface";
|
|
19
|
+
|
|
20
|
+
// src/checkpoint-ipc.ts
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
function getResponsePath(dir, requestId) {
|
|
23
|
+
return join(dir, `res-${requestId}.json`);
|
|
24
|
+
}
|
|
25
|
+
function getResultPath(dir, requestId) {
|
|
26
|
+
return join(dir, `result-${requestId}.json`);
|
|
27
|
+
}
|
|
28
|
+
function getFailurePath(dir, requestId) {
|
|
29
|
+
return join(dir, `failure-${requestId}.json`);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
// src/subprocess-executor.ts
|
|
33
|
+
var EMBEDDED_STEP_EXECUTOR = `
|
|
34
|
+
import { pathToFileURL } from "node:url";
|
|
35
|
+
import { join } from "node:path";
|
|
36
|
+
|
|
37
|
+
function serializeError(err) {
|
|
38
|
+
if (err instanceof Error) {
|
|
39
|
+
return { message: err.message, stack: err.stack, name: err.name };
|
|
40
|
+
}
|
|
41
|
+
return { message: String(err), name: "Error" };
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function createCheckpointFunction(checkpointDir) {
|
|
45
|
+
const checkpointSequences = new Map();
|
|
46
|
+
|
|
47
|
+
return async function checkpoint(name, fn) {
|
|
48
|
+
if (!checkpointDir) return fn();
|
|
49
|
+
|
|
50
|
+
const seq = checkpointSequences.get(name) ?? 0;
|
|
51
|
+
checkpointSequences.set(name, seq + 1);
|
|
52
|
+
const requestId = Date.now() + "-" + Math.random().toString(36).slice(2);
|
|
53
|
+
|
|
54
|
+
// Write request
|
|
55
|
+
await Bun.write(
|
|
56
|
+
join(checkpointDir, "req-" + requestId + ".json"),
|
|
57
|
+
JSON.stringify({ requestId, name, sequenceNumber: seq })
|
|
58
|
+
);
|
|
59
|
+
|
|
60
|
+
// Poll for response
|
|
61
|
+
let response = null;
|
|
62
|
+
const responsePath = join(checkpointDir, "res-" + requestId + ".json");
|
|
63
|
+
while (!response) {
|
|
64
|
+
try {
|
|
65
|
+
response = JSON.parse(await Bun.file(responsePath).text());
|
|
66
|
+
} catch {
|
|
67
|
+
await Bun.sleep(5);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
if (response.hit && response.data !== undefined) {
|
|
72
|
+
return JSON.parse(response.data);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
// Cache miss: execute function
|
|
76
|
+
try {
|
|
77
|
+
const result = await fn();
|
|
78
|
+
await Bun.write(
|
|
79
|
+
join(checkpointDir, "result-" + requestId + ".json"),
|
|
80
|
+
JSON.stringify({ requestId, data: JSON.stringify(result) })
|
|
81
|
+
);
|
|
82
|
+
return result;
|
|
83
|
+
} catch (err) {
|
|
84
|
+
// Write failure file for parent to consume before re-throwing
|
|
85
|
+
await Bun.write(
|
|
86
|
+
join(checkpointDir, "failure-" + requestId + ".json"),
|
|
87
|
+
JSON.stringify({
|
|
88
|
+
requestId,
|
|
89
|
+
name,
|
|
90
|
+
sequenceNumber: seq,
|
|
91
|
+
error: JSON.stringify(serializeError(err)),
|
|
92
|
+
})
|
|
93
|
+
);
|
|
94
|
+
throw err;
|
|
95
|
+
}
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
async function main() {
|
|
100
|
+
try {
|
|
101
|
+
const inputFile = process.env.CF_STEP_INPUT_FILE;
|
|
102
|
+
const outputFile = process.env.STEP_OUTPUT_FILE;
|
|
103
|
+
const checkpointDir = process.env.CF_CHECKPOINT_DIR;
|
|
104
|
+
|
|
105
|
+
if (!inputFile) {
|
|
106
|
+
throw new Error("CF_STEP_INPUT_FILE environment variable is required");
|
|
107
|
+
}
|
|
108
|
+
if (!outputFile) {
|
|
109
|
+
throw new Error("STEP_OUTPUT_FILE environment variable is required");
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
const input = await Bun.file(inputFile).text();
|
|
113
|
+
const { stepPath, dependencies, ctx } = JSON.parse(input);
|
|
114
|
+
|
|
115
|
+
const checkpoint = createCheckpointFunction(checkpointDir);
|
|
116
|
+
const reconstructedCtx = { ...ctx, log: console.log, checkpoint };
|
|
117
|
+
|
|
118
|
+
// Load and execute the step
|
|
119
|
+
const mod = await import(pathToFileURL(stepPath).toString());
|
|
120
|
+
const stepDef = mod.step;
|
|
121
|
+
|
|
122
|
+
if (!stepDef || typeof stepDef.fn !== "function") {
|
|
123
|
+
throw new Error("Invalid step module at " + stepPath);
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
const result = await stepDef.fn({ dependencies, ctx: reconstructedCtx });
|
|
127
|
+
await Bun.write(outputFile, JSON.stringify(result, null, 2));
|
|
128
|
+
process.exit(0);
|
|
129
|
+
} catch (error) {
|
|
130
|
+
const errObj = error instanceof Error
|
|
131
|
+
? { message: error.message, stack: error.stack, name: error.name }
|
|
132
|
+
: { message: String(error), name: "Error" };
|
|
133
|
+
process.stderr.write(JSON.stringify(errObj));
|
|
134
|
+
process.exit(1);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
main();
|
|
139
|
+
`;
|
|
18
140
|
function createStreamHandler(streamType, attemptNumber, emitLog) {
|
|
19
141
|
let buffer = "";
|
|
20
142
|
const handler = (chunk) => {
|
|
@@ -48,15 +170,124 @@ function createStreamHandler(streamType, attemptNumber, emitLog) {
|
|
|
48
170
|
};
|
|
49
171
|
return { handler, getBuffer, flushBuffer };
|
|
50
172
|
}
|
|
51
|
-
|
|
52
|
-
const
|
|
173
|
+
function startCheckpointWatcher(dir, existing, onCheckpoint, onCheckpointFailed) {
|
|
174
|
+
const processed = new Set;
|
|
175
|
+
let running = true;
|
|
176
|
+
const poll = async () => {
|
|
177
|
+
while (running) {
|
|
178
|
+
try {
|
|
179
|
+
const files = await readdir(dir);
|
|
180
|
+
for (const file of files) {
|
|
181
|
+
if (file.startsWith("failure-") && !processed.has(file)) {
|
|
182
|
+
processed.add(file);
|
|
183
|
+
try {
|
|
184
|
+
const content = await readFile(join2(dir, file), "utf-8");
|
|
185
|
+
const failure = JSON.parse(content);
|
|
186
|
+
if (onCheckpointFailed) {
|
|
187
|
+
await onCheckpointFailed({
|
|
188
|
+
name: failure.name,
|
|
189
|
+
sequenceNumber: failure.sequenceNumber,
|
|
190
|
+
error: failure.error
|
|
191
|
+
});
|
|
192
|
+
}
|
|
193
|
+
} catch {}
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
if (!file.startsWith("req-") || processed.has(file))
|
|
197
|
+
continue;
|
|
198
|
+
processed.add(file);
|
|
199
|
+
try {
|
|
200
|
+
const content = await readFile(join2(dir, file), "utf-8");
|
|
201
|
+
const request = JSON.parse(content);
|
|
202
|
+
const cached = existing?.get(request.name)?.[request.sequenceNumber];
|
|
203
|
+
if (cached !== undefined) {
|
|
204
|
+
const response = {
|
|
205
|
+
requestId: request.requestId,
|
|
206
|
+
hit: true,
|
|
207
|
+
data: cached
|
|
208
|
+
};
|
|
209
|
+
await writeFile(getResponsePath(dir, request.requestId), JSON.stringify(response));
|
|
210
|
+
} else {
|
|
211
|
+
const response = {
|
|
212
|
+
requestId: request.requestId,
|
|
213
|
+
hit: false
|
|
214
|
+
};
|
|
215
|
+
await writeFile(getResponsePath(dir, request.requestId), JSON.stringify(response));
|
|
216
|
+
let result = null;
|
|
217
|
+
let failed = false;
|
|
218
|
+
const resultPath = getResultPath(dir, request.requestId);
|
|
219
|
+
const failurePath = getFailurePath(dir, request.requestId);
|
|
220
|
+
while (running && !result && !failed) {
|
|
221
|
+
try {
|
|
222
|
+
const resultContent = await readFile(resultPath, "utf-8");
|
|
223
|
+
result = JSON.parse(resultContent);
|
|
224
|
+
} catch {
|
|
225
|
+
try {
|
|
226
|
+
await readFile(failurePath, "utf-8");
|
|
227
|
+
failed = true;
|
|
228
|
+
} catch {
|
|
229
|
+
await new Promise((resolve2) => setTimeout(resolve2, 5));
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
if (result && onCheckpoint) {
|
|
234
|
+
if (!existing.has(request.name)) {
|
|
235
|
+
existing.set(request.name, []);
|
|
236
|
+
}
|
|
237
|
+
const arr = existing.get(request.name);
|
|
238
|
+
arr[request.sequenceNumber] = result.data;
|
|
239
|
+
await onCheckpoint({
|
|
240
|
+
name: request.name,
|
|
241
|
+
sequenceNumber: request.sequenceNumber,
|
|
242
|
+
data: result.data
|
|
243
|
+
});
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
} catch (e) {}
|
|
247
|
+
}
|
|
248
|
+
} catch {}
|
|
249
|
+
await new Promise((resolve2) => setTimeout(resolve2, 10));
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
poll().catch(() => {});
|
|
253
|
+
return {
|
|
254
|
+
stop: () => {
|
|
255
|
+
running = false;
|
|
256
|
+
}
|
|
257
|
+
};
|
|
258
|
+
}
|
|
259
|
+
async function executeStepInSubprocess(stepFile, stepId, dependencies, ctx, attemptNumber, outputPath, onLog, onCheckpoint, onCheckpointFailed, existingCheckpoints, options) {
|
|
53
260
|
await mkdir(dirname(outputPath), { recursive: true });
|
|
261
|
+
const executorPath = resolve(dirname(fileURLToPath(import.meta.url)), "step-executor.ts");
|
|
262
|
+
let spawnArgs;
|
|
263
|
+
let tempExecutorPath = null;
|
|
264
|
+
const stepInput = JSON.stringify({ stepPath: stepFile, dependencies, ctx });
|
|
265
|
+
const inputFilePath = join2(tmpdir(), `cf-step-input-${Date.now()}-${Math.random().toString(36).slice(2)}.json`);
|
|
266
|
+
await writeFile(inputFilePath, stepInput);
|
|
267
|
+
try {
|
|
268
|
+
await access(executorPath);
|
|
269
|
+
spawnArgs = ["bun", [executorPath]];
|
|
270
|
+
} catch {
|
|
271
|
+
tempExecutorPath = join2(tmpdir(), `cf-step-executor-${Date.now()}-${Math.random().toString(36).slice(2)}.ts`);
|
|
272
|
+
await writeFile(tempExecutorPath, EMBEDDED_STEP_EXECUTOR);
|
|
273
|
+
spawnArgs = ["bun", [tempExecutorPath]];
|
|
274
|
+
}
|
|
275
|
+
let checkpointDir;
|
|
276
|
+
let checkpointWatcher;
|
|
277
|
+
const checkpointCache = existingCheckpoints ?? new Map;
|
|
278
|
+
if (onCheckpoint || onCheckpointFailed || existingCheckpoints) {
|
|
279
|
+
checkpointDir = join2(tmpdir(), `cf-checkpoint-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
280
|
+
await mkdir(checkpointDir, { recursive: true });
|
|
281
|
+
checkpointWatcher = startCheckpointWatcher(checkpointDir, checkpointCache, onCheckpoint, onCheckpointFailed);
|
|
282
|
+
}
|
|
54
283
|
return new Promise((resolve2, reject) => {
|
|
55
|
-
const child = spawn(
|
|
284
|
+
const child = spawn(spawnArgs[0], spawnArgs[1], {
|
|
56
285
|
stdio: ["pipe", "pipe", "pipe"],
|
|
57
286
|
env: {
|
|
58
287
|
...process.env,
|
|
59
|
-
STEP_OUTPUT_FILE: outputPath
|
|
288
|
+
STEP_OUTPUT_FILE: outputPath,
|
|
289
|
+
CF_STEP_INPUT_FILE: inputFilePath,
|
|
290
|
+
...checkpointDir ? { CF_CHECKPOINT_DIR: checkpointDir } : {}
|
|
60
291
|
}
|
|
61
292
|
});
|
|
62
293
|
const logs = [];
|
|
@@ -107,6 +338,22 @@ async function executeStepInSubprocess(stepFile, stepId, dependencies, ctx, atte
|
|
|
107
338
|
});
|
|
108
339
|
child.on("close", async (code, signal2) => {
|
|
109
340
|
cleanup();
|
|
341
|
+
if (checkpointWatcher) {
|
|
342
|
+
checkpointWatcher.stop();
|
|
343
|
+
}
|
|
344
|
+
if (tempExecutorPath) {
|
|
345
|
+
try {
|
|
346
|
+
await unlink(tempExecutorPath);
|
|
347
|
+
} catch {}
|
|
348
|
+
}
|
|
349
|
+
try {
|
|
350
|
+
await unlink(inputFilePath);
|
|
351
|
+
} catch {}
|
|
352
|
+
if (checkpointDir) {
|
|
353
|
+
try {
|
|
354
|
+
await rm(checkpointDir, { recursive: true, force: true });
|
|
355
|
+
} catch {}
|
|
356
|
+
}
|
|
110
357
|
stdoutHandler.flushBuffer();
|
|
111
358
|
stderrHandler.flushBuffer();
|
|
112
359
|
try {
|
|
@@ -156,8 +403,6 @@ async function executeStepInSubprocess(stepFile, stepId, dependencies, ctx, atte
|
|
|
156
403
|
}
|
|
157
404
|
}
|
|
158
405
|
});
|
|
159
|
-
const input = JSON.stringify({ stepPath: stepFile, dependencies, ctx });
|
|
160
|
-
child.stdin.write(input);
|
|
161
406
|
child.stdin.end();
|
|
162
407
|
});
|
|
163
408
|
}
|
|
@@ -12778,22 +13023,22 @@ async function discoverSteps(root = path.resolve("steps")) {
|
|
|
12778
13023
|
}
|
|
12779
13024
|
// src/versioning.ts
|
|
12780
13025
|
import { createHash } from "node:crypto";
|
|
12781
|
-
import { readFile as readFile2, readdir } from "node:fs/promises";
|
|
12782
|
-
import { join } from "node:path";
|
|
13026
|
+
import { readFile as readFile2, readdir as readdir2 } from "node:fs/promises";
|
|
13027
|
+
import { join as join3 } from "node:path";
|
|
12783
13028
|
import { exec } from "node:child_process";
|
|
12784
13029
|
import { promisify } from "node:util";
|
|
12785
13030
|
var execAsync = promisify(exec);
|
|
12786
13031
|
async function calculateWorkflowHash(workflow) {
|
|
12787
13032
|
const hash2 = createHash("sha256");
|
|
12788
13033
|
try {
|
|
12789
|
-
const workflowJsonPath =
|
|
13034
|
+
const workflowJsonPath = join3(workflow.dir, "workflow.json");
|
|
12790
13035
|
const workflowJsonContent = await readFile2(workflowJsonPath, "utf-8");
|
|
12791
13036
|
hash2.update(`workflow.json:${workflowJsonContent}`);
|
|
12792
13037
|
} catch (error46) {
|
|
12793
13038
|
throw new Error(`Failed to read workflow.json for ${workflow.slug}: ${error46 instanceof Error ? error46.message : "Unknown error"}`);
|
|
12794
13039
|
}
|
|
12795
13040
|
try {
|
|
12796
|
-
const inputSchemaPath =
|
|
13041
|
+
const inputSchemaPath = join3(workflow.dir, "input-schema.ts");
|
|
12797
13042
|
const inputSchemaContent = await readFile2(inputSchemaPath, "utf-8");
|
|
12798
13043
|
hash2.update(`input-schema.ts:${inputSchemaContent}`);
|
|
12799
13044
|
} catch (error46) {
|
|
@@ -12818,9 +13063,9 @@ async function calculateWorkflowHash(workflow) {
|
|
|
12818
13063
|
async function collectStepFiles(stepsDir) {
|
|
12819
13064
|
const files = [];
|
|
12820
13065
|
async function scan(dir) {
|
|
12821
|
-
const entries = await
|
|
13066
|
+
const entries = await readdir2(dir, { withFileTypes: true });
|
|
12822
13067
|
for (const entry of entries) {
|
|
12823
|
-
const fullPath =
|
|
13068
|
+
const fullPath = join3(dir, entry.name);
|
|
12824
13069
|
if (entry.isDirectory()) {
|
|
12825
13070
|
await scan(fullPath);
|
|
12826
13071
|
} else if (entry.isFile() && entry.name === "step.ts") {
|
|
@@ -12904,9 +13149,9 @@ async function validateWorkflowVersion(workflowSlug, parentRunId, currentVersion
|
|
|
12904
13149
|
}
|
|
12905
13150
|
|
|
12906
13151
|
// src/index.ts
|
|
12907
|
-
async function executeStepInProcess(stepFile, stepId, dependencies, ctx, attemptNumber, backend, onLog, options) {
|
|
13152
|
+
async function executeStepInProcess(stepFile, stepId, dependencies, ctx, attemptNumber, backend, onLog, onCheckpoint, onCheckpointFailed, existingCheckpoints, options) {
|
|
12908
13153
|
const outputPath = backend.getStepOutputPath(ctx.workflow.slug, ctx.runId, stepId, attemptNumber);
|
|
12909
|
-
return executeStepInSubprocess(stepFile, stepId, dependencies, ctx, attemptNumber, outputPath, onLog, options);
|
|
13154
|
+
return executeStepInSubprocess(stepFile, stepId, dependencies, ctx, attemptNumber, outputPath, onLog, onCheckpoint, onCheckpointFailed, existingCheckpoints, options);
|
|
12910
13155
|
}
|
|
12911
13156
|
export {
|
|
12912
13157
|
validateWorkflowVersion,
|
|
@@ -12918,4 +13163,4 @@ export {
|
|
|
12918
13163
|
calculateWorkflowHash
|
|
12919
13164
|
};
|
|
12920
13165
|
|
|
12921
|
-
//# debugId=
|
|
13166
|
+
//# debugId=922C5B57E7B7118364756E2164756E21
|