@protobase/workflows 0.0.0-stage → 0.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/dist/definition/check-definition.d.ts +21 -0
- package/dist/definition/check-definition.js +128 -0
- package/dist/definition/check-reads.js +46 -0
- package/dist/definition/check-schema.js +74 -0
- package/dist/definition/check-trigger.d.ts +7 -0
- package/dist/definition/check-trigger.js +27 -0
- package/dist/definition/expression-fields.js +68 -0
- package/dist/definition/expressions.js +35 -0
- package/dist/definition/graph.js +39 -0
- package/dist/definition/names.js +14 -0
- package/dist/definition/problem.d.ts +13 -0
- package/dist/definition/problem.js +11 -0
- package/dist/definition/reads.js +81 -0
- package/dist/definition/schema.d.ts +259 -0
- package/dist/definition/schema.js +258 -0
- package/dist/definition/types.d.ts +92 -0
- package/dist/definition/visible.js +52 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +16 -0
- package/dist/limits.d.ts +39 -0
- package/dist/limits.js +40 -0
- package/dist/nesting.d.ts +7 -0
- package/dist/nesting.js +19 -0
- package/dist/plugin/connection-error.d.ts +10 -0
- package/dist/plugin/connection-error.js +15 -0
- package/dist/plugin/context.d.ts +99 -0
- package/dist/plugin/define-plugin.d.ts +7 -0
- package/dist/plugin/define-plugin.js +23 -0
- package/dist/plugin/fetch-errors.d.ts +14 -0
- package/dist/plugin/fetch-errors.js +18 -0
- package/dist/plugin/file-value.d.ts +5 -0
- package/dist/plugin/file-value.js +9 -0
- package/dist/plugin/index.d.ts +8 -0
- package/dist/plugin/index.js +6 -0
- package/dist/plugin/load-plugins.d.ts +10 -0
- package/dist/plugin/load-plugins.js +22 -0
- package/dist/plugin/step-handlers.d.ts +12 -0
- package/dist/plugin/step-handlers.js +33 -0
- package/dist/plugin/trigger-checks.d.ts +6 -0
- package/dist/plugin/trigger-checks.js +28 -0
- package/dist/plugin/type-catalog.d.ts +19 -0
- package/dist/plugin/type-catalog.js +26 -0
- package/dist/plugin/type-list.d.ts +26 -0
- package/dist/plugin/type-list.js +26 -0
- package/dist/plugin/types.d.ts +123 -0
- package/dist/run/context.d.ts +18 -0
- package/dist/run/errors.d.ts +20 -0
- package/dist/run/errors.js +39 -0
- package/dist/run/evaluate.js +59 -0
- package/dist/run/fan-out.js +39 -0
- package/dist/run/json-bytes.js +5 -0
- package/dist/run/port.d.ts +105 -0
- package/dist/run/position.js +54 -0
- package/dist/run/recovery.js +76 -0
- package/dist/run/redact.d.ts +16 -0
- package/dist/run/redact.js +39 -0
- package/dist/run/rerun.d.ts +18 -0
- package/dist/run/rerun.js +62 -0
- package/dist/run/result-states.js +115 -0
- package/dist/run/runner.d.ts +29 -0
- package/dist/run/runner.js +60 -0
- package/dist/run/scope.js +127 -0
- package/dist/run/segment.d.ts +35 -0
- package/dist/run/segment.js +58 -0
- package/dist/run/simple-states.js +108 -0
- package/dist/run/step-input.js +22 -0
- package/dist/run/task.js +39 -0
- package/package.json +34 -4
- package/README.md +0 -3
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
//#region src/run/redact.ts
|
|
2
|
+
const secretKey = /^pass$|pass(word|phrase|code|wd)|secret|token|authorization|api[-_]?key|cookie|credential|private[-_]?key|signature/i;
|
|
3
|
+
/** A copy of a value with each field whose name says it holds a secret, such as `Authorization` or `apiKey`, replaced. */
|
|
4
|
+
const redacted = (value) => {
|
|
5
|
+
if (Array.isArray(value)) return value.map(redacted);
|
|
6
|
+
if (typeof value !== "object" || value === null) return value;
|
|
7
|
+
return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, secretKey.test(key) ? "[redacted]" : redacted(item)]));
|
|
8
|
+
};
|
|
9
|
+
/** Secret values shorter than this are refused, so blanking them never blanks ordinary text. */
|
|
10
|
+
const shortestSecret = 8;
|
|
11
|
+
/**
|
|
12
|
+
* The redactor of the app's secret values, wherever they appear inside a text, such as `Bearer <token>` or a token in
|
|
13
|
+
* a URL's query; the longest first, so a secret inside another is blanked whole.
|
|
14
|
+
*/
|
|
15
|
+
const secretRedactor = (secrets) => {
|
|
16
|
+
if (secrets.find((secret) => secret.length < 8) !== void 0) throw new Error(`A secret workflows read is shorter than 8 characters, too short to blank out of what runs record`);
|
|
17
|
+
const ordered = [...new Set(secrets)].sort((a, b) => b.length - a.length);
|
|
18
|
+
const text = (input) => ordered.reduce((result, secret) => result.replaceAll(secret, "[redacted]"), input);
|
|
19
|
+
const value = (input) => {
|
|
20
|
+
if (ordered.length === 0) return input;
|
|
21
|
+
if (typeof input === "string") return text(input);
|
|
22
|
+
if (Array.isArray(input)) return input.map(value);
|
|
23
|
+
if (typeof input !== "object" || input === null || input instanceof Date) return input;
|
|
24
|
+
const kept = {};
|
|
25
|
+
for (const [key, item] of Object.entries(input)) {
|
|
26
|
+
const blanked = text(key);
|
|
27
|
+
let name = blanked;
|
|
28
|
+
for (let count = 2; Object.hasOwn(kept, name); count++) name = `${blanked} (${count})`;
|
|
29
|
+
kept[name] = value(item);
|
|
30
|
+
}
|
|
31
|
+
return kept;
|
|
32
|
+
};
|
|
33
|
+
return {
|
|
34
|
+
text,
|
|
35
|
+
value
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
//#endregion
|
|
39
|
+
export { redacted, secretRedactor, shortestSecret };
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { Machine } from "../definition/types.js";
|
|
2
|
+
import { SavedStep } from "./port.js";
|
|
3
|
+
//#region src/run/rerun.d.ts
|
|
4
|
+
/** What a re-run reads of a failed run's step to know whether to keep it. */
|
|
5
|
+
export type EarlierStep = Pick<SavedStep, 'scope' | 'seq' | 'state' | 'status' | 'next'>;
|
|
6
|
+
/**
|
|
7
|
+
* The steps of a failed run that a re-run of it keeps, to run on from where they leave it with `machine`, the workflow's
|
|
8
|
+
* version now: every step but, in each scope, the try that failed it and that state's retried tries before it, so the
|
|
9
|
+
* failed state runs again from its first try. A finished Map's items are kept as they ran; the items of a Map that runs
|
|
10
|
+
* again are kept where they stand, each without its own failed try, and those of a Map's earlier tries are not.
|
|
11
|
+
* Returns the problem instead when a scope would go on at a state the version does not have.
|
|
12
|
+
*/
|
|
13
|
+
export declare const rerunSteps: <S extends EarlierStep>(machine: Machine, steps: readonly S[]) => {
|
|
14
|
+
steps: S[];
|
|
15
|
+
} | {
|
|
16
|
+
problem: string;
|
|
17
|
+
};
|
|
18
|
+
//#endregion
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
//#region src/run/rerun.ts
|
|
2
|
+
const itemOf = (scope) => {
|
|
3
|
+
const slash = scope.lastIndexOf("/");
|
|
4
|
+
const match = /^(.*)\.(\d+)\[\d+\]$/.exec(scope.slice(slash + 1));
|
|
5
|
+
if (!match) throw new Error(`"${scope}" is not the scope of a Map item`);
|
|
6
|
+
return {
|
|
7
|
+
parent: slash < 0 ? "" : scope.slice(0, slash),
|
|
8
|
+
map: decodeURIComponent(match[1]),
|
|
9
|
+
seq: Number(match[2])
|
|
10
|
+
};
|
|
11
|
+
};
|
|
12
|
+
const withoutFailure = (steps) => {
|
|
13
|
+
if (steps.at(-1)?.status !== "failed") return [...steps];
|
|
14
|
+
const failed = steps.at(-1);
|
|
15
|
+
let end = steps.length - 1;
|
|
16
|
+
while (end > 0 && steps[end - 1].status === "retrying" && steps[end - 1].state === failed.state) end--;
|
|
17
|
+
return steps.slice(0, end);
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* The steps of a failed run that a re-run of it keeps, to run on from where they leave it with `machine`, the workflow's
|
|
21
|
+
* version now: every step but, in each scope, the try that failed it and that state's retried tries before it, so the
|
|
22
|
+
* failed state runs again from its first try. A finished Map's items are kept as they ran; the items of a Map that runs
|
|
23
|
+
* again are kept where they stand, each without its own failed try, and those of a Map's earlier tries are not.
|
|
24
|
+
* Returns the problem instead when a scope would go on at a state the version does not have.
|
|
25
|
+
*/
|
|
26
|
+
const rerunSteps = (machine, steps) => {
|
|
27
|
+
const byScope = /* @__PURE__ */ new Map();
|
|
28
|
+
for (const step of [...steps].sort((a, b) => a.seq - b.seq)) byScope.set(step.scope, [...byScope.get(step.scope) ?? [], step]);
|
|
29
|
+
const children = /* @__PURE__ */ new Map();
|
|
30
|
+
for (const scope of byScope.keys()) {
|
|
31
|
+
if (scope === "") continue;
|
|
32
|
+
const { parent } = itemOf(scope);
|
|
33
|
+
children.set(parent, [...children.get(parent) ?? [], scope]);
|
|
34
|
+
}
|
|
35
|
+
const kept = [];
|
|
36
|
+
const keepAll = (scope) => {
|
|
37
|
+
kept.push(...byScope.get(scope) ?? []);
|
|
38
|
+
for (const child of children.get(scope) ?? []) keepAll(child);
|
|
39
|
+
};
|
|
40
|
+
const keepFrom = (scope, inner) => {
|
|
41
|
+
const own = withoutFailure(byScope.get(scope) ?? []);
|
|
42
|
+
kept.push(...own);
|
|
43
|
+
const last = own.at(-1);
|
|
44
|
+
const at = last === void 0 ? inner.StartAt : last.status === "retrying" ? last.state : last.next;
|
|
45
|
+
if (at !== null && !Object.hasOwn(inner.States, at)) return `The run goes on at state "${at}"${scope ? ` in ${scope}` : ""}, which the workflow no longer has`;
|
|
46
|
+
const state = at === null ? void 0 : inner.States[at];
|
|
47
|
+
const ran = new Set(own.map((step) => step.seq));
|
|
48
|
+
const next = last === void 0 ? 0 : last.seq + 1;
|
|
49
|
+
for (const child of children.get(scope) ?? []) {
|
|
50
|
+
const item = itemOf(child);
|
|
51
|
+
if (ran.has(item.seq)) keepAll(child);
|
|
52
|
+
else if (item.seq === next && item.map === at && state?.Type === "Map") {
|
|
53
|
+
const problem = keepFrom(child, state.ItemProcessor);
|
|
54
|
+
if (problem) return problem;
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
const problem = keepFrom("", machine);
|
|
59
|
+
return problem ? { problem } : { steps: kept };
|
|
60
|
+
};
|
|
61
|
+
//#endregion
|
|
62
|
+
export { rerunSteps };
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
import { limits } from "../limits.js";
|
|
2
|
+
import { StepError, errorOutput } from "./errors.js";
|
|
3
|
+
import { redacted } from "./redact.js";
|
|
4
|
+
import { jsonBytes } from "./json-bytes.js";
|
|
5
|
+
import { evaluateAssign, evaluateKept, evaluateTemplate, recordedArguments } from "./evaluate.js";
|
|
6
|
+
import { fanOut } from "./fan-out.js";
|
|
7
|
+
import { recover } from "./recovery.js";
|
|
8
|
+
import { innerScope, visibleVariables } from "./step-input.js";
|
|
9
|
+
import { runTask } from "./task.js";
|
|
10
|
+
//#region src/run/result-states.ts
|
|
11
|
+
const task = async (state, call) => {
|
|
12
|
+
const args = await evaluateTemplate(state.Arguments ?? {}, call.bindings, "Arguments");
|
|
13
|
+
const key = `${call.context.runId}/${call.scope}/${call.position.seq - call.position.attempt + 1}`;
|
|
14
|
+
const input = redacted(recordedArguments(state.Arguments ?? {}, args));
|
|
15
|
+
return runTask(call.context, state, args, key, call.logs).then((result) => ({
|
|
16
|
+
result,
|
|
17
|
+
input
|
|
18
|
+
}), (error) => {
|
|
19
|
+
if (!(error instanceof StepError)) throw error;
|
|
20
|
+
return {
|
|
21
|
+
failure: errorOutput(error),
|
|
22
|
+
input
|
|
23
|
+
};
|
|
24
|
+
});
|
|
25
|
+
};
|
|
26
|
+
const map = async (state, call) => {
|
|
27
|
+
const items = await evaluateKept(state.Items, call.bindings, "Items");
|
|
28
|
+
if (!Array.isArray(items)) throw new StepError("States.QueryEvaluationError", `Items: gave ${JSON.stringify(items) ?? "nothing"}, not an array`);
|
|
29
|
+
if (items.length > limits.mapItems) throw new StepError("States.DataLimitExceeded", `Items: ${items.length} items, more than the ${limits.mapItems} a Map takes`);
|
|
30
|
+
const outer = visibleVariables(call.bindings);
|
|
31
|
+
let kept = 0;
|
|
32
|
+
const scopes = items.map((item, index) => async () => {
|
|
33
|
+
const outcome = await call.runScope(state.ItemProcessor, innerScope(call.scope, call.name, call.position.seq, index), item, outer);
|
|
34
|
+
if (outcome.kind !== "ended") return outcome;
|
|
35
|
+
kept += jsonBytes(outcome.output);
|
|
36
|
+
if (kept <= limits.valueBytes) return outcome;
|
|
37
|
+
return {
|
|
38
|
+
kind: "failed",
|
|
39
|
+
error: "States.DataLimitExceeded",
|
|
40
|
+
cause: `The items' outputs are larger than ${limits.valueBytes / 1024 / 1024} MiB together`
|
|
41
|
+
};
|
|
42
|
+
});
|
|
43
|
+
const outcome = await fanOut(scopes, state.MaxConcurrency || limits.mapConcurrency);
|
|
44
|
+
if (outcome.kind === "blocked" || outcome.kind === "yielded") return outcome;
|
|
45
|
+
const input = { items: items.length };
|
|
46
|
+
if (outcome.kind === "failed") return {
|
|
47
|
+
failure: {
|
|
48
|
+
Error: outcome.error,
|
|
49
|
+
Cause: outcome.cause
|
|
50
|
+
},
|
|
51
|
+
input
|
|
52
|
+
};
|
|
53
|
+
call.logs.push({
|
|
54
|
+
at: call.context.now().toISOString(),
|
|
55
|
+
message: `${items.length} items`
|
|
56
|
+
});
|
|
57
|
+
return {
|
|
58
|
+
result: outcome.outputs,
|
|
59
|
+
input
|
|
60
|
+
};
|
|
61
|
+
};
|
|
62
|
+
const attempt = async (state, call) => {
|
|
63
|
+
try {
|
|
64
|
+
if (state.Type === "Task") return await task(state, call);
|
|
65
|
+
return await map(state, call);
|
|
66
|
+
} catch (error) {
|
|
67
|
+
if (!(error instanceof StepError)) throw error;
|
|
68
|
+
return {
|
|
69
|
+
failure: errorOutput(error),
|
|
70
|
+
input: void 0
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
};
|
|
74
|
+
/**
|
|
75
|
+
* A Task or a Map: its result shaped by `Output` and `Assign`, which read it as `$states.result`; a failure,
|
|
76
|
+
* its own or an item's, goes to its retriers and catchers.
|
|
77
|
+
*/
|
|
78
|
+
const runResultState = async (state, call) => {
|
|
79
|
+
const ran = await attempt(state, call);
|
|
80
|
+
if ("kind" in ran) return ran;
|
|
81
|
+
const now = call.context.now();
|
|
82
|
+
if ("failure" in ran) return {
|
|
83
|
+
...await recover(state, call.position, ran.failure, call.bindings, now, call.logs),
|
|
84
|
+
...ran.input !== void 0 && { input: ran.input }
|
|
85
|
+
};
|
|
86
|
+
const after = {
|
|
87
|
+
...call.bindings,
|
|
88
|
+
states: {
|
|
89
|
+
...call.bindings.states,
|
|
90
|
+
result: ran.result
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
const shaped = async () => ({
|
|
94
|
+
output: state.Output === void 0 ? ran.result : await evaluateKept(state.Output, after, "Output"),
|
|
95
|
+
assigned: await evaluateAssign(state.Assign, after)
|
|
96
|
+
});
|
|
97
|
+
const done = await shaped().catch((error) => {
|
|
98
|
+
if (!(error instanceof StepError)) throw error;
|
|
99
|
+
return { failure: errorOutput(error) };
|
|
100
|
+
});
|
|
101
|
+
if ("failure" in done) return {
|
|
102
|
+
...await recover(state, call.position, done.failure, call.bindings, now, call.logs),
|
|
103
|
+
input: ran.input
|
|
104
|
+
};
|
|
105
|
+
return {
|
|
106
|
+
kind: "step",
|
|
107
|
+
status: "succeeded",
|
|
108
|
+
next: state.Next ?? null,
|
|
109
|
+
input: ran.input,
|
|
110
|
+
...done,
|
|
111
|
+
logs: call.logs
|
|
112
|
+
};
|
|
113
|
+
};
|
|
114
|
+
//#endregion
|
|
115
|
+
export { runResultState };
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { RunStore, SegmentQueue, TaskHandlers } from "./port.js";
|
|
2
|
+
//#region src/run/runner.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* The runner's port: where runs are kept, the queue that runs their segments, the step types, and the secrets
|
|
5
|
+
* expressions may read by name, which the runner blanks out of all a run records.
|
|
6
|
+
*/
|
|
7
|
+
export type RunnerPort<Db> = {
|
|
8
|
+
store: RunStore<Db>;
|
|
9
|
+
queue: SegmentQueue<Db>;
|
|
10
|
+
tasks: TaskHandlers;
|
|
11
|
+
secrets?: Readonly<Record<string, string>>;
|
|
12
|
+
now?: () => Date;
|
|
13
|
+
budgetMs?: number;
|
|
14
|
+
maxSteps?: number;
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* The workflow runner over a port. `start` queues a run's first segment in `db`, which may be the caller's
|
|
18
|
+
* transaction, so the run starts when it commits; `segment` is what the queue's job runs.
|
|
19
|
+
*/
|
|
20
|
+
export declare const createRunner: <Db>({ store, queue, tasks, secrets, now, budgetMs, maxSteps }: RunnerPort<Db>) => {
|
|
21
|
+
start: (db: Db, runId: string) => Promise<void>;
|
|
22
|
+
/**
|
|
23
|
+
* Runs the run from where it stands to its next wait, then queues the next segment or ends the run. Run again after
|
|
24
|
+
* a crash, it takes up from the last saved step: only the step that was in progress runs again.
|
|
25
|
+
*/
|
|
26
|
+
segment: (db: Db, runId: string, signal: AbortSignal) => Promise<void>;
|
|
27
|
+
};
|
|
28
|
+
export type Runner<Db> = ReturnType<typeof createRunner<Db>>;
|
|
29
|
+
//#endregion
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { limits } from "../limits.js";
|
|
2
|
+
import { secretRedactor } from "./redact.js";
|
|
3
|
+
import { runSegment } from "./segment.js";
|
|
4
|
+
//#region src/run/runner.ts
|
|
5
|
+
const ended = /* @__PURE__ */ new Set(["succeeded", "failed"]);
|
|
6
|
+
/**
|
|
7
|
+
* The workflow runner over a port. `start` queues a run's first segment in `db`, which may be the caller's
|
|
8
|
+
* transaction, so the run starts when it commits; `segment` is what the queue's job runs.
|
|
9
|
+
*/
|
|
10
|
+
const createRunner = ({ store, queue, tasks, secrets = {}, now = () => /* @__PURE__ */ new Date(), budgetMs = limits.segmentMs, maxSteps = limits.runSteps }) => {
|
|
11
|
+
const redactor = secretRedactor(Object.values(secrets));
|
|
12
|
+
return {
|
|
13
|
+
start: (db, runId) => queue.schedule(db, runId),
|
|
14
|
+
/**
|
|
15
|
+
* Runs the run from where it stands to its next wait, then queues the next segment or ends the run. Run again after
|
|
16
|
+
* a crash, it takes up from the last saved step: only the step that was in progress runs again.
|
|
17
|
+
*/
|
|
18
|
+
segment: async (db, runId, signal) => {
|
|
19
|
+
const run = await store.load(db, runId);
|
|
20
|
+
if (!run || ended.has(run.status)) return;
|
|
21
|
+
await store.update(db, runId, { status: "running" });
|
|
22
|
+
const segment = {
|
|
23
|
+
runId,
|
|
24
|
+
definition: run.definition,
|
|
25
|
+
input: run.input,
|
|
26
|
+
startedAt: run.startedAt,
|
|
27
|
+
secrets,
|
|
28
|
+
redactor,
|
|
29
|
+
tasks,
|
|
30
|
+
signal,
|
|
31
|
+
now,
|
|
32
|
+
budgetMs,
|
|
33
|
+
maxSteps
|
|
34
|
+
};
|
|
35
|
+
const outcome = await runSegment({
|
|
36
|
+
...segment,
|
|
37
|
+
saved: await store.steps(db, runId),
|
|
38
|
+
save: (steps) => store.save(db, runId, steps)
|
|
39
|
+
});
|
|
40
|
+
if (outcome.kind === "superseded") return;
|
|
41
|
+
if (outcome.kind === "ended") return store.update(db, runId, {
|
|
42
|
+
status: "succeeded",
|
|
43
|
+
output: redactor.value(outcome.output)
|
|
44
|
+
});
|
|
45
|
+
if (outcome.kind === "failed") return store.update(db, runId, {
|
|
46
|
+
status: "failed",
|
|
47
|
+
error: redactor.text(outcome.error),
|
|
48
|
+
cause: redactor.text(outcome.cause)
|
|
49
|
+
});
|
|
50
|
+
if (outcome.kind === "yielded") return queue.schedule(db, runId);
|
|
51
|
+
await store.update(db, runId, {
|
|
52
|
+
status: "waiting",
|
|
53
|
+
resumeAt: outcome.until
|
|
54
|
+
});
|
|
55
|
+
await queue.schedule(db, runId, outcome.until);
|
|
56
|
+
}
|
|
57
|
+
};
|
|
58
|
+
};
|
|
59
|
+
//#endregion
|
|
60
|
+
export { createRunner };
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { limits } from "../limits.js";
|
|
2
|
+
import { StepError, SupersededError } from "./errors.js";
|
|
3
|
+
import { jsonBytes } from "./json-bytes.js";
|
|
4
|
+
import { advance, foldPosition } from "./position.js";
|
|
5
|
+
import { bindingsOf } from "./step-input.js";
|
|
6
|
+
import { runResultState } from "./result-states.js";
|
|
7
|
+
import { runChoice, runFail, runPass, runWait } from "./simple-states.js";
|
|
8
|
+
//#region src/run/scope.ts
|
|
9
|
+
const runState = (state, call) => {
|
|
10
|
+
if (state.Type === "Pass" || state.Type === "Succeed") return runPass(state, call);
|
|
11
|
+
if (state.Type === "Choice") return runChoice(state, call);
|
|
12
|
+
if (state.Type === "Wait") return runWait(state, call);
|
|
13
|
+
if (state.Type === "Fail") return runFail(state, call);
|
|
14
|
+
return runResultState(state, call);
|
|
15
|
+
};
|
|
16
|
+
const ranOrFailed = async (state, call) => {
|
|
17
|
+
try {
|
|
18
|
+
return await runState(state, call);
|
|
19
|
+
} catch (error) {
|
|
20
|
+
if (!(error instanceof StepError)) throw error;
|
|
21
|
+
return {
|
|
22
|
+
kind: "step",
|
|
23
|
+
status: "failed",
|
|
24
|
+
next: null,
|
|
25
|
+
input: call.position.input,
|
|
26
|
+
error: error.error,
|
|
27
|
+
cause: error.reason,
|
|
28
|
+
logs: call.logs
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
};
|
|
32
|
+
const keptLogs = (logs, redact) => {
|
|
33
|
+
const cut = (message) => {
|
|
34
|
+
const text = redact(message);
|
|
35
|
+
return text.length > limits.logLineLength ? `${text.slice(0, limits.logLineLength)}…` : text;
|
|
36
|
+
};
|
|
37
|
+
const kept = logs.slice(0, limits.logLines).map((line) => ({
|
|
38
|
+
...line,
|
|
39
|
+
message: cut(line.message)
|
|
40
|
+
}));
|
|
41
|
+
if (logs.length <= limits.logLines) return kept;
|
|
42
|
+
return [...kept, {
|
|
43
|
+
at: logs.at(-1).at,
|
|
44
|
+
message: `… ${logs.length - limits.logLines} more lines not kept`
|
|
45
|
+
}];
|
|
46
|
+
};
|
|
47
|
+
const withinLimits = (record) => {
|
|
48
|
+
const field = [
|
|
49
|
+
"input",
|
|
50
|
+
"output",
|
|
51
|
+
"assigned"
|
|
52
|
+
].find((name) => jsonBytes(record[name]) > limits.valueBytes);
|
|
53
|
+
if (!field) return record;
|
|
54
|
+
const { output: _, assigned: __, input: ___, ...rest } = record;
|
|
55
|
+
return {
|
|
56
|
+
...rest,
|
|
57
|
+
status: "failed",
|
|
58
|
+
next: null,
|
|
59
|
+
resumeAt: null,
|
|
60
|
+
error: "States.DataLimitExceeded",
|
|
61
|
+
cause: `The step's ${field} is larger than ${limits.valueBytes / 1024 / 1024} MiB`
|
|
62
|
+
};
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* Runs one scope (the run or a Map item) from where its saved steps leave it, saving each step as
|
|
66
|
+
* the segment's checkpoints come, until the scope ends or fails, waits for a time, or the segment's time is up.
|
|
67
|
+
*/
|
|
68
|
+
const runScope = async (context, machine, scope, input, outer) => {
|
|
69
|
+
let position = foldPosition(machine, input, context.saved.get(scope) ?? []);
|
|
70
|
+
for (;;) {
|
|
71
|
+
if (position.kind === "ended") return {
|
|
72
|
+
kind: "ended",
|
|
73
|
+
output: position.output
|
|
74
|
+
};
|
|
75
|
+
if (position.kind === "failed") return {
|
|
76
|
+
kind: "failed",
|
|
77
|
+
error: position.error,
|
|
78
|
+
cause: position.cause
|
|
79
|
+
};
|
|
80
|
+
if (context.superseded()) throw new SupersededError(`Scope "${scope}" stopped: another segment saved a step of this run first`);
|
|
81
|
+
if (position.resumeAt && position.resumeAt > context.now()) return {
|
|
82
|
+
kind: "blocked",
|
|
83
|
+
until: position.resumeAt
|
|
84
|
+
};
|
|
85
|
+
if (Date.now() >= context.deadline) return { kind: "yielded" };
|
|
86
|
+
if (context.steps.count >= context.maxSteps) return {
|
|
87
|
+
kind: "failed",
|
|
88
|
+
error: "States.Runtime",
|
|
89
|
+
cause: `The run took more than ${context.maxSteps} steps`
|
|
90
|
+
};
|
|
91
|
+
const name = position.state;
|
|
92
|
+
const state = machine.States[name];
|
|
93
|
+
const startedAt = context.now();
|
|
94
|
+
const call = {
|
|
95
|
+
context,
|
|
96
|
+
name,
|
|
97
|
+
scope,
|
|
98
|
+
position,
|
|
99
|
+
bindings: bindingsOf(context, outer, position),
|
|
100
|
+
logs: [],
|
|
101
|
+
runScope: (inner, key, value, seen) => runScope(context, inner, key, value, seen)
|
|
102
|
+
};
|
|
103
|
+
context.steps.count++;
|
|
104
|
+
const result = await ranOrFailed(state, call);
|
|
105
|
+
if (result.kind !== "step") {
|
|
106
|
+
context.steps.count--;
|
|
107
|
+
return result;
|
|
108
|
+
}
|
|
109
|
+
const { kind: _, logs, ...step } = result;
|
|
110
|
+
const shaped = {
|
|
111
|
+
...step,
|
|
112
|
+
logs: keptLogs(logs, context.redactor.text),
|
|
113
|
+
scope,
|
|
114
|
+
seq: position.seq,
|
|
115
|
+
state: name,
|
|
116
|
+
type: state.Type,
|
|
117
|
+
attempt: position.attempt,
|
|
118
|
+
startedAt,
|
|
119
|
+
finishedAt: context.now()
|
|
120
|
+
};
|
|
121
|
+
const record = withinLimits(context.redactor.value(shaped));
|
|
122
|
+
await context.save(record, state.Type === "Task" || Boolean(record.resumeAt));
|
|
123
|
+
position = advance(position, record);
|
|
124
|
+
}
|
|
125
|
+
};
|
|
126
|
+
//#endregion
|
|
127
|
+
export { runScope };
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { Definition } from "../definition/types.js";
|
|
2
|
+
import { Redactor } from "./redact.js";
|
|
3
|
+
import { SavedStep, StepRecord, TaskHandlers } from "./port.js";
|
|
4
|
+
import { ScopeOutcome } from "./context.js";
|
|
5
|
+
//#region src/run/segment.d.ts
|
|
6
|
+
export type SegmentInput = {
|
|
7
|
+
runId: string;
|
|
8
|
+
definition: Definition;
|
|
9
|
+
input: unknown;
|
|
10
|
+
/** When the run started, which expressions read as `$states.context.Execution.StartTime`. */
|
|
11
|
+
startedAt: Date;
|
|
12
|
+
/** The secrets expressions may read as `$states.context.Secrets.<NAME>`, by name. */
|
|
13
|
+
secrets: Readonly<Record<string, string>>;
|
|
14
|
+
/** Blanks the secrets out of what the run records. */
|
|
15
|
+
redactor: Redactor;
|
|
16
|
+
/** The steps the run saved so far. */
|
|
17
|
+
saved: readonly SavedStep[];
|
|
18
|
+
/** Saves steps in one guarded statement: false when any of their scopes and seqs is taken. */
|
|
19
|
+
save: (steps: readonly StepRecord[]) => Promise<boolean>;
|
|
20
|
+
tasks: TaskHandlers;
|
|
21
|
+
signal: AbortSignal;
|
|
22
|
+
now?: () => Date;
|
|
23
|
+
/** How long the segment takes new steps. Default `limits.segmentMs`. */
|
|
24
|
+
budgetMs?: number;
|
|
25
|
+
/** The most steps the run takes. Default `limits.runSteps`. */
|
|
26
|
+
maxSteps?: number;
|
|
27
|
+
};
|
|
28
|
+
/**
|
|
29
|
+
* Runs a run's steps from where its saved steps leave it to its next wait: it ends or fails, waits until a time
|
|
30
|
+
* (a Wait, a retry's delay), or yields once its time is up. `superseded` when another segment saved a step first.
|
|
31
|
+
*/
|
|
32
|
+
export declare const runSegment: ({ runId, definition, input, startedAt, secrets, redactor, saved, save, tasks, signal, now, budgetMs, maxSteps }: SegmentInput) => Promise<ScopeOutcome | {
|
|
33
|
+
kind: "superseded";
|
|
34
|
+
}>;
|
|
35
|
+
//#endregion
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { limits } from "../limits.js";
|
|
2
|
+
import { SupersededError } from "./errors.js";
|
|
3
|
+
import { runScope } from "./scope.js";
|
|
4
|
+
//#region src/run/segment.ts
|
|
5
|
+
/**
|
|
6
|
+
* Runs a run's steps from where its saved steps leave it to its next wait: it ends or fails, waits until a time
|
|
7
|
+
* (a Wait, a retry's delay), or yields once its time is up. `superseded` when another segment saved a step first.
|
|
8
|
+
*/
|
|
9
|
+
const runSegment = async ({ runId, definition, input, startedAt, secrets, redactor, saved, save, tasks, signal, now = () => /* @__PURE__ */ new Date(), budgetMs = limits.segmentMs, maxSteps = limits.runSteps }) => {
|
|
10
|
+
const byScope = /* @__PURE__ */ new Map();
|
|
11
|
+
for (const step of [...saved].sort((a, b) => a.seq - b.seq)) {
|
|
12
|
+
const steps = byScope.get(step.scope);
|
|
13
|
+
if (steps) steps.push(step);
|
|
14
|
+
else byScope.set(step.scope, [step]);
|
|
15
|
+
}
|
|
16
|
+
let superseded = false;
|
|
17
|
+
let kept = [];
|
|
18
|
+
const write = async () => {
|
|
19
|
+
const steps = kept;
|
|
20
|
+
kept = [];
|
|
21
|
+
if (steps.length === 0 || await save(steps)) return;
|
|
22
|
+
superseded = true;
|
|
23
|
+
throw new SupersededError(`Step ${steps[0].seq} of scope "${steps[0].scope}" was saved by another segment`);
|
|
24
|
+
};
|
|
25
|
+
const context = {
|
|
26
|
+
runId,
|
|
27
|
+
stateContext: {
|
|
28
|
+
Execution: {
|
|
29
|
+
Input: input,
|
|
30
|
+
StartTime: startedAt.toISOString()
|
|
31
|
+
},
|
|
32
|
+
Secrets: secrets
|
|
33
|
+
},
|
|
34
|
+
redactor,
|
|
35
|
+
tasks,
|
|
36
|
+
signal,
|
|
37
|
+
now,
|
|
38
|
+
deadline: Date.now() + budgetMs,
|
|
39
|
+
saved: byScope,
|
|
40
|
+
save: async (step, checkpoint) => {
|
|
41
|
+
kept.push(step);
|
|
42
|
+
if (checkpoint) await write();
|
|
43
|
+
},
|
|
44
|
+
superseded: () => superseded,
|
|
45
|
+
steps: { count: saved.length },
|
|
46
|
+
maxSteps
|
|
47
|
+
};
|
|
48
|
+
try {
|
|
49
|
+
const outcome = await runScope(context, definition.stateMachine, "", input, {});
|
|
50
|
+
await write();
|
|
51
|
+
return outcome;
|
|
52
|
+
} catch (error) {
|
|
53
|
+
if (!(error instanceof SupersededError)) throw error;
|
|
54
|
+
return { kind: "superseded" };
|
|
55
|
+
}
|
|
56
|
+
};
|
|
57
|
+
//#endregion
|
|
58
|
+
export { runSegment };
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
import { limits } from "../limits.js";
|
|
2
|
+
import { StepError } from "./errors.js";
|
|
3
|
+
import { evaluateAssign, evaluateKept, evaluateTemplate } from "./evaluate.js";
|
|
4
|
+
//#region src/run/simple-states.ts
|
|
5
|
+
const nextOf = (state) => state.Next ?? null;
|
|
6
|
+
const outputOf = async (state, call) => state.Output === void 0 ? call.position.input : evaluateKept(state.Output, call.bindings, "Output");
|
|
7
|
+
const runPass = async (state, call) => ({
|
|
8
|
+
kind: "step",
|
|
9
|
+
status: "succeeded",
|
|
10
|
+
next: state.Type === "Pass" ? nextOf(state) : null,
|
|
11
|
+
input: call.position.input,
|
|
12
|
+
output: await outputOf(state, call),
|
|
13
|
+
assigned: await evaluateAssign("Assign" in state ? state.Assign : void 0, call.bindings),
|
|
14
|
+
logs: call.logs
|
|
15
|
+
});
|
|
16
|
+
const runChoice = async (state, call) => {
|
|
17
|
+
for (const [index, rule] of state.Choices.entries()) {
|
|
18
|
+
const field = `Choices[${index}].Condition`;
|
|
19
|
+
const holds = await evaluateTemplate(rule.Condition, call.bindings, field);
|
|
20
|
+
if (typeof holds !== "boolean") throw new StepError("States.QueryEvaluationError", `${field}: gave ${JSON.stringify(holds) ?? "nothing"}, not true or false`);
|
|
21
|
+
if (!holds) continue;
|
|
22
|
+
call.logs.push({
|
|
23
|
+
at: call.context.now().toISOString(),
|
|
24
|
+
message: `Choice ${index + 1} holds`
|
|
25
|
+
});
|
|
26
|
+
const shaped = {
|
|
27
|
+
Output: rule.Output === void 0 ? state.Output : rule.Output,
|
|
28
|
+
Assign: rule.Assign ?? state.Assign
|
|
29
|
+
};
|
|
30
|
+
return {
|
|
31
|
+
kind: "step",
|
|
32
|
+
status: "succeeded",
|
|
33
|
+
next: rule.Next,
|
|
34
|
+
input: call.position.input,
|
|
35
|
+
output: await outputOf(shaped, call),
|
|
36
|
+
assigned: await evaluateAssign(shaped.Assign, call.bindings),
|
|
37
|
+
logs: call.logs
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
if (state.Default === void 0) throw new StepError("States.NoChoiceMatched", "No choice holds and the state has no Default");
|
|
41
|
+
call.logs.push({
|
|
42
|
+
at: call.context.now().toISOString(),
|
|
43
|
+
message: "No choice holds: Default"
|
|
44
|
+
});
|
|
45
|
+
return {
|
|
46
|
+
kind: "step",
|
|
47
|
+
status: "succeeded",
|
|
48
|
+
next: state.Default,
|
|
49
|
+
input: call.position.input,
|
|
50
|
+
output: await outputOf(state, call),
|
|
51
|
+
assigned: await evaluateAssign(state.Assign, call.bindings),
|
|
52
|
+
logs: call.logs
|
|
53
|
+
};
|
|
54
|
+
};
|
|
55
|
+
const isoPattern = /^(\d{4})-(\d{2})-(\d{2})T([01]\d|2[0-3]):[0-5]\d(:[0-5]\d(\.\d+)?)?(Z|[+-]([01]\d|2[0-3]):[0-5]\d)$/;
|
|
56
|
+
const isoTime = (text) => {
|
|
57
|
+
const match = isoPattern.exec(text);
|
|
58
|
+
if (!match) return void 0;
|
|
59
|
+
const [year, month, day] = [
|
|
60
|
+
Number(match[1]),
|
|
61
|
+
Number(match[2]),
|
|
62
|
+
Number(match[3])
|
|
63
|
+
];
|
|
64
|
+
const date = new Date(Date.UTC(year, month - 1, day));
|
|
65
|
+
if (date.getUTCFullYear() !== year || date.getUTCMonth() !== month - 1 || date.getUTCDate() !== day) return void 0;
|
|
66
|
+
return new Date(text);
|
|
67
|
+
};
|
|
68
|
+
const waitUntil = async (state, call) => {
|
|
69
|
+
const now = call.context.now();
|
|
70
|
+
if (state.Seconds !== void 0) {
|
|
71
|
+
const seconds = await evaluateTemplate(state.Seconds, call.bindings, "Seconds");
|
|
72
|
+
if (typeof seconds !== "number" || !(seconds >= 0) || seconds > limits.waitSeconds) throw new StepError("States.QueryEvaluationError", `Seconds: gave ${JSON.stringify(seconds) ?? "nothing"}, not a number of seconds from 0 to ${limits.waitSeconds}`);
|
|
73
|
+
return new Date(now.getTime() + seconds * 1e3);
|
|
74
|
+
}
|
|
75
|
+
const timestamp = await evaluateTemplate(state.Timestamp, call.bindings, "Timestamp");
|
|
76
|
+
const at = typeof timestamp === "string" ? isoTime(timestamp) : void 0;
|
|
77
|
+
if (!at) throw new StepError("States.QueryEvaluationError", `Timestamp: gave ${JSON.stringify(timestamp) ?? "nothing"}, not an ISO 8601 time such as 2026-11-04T12:00:00Z`);
|
|
78
|
+
if (at.getTime() - now.getTime() > limits.waitSeconds * 1e3) throw new StepError("States.QueryEvaluationError", `Timestamp: gave ${JSON.stringify(timestamp)}, more than ${limits.waitSeconds} seconds from now`);
|
|
79
|
+
return at;
|
|
80
|
+
};
|
|
81
|
+
/** A Wait saves its step at once, with the time the next state may run: the segment ends there, and the queue wakes the run. */
|
|
82
|
+
const runWait = async (state, call) => {
|
|
83
|
+
const resumeAt = await waitUntil(state, call);
|
|
84
|
+
return {
|
|
85
|
+
kind: "step",
|
|
86
|
+
status: "succeeded",
|
|
87
|
+
next: nextOf(state),
|
|
88
|
+
input: call.position.input,
|
|
89
|
+
output: await outputOf(state, call),
|
|
90
|
+
assigned: await evaluateAssign(state.Assign, call.bindings),
|
|
91
|
+
resumeAt,
|
|
92
|
+
logs: call.logs
|
|
93
|
+
};
|
|
94
|
+
};
|
|
95
|
+
const runFail = async (state, call) => {
|
|
96
|
+
const text = async (value, field) => value === void 0 ? void 0 : String(await evaluateKept(value, call.bindings, field));
|
|
97
|
+
return {
|
|
98
|
+
kind: "step",
|
|
99
|
+
status: "failed",
|
|
100
|
+
next: null,
|
|
101
|
+
input: call.position.input,
|
|
102
|
+
error: await text(state.Error, "Error") ?? "States.Fail",
|
|
103
|
+
cause: await text(state.Cause, "Cause") ?? "",
|
|
104
|
+
logs: call.logs
|
|
105
|
+
};
|
|
106
|
+
};
|
|
107
|
+
//#endregion
|
|
108
|
+
export { runChoice, runFail, runPass, runWait };
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
//#region src/run/step-input.ts
|
|
2
|
+
/**
|
|
3
|
+
* The names an expression reads in a scope: the outer scopes' variables, this one's, and `$states` with the run's
|
|
4
|
+
* context and the state's input as `$states.input`, which the check lets only a Map item's first state read.
|
|
5
|
+
*/
|
|
6
|
+
const bindingsOf = (context, outer, position) => ({
|
|
7
|
+
...outer,
|
|
8
|
+
...position.variables,
|
|
9
|
+
states: {
|
|
10
|
+
input: position.input,
|
|
11
|
+
context: context.stateContext
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
/** The variables an inner scope sees: every one in scope here, without `$states`, which each scope adds itself. */
|
|
15
|
+
const visibleVariables = ({ states: _, ...variables }) => variables;
|
|
16
|
+
/**
|
|
17
|
+
* The key of an inner scope: item `index` of the Map `name` that was step `seq` of `scope`, such as `Each.3[0]`. The
|
|
18
|
+
* name is URI-encoded, so a `/` or `[` in a state's name cannot make two scopes' keys the same.
|
|
19
|
+
*/
|
|
20
|
+
const innerScope = (scope, name, seq, index) => `${scope ? `${scope}/` : ""}${encodeURIComponent(name)}.${seq}[${index}]`;
|
|
21
|
+
//#endregion
|
|
22
|
+
export { bindingsOf, innerScope, visibleVariables };
|