@protobase/workflows 0.0.0-stage → 0.68.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/dist/definition/check-definition.d.ts +21 -0
- package/dist/definition/check-definition.js +128 -0
- package/dist/definition/check-reads.js +46 -0
- package/dist/definition/check-schema.js +74 -0
- package/dist/definition/check-trigger.d.ts +7 -0
- package/dist/definition/check-trigger.js +27 -0
- package/dist/definition/expression-fields.js +68 -0
- package/dist/definition/expressions.js +35 -0
- package/dist/definition/graph.js +39 -0
- package/dist/definition/names.js +14 -0
- package/dist/definition/problem.d.ts +13 -0
- package/dist/definition/problem.js +11 -0
- package/dist/definition/reads.js +81 -0
- package/dist/definition/schema.d.ts +259 -0
- package/dist/definition/schema.js +258 -0
- package/dist/definition/types.d.ts +92 -0
- package/dist/definition/visible.js +52 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +16 -0
- package/dist/limits.d.ts +39 -0
- package/dist/limits.js +40 -0
- package/dist/nesting.d.ts +7 -0
- package/dist/nesting.js +19 -0
- package/dist/plugin/connection-error.d.ts +10 -0
- package/dist/plugin/connection-error.js +15 -0
- package/dist/plugin/context.d.ts +99 -0
- package/dist/plugin/define-plugin.d.ts +7 -0
- package/dist/plugin/define-plugin.js +23 -0
- package/dist/plugin/fetch-errors.d.ts +14 -0
- package/dist/plugin/fetch-errors.js +18 -0
- package/dist/plugin/file-value.d.ts +5 -0
- package/dist/plugin/file-value.js +9 -0
- package/dist/plugin/index.d.ts +8 -0
- package/dist/plugin/index.js +6 -0
- package/dist/plugin/load-plugins.d.ts +10 -0
- package/dist/plugin/load-plugins.js +22 -0
- package/dist/plugin/step-handlers.d.ts +12 -0
- package/dist/plugin/step-handlers.js +33 -0
- package/dist/plugin/trigger-checks.d.ts +6 -0
- package/dist/plugin/trigger-checks.js +28 -0
- package/dist/plugin/type-catalog.d.ts +19 -0
- package/dist/plugin/type-catalog.js +26 -0
- package/dist/plugin/type-list.d.ts +26 -0
- package/dist/plugin/type-list.js +26 -0
- package/dist/plugin/types.d.ts +123 -0
- package/dist/run/context.d.ts +18 -0
- package/dist/run/errors.d.ts +20 -0
- package/dist/run/errors.js +39 -0
- package/dist/run/evaluate.js +59 -0
- package/dist/run/fan-out.js +39 -0
- package/dist/run/json-bytes.js +5 -0
- package/dist/run/port.d.ts +105 -0
- package/dist/run/position.js +54 -0
- package/dist/run/recovery.js +76 -0
- package/dist/run/redact.d.ts +16 -0
- package/dist/run/redact.js +39 -0
- package/dist/run/rerun.d.ts +18 -0
- package/dist/run/rerun.js +62 -0
- package/dist/run/result-states.js +115 -0
- package/dist/run/runner.d.ts +29 -0
- package/dist/run/runner.js +60 -0
- package/dist/run/scope.js +127 -0
- package/dist/run/segment.d.ts +35 -0
- package/dist/run/segment.js +58 -0
- package/dist/run/simple-states.js +108 -0
- package/dist/run/step-input.js +22 -0
- package/dist/run/task.js +39 -0
- package/package.json +34 -4
- package/README.md +0 -3
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { TypeEntry } from "./type-list.js";
|
|
2
|
+
//#region src/plugin/type-catalog.d.ts
|
|
3
|
+
/** A type as a model or the editor reads it: its declaration as its plugin gives it, with whether this app can run it. */
|
|
4
|
+
export type CatalogEntry = Omit<TypeEntry, 'kind' | 'plugin'> & {
|
|
5
|
+
kind: TypeEntry['kind'] | 'manual';
|
|
6
|
+
};
|
|
7
|
+
/**
|
|
8
|
+
* What `list_workflow_types` answers and the editor reads: every step type a Task names as its `Resource` and every
|
|
9
|
+
* trigger a definition's `trigger` holds as `{ "<id>": <input> }`, built-in and plugin types alike, each with its JSON
|
|
10
|
+
* Schemas.
|
|
11
|
+
*/
|
|
12
|
+
export declare const typeCatalog: (types: readonly TypeEntry[]) => TypeCatalog;
|
|
13
|
+
/** The app's types, `manual` first among the triggers, as `typeCatalog` lists them. */
|
|
14
|
+
export type TypeCatalog = {
|
|
15
|
+
step_types: CatalogEntry[];
|
|
16
|
+
triggers: CatalogEntry[];
|
|
17
|
+
note: string;
|
|
18
|
+
};
|
|
19
|
+
//#endregion
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
//#region src/plugin/type-catalog.ts
|
|
2
|
+
const entry = ({ plugin: _, ...type }) => type;
|
|
3
|
+
/** `manual`, the trigger of a workflow that runs only when someone starts it. */
|
|
4
|
+
const manual = {
|
|
5
|
+
id: "manual",
|
|
6
|
+
kind: "manual",
|
|
7
|
+
label: "By hand",
|
|
8
|
+
description: "Runs when someone starts it, such as with run_workflow, with the input they give.",
|
|
9
|
+
input: {
|
|
10
|
+
type: "object",
|
|
11
|
+
additionalProperties: false
|
|
12
|
+
},
|
|
13
|
+
available: true
|
|
14
|
+
};
|
|
15
|
+
/**
|
|
16
|
+
* What `list_workflow_types` answers and the editor reads: every step type a Task names as its `Resource` and every
|
|
17
|
+
* trigger a definition's `trigger` holds as `{ "<id>": <input> }`, built-in and plugin types alike, each with its JSON
|
|
18
|
+
* Schemas.
|
|
19
|
+
*/
|
|
20
|
+
const typeCatalog = (types) => ({
|
|
21
|
+
step_types: types.filter((type) => type.kind === "step").map(entry),
|
|
22
|
+
triggers: [manual, ...types.filter((type) => type.kind !== "step").map(entry)],
|
|
23
|
+
note: "A definition holds its trigger as { \"<id>\": <input> }, such as { \"app:record\": { \"resource\": \"orders\" } }. A type with available false cannot be saved here; its reason says why. A trigger other than manual starts runs only while its workflow is enabled, its enabled_version set; run_workflow starts a run of any workflow, enabled or not, so a manual workflow needs no enabled_version."
|
|
24
|
+
});
|
|
25
|
+
//#endregion
|
|
26
|
+
export { typeCatalog };
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { JsonSchema } from "./context.js";
|
|
2
|
+
import { ConnectionDeclaration, Plugin, TriggerType } from "./types.js";
|
|
3
|
+
//#region src/plugin/type-list.d.ts
|
|
4
|
+
/** A type as the MCP tools and the editor list it: built-in and plugin types alike. */
|
|
5
|
+
export type TypeEntry = {
|
|
6
|
+
/** `<plugin>:<key>`: a Task's `Resource`, or the key of a definition's `trigger`. */
|
|
7
|
+
id: string;
|
|
8
|
+
kind: 'step' | TriggerType['kind'];
|
|
9
|
+
plugin: string;
|
|
10
|
+
label: string;
|
|
11
|
+
description: string;
|
|
12
|
+
input: JsonSchema;
|
|
13
|
+
output?: JsonSchema;
|
|
14
|
+
errors?: Record<string, string>;
|
|
15
|
+
connection?: {
|
|
16
|
+
name: string;
|
|
17
|
+
} & ConnectionDeclaration;
|
|
18
|
+
/** False when this app cannot run it, with the reason; a definition that uses it is not saved. */
|
|
19
|
+
available: boolean;
|
|
20
|
+
reason?: string;
|
|
21
|
+
};
|
|
22
|
+
/** Why the app cannot run a type, or undefined when it can. */
|
|
23
|
+
export type Unavailable = (type: Omit<TypeEntry, 'available'>) => string | undefined;
|
|
24
|
+
/** Every type of `plugins`, steps first, each marked available or not by `unavailable`. */
|
|
25
|
+
export declare const typeList: (plugins: readonly Plugin[], unavailable: Unavailable) => TypeEntry[];
|
|
26
|
+
//#endregion
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
//#region src/plugin/type-list.ts
|
|
2
|
+
const entry = (plugin, key, type) => ({
|
|
3
|
+
id: `${plugin.name}:${key}`,
|
|
4
|
+
kind: "kind" in type ? type.kind : "step",
|
|
5
|
+
plugin: plugin.name,
|
|
6
|
+
label: type.label,
|
|
7
|
+
description: type.description,
|
|
8
|
+
input: type.input,
|
|
9
|
+
...type.output && { output: type.output },
|
|
10
|
+
..."errors" in type && type.errors && { errors: Object.fromEntries(Object.entries(type.errors).map(([name, text]) => [`${type.errorPrefix ?? "Step"}.${name}`, text])) },
|
|
11
|
+
...type.connection !== void 0 && { connection: {
|
|
12
|
+
name: type.connection,
|
|
13
|
+
...plugin.connections[type.connection]
|
|
14
|
+
} }
|
|
15
|
+
});
|
|
16
|
+
/** Every type of `plugins`, steps first, each marked available or not by `unavailable`. */
|
|
17
|
+
const typeList = (plugins, unavailable) => plugins.flatMap((plugin) => [...Object.entries(plugin.steps ?? {}), ...Object.entries(plugin.triggers ?? {})].map(([key, type]) => entry(plugin, key, type))).map((type) => {
|
|
18
|
+
const reason = unavailable(type);
|
|
19
|
+
return {
|
|
20
|
+
...type,
|
|
21
|
+
available: reason === void 0,
|
|
22
|
+
...reason !== void 0 && { reason }
|
|
23
|
+
};
|
|
24
|
+
});
|
|
25
|
+
//#endregion
|
|
26
|
+
export { typeList };
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { JsonSchema, PluginConnection, PluginFetch, StepContext } from "./context.js";
|
|
2
|
+
//#region src/plugin/types.d.ts
|
|
3
|
+
/** What a type shows in the type list the MCP tools and the editor read. */
|
|
4
|
+
type Described = {
|
|
5
|
+
label: string;
|
|
6
|
+
description: string;
|
|
7
|
+
/** What it takes, checked before it runs. */
|
|
8
|
+
input: JsonSchema;
|
|
9
|
+
/** What it gives: a step's result, or a trigger's run input. */
|
|
10
|
+
output?: JsonSchema;
|
|
11
|
+
/** The name of a connection the plugin declares, which it signs in with. */
|
|
12
|
+
connection?: string;
|
|
13
|
+
};
|
|
14
|
+
/** A step type, which a Task names as `<plugin>:<key>` in its `Resource`. */
|
|
15
|
+
export type StepType = Described & {
|
|
16
|
+
/** The errors it fails with, by name, as `<Prefix>.<Name>`, each with what it means. */
|
|
17
|
+
errors?: Record<string, string>;
|
|
18
|
+
/** The prefix of its error names, such as `Http`; an input that does not fit fails as `<prefix>.InvalidArguments`. Default `Step`. */
|
|
19
|
+
errorPrefix?: string;
|
|
20
|
+
run: (context: StepContext) => Promise<unknown>;
|
|
21
|
+
};
|
|
22
|
+
/** What every trigger type has: its description, and a check of its input beyond its schema, such as a cron's fields. */
|
|
23
|
+
type TriggerBase = Described & {
|
|
24
|
+
check?: (input: Record<string, unknown>) => {
|
|
25
|
+
field?: string;
|
|
26
|
+
message: string;
|
|
27
|
+
}[];
|
|
28
|
+
};
|
|
29
|
+
/** What a trigger's hooks get: its checked input, its connection, and the guarded `fetch`. */
|
|
30
|
+
export type TriggerContext = {
|
|
31
|
+
input: Record<string, unknown>;
|
|
32
|
+
connection: PluginConnection | undefined;
|
|
33
|
+
fetch: PluginFetch;
|
|
34
|
+
signal: AbortSignal;
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* A trigger that starts one run per record that becomes ready: `when` is a filter on `resource` in the filter language,
|
|
38
|
+
* whose `<timestamp field> <= now() - <duration>` terms make the start wait until that time. A record starts once per
|
|
39
|
+
* ready key: its record key and the values of the `key` fields.
|
|
40
|
+
*/
|
|
41
|
+
export type ReadyTrigger = TriggerBase & {
|
|
42
|
+
kind: 'ready';
|
|
43
|
+
ready: (input: Record<string, unknown>) => {
|
|
44
|
+
resource: string;
|
|
45
|
+
when?: string;
|
|
46
|
+
key?: string[];
|
|
47
|
+
};
|
|
48
|
+
};
|
|
49
|
+
/** One new thing a poll or a webhook call found: `key` is unique for the trigger, `data` becomes the run's input. */
|
|
50
|
+
export type TriggerItem = {
|
|
51
|
+
key: string;
|
|
52
|
+
data: unknown;
|
|
53
|
+
};
|
|
54
|
+
/** A trigger that asks a service for new items on a schedule. */
|
|
55
|
+
export type PollingTrigger = TriggerBase & {
|
|
56
|
+
kind: 'polling';
|
|
57
|
+
/** A duration such as `1 minute`, or a cron in an IANA time zone. */
|
|
58
|
+
every: string | {
|
|
59
|
+
cron: string;
|
|
60
|
+
timezone?: string;
|
|
61
|
+
};
|
|
62
|
+
/** The most items one poll returns, at most 100. */
|
|
63
|
+
limit?: number;
|
|
64
|
+
start: (context: TriggerContext) => Promise<unknown>;
|
|
65
|
+
/** Items after `cursor`, and the next cursor, which must move forward. */
|
|
66
|
+
poll: (context: TriggerContext & {
|
|
67
|
+
cursor: unknown;
|
|
68
|
+
store: unknown;
|
|
69
|
+
limit: number;
|
|
70
|
+
}) => Promise<{
|
|
71
|
+
items: TriggerItem[];
|
|
72
|
+
cursor: unknown;
|
|
73
|
+
store?: unknown;
|
|
74
|
+
}>;
|
|
75
|
+
};
|
|
76
|
+
/** A trigger that the app's webhook URL starts, after the route checked the call's signature. */
|
|
77
|
+
export type WebhookTrigger = TriggerBase & {
|
|
78
|
+
kind: 'webhook';
|
|
79
|
+
/** Registers the URL with the service, returning what `unsubscribe` and `renew` need. */
|
|
80
|
+
subscribe?: (context: TriggerContext & {
|
|
81
|
+
url: string;
|
|
82
|
+
}) => Promise<unknown>;
|
|
83
|
+
unsubscribe?: (context: TriggerContext & {
|
|
84
|
+
state: unknown;
|
|
85
|
+
}) => Promise<void>;
|
|
86
|
+
renew?: (context: TriggerContext & {
|
|
87
|
+
state: unknown;
|
|
88
|
+
}) => Promise<unknown>;
|
|
89
|
+
renewEvery?: string;
|
|
90
|
+
receive: (context: {
|
|
91
|
+
request: Request;
|
|
92
|
+
input: Record<string, unknown>;
|
|
93
|
+
}) => Promise<{
|
|
94
|
+
items: TriggerItem[];
|
|
95
|
+
response?: Response;
|
|
96
|
+
}>;
|
|
97
|
+
};
|
|
98
|
+
export type TriggerType = ReadyTrigger | PollingTrigger | WebhookTrigger;
|
|
99
|
+
/** A sign-in a plugin's types use: one of the app's sign-in providers, or an OAuth provider of its own. */
|
|
100
|
+
export type ConnectionDeclaration = {
|
|
101
|
+
label: string;
|
|
102
|
+
scopes?: string[];
|
|
103
|
+
} & ({
|
|
104
|
+
signIn: string;
|
|
105
|
+
} | {
|
|
106
|
+
oauth: {
|
|
107
|
+
authorization_url: string;
|
|
108
|
+
token_url: string;
|
|
109
|
+
user_info_url?: string;
|
|
110
|
+
revocation_url?: string;
|
|
111
|
+
pkce?: boolean;
|
|
112
|
+
};
|
|
113
|
+
});
|
|
114
|
+
/** A workflow plugin: the trigger and step types it adds, and the sign-ins they use. */
|
|
115
|
+
export type Plugin = {
|
|
116
|
+
/** Its types are named `<name>:<key>`. */
|
|
117
|
+
name: string;
|
|
118
|
+
label: string;
|
|
119
|
+
connections?: Record<string, ConnectionDeclaration>;
|
|
120
|
+
triggers?: Record<string, TriggerType>;
|
|
121
|
+
steps?: Record<string, StepType>;
|
|
122
|
+
};
|
|
123
|
+
//#endregion
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
//#region src/run/context.d.ts
|
|
2
|
+
/** A scope that cannot go on in this segment: it waits until a time, or the segment's time is up. */
|
|
3
|
+
export type Paused = {
|
|
4
|
+
kind: 'blocked';
|
|
5
|
+
until: Date;
|
|
6
|
+
} | {
|
|
7
|
+
kind: 'yielded';
|
|
8
|
+
};
|
|
9
|
+
/** How a scope ended in this segment. */
|
|
10
|
+
export type ScopeOutcome = {
|
|
11
|
+
kind: 'ended';
|
|
12
|
+
output: unknown;
|
|
13
|
+
} | {
|
|
14
|
+
kind: 'failed';
|
|
15
|
+
error: string;
|
|
16
|
+
cause: string;
|
|
17
|
+
} | Paused;
|
|
18
|
+
//#endregion
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
//#region src/run/errors.d.ts
|
|
2
|
+
/**
|
|
3
|
+
* A step's failure by name, as `Catch` and `Retry` match it in `ErrorEquals`: a step type throws one with its own
|
|
4
|
+
* names (such as `Http.Status`); any other error a step throws counts as `States.TaskFailed`.
|
|
5
|
+
*/
|
|
6
|
+
declare const brand: unique symbol;
|
|
7
|
+
export declare class StepError extends Error {
|
|
8
|
+
readonly error: string;
|
|
9
|
+
readonly reason: string;
|
|
10
|
+
name: string;
|
|
11
|
+
readonly [brand] = true;
|
|
12
|
+
constructor(error: string, reason: string);
|
|
13
|
+
static [Symbol.hasInstance](value: unknown): boolean;
|
|
14
|
+
}
|
|
15
|
+
/** The error and cause a failed step records, which `$states.errorOutput` holds in a catcher. */
|
|
16
|
+
export type ErrorOutput = {
|
|
17
|
+
Error: string;
|
|
18
|
+
Cause: string;
|
|
19
|
+
};
|
|
20
|
+
//#endregion
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
//#region src/run/errors.ts
|
|
2
|
+
/**
|
|
3
|
+
* A step's failure by name, as `Catch` and `Retry` match it in `ErrorEquals`: a step type throws one with its own
|
|
4
|
+
* names (such as `Http.Status`); any other error a step throws counts as `States.TaskFailed`.
|
|
5
|
+
*/
|
|
6
|
+
const brand = Symbol.for("protobase.workflows.StepError");
|
|
7
|
+
var StepError = class extends Error {
|
|
8
|
+
error;
|
|
9
|
+
reason;
|
|
10
|
+
name = "StepError";
|
|
11
|
+
[brand] = true;
|
|
12
|
+
constructor(error, reason) {
|
|
13
|
+
super(`${error}: ${reason}`);
|
|
14
|
+
this.error = error;
|
|
15
|
+
this.reason = reason;
|
|
16
|
+
}
|
|
17
|
+
static [Symbol.hasInstance](value) {
|
|
18
|
+
return typeof value === "object" && value !== null && value[brand] === true;
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
const errorOutput = (thrown) => {
|
|
22
|
+
if (thrown instanceof StepError) return {
|
|
23
|
+
Error: thrown.error,
|
|
24
|
+
Cause: thrown.reason
|
|
25
|
+
};
|
|
26
|
+
return {
|
|
27
|
+
Error: "States.TaskFailed",
|
|
28
|
+
Cause: thrown instanceof Error ? thrown.message : String(thrown)
|
|
29
|
+
};
|
|
30
|
+
};
|
|
31
|
+
const terminal = /* @__PURE__ */ new Set(["States.Runtime", "States.DataLimitExceeded"]);
|
|
32
|
+
/** Whether `ErrorEquals` names the error, or `States.ALL` does, for an error that does not end the run. */
|
|
33
|
+
const matches = (names, error) => !terminal.has(error) && (names.includes(error) || names.includes("States.ALL"));
|
|
34
|
+
/** Thrown when another segment saved a step first: this one stops, since the run went on without it. */
|
|
35
|
+
var SupersededError = class extends Error {
|
|
36
|
+
name = "SupersededError";
|
|
37
|
+
};
|
|
38
|
+
//#endregion
|
|
39
|
+
export { StepError, SupersededError, errorOutput, matches };
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { compile, expressionSource, isJsonataError, parse } from "../definition/expressions.js";
|
|
2
|
+
import { expressionReads } from "../definition/reads.js";
|
|
3
|
+
import { StepError } from "./errors.js";
|
|
4
|
+
//#region src/run/evaluate.ts
|
|
5
|
+
const evaluated = async (source, bindings, field) => {
|
|
6
|
+
try {
|
|
7
|
+
return await compile(source).evaluate(void 0, bindings);
|
|
8
|
+
} catch (error) {
|
|
9
|
+
if (!isJsonataError(error)) throw error;
|
|
10
|
+
throw new StepError("States.QueryEvaluationError", `${field}: ${error.message}`);
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
/**
|
|
14
|
+
* Whether an expression reads one of the app's secrets, from its syntax tree, so no spelling (a quoted name, a comment)
|
|
15
|
+
* hides one. The check lets a secret be read only as `$states.context.Secrets.<NAME>`; a read of `$states` or its
|
|
16
|
+
* `context` whole counts too, and so does an expression that does not parse, rather than being trusted.
|
|
17
|
+
*/
|
|
18
|
+
const readsSecret = (source) => {
|
|
19
|
+
const parsed = parse(source);
|
|
20
|
+
if ("error" in parsed) return true;
|
|
21
|
+
return expressionReads(parsed.expression).states.some(([field, next]) => field === void 0 || field === "context" && (next === void 0 || next === "Secrets"));
|
|
22
|
+
};
|
|
23
|
+
/** What a value computed from a secret is kept as, however the expression changed it, such as encoded or cut apart. */
|
|
24
|
+
const secretValue = "[redacted]";
|
|
25
|
+
/**
|
|
26
|
+
* A template with each `{% ... %}` replaced by its value; an evaluation that fails is `States.QueryEvaluationError`.
|
|
27
|
+
* With `kept`, for a value the run keeps (an Output, a variable, a Map's items, a Fail's error), an expression that
|
|
28
|
+
* reads a secret gives `[redacted]`: what it computes is never saved, and later states read what was saved.
|
|
29
|
+
*/
|
|
30
|
+
const evaluateTemplate = async (template, bindings, field, kept = false) => {
|
|
31
|
+
const source = expressionSource(template);
|
|
32
|
+
if (source !== void 0) return kept && readsSecret(source) ? secretValue : evaluated(source, bindings, field);
|
|
33
|
+
if (Array.isArray(template)) return Promise.all(template.map((item, index) => evaluateTemplate(item, bindings, `${field}[${index}]`, kept)));
|
|
34
|
+
if (typeof template === "object" && template !== null) {
|
|
35
|
+
const entries = await Promise.all(Object.entries(template).map(async ([key, value]) => [key, await evaluateTemplate(value, bindings, `${field}.${key}`, kept)]));
|
|
36
|
+
return Object.fromEntries(entries.filter(([, value]) => value !== void 0));
|
|
37
|
+
}
|
|
38
|
+
return template;
|
|
39
|
+
};
|
|
40
|
+
/** A value the run keeps: an Output, a Map's items or a Fail's error, an expression reading a secret kept as `[redacted]`. */
|
|
41
|
+
const evaluateKept = (template, bindings, field) => evaluateTemplate(template, bindings, field, true);
|
|
42
|
+
/** The values of an `Assign`, each by its name, an expression reading a secret kept as `[redacted]`. */
|
|
43
|
+
const evaluateAssign = async (assign, bindings, field = "Assign") => {
|
|
44
|
+
if (!assign) return {};
|
|
45
|
+
return await evaluateKept(assign, bindings, field);
|
|
46
|
+
};
|
|
47
|
+
/**
|
|
48
|
+
* A Task's arguments as its step records them: each one an expression computed from a secret is `[redacted]`, while
|
|
49
|
+
* the step type is given the value itself.
|
|
50
|
+
*/
|
|
51
|
+
const recordedArguments = (template, args) => {
|
|
52
|
+
const source = expressionSource(template);
|
|
53
|
+
if (source !== void 0) return readsSecret(source) ? secretValue : args;
|
|
54
|
+
if (Array.isArray(template) && Array.isArray(args)) return args.map((item, index) => recordedArguments(template[index], item));
|
|
55
|
+
if (typeof template === "object" && template !== null && typeof args === "object" && args !== null && !Array.isArray(args)) return Object.fromEntries(Object.entries(args).map(([key, value]) => [key, recordedArguments(template[key], value)]));
|
|
56
|
+
return args;
|
|
57
|
+
};
|
|
58
|
+
//#endregion
|
|
59
|
+
export { evaluateAssign, evaluateKept, evaluateTemplate, readsSecret, recordedArguments, secretValue };
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
//#region src/run/fan-out.ts
|
|
2
|
+
/**
|
|
3
|
+
* Runs the scopes, at most `concurrency` at once. Once one fails, throws, or the segment's time is up, no more start and
|
|
4
|
+
* the running ones settle, so none runs on after the segment ends; one that waits leaves the others running, and the
|
|
5
|
+
* whole waits for the earliest time. A scope's error is thrown once all have settled.
|
|
6
|
+
*/
|
|
7
|
+
const fanOut = async (scopes, concurrency) => {
|
|
8
|
+
const outcomes = new Array(scopes.length);
|
|
9
|
+
let next = 0;
|
|
10
|
+
let stop = false;
|
|
11
|
+
const lane = async () => {
|
|
12
|
+
while (!stop && next < scopes.length) {
|
|
13
|
+
const index = next++;
|
|
14
|
+
const outcome = await scopes[index]().catch((error) => {
|
|
15
|
+
stop = true;
|
|
16
|
+
throw error;
|
|
17
|
+
});
|
|
18
|
+
outcomes[index] = outcome;
|
|
19
|
+
if (outcome.kind === "failed" || outcome.kind === "yielded") stop = true;
|
|
20
|
+
}
|
|
21
|
+
};
|
|
22
|
+
const thrown = (await Promise.allSettled(Array.from({ length: Math.min(concurrency, scopes.length) }, lane))).find((settled) => settled.status === "rejected");
|
|
23
|
+
if (thrown) throw thrown.reason;
|
|
24
|
+
const settled = outcomes.filter(Boolean);
|
|
25
|
+
const failed = settled.find((outcome) => outcome.kind === "failed");
|
|
26
|
+
if (failed) return failed;
|
|
27
|
+
if (settled.some((outcome) => outcome.kind === "yielded") || settled.length < scopes.length) return { kind: "yielded" };
|
|
28
|
+
const waits = settled.flatMap((outcome) => outcome.kind === "blocked" ? [outcome.until.getTime()] : []);
|
|
29
|
+
if (waits.length > 0) return {
|
|
30
|
+
kind: "blocked",
|
|
31
|
+
until: new Date(Math.min(...waits))
|
|
32
|
+
};
|
|
33
|
+
return {
|
|
34
|
+
kind: "ended",
|
|
35
|
+
outputs: settled.map((outcome) => outcome.output)
|
|
36
|
+
};
|
|
37
|
+
};
|
|
38
|
+
//#endregion
|
|
39
|
+
export { fanOut };
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { Definition } from "../definition/types.js";
|
|
2
|
+
//#region src/run/port.d.ts
|
|
3
|
+
/**
|
|
4
|
+
* How a step ended: `succeeded`, `retrying` (it failed and a retrier tries it again), `caught` (it failed and a
|
|
5
|
+
* catcher went on elsewhere) or `failed` (nothing caught it, so its scope failed).
|
|
6
|
+
*/
|
|
7
|
+
export type StepStatus = 'succeeded' | 'retrying' | 'caught' | 'failed';
|
|
8
|
+
/** What the runner reads of a saved step to know where its scope stands. */
|
|
9
|
+
export type SavedStep = {
|
|
10
|
+
/** The scope the step ran in: `''` for the run itself, `Each.3[0]` for item 0 of the Map `Each` that was step 3. */
|
|
11
|
+
scope: string;
|
|
12
|
+
/** The step's number within its scope, from 0: with the scope, the step's key, saved once. */
|
|
13
|
+
seq: number;
|
|
14
|
+
state: string;
|
|
15
|
+
status: StepStatus;
|
|
16
|
+
attempt: number;
|
|
17
|
+
output?: unknown;
|
|
18
|
+
/** The variables the step assigned. */
|
|
19
|
+
assigned?: Record<string, unknown>;
|
|
20
|
+
/** The state its scope goes on to; `null` when the scope ended or failed, or for a try that is retried. */
|
|
21
|
+
next: string | null;
|
|
22
|
+
/** The next state may not run before this time: after a Wait, or a retry's delay. */
|
|
23
|
+
resumeAt?: Date | null;
|
|
24
|
+
error?: string | null;
|
|
25
|
+
cause?: string | null;
|
|
26
|
+
};
|
|
27
|
+
export type StepLog = {
|
|
28
|
+
at: string;
|
|
29
|
+
message: string;
|
|
30
|
+
};
|
|
31
|
+
/** All a step records: what the runner reads, and its type, input, logs and timing, with every secret redacted. */
|
|
32
|
+
export type StepRecord = SavedStep & {
|
|
33
|
+
type: string;
|
|
34
|
+
input?: unknown;
|
|
35
|
+
logs: StepLog[];
|
|
36
|
+
startedAt: Date;
|
|
37
|
+
finishedAt: Date;
|
|
38
|
+
};
|
|
39
|
+
/** What a step type's handler gets beside its arguments. */
|
|
40
|
+
export type TaskContext = {
|
|
41
|
+
/** The run's id: files a step stores for the run are kept under it. */
|
|
42
|
+
runId: string;
|
|
43
|
+
/** Aborted when the step's timeout passes, or when the worker stops. */
|
|
44
|
+
signal: AbortSignal;
|
|
45
|
+
/** The same for every retry of one visit of this step in this run, so a receiver that dedupes by it sees a repeated try once; a later visit gets a new one. */
|
|
46
|
+
idempotencyKey: string;
|
|
47
|
+
/** Adds a line to the step's logs, with the app's secrets blanked out; a step keeps `limits.logLines` lines. */
|
|
48
|
+
log: (message: string) => void;
|
|
49
|
+
/**
|
|
50
|
+
* The text with the app's secrets blanked out. The runner blanks them out of all a step records; a step that cuts a
|
|
51
|
+
* text short, such as an answer's body in an error, blanks it first, so the cut leaves no part of a secret.
|
|
52
|
+
*/
|
|
53
|
+
redact: (text: string) => string;
|
|
54
|
+
};
|
|
55
|
+
/**
|
|
56
|
+
* A step type, which a Task names as its `Resource`: it takes the Task's evaluated `Arguments` and returns its result.
|
|
57
|
+
* It throws a `StepError` to fail with an error name of its own; any other error is `States.TaskFailed`.
|
|
58
|
+
*/
|
|
59
|
+
export type TaskHandler = (args: unknown, context: TaskContext) => Promise<unknown>;
|
|
60
|
+
/** The interpreter's back end: the step types by the name a Task's `Resource` gives. */
|
|
61
|
+
export type TaskHandlers = Readonly<Record<string, TaskHandler>>;
|
|
62
|
+
/** A run as the runner loads it: its pinned definition, its input and when it started. */
|
|
63
|
+
export type LoadedRun = {
|
|
64
|
+
id: string;
|
|
65
|
+
definition: Definition;
|
|
66
|
+
input: unknown;
|
|
67
|
+
status: RunStatus;
|
|
68
|
+
startedAt: Date;
|
|
69
|
+
};
|
|
70
|
+
export type RunStatus = 'queued' | 'running' | 'waiting' | 'succeeded' | 'failed';
|
|
71
|
+
/** What changes on a run as a segment ends. */
|
|
72
|
+
export type RunChange = {
|
|
73
|
+
status: 'running';
|
|
74
|
+
} | {
|
|
75
|
+
status: 'waiting';
|
|
76
|
+
resumeAt: Date;
|
|
77
|
+
} | {
|
|
78
|
+
status: 'succeeded';
|
|
79
|
+
output: unknown;
|
|
80
|
+
} | {
|
|
81
|
+
status: 'failed';
|
|
82
|
+
error: string;
|
|
83
|
+
cause: string;
|
|
84
|
+
};
|
|
85
|
+
/**
|
|
86
|
+
* Where runs and their steps are kept, in the database or transaction `Db` the caller gives: the first part of the
|
|
87
|
+
* runner's port.
|
|
88
|
+
*/
|
|
89
|
+
export type RunStore<Db> = {
|
|
90
|
+
load(db: Db, runId: string): Promise<LoadedRun | undefined>;
|
|
91
|
+
/** Every step the run saved, in each scope's order; the record's input and logs are not needed. */
|
|
92
|
+
steps(db: Db, runId: string): Promise<SavedStep[]>;
|
|
93
|
+
/**
|
|
94
|
+
* Saves steps in one guarded statement: false when any of their scopes and seqs is taken already, and those are left
|
|
95
|
+
* as the other segment saved them.
|
|
96
|
+
*/
|
|
97
|
+
save(db: Db, runId: string, steps: readonly StepRecord[]): Promise<boolean>;
|
|
98
|
+
/** Changes the run unless it has ended already. */
|
|
99
|
+
update(db: Db, runId: string, change: RunChange): Promise<void>;
|
|
100
|
+
};
|
|
101
|
+
/** The jobs queue: the second part of the port, which runs a segment of the run at `at`, or as soon as it can. */
|
|
102
|
+
export type SegmentQueue<Db> = {
|
|
103
|
+
schedule(db: Db, runId: string, at?: Date): Promise<void>;
|
|
104
|
+
};
|
|
105
|
+
//#endregion
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
//#region src/run/position.ts
|
|
2
|
+
const startPosition = (machine, input) => ({
|
|
3
|
+
kind: "at",
|
|
4
|
+
seq: 0,
|
|
5
|
+
state: machine.StartAt,
|
|
6
|
+
input,
|
|
7
|
+
variables: {},
|
|
8
|
+
attempt: 1,
|
|
9
|
+
failures: []
|
|
10
|
+
});
|
|
11
|
+
/** The position after a saved step. */
|
|
12
|
+
const advance = (position, step) => {
|
|
13
|
+
if (position.kind !== "at") return position;
|
|
14
|
+
const seq = step.seq + 1;
|
|
15
|
+
const variables = {
|
|
16
|
+
...position.variables,
|
|
17
|
+
...step.assigned
|
|
18
|
+
};
|
|
19
|
+
if (step.status === "failed") return {
|
|
20
|
+
kind: "failed",
|
|
21
|
+
seq,
|
|
22
|
+
error: step.error ?? "States.Runtime",
|
|
23
|
+
cause: step.cause ?? ""
|
|
24
|
+
};
|
|
25
|
+
if (step.status === "retrying") return {
|
|
26
|
+
...position,
|
|
27
|
+
seq,
|
|
28
|
+
variables,
|
|
29
|
+
attempt: position.attempt + 1,
|
|
30
|
+
failures: [...position.failures, step.error ?? ""],
|
|
31
|
+
...step.resumeAt && { resumeAt: step.resumeAt }
|
|
32
|
+
};
|
|
33
|
+
const output = step.output === void 0 ? null : step.output;
|
|
34
|
+
if (step.next === null) return {
|
|
35
|
+
kind: "ended",
|
|
36
|
+
seq,
|
|
37
|
+
output,
|
|
38
|
+
variables
|
|
39
|
+
};
|
|
40
|
+
return {
|
|
41
|
+
kind: "at",
|
|
42
|
+
seq,
|
|
43
|
+
state: step.next,
|
|
44
|
+
input: output,
|
|
45
|
+
variables,
|
|
46
|
+
attempt: 1,
|
|
47
|
+
failures: [],
|
|
48
|
+
...step.resumeAt && { resumeAt: step.resumeAt }
|
|
49
|
+
};
|
|
50
|
+
};
|
|
51
|
+
/** Where a scope stands after the steps it saved, in their order. */
|
|
52
|
+
const foldPosition = (machine, input, steps) => steps.reduce(advance, startPosition(machine, input));
|
|
53
|
+
//#endregion
|
|
54
|
+
export { advance, foldPosition, startPosition };
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { limits } from "../limits.js";
|
|
2
|
+
import { StepError, matches } from "./errors.js";
|
|
3
|
+
import { evaluateAssign, evaluateKept } from "./evaluate.js";
|
|
4
|
+
//#region src/run/recovery.ts
|
|
5
|
+
/**
|
|
6
|
+
* The wait before a retry: `IntervalSeconds` times `BackoffRate` for each earlier retry, up to `MaxDelaySeconds`, and
|
|
7
|
+
* never more than `limits.retryDelaySeconds`, which is also the cap when the retrier names none.
|
|
8
|
+
*/
|
|
9
|
+
const retryDelaySeconds = (retrier, retries) => {
|
|
10
|
+
const delay = (retrier.IntervalSeconds ?? 1) * (retrier.BackoffRate ?? 2) ** retries;
|
|
11
|
+
return Math.min(delay, retrier.MaxDelaySeconds ?? limits.retryDelaySeconds);
|
|
12
|
+
};
|
|
13
|
+
const caught = async (catcher, failure, bindings, logs) => {
|
|
14
|
+
const after = {
|
|
15
|
+
...bindings,
|
|
16
|
+
states: {
|
|
17
|
+
...bindings.states,
|
|
18
|
+
errorOutput: failure
|
|
19
|
+
}
|
|
20
|
+
};
|
|
21
|
+
const output = catcher.Output === void 0 ? failure : await evaluateKept(catcher.Output, after, "Output");
|
|
22
|
+
return {
|
|
23
|
+
kind: "step",
|
|
24
|
+
status: "caught",
|
|
25
|
+
next: catcher.Next,
|
|
26
|
+
output,
|
|
27
|
+
assigned: await evaluateAssign(catcher.Assign, after),
|
|
28
|
+
error: failure.Error,
|
|
29
|
+
cause: failure.Cause,
|
|
30
|
+
logs
|
|
31
|
+
};
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* What a failed Task or Map does: the first retrier that names its error tries it again after its delay
|
|
35
|
+
* while it has tries left; else the first catcher that names it goes on to its `Next`; else the state fails.
|
|
36
|
+
*/
|
|
37
|
+
const recover = async (state, position, failure, bindings, now, logs) => {
|
|
38
|
+
const retriers = state.Retry ?? [];
|
|
39
|
+
const retrierOf = (error) => retriers.findIndex((retrier) => matches(retrier.ErrorEquals, error));
|
|
40
|
+
const index = retrierOf(failure.Error);
|
|
41
|
+
if (index >= 0) {
|
|
42
|
+
const retrier = retriers[index];
|
|
43
|
+
const retries = position.failures.filter((error) => retrierOf(error) === index).length;
|
|
44
|
+
if (retries < (retrier.MaxAttempts ?? 3)) return {
|
|
45
|
+
kind: "step",
|
|
46
|
+
status: "retrying",
|
|
47
|
+
next: null,
|
|
48
|
+
resumeAt: new Date(now.getTime() + retryDelaySeconds(retrier, retries) * 1e3),
|
|
49
|
+
error: failure.Error,
|
|
50
|
+
cause: failure.Cause,
|
|
51
|
+
logs
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
const catcher = (state.Catch ?? []).find((candidate) => matches(candidate.ErrorEquals, failure.Error));
|
|
55
|
+
if (!catcher) return {
|
|
56
|
+
kind: "step",
|
|
57
|
+
status: "failed",
|
|
58
|
+
next: null,
|
|
59
|
+
error: failure.Error,
|
|
60
|
+
cause: failure.Cause,
|
|
61
|
+
logs
|
|
62
|
+
};
|
|
63
|
+
return caught(catcher, failure, bindings, logs).catch((error) => {
|
|
64
|
+
if (!(error instanceof StepError)) throw error;
|
|
65
|
+
return {
|
|
66
|
+
kind: "step",
|
|
67
|
+
status: "failed",
|
|
68
|
+
next: null,
|
|
69
|
+
error: error.error,
|
|
70
|
+
cause: error.reason,
|
|
71
|
+
logs
|
|
72
|
+
};
|
|
73
|
+
});
|
|
74
|
+
};
|
|
75
|
+
//#endregion
|
|
76
|
+
export { recover, retryDelaySeconds };
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
//#region src/run/redact.d.ts
|
|
2
|
+
/** A copy of a value with each field whose name says it holds a secret, such as `Authorization` or `apiKey`, replaced. */
|
|
3
|
+
export declare const redacted: (value: unknown) => unknown;
|
|
4
|
+
/** Secret values shorter than this are refused, so blanking them never blanks ordinary text. */
|
|
5
|
+
export declare const shortestSecret = 8;
|
|
6
|
+
/** Replaces every secret value inside a text, and inside every text of a value, by `[redacted]`. */
|
|
7
|
+
export type Redactor = {
|
|
8
|
+
text: (text: string) => string;
|
|
9
|
+
value: <T>(value: T) => T;
|
|
10
|
+
};
|
|
11
|
+
/**
|
|
12
|
+
* The redactor of the app's secret values, wherever they appear inside a text, such as `Bearer <token>` or a token in
|
|
13
|
+
* a URL's query; the longest first, so a secret inside another is blanked whole.
|
|
14
|
+
*/
|
|
15
|
+
export declare const secretRedactor: (secrets: readonly string[]) => Redactor;
|
|
16
|
+
//#endregion
|