@happyvertical/smrt-workflows 0.0.0-stage → 0.62.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +33 -0
- package/CLAUDE.md +1 -0
- package/LICENSE +7 -0
- package/README.md +119 -2
- package/agents/workflow-runs.md +125 -0
- package/dist/checkpoint-store.d.ts +61 -0
- package/dist/checkpoint-store.d.ts.map +1 -0
- package/dist/claim-fence.d.ts +25 -0
- package/dist/claim-fence.d.ts.map +1 -0
- package/dist/driver.d.ts +154 -0
- package/dist/driver.d.ts.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1136 -0
- package/dist/index.js.map +1 -0
- package/dist/jobs.d.ts +121 -0
- package/dist/jobs.d.ts.map +1 -0
- package/dist/manifest.json +15 -0
- package/dist/smrt-knowledge.json +88 -0
- package/dist/smrt-knowledge.okf.md +51 -0
- package/dist/types.d.ts +216 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +7 -0
- package/dist/types.js.map +1 -0
- package/package.json +60 -3
package/dist/index.js
ADDED
|
@@ -0,0 +1,1136 @@
|
|
|
1
|
+
import { WORKFLOW_RUN_TYPE, WORKFLOW_STATE_VERSION } from "./types.js";
|
|
2
|
+
import { AiCallLimitError, AiRunCheckpointConflictError, AiRunClaimLostError, AiRunResumeRefusedError, AiRunStoppedError, aiRunEngine, appendAiRunEvent, cancelAiRun, createTrackedAiClient, finishAiRun, getAiRun, getRunTrajectory, lockAiRunChain, readAiRunCheckpoint, reconcileAiRunOnClaim, resumeAiRun, saveAiRunCheckpoint, startAiRunStep, stopAiRun, waitAiRunApproval, withAiRunTransaction } from "@happyvertical/smrt-core";
|
|
3
|
+
import { JobClaimLostError, JobHandle, SmrtJobCollection, SmrtJobEventCollection, assertFailedJobDispatch, assertJobClaim, assertJobClaimForRelease, getJobClaim, isRunnerExecutionContext, jobHandlerTarget, listJobHandlers, registerJobHandler, unregisterJobHandler } from "@happyvertical/smrt-jobs";
|
|
4
|
+
import { promptMessageOptions, resolvePrompt, trackedPromptCall } from "@happyvertical/smrt-prompts";
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
//#region src/checkpoint-store.ts
|
|
7
|
+
var RunConflictError = class extends Error {
|
|
8
|
+
constructor(message = "Workflow run changed; reload its current status") {
|
|
9
|
+
super(message);
|
|
10
|
+
this.name = "RunConflictError";
|
|
11
|
+
}
|
|
12
|
+
};
|
|
13
|
+
var WorkflowRunNotFoundError = class extends Error {
|
|
14
|
+
constructor(runId) {
|
|
15
|
+
super("Workflow run not found");
|
|
16
|
+
this.runId = runId;
|
|
17
|
+
this.name = "WorkflowRunNotFoundError";
|
|
18
|
+
}
|
|
19
|
+
runId;
|
|
20
|
+
};
|
|
21
|
+
function isWorkflowCheckpointState(value) {
|
|
22
|
+
const state = value;
|
|
23
|
+
return Boolean(state && typeof state === "object" && state.v === 1 && Number.isSafeInteger(state.stepIndex) && state.stepIndex >= 0 && (state.phase === "call" || state.phase === "check") && Number.isSafeInteger(state.repairs) && Number.isSafeInteger(state.repeats) && Array.isArray(state.outputs) && (state.inputRef === null || typeof state.inputRef === "string"));
|
|
24
|
+
}
|
|
25
|
+
function createRunCheckpointStore(options) {
|
|
26
|
+
if (!options.ownerId) throw new Error("Workflow runs require an owner");
|
|
27
|
+
const tenantId = options.tenantId ?? null;
|
|
28
|
+
return {
|
|
29
|
+
db: options.db,
|
|
30
|
+
ownerId: options.ownerId,
|
|
31
|
+
tenantId,
|
|
32
|
+
async load(runId) {
|
|
33
|
+
const run = await getAiRun(options.db, runId);
|
|
34
|
+
if (!run || run.type !== "workflow" || run.ownerId !== options.ownerId || (run.tenantId ?? null) !== tenantId) return null;
|
|
35
|
+
const checkpoint = await readAiRunCheckpoint(options.db, runId);
|
|
36
|
+
if (!checkpoint) return null;
|
|
37
|
+
if (!isWorkflowCheckpointState(checkpoint.state)) throw new TypeError(`Workflow run ${runId} has a checkpoint this package cannot read`);
|
|
38
|
+
return {
|
|
39
|
+
run,
|
|
40
|
+
checkpoint,
|
|
41
|
+
state: checkpoint.state
|
|
42
|
+
};
|
|
43
|
+
},
|
|
44
|
+
async save(runId, input) {
|
|
45
|
+
try {
|
|
46
|
+
return await saveAiRunCheckpoint(options.db, runId, {
|
|
47
|
+
expectedRevision: input.expectedRevision,
|
|
48
|
+
state: input.state,
|
|
49
|
+
...input.lease !== void 0 ? { lease: input.lease } : {},
|
|
50
|
+
...input.definitionVersion !== void 0 ? { definitionVersion: input.definitionVersion } : {},
|
|
51
|
+
...options.fence ? { fence: options.fence } : {}
|
|
52
|
+
});
|
|
53
|
+
} catch (error) {
|
|
54
|
+
if (error instanceof AiRunCheckpointConflictError) throw new RunConflictError();
|
|
55
|
+
throw error;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
//#endregion
|
|
61
|
+
//#region src/claim-fence.ts
|
|
62
|
+
function workflowClaimFence(claim) {
|
|
63
|
+
return (tx) => assertJobClaim(tx, claim);
|
|
64
|
+
}
|
|
65
|
+
function workflowStopFence(claim) {
|
|
66
|
+
return async (tx, durableRunId) => {
|
|
67
|
+
await assertJobClaimForRelease(tx, claim);
|
|
68
|
+
await lockAiRunChain({
|
|
69
|
+
db: tx,
|
|
70
|
+
engine: aiRunEngine(tx),
|
|
71
|
+
now: /* @__PURE__ */ new Date()
|
|
72
|
+
}, durableRunId);
|
|
73
|
+
await tx.query("UPDATE _smrt_ai_runs SET retain = retain WHERE id = ?", durableRunId);
|
|
74
|
+
const jobs = await SmrtJobCollection.create({ db: tx });
|
|
75
|
+
const own = await jobs.get({ id: claim.jobId });
|
|
76
|
+
if (!own?.dedupeKey) return;
|
|
77
|
+
if ((await jobs.list({ where: {
|
|
78
|
+
objectType: claim.objectType,
|
|
79
|
+
dedupeKey: own.dedupeKey,
|
|
80
|
+
tenantId: claim.tenantId,
|
|
81
|
+
status: ["pending", "running"]
|
|
82
|
+
} })).some((job) => String(job.id) !== claim.jobId)) throw new JobClaimLostError();
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
//#endregion
|
|
86
|
+
//#region src/driver.ts
|
|
87
|
+
var WorkflowOutputError = class extends Error {
|
|
88
|
+
constructor(message) {
|
|
89
|
+
super(message);
|
|
90
|
+
this.name = "WorkflowOutputError";
|
|
91
|
+
}
|
|
92
|
+
};
|
|
93
|
+
function readWorkflowRunLease(checkpoint) {
|
|
94
|
+
if (!checkpoint.lease) return null;
|
|
95
|
+
try {
|
|
96
|
+
const parsed = JSON.parse(checkpoint.lease);
|
|
97
|
+
return typeof parsed.holder === "string" && typeof parsed.expiresAt === "string" ? {
|
|
98
|
+
holder: parsed.holder,
|
|
99
|
+
expiresAt: parsed.expiresAt
|
|
100
|
+
} : null;
|
|
101
|
+
} catch {
|
|
102
|
+
return null;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
function initialWorkflowState(input, inputRef) {
|
|
106
|
+
return {
|
|
107
|
+
v: 1,
|
|
108
|
+
stepIndex: 0,
|
|
109
|
+
phase: "call",
|
|
110
|
+
repairs: 0,
|
|
111
|
+
repeats: 0,
|
|
112
|
+
input: input ?? null,
|
|
113
|
+
inputRef,
|
|
114
|
+
outputs: [],
|
|
115
|
+
pending: null,
|
|
116
|
+
invalid: null,
|
|
117
|
+
approval: null
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
function parseOutput(raw) {
|
|
121
|
+
let value = raw;
|
|
122
|
+
if (value && typeof value === "object" && typeof value.content === "string") value = value.content;
|
|
123
|
+
if (typeof value !== "string") return value;
|
|
124
|
+
const text = value.trim().replace(/^```(?:json)?\s*/i, "").replace(/\s*```$/, "");
|
|
125
|
+
try {
|
|
126
|
+
return JSON.parse(text);
|
|
127
|
+
} catch {
|
|
128
|
+
throw new WorkflowOutputError("The step output is not valid JSON");
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
function serial() {
|
|
132
|
+
let tail = Promise.resolve();
|
|
133
|
+
return (operation) => {
|
|
134
|
+
const next = tail.then(operation, operation);
|
|
135
|
+
tail = next.catch(() => void 0);
|
|
136
|
+
return next;
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
function createWorkflowRunDriver(options) {
|
|
140
|
+
const { ai, definition } = options;
|
|
141
|
+
const steps = definition.steps;
|
|
142
|
+
if (!steps.length) throw new Error("A workflow needs steps");
|
|
143
|
+
if (!definition.key.trim() || !definition.version.trim()) throw new Error("A workflow definition needs a key and a version");
|
|
144
|
+
const db = ai.db;
|
|
145
|
+
const tenantId = ai.tenantId ?? null;
|
|
146
|
+
const leaseTtlMs = options.leaseTtlMs ?? 3e4;
|
|
147
|
+
const now = options.now ?? (() => /* @__PURE__ */ new Date());
|
|
148
|
+
const active = /* @__PURE__ */ new Map();
|
|
149
|
+
const acceptsVersion = (version) => version === definition.version || Boolean(definition.olderCheckpoints?.accepts(version));
|
|
150
|
+
const storeFor = (ownerId, fence) => createRunCheckpointStore({
|
|
151
|
+
db,
|
|
152
|
+
ownerId,
|
|
153
|
+
tenantId,
|
|
154
|
+
...fence ? { fence } : {}
|
|
155
|
+
});
|
|
156
|
+
const mustLoad = async (store, runId) => {
|
|
157
|
+
const loaded = await store.load(runId);
|
|
158
|
+
if (!loaded || loaded.run.definitionKey !== definition.key) throw new WorkflowRunNotFoundError(runId);
|
|
159
|
+
return loaded;
|
|
160
|
+
};
|
|
161
|
+
const stepInfo = (index) => {
|
|
162
|
+
const at = Math.min(index, steps.length - 1);
|
|
163
|
+
return {
|
|
164
|
+
index: Math.min(index + 1, steps.length),
|
|
165
|
+
count: steps.length,
|
|
166
|
+
name: steps[at].name
|
|
167
|
+
};
|
|
168
|
+
};
|
|
169
|
+
const snapshot = (loaded) => {
|
|
170
|
+
const { run, checkpoint, state } = loaded;
|
|
171
|
+
return {
|
|
172
|
+
id: run.id,
|
|
173
|
+
ownerId: run.ownerId,
|
|
174
|
+
tenantId: run.tenantId,
|
|
175
|
+
definitionKey: run.definitionKey,
|
|
176
|
+
definitionVersion: checkpoint.definitionVersion,
|
|
177
|
+
status: run.status,
|
|
178
|
+
stopReason: run.stopReason,
|
|
179
|
+
stoppedByRunId: run.stoppedByRunId,
|
|
180
|
+
stopDetail: run.stopDetail,
|
|
181
|
+
retryOfRunId: run.retryOfRunId,
|
|
182
|
+
budget: run.budget,
|
|
183
|
+
revision: checkpoint.revision,
|
|
184
|
+
step: stepInfo(state.stepIndex),
|
|
185
|
+
phase: state.phase,
|
|
186
|
+
input: state.input,
|
|
187
|
+
inputRef: state.inputRef,
|
|
188
|
+
outputs: [...state.outputs],
|
|
189
|
+
pending: state.pending?.output ?? null,
|
|
190
|
+
totals: {
|
|
191
|
+
tokens: run.tokens,
|
|
192
|
+
calls: run.calls,
|
|
193
|
+
costUsed: run.costUsed,
|
|
194
|
+
stepsUsed: run.stepsUsed,
|
|
195
|
+
inclusiveTokens: run.inclusiveTokens,
|
|
196
|
+
inclusiveCalls: run.inclusiveCalls,
|
|
197
|
+
inclusiveCost: run.inclusiveCost
|
|
198
|
+
}
|
|
199
|
+
};
|
|
200
|
+
};
|
|
201
|
+
const emit = async (state) => {
|
|
202
|
+
try {
|
|
203
|
+
await options.onProgress?.({
|
|
204
|
+
runId: state.id,
|
|
205
|
+
step: state.step,
|
|
206
|
+
status: state.status,
|
|
207
|
+
stopReason: state.stopReason,
|
|
208
|
+
phase: state.phase
|
|
209
|
+
});
|
|
210
|
+
} catch {}
|
|
211
|
+
};
|
|
212
|
+
const current = async (store, runId) => {
|
|
213
|
+
const state = snapshot(await mustLoad(store, runId));
|
|
214
|
+
await emit(state);
|
|
215
|
+
return state;
|
|
216
|
+
};
|
|
217
|
+
const judgedEvent = async (runId, afterSeq, stepIndex) => {
|
|
218
|
+
const candidates = ((await getRunTrajectory(db, runId))?.entries ?? []).map((entry) => entry.event).filter((event) => Boolean(event) && (event?.seq ?? 0) > afterSeq && (event?.type === "call" || event?.type === "handoff") && (event?.stepIndex === null || event?.stepIndex === stepIndex));
|
|
219
|
+
return candidates[candidates.length - 1]?.id ?? null;
|
|
220
|
+
};
|
|
221
|
+
const lastSeq = async (runId) => {
|
|
222
|
+
return ((await getRunTrajectory(db, runId))?.entries ?? []).reduce((max, entry) => Math.max(max, entry.event?.seq ?? 0), 0);
|
|
223
|
+
};
|
|
224
|
+
const approvalDecision = async (runId, requestEventId) => {
|
|
225
|
+
const trajectory = await getRunTrajectory(db, runId);
|
|
226
|
+
for (const entry of [...trajectory?.entries ?? []].reverse()) {
|
|
227
|
+
const event = entry.event;
|
|
228
|
+
if (event?.type === "approval" && event.parentEventId === requestEventId && (event.status === "approved" || event.status === "refused")) return event.status;
|
|
229
|
+
}
|
|
230
|
+
return null;
|
|
231
|
+
};
|
|
232
|
+
const resolveInput = async (state, ownerId) => {
|
|
233
|
+
if (state.inputRef === null) return state.input;
|
|
234
|
+
if (!options.readInput) throw new Error("This workflow passes input by reference: set readInput");
|
|
235
|
+
return options.readInput(state.inputRef, {
|
|
236
|
+
ownerId,
|
|
237
|
+
tenantId
|
|
238
|
+
});
|
|
239
|
+
};
|
|
240
|
+
const defaultVariables = (step, context) => ({
|
|
241
|
+
input: JSON.stringify(context.input ?? null),
|
|
242
|
+
outputs: JSON.stringify(context.outputs),
|
|
243
|
+
schema: JSON.stringify(step.schema),
|
|
244
|
+
attempt: String(context.attempt),
|
|
245
|
+
repairOutput: context.repair ? JSON.stringify(context.repair.output) : "",
|
|
246
|
+
repairErrors: context.repair ? context.repair.errors.join("\n") : ""
|
|
247
|
+
});
|
|
248
|
+
const callStep = async (step, context, runContext) => {
|
|
249
|
+
if (step.execute) return parseOutput(await step.execute({
|
|
250
|
+
...context,
|
|
251
|
+
runContext
|
|
252
|
+
}));
|
|
253
|
+
const variables = step.variables ? await step.variables(context) : defaultVariables(step, context);
|
|
254
|
+
const resolved = await resolvePrompt(step.promptKey, {
|
|
255
|
+
db,
|
|
256
|
+
tenantId,
|
|
257
|
+
variables
|
|
258
|
+
});
|
|
259
|
+
return parseOutput(await (await ai.client(trackedPromptCall(resolved), { runContext })).message(resolved.text, {
|
|
260
|
+
...promptMessageOptions(resolved.ai, { routeTier: false }),
|
|
261
|
+
signal: context.signal,
|
|
262
|
+
...step.expectedOutputTokens !== void 0 ? { expectedOutputTokens: step.expectedOutputTokens } : {}
|
|
263
|
+
}));
|
|
264
|
+
};
|
|
265
|
+
return {
|
|
266
|
+
definition,
|
|
267
|
+
/**
|
|
268
|
+
* Open the durable `workflow` run with its first checkpoint, in one
|
|
269
|
+
* transaction. Throws `AiRunTrackingDisabledError` while usage tracking
|
|
270
|
+
* is off. Schedule `run` on a background job; never on a request.
|
|
271
|
+
*/
|
|
272
|
+
async start(input) {
|
|
273
|
+
if (!input.ownerId) throw new Error("Workflow runs require an owner");
|
|
274
|
+
const { run } = await ai.openRun({
|
|
275
|
+
...input.id ? { id: input.id } : {},
|
|
276
|
+
type: WORKFLOW_RUN_TYPE,
|
|
277
|
+
definitionKey: definition.key,
|
|
278
|
+
definitionVersion: definition.version,
|
|
279
|
+
ownerId: input.ownerId,
|
|
280
|
+
durable: true,
|
|
281
|
+
checkpoint: { state: initialWorkflowState(input.inputRef ? null : input.input ?? null, input.inputRef ?? null) },
|
|
282
|
+
...input.budget ? { budget: input.budget } : {},
|
|
283
|
+
...input.tags ? { tags: input.tags } : {},
|
|
284
|
+
...input.experimentId ? { experimentId: input.experimentId } : {},
|
|
285
|
+
...input.datasetItem ? { datasetItem: input.datasetItem } : {},
|
|
286
|
+
...input.parentRunId ? { parentRunId: input.parentRunId } : {}
|
|
287
|
+
});
|
|
288
|
+
return current(storeFor(input.ownerId), run.id);
|
|
289
|
+
},
|
|
290
|
+
/** The run as this owner sees it. */
|
|
291
|
+
get(runId, ownerId) {
|
|
292
|
+
return mustLoad(storeFor(ownerId), runId).then(snapshot);
|
|
293
|
+
},
|
|
294
|
+
/**
|
|
295
|
+
* Work the run until it finishes, stops, fails or waits for approval.
|
|
296
|
+
* Run it only from the job that holds the run, with that job's signal
|
|
297
|
+
* and claim fence. Reconciles reservations a dead worker left behind
|
|
298
|
+
* before the first call.
|
|
299
|
+
*/
|
|
300
|
+
async run(runId, ownerId, runOptions = {}) {
|
|
301
|
+
const store = storeFor(ownerId, runOptions.fence);
|
|
302
|
+
let loaded = await mustLoad(store, runId);
|
|
303
|
+
if (loaded.run.status !== "running") return snapshot(loaded);
|
|
304
|
+
if (active.has(runId)) throw new RunConflictError("This workflow run is already working");
|
|
305
|
+
const controller = new AbortController();
|
|
306
|
+
const abort = () => controller.abort(runOptions.signal?.reason);
|
|
307
|
+
runOptions.signal?.addEventListener("abort", abort, { once: true });
|
|
308
|
+
if (runOptions.signal?.aborted) abort();
|
|
309
|
+
active.set(runId, controller);
|
|
310
|
+
const queue = serial();
|
|
311
|
+
const fence = runOptions.fence;
|
|
312
|
+
const fenced = fence ? { fence } : {};
|
|
313
|
+
const stopFence = runOptions.stopFence ?? fence;
|
|
314
|
+
let revision = loaded.checkpoint.revision;
|
|
315
|
+
let state = loaded.state;
|
|
316
|
+
const lease = () => runOptions.holder ? JSON.stringify({
|
|
317
|
+
holder: runOptions.holder,
|
|
318
|
+
expiresAt: new Date(now().getTime() + leaseTtlMs).toISOString()
|
|
319
|
+
}) : void 0;
|
|
320
|
+
const persist = (next, extra = {}) => queue(async () => {
|
|
321
|
+
revision = await store.save(runId, {
|
|
322
|
+
expectedRevision: revision,
|
|
323
|
+
state: next,
|
|
324
|
+
...extra
|
|
325
|
+
});
|
|
326
|
+
state = next;
|
|
327
|
+
});
|
|
328
|
+
let renewal;
|
|
329
|
+
const stopIfRunning = async (reason) => {
|
|
330
|
+
await stopAiRun(db, runId, {
|
|
331
|
+
reason,
|
|
332
|
+
...stopFence ? { fence: stopFence } : {}
|
|
333
|
+
}).catch((error) => {
|
|
334
|
+
if (error instanceof AiRunClaimLostError) throw error;
|
|
335
|
+
return null;
|
|
336
|
+
});
|
|
337
|
+
};
|
|
338
|
+
const fail = async (stopReason) => {
|
|
339
|
+
await finishAiRun(db, runId, {
|
|
340
|
+
status: "failed",
|
|
341
|
+
stopReason
|
|
342
|
+
}, fenced);
|
|
343
|
+
};
|
|
344
|
+
try {
|
|
345
|
+
await reconcileAiRunOnClaim(db, runId);
|
|
346
|
+
const saved = loaded.checkpoint.definitionVersion;
|
|
347
|
+
if (saved !== definition.version) {
|
|
348
|
+
let migrated = null;
|
|
349
|
+
if (definition.olderCheckpoints?.accepts(saved)) try {
|
|
350
|
+
const candidate = definition.olderCheckpoints.migrate(state, saved);
|
|
351
|
+
if (isWorkflowCheckpointState(candidate)) migrated = candidate;
|
|
352
|
+
} catch {
|
|
353
|
+
migrated = null;
|
|
354
|
+
}
|
|
355
|
+
if (!migrated) {
|
|
356
|
+
await stopIfRunning("definition_changed");
|
|
357
|
+
return await current(store, runId);
|
|
358
|
+
}
|
|
359
|
+
await persist(migrated, { definitionVersion: definition.version });
|
|
360
|
+
}
|
|
361
|
+
const holderLease = lease();
|
|
362
|
+
if (holderLease) {
|
|
363
|
+
await queue(async () => {
|
|
364
|
+
revision = await store.save(runId, {
|
|
365
|
+
expectedRevision: revision,
|
|
366
|
+
state,
|
|
367
|
+
lease: holderLease
|
|
368
|
+
});
|
|
369
|
+
});
|
|
370
|
+
renewal = setInterval(() => {
|
|
371
|
+
const renewed = lease();
|
|
372
|
+
queue(async () => {
|
|
373
|
+
revision = await store.save(runId, {
|
|
374
|
+
expectedRevision: revision,
|
|
375
|
+
state,
|
|
376
|
+
lease: renewed ?? null
|
|
377
|
+
});
|
|
378
|
+
}).catch(() => {
|
|
379
|
+
if (renewal) clearInterval(renewal);
|
|
380
|
+
});
|
|
381
|
+
}, Math.max(10, Math.floor(leaseTtlMs / 3)));
|
|
382
|
+
renewal.unref?.();
|
|
383
|
+
}
|
|
384
|
+
let entered = false;
|
|
385
|
+
let stepInstanceId;
|
|
386
|
+
while (true) {
|
|
387
|
+
if (state.stepIndex >= steps.length) {
|
|
388
|
+
await finishAiRun(db, runId, { status: "done" }, fenced);
|
|
389
|
+
break;
|
|
390
|
+
}
|
|
391
|
+
if (controller.signal.aborted) {
|
|
392
|
+
await stopIfRunning("interrupted");
|
|
393
|
+
break;
|
|
394
|
+
}
|
|
395
|
+
const step = steps[state.stepIndex];
|
|
396
|
+
const base = {
|
|
397
|
+
runId,
|
|
398
|
+
step: stepInfo(state.stepIndex),
|
|
399
|
+
input: await resolveInput(state, ownerId),
|
|
400
|
+
outputs: state.outputs.slice(0, state.stepIndex),
|
|
401
|
+
attempt: state.repairs + 1,
|
|
402
|
+
repair: state.invalid,
|
|
403
|
+
signal: controller.signal
|
|
404
|
+
};
|
|
405
|
+
if (state.phase === "call") {
|
|
406
|
+
if (!entered) {
|
|
407
|
+
const started = await queue(() => startAiRunStep(db, runId, {
|
|
408
|
+
stepIndex: state.stepIndex,
|
|
409
|
+
...fence ? { fence } : {},
|
|
410
|
+
checkpoint: {
|
|
411
|
+
expectedRevision: revision,
|
|
412
|
+
state
|
|
413
|
+
}
|
|
414
|
+
}));
|
|
415
|
+
if (started.revision !== null) revision = started.revision;
|
|
416
|
+
stepInstanceId = started.stepInstanceId;
|
|
417
|
+
entered = true;
|
|
418
|
+
await current(store, runId);
|
|
419
|
+
}
|
|
420
|
+
const runContext = {
|
|
421
|
+
runId,
|
|
422
|
+
db,
|
|
423
|
+
tenantId,
|
|
424
|
+
...fence ? { fence } : {},
|
|
425
|
+
stepIndex: state.stepIndex,
|
|
426
|
+
stepName: step.name,
|
|
427
|
+
...stepInstanceId ? { stepInstanceId } : {},
|
|
428
|
+
attempt: base.attempt
|
|
429
|
+
};
|
|
430
|
+
const before = await lastSeq(runId);
|
|
431
|
+
let output;
|
|
432
|
+
let errors = [];
|
|
433
|
+
try {
|
|
434
|
+
output = await callStep(step, base, runContext);
|
|
435
|
+
} catch (error) {
|
|
436
|
+
if (error instanceof AiRunStoppedError || error instanceof AiRunClaimLostError) throw error;
|
|
437
|
+
if (controller.signal.aborted) {
|
|
438
|
+
await stopIfRunning("interrupted");
|
|
439
|
+
break;
|
|
440
|
+
}
|
|
441
|
+
if (error instanceof WorkflowOutputError || error instanceof AiCallLimitError) errors = [error.message];
|
|
442
|
+
else throw error;
|
|
443
|
+
}
|
|
444
|
+
if (!errors.length) errors = [...options.validate(step.schema, output)];
|
|
445
|
+
if (errors.length) {
|
|
446
|
+
const retries = loaded.run.budget.repairRetries ?? 0;
|
|
447
|
+
if (state.repairs >= retries) {
|
|
448
|
+
await fail("repairs_exhausted");
|
|
449
|
+
break;
|
|
450
|
+
}
|
|
451
|
+
await persist({
|
|
452
|
+
...state,
|
|
453
|
+
repairs: state.repairs + 1,
|
|
454
|
+
invalid: {
|
|
455
|
+
output: output ?? null,
|
|
456
|
+
errors
|
|
457
|
+
}
|
|
458
|
+
});
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
461
|
+
await persist({
|
|
462
|
+
...state,
|
|
463
|
+
phase: "check",
|
|
464
|
+
pending: {
|
|
465
|
+
output,
|
|
466
|
+
judgedEventId: await judgedEvent(runId, before, state.stepIndex)
|
|
467
|
+
},
|
|
468
|
+
invalid: null
|
|
469
|
+
});
|
|
470
|
+
continue;
|
|
471
|
+
}
|
|
472
|
+
const pending = state.pending ?? {
|
|
473
|
+
output: null,
|
|
474
|
+
judgedEventId: null
|
|
475
|
+
};
|
|
476
|
+
let approval = null;
|
|
477
|
+
if (state.approval) {
|
|
478
|
+
const decision = await approvalDecision(runId, state.approval.requestEventId);
|
|
479
|
+
if (decision === "refused") {
|
|
480
|
+
await fail("check_failed");
|
|
481
|
+
break;
|
|
482
|
+
}
|
|
483
|
+
if (decision !== "approved") {
|
|
484
|
+
await waitAiRunApproval(db, runId, fenced);
|
|
485
|
+
break;
|
|
486
|
+
}
|
|
487
|
+
approval = "approved";
|
|
488
|
+
await persist({
|
|
489
|
+
...state,
|
|
490
|
+
approval: null,
|
|
491
|
+
approvals: (state.approvals ?? 0) + 1
|
|
492
|
+
});
|
|
493
|
+
}
|
|
494
|
+
const result = step.check ? await step.check(pending.output, {
|
|
495
|
+
...base,
|
|
496
|
+
approval,
|
|
497
|
+
approvals: state.approvals ?? 0
|
|
498
|
+
}) : {};
|
|
499
|
+
if (controller.signal.aborted) {
|
|
500
|
+
await stopIfRunning("interrupted");
|
|
501
|
+
break;
|
|
502
|
+
}
|
|
503
|
+
const input = result.input !== void 0 ? {
|
|
504
|
+
input: result.input,
|
|
505
|
+
inputRef: null
|
|
506
|
+
} : {
|
|
507
|
+
input: state.input,
|
|
508
|
+
inputRef: state.inputRef
|
|
509
|
+
};
|
|
510
|
+
const checkEvent = (status) => appendAiRunEvent(db, runId, {
|
|
511
|
+
type: "check",
|
|
512
|
+
status,
|
|
513
|
+
parentEventId: pending.judgedEventId,
|
|
514
|
+
stepIndex: state.stepIndex,
|
|
515
|
+
name: step.name,
|
|
516
|
+
...result.note || result.failure ? { payload: {
|
|
517
|
+
...result.note ? { note: result.note } : {},
|
|
518
|
+
...result.failure ? { failure: result.failure } : {}
|
|
519
|
+
} } : {}
|
|
520
|
+
}, fenced);
|
|
521
|
+
if (result.failure !== void 0) {
|
|
522
|
+
await checkEvent("failed");
|
|
523
|
+
await fail("check_failed");
|
|
524
|
+
break;
|
|
525
|
+
}
|
|
526
|
+
if (result.waitingApproval) {
|
|
527
|
+
const request = await appendAiRunEvent(db, runId, {
|
|
528
|
+
type: "approval",
|
|
529
|
+
status: "requested",
|
|
530
|
+
parentEventId: pending.judgedEventId,
|
|
531
|
+
stepIndex: state.stepIndex,
|
|
532
|
+
name: step.name,
|
|
533
|
+
...result.note ? { payload: { note: result.note } } : {},
|
|
534
|
+
...result.holdRef ? { payloadRef: result.holdRef } : {}
|
|
535
|
+
}, fenced);
|
|
536
|
+
await persist({
|
|
537
|
+
...state,
|
|
538
|
+
...input,
|
|
539
|
+
approval: { requestEventId: request.id }
|
|
540
|
+
});
|
|
541
|
+
await waitAiRunApproval(db, runId, fenced);
|
|
542
|
+
break;
|
|
543
|
+
}
|
|
544
|
+
if (result.repeat) {
|
|
545
|
+
await checkEvent("failed");
|
|
546
|
+
if (state.repeats >= (step.maxRepeats ?? 0)) {
|
|
547
|
+
await fail("check_failed");
|
|
548
|
+
break;
|
|
549
|
+
}
|
|
550
|
+
await persist({
|
|
551
|
+
...state,
|
|
552
|
+
...input,
|
|
553
|
+
phase: "call",
|
|
554
|
+
repeats: state.repeats + 1,
|
|
555
|
+
repairs: 0,
|
|
556
|
+
pending: null,
|
|
557
|
+
invalid: null,
|
|
558
|
+
approval: null,
|
|
559
|
+
approvals: 0
|
|
560
|
+
});
|
|
561
|
+
entered = false;
|
|
562
|
+
continue;
|
|
563
|
+
}
|
|
564
|
+
await checkEvent("passed");
|
|
565
|
+
const outputs = state.outputs.slice(0, state.stepIndex);
|
|
566
|
+
outputs[state.stepIndex] = pending.output;
|
|
567
|
+
if (result.complete) {
|
|
568
|
+
await persist({
|
|
569
|
+
...state,
|
|
570
|
+
...input,
|
|
571
|
+
outputs,
|
|
572
|
+
pending: null,
|
|
573
|
+
approval: null,
|
|
574
|
+
approvals: 0
|
|
575
|
+
});
|
|
576
|
+
await finishAiRun(db, runId, { status: "done" }, fenced);
|
|
577
|
+
break;
|
|
578
|
+
}
|
|
579
|
+
await persist({
|
|
580
|
+
...state,
|
|
581
|
+
...input,
|
|
582
|
+
stepIndex: state.stepIndex + 1,
|
|
583
|
+
phase: "call",
|
|
584
|
+
repairs: 0,
|
|
585
|
+
repeats: 0,
|
|
586
|
+
outputs,
|
|
587
|
+
pending: null,
|
|
588
|
+
invalid: null,
|
|
589
|
+
approval: null,
|
|
590
|
+
approvals: 0
|
|
591
|
+
});
|
|
592
|
+
entered = false;
|
|
593
|
+
}
|
|
594
|
+
} catch (error) {
|
|
595
|
+
if (error instanceof AiRunClaimLostError || error instanceof RunConflictError) throw error;
|
|
596
|
+
if (!(error instanceof AiRunStoppedError)) if (controller.signal.aborted) await stopIfRunning("interrupted");
|
|
597
|
+
else await fail("error");
|
|
598
|
+
} finally {
|
|
599
|
+
if (renewal) clearInterval(renewal);
|
|
600
|
+
runOptions.signal?.removeEventListener("abort", abort);
|
|
601
|
+
if (active.get(runId) === controller) active.delete(runId);
|
|
602
|
+
}
|
|
603
|
+
loaded = await mustLoad(store, runId);
|
|
604
|
+
const finalState = snapshot(loaded);
|
|
605
|
+
await emit(finalState);
|
|
606
|
+
return finalState;
|
|
607
|
+
},
|
|
608
|
+
/** Pause the run (`stopped`, `interrupted`): resumable. */
|
|
609
|
+
async stop(runId, ownerId) {
|
|
610
|
+
const store = storeFor(ownerId);
|
|
611
|
+
const { run } = await mustLoad(store, runId);
|
|
612
|
+
if (run.status === "running" || run.status === "waiting-approval") await stopAiRun(db, runId, {
|
|
613
|
+
reason: "interrupted",
|
|
614
|
+
from: run.status
|
|
615
|
+
});
|
|
616
|
+
active.get(runId)?.abort(/* @__PURE__ */ new Error("Workflow run stopped"));
|
|
617
|
+
return current(store, runId);
|
|
618
|
+
},
|
|
619
|
+
/** End the run for good, with every child run: never resumed or retried. */
|
|
620
|
+
async cancel(runId, ownerId) {
|
|
621
|
+
const store = storeFor(ownerId);
|
|
622
|
+
await mustLoad(store, runId);
|
|
623
|
+
await cancelAiRun(db, runId);
|
|
624
|
+
active.get(runId)?.abort(/* @__PURE__ */ new Error("Workflow run cancelled"));
|
|
625
|
+
return current(store, runId);
|
|
626
|
+
},
|
|
627
|
+
/**
|
|
628
|
+
* Resume a stopped run (page section 7): under this definition version
|
|
629
|
+
* (or one that accepts the checkpoint), with limits only raised and at
|
|
630
|
+
* most to the app's ceiling; a budget stop needs the limit that stopped
|
|
631
|
+
* it raised. Reconciles the run's stale reservations first. Then admit
|
|
632
|
+
* its job again: with `admit`, the status change and `admit`'s writes
|
|
633
|
+
* (the new job) commit in one transaction, so no stale worker's stop
|
|
634
|
+
* can land between them.
|
|
635
|
+
*/
|
|
636
|
+
async resume(runId, ownerId, resumeOptions = {}) {
|
|
637
|
+
const store = storeFor(ownerId);
|
|
638
|
+
await mustLoad(store, runId);
|
|
639
|
+
if (active.has(runId)) throw new RunConflictError("The previous run is still stopping. Try again when it has stopped.");
|
|
640
|
+
const resumeWith = (database, reconcile) => resumeAiRun(database, runId, {
|
|
641
|
+
...resumeOptions.budget ? { budget: resumeOptions.budget } : {},
|
|
642
|
+
...options.ceiling ? { ceiling: options.ceiling } : {},
|
|
643
|
+
definitionVersion: definition.version,
|
|
644
|
+
acceptsVersion,
|
|
645
|
+
...resumeOptions.fence ? { fence: resumeOptions.fence } : {},
|
|
646
|
+
...reconcile ? { reconcile: async (target, id) => {
|
|
647
|
+
await reconcileAiRunOnClaim(target, id);
|
|
648
|
+
} } : {}
|
|
649
|
+
});
|
|
650
|
+
const admit = resumeOptions.admit;
|
|
651
|
+
if (!admit) {
|
|
652
|
+
await resumeWith(db, true);
|
|
653
|
+
return current(store, runId);
|
|
654
|
+
}
|
|
655
|
+
await reconcileAiRunOnClaim(db, runId);
|
|
656
|
+
const refused = await withAiRunTransaction(db, async (tx) => {
|
|
657
|
+
let resumed;
|
|
658
|
+
try {
|
|
659
|
+
resumed = await resumeWith(tx.db, false);
|
|
660
|
+
} catch (error) {
|
|
661
|
+
if (error instanceof AiRunResumeRefusedError) return error;
|
|
662
|
+
throw error;
|
|
663
|
+
}
|
|
664
|
+
await admit(tx.db, resumed.budget);
|
|
665
|
+
return null;
|
|
666
|
+
});
|
|
667
|
+
if (refused) throw refused;
|
|
668
|
+
return current(store, runId);
|
|
669
|
+
},
|
|
670
|
+
/**
|
|
671
|
+
* Record a person's decision on the approval the run waits for (an
|
|
672
|
+
* `approval` event whose parent is the request). Approved: the run goes
|
|
673
|
+
* back to `running` and its next `run` re-runs the check with the
|
|
674
|
+
* approval; refused: the run fails with `check_failed`. A run stopped
|
|
675
|
+
* while it waited keeps the decision and stays stopped: `resume` then
|
|
676
|
+
* continues with it.
|
|
677
|
+
*/
|
|
678
|
+
async decide(runId, ownerId, decision) {
|
|
679
|
+
const store = storeFor(ownerId);
|
|
680
|
+
const loaded = await mustLoad(store, runId);
|
|
681
|
+
const request = loaded.state.approval?.requestEventId;
|
|
682
|
+
const stopped = loaded.run.status === "stopped";
|
|
683
|
+
if (loaded.run.status !== "waiting-approval" && !stopped || !request) throw new RunConflictError("This workflow run is not waiting for approval");
|
|
684
|
+
if (stopped && await approvalDecision(runId, request)) throw new RunConflictError("This approval was already decided");
|
|
685
|
+
await appendAiRunEvent(db, runId, {
|
|
686
|
+
type: "approval",
|
|
687
|
+
status: decision.approved ? "approved" : "refused",
|
|
688
|
+
parentEventId: request,
|
|
689
|
+
stepIndex: loaded.state.stepIndex,
|
|
690
|
+
name: stepInfo(loaded.state.stepIndex).name,
|
|
691
|
+
...decision.note ? { payload: { note: decision.note } } : {}
|
|
692
|
+
});
|
|
693
|
+
if (stopped) {} else if (decision.approved) await resumeAiRun(db, runId, {
|
|
694
|
+
definitionVersion: definition.version,
|
|
695
|
+
acceptsVersion
|
|
696
|
+
});
|
|
697
|
+
else await finishAiRun(db, runId, {
|
|
698
|
+
status: "failed",
|
|
699
|
+
stopReason: "check_failed"
|
|
700
|
+
}, { from: "waiting-approval" });
|
|
701
|
+
return current(store, runId);
|
|
702
|
+
},
|
|
703
|
+
/**
|
|
704
|
+
* Retry a failed run from its last accepted step (page section 7): a new
|
|
705
|
+
* durable run naming it in `retry_of_run_id`, inheriting only the
|
|
706
|
+
* accepted outputs of its completed steps; its totals start at zero and
|
|
707
|
+
* the inherited steps are not counted against its `maxSteps`. Refused
|
|
708
|
+
* with `AiRunResumeRefusedError` (`not_retryable`, `definition_changed`,
|
|
709
|
+
* `retry_source_pruned`) and nothing is opened.
|
|
710
|
+
*/
|
|
711
|
+
async retry(runId, ownerId, retryOptions = {}) {
|
|
712
|
+
const store = storeFor(ownerId);
|
|
713
|
+
const failed = await store.load(runId);
|
|
714
|
+
if (failed && failed.run.definitionKey !== definition.key) throw new WorkflowRunNotFoundError(runId);
|
|
715
|
+
if (!failed) {
|
|
716
|
+
const visible = await getAiRun(db, runId);
|
|
717
|
+
if (visible && (visible.type !== "workflow" || visible.definitionKey !== definition.key || visible.ownerId !== ownerId || (visible.tenantId ?? null) !== tenantId)) throw new WorkflowRunNotFoundError(runId);
|
|
718
|
+
}
|
|
719
|
+
const { run } = await ai.openRun({
|
|
720
|
+
...retryOptions.id ? { id: retryOptions.id } : {},
|
|
721
|
+
type: WORKFLOW_RUN_TYPE,
|
|
722
|
+
definitionKey: definition.key,
|
|
723
|
+
definitionVersion: definition.version,
|
|
724
|
+
ownerId,
|
|
725
|
+
retryOfRunId: runId,
|
|
726
|
+
...retryOptions.budget ? { budget: retryOptions.budget } : {},
|
|
727
|
+
retry: {
|
|
728
|
+
acceptsVersion,
|
|
729
|
+
carry: (checkpoint) => {
|
|
730
|
+
let source = checkpoint.state;
|
|
731
|
+
if (checkpoint.definitionVersion !== definition.version && definition.olderCheckpoints && isWorkflowCheckpointState(source)) source = definition.olderCheckpoints.migrate(source, checkpoint.definitionVersion);
|
|
732
|
+
if (!isWorkflowCheckpointState(source)) throw new TypeError(`Workflow run ${runId} has a checkpoint this package cannot read`);
|
|
733
|
+
const stepIndex = source.stepIndex;
|
|
734
|
+
return {
|
|
735
|
+
state: {
|
|
736
|
+
...initialWorkflowState(source.input, source.inputRef),
|
|
737
|
+
stepIndex,
|
|
738
|
+
outputs: source.outputs.slice(0, stepIndex)
|
|
739
|
+
},
|
|
740
|
+
highestStepIndex: stepIndex > 0 ? stepIndex - 1 : null
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
}
|
|
744
|
+
});
|
|
745
|
+
return current(store, run.id);
|
|
746
|
+
}
|
|
747
|
+
};
|
|
748
|
+
}
|
|
749
|
+
//#endregion
|
|
750
|
+
//#region src/jobs.ts
|
|
751
|
+
var WORKFLOW_RUN_HANDLER = "@happyvertical/smrt-workflows:workflow-run";
|
|
752
|
+
var WorkflowRunKeyConflictError = class extends Error {
|
|
753
|
+
constructor(runId) {
|
|
754
|
+
super("This workflow key was already used with a different request");
|
|
755
|
+
this.runId = runId;
|
|
756
|
+
this.name = "WorkflowRunKeyConflictError";
|
|
757
|
+
}
|
|
758
|
+
runId;
|
|
759
|
+
};
|
|
760
|
+
var WorkflowRunJobError = class extends Error {
|
|
761
|
+
/** Read by the job runner's retry hints: never retried. */
|
|
762
|
+
retryable = false;
|
|
763
|
+
constructor(message) {
|
|
764
|
+
super(message);
|
|
765
|
+
this.name = "WorkflowRunJobError";
|
|
766
|
+
}
|
|
767
|
+
};
|
|
768
|
+
var hosts = /* @__PURE__ */ new Map();
|
|
769
|
+
function readPayload(payload) {
|
|
770
|
+
const value = payload;
|
|
771
|
+
if (!value || typeof value.runId !== "string" || !value.runId || typeof value.definition !== "string" || !value.definition) throw new WorkflowRunJobError("A workflow run job needs a run id");
|
|
772
|
+
return {
|
|
773
|
+
runId: value.runId,
|
|
774
|
+
definition: value.definition
|
|
775
|
+
};
|
|
776
|
+
}
|
|
777
|
+
async function workflowRunHandler(payload, context) {
|
|
778
|
+
if (!isRunnerExecutionContext(context)) throw new WorkflowRunJobError("Workflow runs need TaskRunner context");
|
|
779
|
+
const job = readPayload(payload);
|
|
780
|
+
const host = hosts.get(job.definition);
|
|
781
|
+
if (!host) throw new WorkflowRunJobError("No workflow definition is registered for this job");
|
|
782
|
+
return host.execute(job, context);
|
|
783
|
+
}
|
|
784
|
+
function registerWorkflowRunHandler() {
|
|
785
|
+
if (listJobHandlers().includes("@happyvertical/smrt-workflows:workflow-run")) unregisterJobHandler(WORKFLOW_RUN_HANDLER);
|
|
786
|
+
registerJobHandler(WORKFLOW_RUN_HANDLER, workflowRunHandler);
|
|
787
|
+
}
|
|
788
|
+
function canonical(value) {
|
|
789
|
+
if (Array.isArray(value)) return value.map(canonical);
|
|
790
|
+
if (value && typeof value === "object") return Object.fromEntries(Object.entries(value).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0).map(([key, item]) => [key, canonical(item)]));
|
|
791
|
+
return value;
|
|
792
|
+
}
|
|
793
|
+
var ACTIVE = ["pending", "running"];
|
|
794
|
+
function createWorkflowRunJobs(options) {
|
|
795
|
+
const { db, definition } = options;
|
|
796
|
+
const key = definition.key;
|
|
797
|
+
if (hosts.has(key)) throw new Error(`Workflow jobs already exist for definition ${key}`);
|
|
798
|
+
const target = jobHandlerTarget(WORKFLOW_RUN_HANDLER).objectType;
|
|
799
|
+
const queue = options.queue ?? "default";
|
|
800
|
+
const jobsReady = options.jobs ? Promise.resolve(options.jobs) : SmrtJobCollection.create({ db });
|
|
801
|
+
const controllers = /* @__PURE__ */ new Map();
|
|
802
|
+
const transaction = async (operation) => {
|
|
803
|
+
if (!db.transaction) throw new Error("Workflow jobs require transactions");
|
|
804
|
+
return db.transaction(operation);
|
|
805
|
+
};
|
|
806
|
+
const sameOwner = (owner, driver) => {
|
|
807
|
+
if (driver.ai.db !== db || (driver.ai.tenantId ?? null) !== owner.tenantId) throw new Error("configure must return the owner’s tenant client on db");
|
|
808
|
+
};
|
|
809
|
+
const driverFor = (owner, input) => {
|
|
810
|
+
const configured = options.configure(owner, input);
|
|
811
|
+
sameOwner(owner, configured);
|
|
812
|
+
return createWorkflowRunDriver({
|
|
813
|
+
...configured,
|
|
814
|
+
definition
|
|
815
|
+
});
|
|
816
|
+
};
|
|
817
|
+
const lightDriver = (owner) => createWorkflowRunDriver({
|
|
818
|
+
ai: createTrackedAiClient({
|
|
819
|
+
db,
|
|
820
|
+
tenantId: owner.tenantId
|
|
821
|
+
}),
|
|
822
|
+
definition,
|
|
823
|
+
validate: () => []
|
|
824
|
+
});
|
|
825
|
+
const jobsOf = async (owner, runId, tx, status) => {
|
|
826
|
+
return (tx ? await SmrtJobCollection.create({ db: tx }) : await jobsReady).list({
|
|
827
|
+
where: {
|
|
828
|
+
objectType: target,
|
|
829
|
+
dedupeKey: runId,
|
|
830
|
+
tenantId: owner.tenantId,
|
|
831
|
+
...status ? { status } : {}
|
|
832
|
+
},
|
|
833
|
+
orderBy: "updated_at DESC"
|
|
834
|
+
});
|
|
835
|
+
};
|
|
836
|
+
const enqueue = async (owner, runId, budget, tx) => {
|
|
837
|
+
const payload = {
|
|
838
|
+
runId,
|
|
839
|
+
definition: key
|
|
840
|
+
};
|
|
841
|
+
await (await jobsReady).enqueueHandlerJob({
|
|
842
|
+
handler: WORKFLOW_RUN_HANDLER,
|
|
843
|
+
payload,
|
|
844
|
+
tenantId: owner.tenantId,
|
|
845
|
+
dedupeKey: runId,
|
|
846
|
+
queue,
|
|
847
|
+
maxAttempts: 1,
|
|
848
|
+
...budget.maxDurationMs !== void 0 ? {
|
|
849
|
+
timeout: budget.maxDurationMs,
|
|
850
|
+
timeoutBehavior: "warn"
|
|
851
|
+
} : {}
|
|
852
|
+
}, tx ? { transaction: tx } : {});
|
|
853
|
+
};
|
|
854
|
+
const outcomeOf = async (owner, job) => {
|
|
855
|
+
if (!job.completedAt) return null;
|
|
856
|
+
return (await SmrtJobEventCollection.create({ db })).getTerminalOutcome(String(job.id), {
|
|
857
|
+
tenantId: owner.tenantId,
|
|
858
|
+
attempts: job.attempts,
|
|
859
|
+
completedAt: job.completedAt.toISOString()
|
|
860
|
+
});
|
|
861
|
+
};
|
|
862
|
+
const dispatchOf = (job) => ({
|
|
863
|
+
jobId: String(job.id),
|
|
864
|
+
workerId: job.workerId ?? "",
|
|
865
|
+
tenantId: job.tenantId ?? null,
|
|
866
|
+
attempt: job.attempts,
|
|
867
|
+
startedAt: job.startedAt?.toISOString() ?? "",
|
|
868
|
+
objectType: job.objectType,
|
|
869
|
+
objectId: job.objectId ?? null,
|
|
870
|
+
method: job.method
|
|
871
|
+
});
|
|
872
|
+
const park = async (runId, claim) => {
|
|
873
|
+
try {
|
|
874
|
+
await stopAiRun(db, runId, {
|
|
875
|
+
reason: "interrupted",
|
|
876
|
+
fence: workflowClaimFence(claim)
|
|
877
|
+
});
|
|
878
|
+
} catch {}
|
|
879
|
+
};
|
|
880
|
+
const reconcile = async (owner, runId) => {
|
|
881
|
+
if ((await getAiRun(db, runId))?.status !== "running") return;
|
|
882
|
+
if ((await jobsOf(owner, runId, void 0, ACTIVE)).length) return;
|
|
883
|
+
const [job] = await jobsOf(owner, runId);
|
|
884
|
+
if (!job) return;
|
|
885
|
+
if (job.status === "cancelled") {
|
|
886
|
+
await cancelAiRun(db, runId);
|
|
887
|
+
return;
|
|
888
|
+
}
|
|
889
|
+
if (job.status !== "failed") return;
|
|
890
|
+
const outcome = await outcomeOf(owner, job);
|
|
891
|
+
if (!outcome || outcome.failureKind === "stale-recovery") return;
|
|
892
|
+
if ((await jobsOf(owner, runId, void 0, ACTIVE)).length) return;
|
|
893
|
+
if (outcome.failureKind === "handler-unregistered" || outcome.failureKind === "handler-unclaimed") {
|
|
894
|
+
await stopAiRun(db, runId, { reason: "interrupted" }).catch(() => null);
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
897
|
+
await finishAiRun(db, runId, {
|
|
898
|
+
status: "failed",
|
|
899
|
+
stopReason: "error"
|
|
900
|
+
});
|
|
901
|
+
};
|
|
902
|
+
const execute = async (payload, context) => {
|
|
903
|
+
const claim = getJobClaim(context);
|
|
904
|
+
const runId = payload.runId;
|
|
905
|
+
const run = await getAiRun(db, runId);
|
|
906
|
+
if (!run || run.type !== "workflow" || run.definitionKey !== key || !run.ownerId) throw new WorkflowRunJobError("Workflow run not found");
|
|
907
|
+
const owner = {
|
|
908
|
+
ownerId: run.ownerId,
|
|
909
|
+
tenantId: run.tenantId ?? null
|
|
910
|
+
};
|
|
911
|
+
if (claim.objectType !== target || claim.method !== "handle" || claim.objectId !== null || claim.tenantId !== owner.tenantId) throw new WorkflowRunJobError("Workflow job target changed");
|
|
912
|
+
const controller = new AbortController();
|
|
913
|
+
const abort = () => controller.abort(context.signal.reason);
|
|
914
|
+
context.signal.addEventListener("abort", abort, { once: true });
|
|
915
|
+
if (context.signal.aborted) abort();
|
|
916
|
+
controllers.set(runId, controller);
|
|
917
|
+
let began = false;
|
|
918
|
+
try {
|
|
919
|
+
await options.authorize?.(owner);
|
|
920
|
+
await transaction((tx) => assertJobClaim(tx, claim));
|
|
921
|
+
const loaded = await createRunCheckpointStore({
|
|
922
|
+
db,
|
|
923
|
+
ownerId: owner.ownerId,
|
|
924
|
+
tenantId: owner.tenantId
|
|
925
|
+
}).load(runId);
|
|
926
|
+
if (!loaded) throw new WorkflowRunNotFoundError(runId);
|
|
927
|
+
const driver = driverFor(owner, loaded.state.input);
|
|
928
|
+
began = true;
|
|
929
|
+
return {
|
|
930
|
+
runId,
|
|
931
|
+
status: (await driver.run(runId, owner.ownerId, {
|
|
932
|
+
signal: controller.signal,
|
|
933
|
+
fence: workflowClaimFence(claim),
|
|
934
|
+
stopFence: workflowStopFence(claim),
|
|
935
|
+
holder: `${claim.workerId}:${claim.jobId}`
|
|
936
|
+
})).status
|
|
937
|
+
};
|
|
938
|
+
} catch (error) {
|
|
939
|
+
if (!began && !(error instanceof JobClaimLostError)) await park(runId, claim);
|
|
940
|
+
throw error;
|
|
941
|
+
} finally {
|
|
942
|
+
if (controllers.get(runId) === controller) controllers.delete(runId);
|
|
943
|
+
context.signal.removeEventListener("abort", abort);
|
|
944
|
+
}
|
|
945
|
+
};
|
|
946
|
+
registerWorkflowRunHandler();
|
|
947
|
+
hosts.set(key, { execute });
|
|
948
|
+
return {
|
|
949
|
+
/** The registered handler's name. */
|
|
950
|
+
handler: WORKFLOW_RUN_HANDLER,
|
|
951
|
+
/**
|
|
952
|
+
* Forget this definition. When it was the last, unregister the handler:
|
|
953
|
+
* stop every runner first, or this throws and the handler stays.
|
|
954
|
+
*/
|
|
955
|
+
dispose() {
|
|
956
|
+
hosts.delete(key);
|
|
957
|
+
if (!hosts.size && listJobHandlers().includes("@happyvertical/smrt-workflows:workflow-run")) unregisterJobHandler(WORKFLOW_RUN_HANDLER);
|
|
958
|
+
},
|
|
959
|
+
/** Operations of one authenticated owner and tenant. */
|
|
960
|
+
forOwner(ownerId, tenantId) {
|
|
961
|
+
if (!ownerId) throw new Error("Workflow runs require an owner");
|
|
962
|
+
const owner = {
|
|
963
|
+
ownerId,
|
|
964
|
+
tenantId
|
|
965
|
+
};
|
|
966
|
+
const light = lightDriver(owner);
|
|
967
|
+
const authorized = async (id) => {
|
|
968
|
+
await options.authorize?.(owner);
|
|
969
|
+
await light.get(id, ownerId);
|
|
970
|
+
await reconcile(owner, id);
|
|
971
|
+
return light.get(id, ownerId);
|
|
972
|
+
};
|
|
973
|
+
const cancelActive = async (id) => {
|
|
974
|
+
controllers.get(id)?.abort(/* @__PURE__ */ new Error("Workflow run stopped"));
|
|
975
|
+
for (const job of await jobsOf(owner, id, void 0, ACTIVE)) try {
|
|
976
|
+
await job.cancel();
|
|
977
|
+
} catch (error) {
|
|
978
|
+
const current = await (await jobsReady).get({ id: String(job.id) });
|
|
979
|
+
if (!current || ACTIVE.includes(current.status)) throw error;
|
|
980
|
+
}
|
|
981
|
+
};
|
|
982
|
+
const settleActive = async (id) => {
|
|
983
|
+
for (const job of await jobsOf(owner, id, void 0, ACTIVE)) if (job.status === "running") await new JobHandle(String(job.id), await jobsReady).wait({
|
|
984
|
+
timeout: 3e4,
|
|
985
|
+
pollInterval: 25
|
|
986
|
+
});
|
|
987
|
+
};
|
|
988
|
+
const admit = async (id, budget) => {
|
|
989
|
+
await transaction(async (tx) => {
|
|
990
|
+
const [latest] = await jobsOf(owner, id, tx);
|
|
991
|
+
if (latest?.status === "failed") await assertFailedJobDispatch(tx, dispatchOf(latest));
|
|
992
|
+
if ((await jobsOf(owner, id, tx, ACTIVE)).length) throw new RunConflictError("The recovered job changed");
|
|
993
|
+
await enqueue(owner, id, budget, tx);
|
|
994
|
+
});
|
|
995
|
+
};
|
|
996
|
+
const recover = async (state) => {
|
|
997
|
+
const [latest] = await jobsOf(owner, state.id);
|
|
998
|
+
if (latest) {
|
|
999
|
+
if ((latest.status === "failed" ? await outcomeOf(owner, latest) : null)?.failureKind !== "stale-recovery") throw new RunConflictError("The previous job has not ended in a recoverable way");
|
|
1000
|
+
}
|
|
1001
|
+
await admit(state.id, state.budget);
|
|
1002
|
+
return light.get(state.id, ownerId);
|
|
1003
|
+
};
|
|
1004
|
+
const resume = async (id, budget) => {
|
|
1005
|
+
const state = await authorized(id);
|
|
1006
|
+
if ((await jobsOf(owner, id, void 0, ACTIVE)).length) throw new RunConflictError("The previous job is still settling");
|
|
1007
|
+
if (state.status === "waiting-approval") throw new RunConflictError("This workflow run is waiting for approval: decide it first");
|
|
1008
|
+
if (state.status === "running") {
|
|
1009
|
+
if (budget) throw new RunConflictError("Recover the job first; raise its budget after a budget stop");
|
|
1010
|
+
return recover(state);
|
|
1011
|
+
}
|
|
1012
|
+
const approved = options.resumeBudget ? await options.resumeBudget(owner, state, Object.freeze({
|
|
1013
|
+
...state.budget,
|
|
1014
|
+
...budget
|
|
1015
|
+
})) : budget;
|
|
1016
|
+
return driverFor(owner, state.input).resume(id, ownerId, {
|
|
1017
|
+
...approved ? { budget: approved } : {},
|
|
1018
|
+
admit: (tx, resumedBudget) => enqueue(owner, id, resumedBudget, tx)
|
|
1019
|
+
});
|
|
1020
|
+
};
|
|
1021
|
+
const continueAfterApproval = async (id) => {
|
|
1022
|
+
await options.authorize?.(owner);
|
|
1023
|
+
await settleActive(id);
|
|
1024
|
+
const state = await authorized(id);
|
|
1025
|
+
if (state.status !== "running") return state;
|
|
1026
|
+
const [latest] = await jobsOf(owner, id);
|
|
1027
|
+
if (latest?.status !== "completed") return state;
|
|
1028
|
+
try {
|
|
1029
|
+
await admit(id, state.budget);
|
|
1030
|
+
} catch (error) {
|
|
1031
|
+
if (!(error instanceof RunConflictError)) throw error;
|
|
1032
|
+
}
|
|
1033
|
+
return light.get(id, ownerId);
|
|
1034
|
+
};
|
|
1035
|
+
return {
|
|
1036
|
+
/**
|
|
1037
|
+
* Open the run and admit its job. The same key and request returns the
|
|
1038
|
+
* same run (and admits a job again if a crash lost it); the same key
|
|
1039
|
+
* with another request throws `WorkflowRunKeyConflictError`.
|
|
1040
|
+
*/
|
|
1041
|
+
async start(input) {
|
|
1042
|
+
await options.authorize?.(owner);
|
|
1043
|
+
if (!input.key.trim() || input.key.length > 512) throw new Error("Workflow key must be nonblank and at most 512 characters");
|
|
1044
|
+
const hash = createHash("sha256").update(JSON.stringify([
|
|
1045
|
+
key,
|
|
1046
|
+
ownerId,
|
|
1047
|
+
tenantId,
|
|
1048
|
+
input.key
|
|
1049
|
+
])).digest("hex");
|
|
1050
|
+
const id = `${hash.slice(0, 8)}-${hash.slice(8, 12)}-8${hash.slice(13, 16)}-a${hash.slice(17, 20)}-${hash.slice(20, 32)}`;
|
|
1051
|
+
const digest = createHash("sha256").update(JSON.stringify(canonical(JSON.parse(JSON.stringify({
|
|
1052
|
+
input: input.input ?? null,
|
|
1053
|
+
inputRef: input.inputRef ?? null,
|
|
1054
|
+
budget: input.budget ?? null,
|
|
1055
|
+
tags: input.tags ?? null
|
|
1056
|
+
}))))).digest("hex");
|
|
1057
|
+
const store = createRunCheckpointStore({
|
|
1058
|
+
db,
|
|
1059
|
+
ownerId,
|
|
1060
|
+
tenantId
|
|
1061
|
+
});
|
|
1062
|
+
const replay = async () => {
|
|
1063
|
+
const loaded = await store.load(id);
|
|
1064
|
+
if (!loaded) return null;
|
|
1065
|
+
if (loaded.run.definitionKey !== key) throw new Error("Workflow run definition changed");
|
|
1066
|
+
if (loaded.run.tags.workflowAdmission !== digest) throw new WorkflowRunKeyConflictError(id);
|
|
1067
|
+
if (loaded.run.status === "running" && !(await jobsOf(owner, id)).length) await enqueue(owner, id, loaded.run.budget);
|
|
1068
|
+
return light.get(id, ownerId);
|
|
1069
|
+
};
|
|
1070
|
+
const existing = await replay();
|
|
1071
|
+
if (existing) return existing;
|
|
1072
|
+
let started;
|
|
1073
|
+
try {
|
|
1074
|
+
started = await driverFor(owner, input.input).start({
|
|
1075
|
+
id,
|
|
1076
|
+
ownerId,
|
|
1077
|
+
...input.inputRef ? { inputRef: input.inputRef } : { input: input.input },
|
|
1078
|
+
...input.budget ? { budget: input.budget } : {},
|
|
1079
|
+
tags: {
|
|
1080
|
+
...input.tags,
|
|
1081
|
+
workflowAdmission: digest
|
|
1082
|
+
}
|
|
1083
|
+
});
|
|
1084
|
+
} catch (error) {
|
|
1085
|
+
const raced = await replay();
|
|
1086
|
+
if (raced) return raced;
|
|
1087
|
+
throw error;
|
|
1088
|
+
}
|
|
1089
|
+
await enqueue(owner, id, started.budget);
|
|
1090
|
+
return started;
|
|
1091
|
+
},
|
|
1092
|
+
get: authorized,
|
|
1093
|
+
/** Pause the run (resumable) and cancel its job. */
|
|
1094
|
+
async stop(id) {
|
|
1095
|
+
await authorized(id);
|
|
1096
|
+
const state = await light.stop(id, ownerId);
|
|
1097
|
+
await cancelActive(id);
|
|
1098
|
+
return state;
|
|
1099
|
+
},
|
|
1100
|
+
/** End the run for good and cancel its job. */
|
|
1101
|
+
async cancel(id) {
|
|
1102
|
+
await authorized(id);
|
|
1103
|
+
const state = await light.cancel(id, ownerId);
|
|
1104
|
+
await cancelActive(id);
|
|
1105
|
+
return state;
|
|
1106
|
+
},
|
|
1107
|
+
resume,
|
|
1108
|
+
/**
|
|
1109
|
+
* Record a person's decision on the approval the run waits for. An
|
|
1110
|
+
* approval admits a new job that re-runs the step's check.
|
|
1111
|
+
*/
|
|
1112
|
+
async decide(id, decision) {
|
|
1113
|
+
await options.authorize?.(owner);
|
|
1114
|
+
await settleActive(id);
|
|
1115
|
+
await authorized(id);
|
|
1116
|
+
const decided = await light.decide(id, ownerId, decision);
|
|
1117
|
+
if (decision.approved) return continueAfterApproval(id);
|
|
1118
|
+
return decided;
|
|
1119
|
+
},
|
|
1120
|
+
/** After a decision was recorded elsewhere: admit the job that continues. */
|
|
1121
|
+
continueAfterApproval,
|
|
1122
|
+
/** Retry a failed run from its last accepted step as a new run with a job. */
|
|
1123
|
+
async retry(id, retryOptions = {}) {
|
|
1124
|
+
const state = await authorized(id);
|
|
1125
|
+
const retried = await driverFor(owner, state.input).retry(id, ownerId, retryOptions);
|
|
1126
|
+
await enqueue(owner, retried.id, retried.budget);
|
|
1127
|
+
return retried;
|
|
1128
|
+
}
|
|
1129
|
+
};
|
|
1130
|
+
}
|
|
1131
|
+
};
|
|
1132
|
+
}
|
|
1133
|
+
//#endregion
|
|
1134
|
+
export { RunConflictError, WORKFLOW_RUN_HANDLER, WORKFLOW_RUN_TYPE, WORKFLOW_STATE_VERSION, WorkflowOutputError, WorkflowRunJobError, WorkflowRunKeyConflictError, WorkflowRunNotFoundError, createRunCheckpointStore, createWorkflowRunDriver, createWorkflowRunJobs, initialWorkflowState, isWorkflowCheckpointState, readWorkflowRunLease, registerWorkflowRunHandler, workflowClaimFence, workflowStopFence };
|
|
1135
|
+
|
|
1136
|
+
//# sourceMappingURL=index.js.map
|