@aikirun/worker 0.23.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -4
- package/dist/index.d.ts +10 -27
- package/dist/index.js +89 -1342
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -53,1213 +53,12 @@ var objectOverrider = (defaultObj) => (obj) => {
|
|
|
53
53
|
};
|
|
54
54
|
|
|
55
55
|
// worker.ts
|
|
56
|
-
import {
|
|
56
|
+
import { dbSubscriber } from "@aikirun/subscriber-db";
|
|
57
57
|
import {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError6,
|
|
62
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError5
|
|
63
|
-
} from "@aikirun/types/workflow-run";
|
|
64
|
-
import {
|
|
65
|
-
createEventWaiters,
|
|
66
|
-
createReplayManifest,
|
|
67
|
-
createSleeper,
|
|
68
|
-
workflowRegistry,
|
|
69
|
-
workflowRunHandle as workflowRunHandle2
|
|
58
|
+
executeWorkflowRun,
|
|
59
|
+
getSystemWorkflows,
|
|
60
|
+
workflowRegistry
|
|
70
61
|
} from "@aikirun/workflow";
|
|
71
|
-
|
|
72
|
-
// ../workflow/system/cancel-child-runs.ts
|
|
73
|
-
import { NON_TERMINAL_WORKFLOW_RUN_STATUSES } from "@aikirun/types/workflow-run";
|
|
74
|
-
|
|
75
|
-
// ../../lib/address/index.ts
|
|
76
|
-
function getTaskAddress(name, inputHash) {
|
|
77
|
-
return `${name}:${inputHash}`;
|
|
78
|
-
}
|
|
79
|
-
function getWorkflowRunAddress(name, versionId, referenceId) {
|
|
80
|
-
return `${name}:${versionId}:${referenceId}`;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
// ../../lib/crypto/hash.ts
|
|
84
|
-
import { createHash } from "crypto";
|
|
85
|
-
|
|
86
|
-
// ../../lib/json/stable-stringify.ts
|
|
87
|
-
function stableStringify(value) {
|
|
88
|
-
return stringifyValue(value);
|
|
89
|
-
}
|
|
90
|
-
function stringifyValue(value) {
|
|
91
|
-
if (value === null || value === void 0) {
|
|
92
|
-
return "null";
|
|
93
|
-
}
|
|
94
|
-
if (typeof value !== "object") {
|
|
95
|
-
return JSON.stringify(value);
|
|
96
|
-
}
|
|
97
|
-
if (Array.isArray(value)) {
|
|
98
|
-
return `[${value.map(stringifyValue).join(",")}]`;
|
|
99
|
-
}
|
|
100
|
-
const keys = Object.keys(value).sort();
|
|
101
|
-
const pairs = [];
|
|
102
|
-
for (const key of keys) {
|
|
103
|
-
const keyValue = value[key];
|
|
104
|
-
if (keyValue !== void 0) {
|
|
105
|
-
pairs.push(`${JSON.stringify(key)}:${stringifyValue(keyValue)}`);
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
return `{${pairs.join(",")}}`;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
// ../../lib/crypto/hash.ts
|
|
112
|
-
async function sha256(input) {
|
|
113
|
-
const data = new TextEncoder().encode(input);
|
|
114
|
-
const hashBuffer = await crypto.subtle.digest("SHA-256", data);
|
|
115
|
-
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
|
116
|
-
return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
117
|
-
}
|
|
118
|
-
async function hashInput(input) {
|
|
119
|
-
return sha256(stableStringify({ input }));
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// ../../lib/error/serializable.ts
|
|
123
|
-
function createSerializableError(error) {
|
|
124
|
-
return error instanceof Error ? {
|
|
125
|
-
message: error.message,
|
|
126
|
-
name: error.name,
|
|
127
|
-
stack: error.stack,
|
|
128
|
-
cause: error.cause ? createSerializableError(error.cause) : void 0
|
|
129
|
-
} : {
|
|
130
|
-
message: String(error),
|
|
131
|
-
name: "UnknownError"
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
// ../../lib/retry/strategy.ts
|
|
136
|
-
function withRetry(fn, strategy, options) {
|
|
137
|
-
return {
|
|
138
|
-
run: async (...args) => {
|
|
139
|
-
let attempts = 0;
|
|
140
|
-
while (true) {
|
|
141
|
-
if (options?.abortSignal?.aborted) {
|
|
142
|
-
return {
|
|
143
|
-
state: "aborted",
|
|
144
|
-
reason: options.abortSignal.reason
|
|
145
|
-
};
|
|
146
|
-
}
|
|
147
|
-
attempts++;
|
|
148
|
-
let result;
|
|
149
|
-
try {
|
|
150
|
-
result = await fn(...args);
|
|
151
|
-
if (options?.shouldRetryOnResult === void 0 || !await options.shouldRetryOnResult(result)) {
|
|
152
|
-
return {
|
|
153
|
-
state: "completed",
|
|
154
|
-
result,
|
|
155
|
-
attempts
|
|
156
|
-
};
|
|
157
|
-
}
|
|
158
|
-
} catch (err) {
|
|
159
|
-
if (options?.shouldNotRetryOnError !== void 0 && await options.shouldNotRetryOnError(err)) {
|
|
160
|
-
throw err;
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
const retryParams = getRetryParams(attempts, strategy);
|
|
164
|
-
if (!retryParams.retriesLeft) {
|
|
165
|
-
return {
|
|
166
|
-
state: "timeout"
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
await delay(retryParams.delayMs, { abortSignal: options?.abortSignal });
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
function getRetryParams(attempts, strategy) {
|
|
175
|
-
const strategyType = strategy.type;
|
|
176
|
-
switch (strategyType) {
|
|
177
|
-
case "never":
|
|
178
|
-
return {
|
|
179
|
-
retriesLeft: false
|
|
180
|
-
};
|
|
181
|
-
case "fixed":
|
|
182
|
-
if (attempts >= strategy.maxAttempts) {
|
|
183
|
-
return {
|
|
184
|
-
retriesLeft: false
|
|
185
|
-
};
|
|
186
|
-
}
|
|
187
|
-
return {
|
|
188
|
-
retriesLeft: true,
|
|
189
|
-
delayMs: strategy.delayMs
|
|
190
|
-
};
|
|
191
|
-
case "exponential": {
|
|
192
|
-
if (attempts >= strategy.maxAttempts) {
|
|
193
|
-
return {
|
|
194
|
-
retriesLeft: false
|
|
195
|
-
};
|
|
196
|
-
}
|
|
197
|
-
const delayMs = strategy.baseDelayMs * (strategy.factor ?? 2) ** (attempts - 1);
|
|
198
|
-
return {
|
|
199
|
-
retriesLeft: true,
|
|
200
|
-
delayMs: Math.min(delayMs, strategy.maxDelayMs ?? Number.POSITIVE_INFINITY)
|
|
201
|
-
};
|
|
202
|
-
}
|
|
203
|
-
case "jittered": {
|
|
204
|
-
if (attempts >= strategy.maxAttempts) {
|
|
205
|
-
return {
|
|
206
|
-
retriesLeft: false
|
|
207
|
-
};
|
|
208
|
-
}
|
|
209
|
-
const base = strategy.baseDelayMs * (strategy.jitterFactor ?? 2) ** (attempts - 1);
|
|
210
|
-
const delayMs = Math.random() * base;
|
|
211
|
-
return {
|
|
212
|
-
retriesLeft: true,
|
|
213
|
-
delayMs: Math.min(delayMs, strategy.maxDelayMs ?? Number.POSITIVE_INFINITY)
|
|
214
|
-
};
|
|
215
|
-
}
|
|
216
|
-
default:
|
|
217
|
-
return strategyType;
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
// ../task/task.ts
|
|
222
|
-
import { INTERNAL } from "@aikirun/types/symbols";
|
|
223
|
-
import { TaskFailedError } from "@aikirun/types/task";
|
|
224
|
-
import {
|
|
225
|
-
NonDeterminismError,
|
|
226
|
-
WorkflowRunFailedError,
|
|
227
|
-
WorkflowRunRevisionConflictError,
|
|
228
|
-
WorkflowRunSuspendedError
|
|
229
|
-
} from "@aikirun/types/workflow-run";
|
|
230
|
-
function task(params) {
|
|
231
|
-
return new TaskImpl(params);
|
|
232
|
-
}
|
|
233
|
-
var TaskImpl = class {
|
|
234
|
-
constructor(params) {
|
|
235
|
-
this.params = params;
|
|
236
|
-
this.name = params.name;
|
|
237
|
-
}
|
|
238
|
-
name;
|
|
239
|
-
with() {
|
|
240
|
-
const startOpts = this.params.opts ?? {};
|
|
241
|
-
const startOptsOverrider = objectOverrider(startOpts);
|
|
242
|
-
return new TaskBuilderImpl(this, startOptsOverrider());
|
|
243
|
-
}
|
|
244
|
-
async start(run, ...args) {
|
|
245
|
-
return this.startWithOpts(run, this.params.opts ?? {}, ...args);
|
|
246
|
-
}
|
|
247
|
-
async startWithOpts(run, startOpts, ...args) {
|
|
248
|
-
const handle = run[INTERNAL].handle;
|
|
249
|
-
handle[INTERNAL].assertExecutionAllowed();
|
|
250
|
-
const inputRaw = args[0];
|
|
251
|
-
const input = await this.parse(handle, this.params.schema?.input, inputRaw, run.logger);
|
|
252
|
-
const inputHash = await hashInput(input);
|
|
253
|
-
const address = getTaskAddress(this.name, inputHash);
|
|
254
|
-
const replayManifest = run[INTERNAL].replayManifest;
|
|
255
|
-
if (replayManifest.hasUnconsumedEntries()) {
|
|
256
|
-
const existingTaskInfo = replayManifest.consumeNextTask(address);
|
|
257
|
-
if (existingTaskInfo) {
|
|
258
|
-
return this.getExistingTaskResult(run, handle, startOpts, input, existingTaskInfo);
|
|
259
|
-
}
|
|
260
|
-
await this.throwNonDeterminismError(run, handle, inputHash, replayManifest);
|
|
261
|
-
}
|
|
262
|
-
const attempts = 1;
|
|
263
|
-
const retryStrategy = startOpts.retry ?? { type: "never" };
|
|
264
|
-
const taskInfo = await handle[INTERNAL].transitionTaskState({
|
|
265
|
-
type: "create",
|
|
266
|
-
taskName: this.name,
|
|
267
|
-
options: startOpts,
|
|
268
|
-
taskState: { status: "running", attempts, input }
|
|
269
|
-
});
|
|
270
|
-
const logger = run.logger.child({
|
|
271
|
-
"aiki.component": "task-execution",
|
|
272
|
-
"aiki.taskName": this.name,
|
|
273
|
-
"aiki.taskId": taskInfo.id
|
|
274
|
-
});
|
|
275
|
-
logger.info("Task started", { "aiki.attempts": attempts });
|
|
276
|
-
const { output, lastAttempt } = await this.tryExecuteTask(
|
|
277
|
-
handle,
|
|
278
|
-
input,
|
|
279
|
-
taskInfo.id,
|
|
280
|
-
retryStrategy,
|
|
281
|
-
attempts,
|
|
282
|
-
run[INTERNAL].options.spinThresholdMs,
|
|
283
|
-
logger
|
|
284
|
-
);
|
|
285
|
-
await handle[INTERNAL].transitionTaskState({
|
|
286
|
-
taskId: taskInfo.id,
|
|
287
|
-
taskState: { status: "completed", attempts: lastAttempt, output }
|
|
288
|
-
});
|
|
289
|
-
logger.info("Task complete", { "aiki.attempts": lastAttempt });
|
|
290
|
-
return output;
|
|
291
|
-
}
|
|
292
|
-
async getExistingTaskResult(run, handle, startOpts, input, existingTaskInfo) {
|
|
293
|
-
const existingTaskState = existingTaskInfo.state;
|
|
294
|
-
if (existingTaskState.status === "completed") {
|
|
295
|
-
return this.parse(handle, this.params.schema?.output, existingTaskState.output, run.logger);
|
|
296
|
-
}
|
|
297
|
-
if (existingTaskState.status === "failed") {
|
|
298
|
-
throw new TaskFailedError(
|
|
299
|
-
existingTaskInfo.id,
|
|
300
|
-
existingTaskState.attempts,
|
|
301
|
-
existingTaskState.error.message
|
|
302
|
-
);
|
|
303
|
-
}
|
|
304
|
-
existingTaskState.status;
|
|
305
|
-
const attempts = existingTaskState.attempts;
|
|
306
|
-
const retryStrategy = startOpts.retry ?? { type: "never" };
|
|
307
|
-
this.assertRetryAllowed(existingTaskInfo.id, attempts, retryStrategy, run.logger);
|
|
308
|
-
run.logger.debug("Retrying task", {
|
|
309
|
-
"aiki.taskName": this.name,
|
|
310
|
-
"aiki.taskId": existingTaskInfo.id,
|
|
311
|
-
"aiki.attempts": attempts,
|
|
312
|
-
"aiki.taskStatus": existingTaskState.status
|
|
313
|
-
});
|
|
314
|
-
return this.retryAndExecute(run, handle, input, existingTaskInfo.id, startOpts, retryStrategy, attempts);
|
|
315
|
-
}
|
|
316
|
-
async throwNonDeterminismError(run, handle, inputHash, manifest) {
|
|
317
|
-
const unconsumedManifestEntries = manifest.getUnconsumedEntries();
|
|
318
|
-
run.logger.error("Replay divergence", {
|
|
319
|
-
"aiki.taskName": this.name,
|
|
320
|
-
"aiki.inputHash": inputHash,
|
|
321
|
-
"aiki.unconsumedManifestEntries": unconsumedManifestEntries
|
|
322
|
-
});
|
|
323
|
-
const error = new NonDeterminismError(run.id, handle.run.attempts, unconsumedManifestEntries);
|
|
324
|
-
await handle[INTERNAL].transitionState({
|
|
325
|
-
status: "failed",
|
|
326
|
-
cause: "self",
|
|
327
|
-
error: createSerializableError(error)
|
|
328
|
-
});
|
|
329
|
-
throw error;
|
|
330
|
-
}
|
|
331
|
-
async retryAndExecute(run, handle, input, taskId, startOpts, retryStrategy, previousAttempts) {
|
|
332
|
-
const attempts = previousAttempts + 1;
|
|
333
|
-
const taskInfo = await handle[INTERNAL].transitionTaskState({
|
|
334
|
-
type: "retry",
|
|
335
|
-
taskId,
|
|
336
|
-
options: startOpts,
|
|
337
|
-
taskState: { status: "running", attempts, input }
|
|
338
|
-
});
|
|
339
|
-
const logger = run.logger.child({
|
|
340
|
-
"aiki.component": "task-execution",
|
|
341
|
-
"aiki.taskName": this.name,
|
|
342
|
-
"aiki.taskId": taskInfo.id
|
|
343
|
-
});
|
|
344
|
-
logger.info("Task started", { "aiki.attempts": attempts });
|
|
345
|
-
const { output, lastAttempt } = await this.tryExecuteTask(
|
|
346
|
-
handle,
|
|
347
|
-
input,
|
|
348
|
-
taskInfo.id,
|
|
349
|
-
retryStrategy,
|
|
350
|
-
attempts,
|
|
351
|
-
run[INTERNAL].options.spinThresholdMs,
|
|
352
|
-
logger
|
|
353
|
-
);
|
|
354
|
-
await handle[INTERNAL].transitionTaskState({
|
|
355
|
-
taskId: taskInfo.id,
|
|
356
|
-
taskState: { status: "completed", attempts: lastAttempt, output }
|
|
357
|
-
});
|
|
358
|
-
logger.info("Task complete", { "aiki.attempts": lastAttempt });
|
|
359
|
-
return output;
|
|
360
|
-
}
|
|
361
|
-
async tryExecuteTask(handle, input, taskId, retryStrategy, currentAttempt, spinThresholdMs, logger) {
|
|
362
|
-
let attempts = currentAttempt;
|
|
363
|
-
while (true) {
|
|
364
|
-
try {
|
|
365
|
-
const outputRaw = await this.params.handler(input);
|
|
366
|
-
const output = await this.parse(handle, this.params.schema?.output, outputRaw, logger);
|
|
367
|
-
return { output, lastAttempt: attempts };
|
|
368
|
-
} catch (error) {
|
|
369
|
-
if (error instanceof WorkflowRunSuspendedError || error instanceof WorkflowRunFailedError || error instanceof WorkflowRunRevisionConflictError) {
|
|
370
|
-
throw error;
|
|
371
|
-
}
|
|
372
|
-
const serializableError = createSerializableError(error);
|
|
373
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
374
|
-
if (!retryParams.retriesLeft) {
|
|
375
|
-
logger.error("Task failed", {
|
|
376
|
-
"aiki.attempts": attempts,
|
|
377
|
-
"aiki.reason": serializableError.message
|
|
378
|
-
});
|
|
379
|
-
await handle[INTERNAL].transitionTaskState({
|
|
380
|
-
taskId,
|
|
381
|
-
taskState: { status: "failed", attempts, error: serializableError }
|
|
382
|
-
});
|
|
383
|
-
throw new TaskFailedError(taskId, attempts, serializableError.message);
|
|
384
|
-
}
|
|
385
|
-
logger.debug("Task failed. It will be retried", {
|
|
386
|
-
"aiki.attempts": attempts,
|
|
387
|
-
"aiki.nextAttemptInMs": retryParams.delayMs,
|
|
388
|
-
"aiki.reason": serializableError.message
|
|
389
|
-
});
|
|
390
|
-
if (retryParams.delayMs <= spinThresholdMs) {
|
|
391
|
-
await delay(retryParams.delayMs);
|
|
392
|
-
attempts++;
|
|
393
|
-
continue;
|
|
394
|
-
}
|
|
395
|
-
await handle[INTERNAL].transitionTaskState({
|
|
396
|
-
taskId,
|
|
397
|
-
taskState: {
|
|
398
|
-
status: "awaiting_retry",
|
|
399
|
-
attempts,
|
|
400
|
-
error: serializableError,
|
|
401
|
-
nextAttemptInMs: retryParams.delayMs
|
|
402
|
-
}
|
|
403
|
-
});
|
|
404
|
-
throw new WorkflowRunSuspendedError(handle.run.id);
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
assertRetryAllowed(taskId, attempts, retryStrategy, logger) {
|
|
409
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
410
|
-
if (!retryParams.retriesLeft) {
|
|
411
|
-
logger.error("Task retry not allowed", {
|
|
412
|
-
"aiki.taskName": this.name,
|
|
413
|
-
"aiki.taskId": taskId,
|
|
414
|
-
"aiki.attempts": attempts
|
|
415
|
-
});
|
|
416
|
-
throw new TaskFailedError(taskId, attempts, "Task retry not allowed");
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
async parse(handle, schema, data, logger) {
|
|
420
|
-
if (!schema) {
|
|
421
|
-
return data;
|
|
422
|
-
}
|
|
423
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
424
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
425
|
-
if (!schemaValidationResult.issues) {
|
|
426
|
-
return schemaValidationResult.value;
|
|
427
|
-
}
|
|
428
|
-
logger.error("Invalid task data", { "aiki.issues": schemaValidationResult.issues });
|
|
429
|
-
await handle[INTERNAL].transitionState({
|
|
430
|
-
status: "failed",
|
|
431
|
-
cause: "self",
|
|
432
|
-
error: {
|
|
433
|
-
name: "SchemaValidationError",
|
|
434
|
-
message: JSON.stringify(schemaValidationResult.issues)
|
|
435
|
-
}
|
|
436
|
-
});
|
|
437
|
-
throw new WorkflowRunFailedError(handle.run.id, handle.run.attempts);
|
|
438
|
-
}
|
|
439
|
-
};
|
|
440
|
-
var TaskBuilderImpl = class _TaskBuilderImpl {
|
|
441
|
-
constructor(task2, startOptsBuilder) {
|
|
442
|
-
this.task = task2;
|
|
443
|
-
this.startOptsBuilder = startOptsBuilder;
|
|
444
|
-
}
|
|
445
|
-
opt(path, value) {
|
|
446
|
-
return new _TaskBuilderImpl(this.task, this.startOptsBuilder.with(path, value));
|
|
447
|
-
}
|
|
448
|
-
start(run, ...args) {
|
|
449
|
-
return this.task.startWithOpts(run, this.startOptsBuilder.build(), ...args);
|
|
450
|
-
}
|
|
451
|
-
};
|
|
452
|
-
|
|
453
|
-
// ../workflow/workflow.ts
|
|
454
|
-
import { INTERNAL as INTERNAL6 } from "@aikirun/types/symbols";
|
|
455
|
-
|
|
456
|
-
// ../workflow/workflow-version.ts
|
|
457
|
-
import { INTERNAL as INTERNAL5 } from "@aikirun/types/symbols";
|
|
458
|
-
import { TaskFailedError as TaskFailedError2 } from "@aikirun/types/task";
|
|
459
|
-
import { SchemaValidationError as SchemaValidationError2 } from "@aikirun/types/validator";
|
|
460
|
-
import {
|
|
461
|
-
NonDeterminismError as NonDeterminismError2,
|
|
462
|
-
WorkflowRunFailedError as WorkflowRunFailedError3,
|
|
463
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError5,
|
|
464
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError4
|
|
465
|
-
} from "@aikirun/types/workflow-run";
|
|
466
|
-
|
|
467
|
-
// ../../lib/duration/convert.ts
|
|
468
|
-
var MS_PER_SECOND = 1e3;
|
|
469
|
-
var MS_PER_MINUTE = 60 * MS_PER_SECOND;
|
|
470
|
-
var MS_PER_HOUR = 60 * MS_PER_MINUTE;
|
|
471
|
-
var MS_PER_DAY = 24 * MS_PER_HOUR;
|
|
472
|
-
function toMilliseconds(duration) {
|
|
473
|
-
if (typeof duration === "number") {
|
|
474
|
-
assertIsPositiveNumber(duration);
|
|
475
|
-
return duration;
|
|
476
|
-
}
|
|
477
|
-
let totalMs = 0;
|
|
478
|
-
if (duration.days !== void 0) {
|
|
479
|
-
assertIsPositiveNumber(duration.days, "days");
|
|
480
|
-
totalMs += duration.days * MS_PER_DAY;
|
|
481
|
-
}
|
|
482
|
-
if (duration.hours !== void 0) {
|
|
483
|
-
assertIsPositiveNumber(duration.hours, "hours");
|
|
484
|
-
totalMs += duration.hours * MS_PER_HOUR;
|
|
485
|
-
}
|
|
486
|
-
if (duration.minutes !== void 0) {
|
|
487
|
-
assertIsPositiveNumber(duration.minutes, "minutes");
|
|
488
|
-
totalMs += duration.minutes * MS_PER_MINUTE;
|
|
489
|
-
}
|
|
490
|
-
if (duration.seconds !== void 0) {
|
|
491
|
-
assertIsPositiveNumber(duration.seconds, "seconds");
|
|
492
|
-
totalMs += duration.seconds * MS_PER_SECOND;
|
|
493
|
-
}
|
|
494
|
-
if (duration.milliseconds !== void 0) {
|
|
495
|
-
assertIsPositiveNumber(duration.milliseconds, "milliseconds");
|
|
496
|
-
totalMs += duration.milliseconds;
|
|
497
|
-
}
|
|
498
|
-
return totalMs;
|
|
499
|
-
}
|
|
500
|
-
function assertIsPositiveNumber(value, field) {
|
|
501
|
-
if (!Number.isFinite(value)) {
|
|
502
|
-
throw new Error(
|
|
503
|
-
field !== void 0 ? `'${field}' duration must be finite. Received: ${value}` : `Duration must be finite. Received: ${value}`
|
|
504
|
-
);
|
|
505
|
-
}
|
|
506
|
-
if (value < 0) {
|
|
507
|
-
throw new Error(
|
|
508
|
-
field !== void 0 ? `'${field}' duration must be non-negative. Received: ${value}` : `Duration must be non-negative. Received: ${value}`
|
|
509
|
-
);
|
|
510
|
-
}
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
// ../workflow/run/event.ts
|
|
514
|
-
import { INTERNAL as INTERNAL2 } from "@aikirun/types/symbols";
|
|
515
|
-
import { SchemaValidationError } from "@aikirun/types/validator";
|
|
516
|
-
import {
|
|
517
|
-
WorkflowRunFailedError as WorkflowRunFailedError2,
|
|
518
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError2,
|
|
519
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError2
|
|
520
|
-
} from "@aikirun/types/workflow-run";
|
|
521
|
-
function createEventSenders(api, workflowRunId, eventsDefinition, logger) {
|
|
522
|
-
const senders = {};
|
|
523
|
-
for (const [eventName, eventDefinition] of Object.entries(eventsDefinition)) {
|
|
524
|
-
const sender = createEventSender(
|
|
525
|
-
api,
|
|
526
|
-
workflowRunId,
|
|
527
|
-
eventName,
|
|
528
|
-
eventDefinition.schema,
|
|
529
|
-
logger.child({ "aiki.eventName": eventName })
|
|
530
|
-
);
|
|
531
|
-
senders[eventName] = sender;
|
|
532
|
-
}
|
|
533
|
-
return senders;
|
|
534
|
-
}
|
|
535
|
-
function createEventSender(api, workflowRunId, eventName, schema, logger, options) {
|
|
536
|
-
const optsOverrider = objectOverrider(options ?? {});
|
|
537
|
-
const createBuilder = (optsBuilder) => ({
|
|
538
|
-
opt: (path, value) => createBuilder(optsBuilder.with(path, value)),
|
|
539
|
-
send: (...args) => createEventSender(api, workflowRunId, eventName, schema, logger, optsBuilder.build()).send(...args)
|
|
540
|
-
});
|
|
541
|
-
async function send(...args) {
|
|
542
|
-
let data = args[0];
|
|
543
|
-
if (schema) {
|
|
544
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
545
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
546
|
-
if (schemaValidationResult.issues) {
|
|
547
|
-
logger.error("Invalid event data", { "aiki.issues": schemaValidationResult.issues });
|
|
548
|
-
throw new SchemaValidationError("Invalid event data", schemaValidationResult.issues);
|
|
549
|
-
}
|
|
550
|
-
data = schemaValidationResult.value;
|
|
551
|
-
}
|
|
552
|
-
await api.workflowRun.sendEventV1({
|
|
553
|
-
id: workflowRunId,
|
|
554
|
-
eventName,
|
|
555
|
-
data,
|
|
556
|
-
options
|
|
557
|
-
});
|
|
558
|
-
logger.info("Sent event to workflow", {
|
|
559
|
-
...options?.reference ? { "aiki.referenceId": options.reference.id } : {}
|
|
560
|
-
});
|
|
561
|
-
}
|
|
562
|
-
return {
|
|
563
|
-
with: () => createBuilder(optsOverrider()),
|
|
564
|
-
send
|
|
565
|
-
};
|
|
566
|
-
}
|
|
567
|
-
function createEventMulticasters(workflowName, workflowVersionId, eventsDefinition) {
|
|
568
|
-
const senders = {};
|
|
569
|
-
for (const [eventName, eventDefinition] of Object.entries(eventsDefinition)) {
|
|
570
|
-
const sender = createEventMulticaster(
|
|
571
|
-
workflowName,
|
|
572
|
-
workflowVersionId,
|
|
573
|
-
eventName,
|
|
574
|
-
eventDefinition.schema
|
|
575
|
-
);
|
|
576
|
-
senders[eventName] = sender;
|
|
577
|
-
}
|
|
578
|
-
return senders;
|
|
579
|
-
}
|
|
580
|
-
function createEventMulticaster(workflowName, workflowVersionId, eventName, schema, options) {
|
|
581
|
-
const optsOverrider = objectOverrider(options ?? {});
|
|
582
|
-
const createBuilder = (optsBuilder) => ({
|
|
583
|
-
opt: (path, value) => createBuilder(optsBuilder.with(path, value)),
|
|
584
|
-
send: (client, runId, ...args) => createEventMulticaster(workflowName, workflowVersionId, eventName, schema, optsBuilder.build()).send(
|
|
585
|
-
client,
|
|
586
|
-
runId,
|
|
587
|
-
...args
|
|
588
|
-
),
|
|
589
|
-
sendByReferenceId: (client, referenceId, ...args) => createEventMulticaster(workflowName, workflowVersionId, eventName, schema, optsBuilder.build()).sendByReferenceId(
|
|
590
|
-
client,
|
|
591
|
-
referenceId,
|
|
592
|
-
...args
|
|
593
|
-
)
|
|
594
|
-
});
|
|
595
|
-
async function send(client, runId, ...args) {
|
|
596
|
-
let data = args[0];
|
|
597
|
-
if (schema) {
|
|
598
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
599
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
600
|
-
if (schemaValidationResult.issues) {
|
|
601
|
-
client.logger.error("Invalid event data", {
|
|
602
|
-
"aiki.workflowName": workflowName,
|
|
603
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
604
|
-
"aiki.eventName": eventName,
|
|
605
|
-
"aiki.issues": schemaValidationResult.issues
|
|
606
|
-
});
|
|
607
|
-
throw new SchemaValidationError("Invalid event data", schemaValidationResult.issues);
|
|
608
|
-
}
|
|
609
|
-
data = schemaValidationResult.value;
|
|
610
|
-
}
|
|
611
|
-
const runIds = Array.isArray(runId) ? runId : [runId];
|
|
612
|
-
if (!isNonEmptyArray(runIds)) {
|
|
613
|
-
return;
|
|
614
|
-
}
|
|
615
|
-
await client.api.workflowRun.multicastEventV1({
|
|
616
|
-
ids: runIds,
|
|
617
|
-
eventName,
|
|
618
|
-
data,
|
|
619
|
-
options
|
|
620
|
-
});
|
|
621
|
-
client.logger.info("Multicasted event to workflows", {
|
|
622
|
-
"aiki.workflowName": workflowName,
|
|
623
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
624
|
-
"aiki.workflowRunIds": runIds,
|
|
625
|
-
"aiki.eventName": eventName,
|
|
626
|
-
...options?.reference ? { "aiki.eventReferenceId": options.reference.id } : {}
|
|
627
|
-
});
|
|
628
|
-
}
|
|
629
|
-
async function sendByReferenceId(client, referenceId, ...args) {
|
|
630
|
-
let data = args[0];
|
|
631
|
-
if (schema) {
|
|
632
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
633
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
634
|
-
if (schemaValidationResult.issues) {
|
|
635
|
-
client.logger.error("Invalid event data", {
|
|
636
|
-
"aiki.workflowName": workflowName,
|
|
637
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
638
|
-
"aiki.eventName": eventName,
|
|
639
|
-
"aiki.issues": schemaValidationResult.issues
|
|
640
|
-
});
|
|
641
|
-
throw new SchemaValidationError("Invalid event data", schemaValidationResult.issues);
|
|
642
|
-
}
|
|
643
|
-
data = schemaValidationResult.value;
|
|
644
|
-
}
|
|
645
|
-
const referenceIds = Array.isArray(referenceId) ? referenceId : [referenceId];
|
|
646
|
-
if (!isNonEmptyArray(referenceIds)) {
|
|
647
|
-
return;
|
|
648
|
-
}
|
|
649
|
-
await client.api.workflowRun.multicastEventByReferenceV1({
|
|
650
|
-
references: referenceIds.map((referenceId2) => ({
|
|
651
|
-
name: workflowName,
|
|
652
|
-
versionId: workflowVersionId,
|
|
653
|
-
referenceId: referenceId2
|
|
654
|
-
})),
|
|
655
|
-
eventName,
|
|
656
|
-
data,
|
|
657
|
-
options
|
|
658
|
-
});
|
|
659
|
-
client.logger.info("Multicasted event by reference", {
|
|
660
|
-
"aiki.workflowName": workflowName,
|
|
661
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
662
|
-
"aiki.referenceIds": referenceIds,
|
|
663
|
-
"aiki.eventName": eventName,
|
|
664
|
-
...options?.reference ? { "aiki.eventReferenceId": options.reference.id } : {}
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
|
-
return {
|
|
668
|
-
with: () => createBuilder(optsOverrider()),
|
|
669
|
-
send,
|
|
670
|
-
sendByReferenceId
|
|
671
|
-
};
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
// ../workflow/run/handle.ts
|
|
675
|
-
import { INTERNAL as INTERNAL3 } from "@aikirun/types/symbols";
|
|
676
|
-
import {
|
|
677
|
-
WorkflowRunNotExecutableError,
|
|
678
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError3
|
|
679
|
-
} from "@aikirun/types/workflow-run";
|
|
680
|
-
async function workflowRunHandle(client, runOrId, eventsDefinition, logger) {
|
|
681
|
-
const run = typeof runOrId !== "string" ? runOrId : (await client.api.workflowRun.getByIdV1({ id: runOrId })).run;
|
|
682
|
-
return new WorkflowRunHandleImpl(
|
|
683
|
-
client,
|
|
684
|
-
run,
|
|
685
|
-
eventsDefinition ?? {},
|
|
686
|
-
logger ?? client.logger.child({
|
|
687
|
-
"aiki.workflowName": run.name,
|
|
688
|
-
"aiki.workflowVersionId": run.versionId,
|
|
689
|
-
"aiki.workflowRunId": run.id
|
|
690
|
-
})
|
|
691
|
-
);
|
|
692
|
-
}
|
|
693
|
-
var WorkflowRunHandleImpl = class {
|
|
694
|
-
constructor(client, _run, eventsDefinition, logger) {
|
|
695
|
-
this._run = _run;
|
|
696
|
-
this.logger = logger;
|
|
697
|
-
this.api = client.api;
|
|
698
|
-
this.events = createEventSenders(client.api, this._run.id, eventsDefinition, this.logger);
|
|
699
|
-
this[INTERNAL3] = {
|
|
700
|
-
client,
|
|
701
|
-
transitionState: this.transitionState.bind(this),
|
|
702
|
-
transitionTaskState: this.transitionTaskState.bind(this),
|
|
703
|
-
assertExecutionAllowed: this.assertExecutionAllowed.bind(this)
|
|
704
|
-
};
|
|
705
|
-
}
|
|
706
|
-
api;
|
|
707
|
-
events;
|
|
708
|
-
[INTERNAL3];
|
|
709
|
-
get run() {
|
|
710
|
-
return this._run;
|
|
711
|
-
}
|
|
712
|
-
async refresh() {
|
|
713
|
-
const { run: currentRun } = await this.api.workflowRun.getByIdV1({ id: this.run.id });
|
|
714
|
-
this._run = currentRun;
|
|
715
|
-
}
|
|
716
|
-
async waitForStatus(status, options) {
|
|
717
|
-
return this.waitForStatusByPolling(status, options);
|
|
718
|
-
}
|
|
719
|
-
async waitForStatusByPolling(expectedStatus, options) {
|
|
720
|
-
if (options?.abortSignal?.aborted) {
|
|
721
|
-
return {
|
|
722
|
-
success: false,
|
|
723
|
-
cause: "aborted"
|
|
724
|
-
};
|
|
725
|
-
}
|
|
726
|
-
const delayMs = options?.interval ? toMilliseconds(options.interval) : 1e3;
|
|
727
|
-
const maxAttempts = options?.timeout ? Math.ceil(toMilliseconds(options.timeout) / delayMs) : Number.POSITIVE_INFINITY;
|
|
728
|
-
const retryStrategy = { type: "fixed", maxAttempts, delayMs };
|
|
729
|
-
const afterStateTransitionId = this._run.stateTransitionId;
|
|
730
|
-
const hasTerminated = async () => {
|
|
731
|
-
const { terminated } = await this.api.workflowRun.hasTerminatedV1({
|
|
732
|
-
id: this._run.id,
|
|
733
|
-
afterStateTransitionId
|
|
734
|
-
});
|
|
735
|
-
return terminated;
|
|
736
|
-
};
|
|
737
|
-
const shouldRetryOnResult = async (terminated) => !terminated;
|
|
738
|
-
const maybeResult = options?.abortSignal ? await withRetry(hasTerminated, retryStrategy, {
|
|
739
|
-
abortSignal: options.abortSignal,
|
|
740
|
-
shouldRetryOnResult
|
|
741
|
-
}).run() : await withRetry(hasTerminated, retryStrategy, { shouldRetryOnResult }).run();
|
|
742
|
-
if (maybeResult.state === "timeout") {
|
|
743
|
-
if (!Number.isFinite(maxAttempts)) {
|
|
744
|
-
throw new Error("Something's wrong, this should've never timed out");
|
|
745
|
-
}
|
|
746
|
-
return { success: false, cause: "timeout" };
|
|
747
|
-
}
|
|
748
|
-
if (maybeResult.state === "aborted") {
|
|
749
|
-
return { success: false, cause: "aborted" };
|
|
750
|
-
}
|
|
751
|
-
maybeResult.state;
|
|
752
|
-
await this.refresh();
|
|
753
|
-
if (this._run.state.status === expectedStatus) {
|
|
754
|
-
return {
|
|
755
|
-
success: true,
|
|
756
|
-
state: this._run.state
|
|
757
|
-
};
|
|
758
|
-
}
|
|
759
|
-
return { success: false, cause: "run_terminated" };
|
|
760
|
-
}
|
|
761
|
-
async cancel(reason) {
|
|
762
|
-
await this.transitionState({ status: "cancelled", reason });
|
|
763
|
-
this.logger.info("Workflow cancelled");
|
|
764
|
-
}
|
|
765
|
-
async pause() {
|
|
766
|
-
await this.transitionState({ status: "paused" });
|
|
767
|
-
this.logger.info("Workflow paused");
|
|
768
|
-
}
|
|
769
|
-
async resume() {
|
|
770
|
-
await this.transitionState({ status: "scheduled", scheduledInMs: 0, reason: "resume" });
|
|
771
|
-
this.logger.info("Workflow resumed");
|
|
772
|
-
}
|
|
773
|
-
async awake() {
|
|
774
|
-
await this.transitionState({ status: "scheduled", scheduledInMs: 0, reason: "awake_early" });
|
|
775
|
-
this.logger.info("Workflow awoken");
|
|
776
|
-
}
|
|
777
|
-
async transitionState(targetState) {
|
|
778
|
-
try {
|
|
779
|
-
let response;
|
|
780
|
-
if (targetState.status === "scheduled" && (targetState.reason === "new" || targetState.reason === "resume" || targetState.reason === "awake_early") || targetState.status === "paused" || targetState.status === "cancelled") {
|
|
781
|
-
response = await this.api.workflowRun.transitionStateV1({
|
|
782
|
-
type: "pessimistic",
|
|
783
|
-
id: this.run.id,
|
|
784
|
-
state: targetState
|
|
785
|
-
});
|
|
786
|
-
} else {
|
|
787
|
-
response = await this.api.workflowRun.transitionStateV1({
|
|
788
|
-
type: "optimistic",
|
|
789
|
-
id: this.run.id,
|
|
790
|
-
state: targetState,
|
|
791
|
-
expectedRevision: this.run.revision
|
|
792
|
-
});
|
|
793
|
-
}
|
|
794
|
-
this._run.revision = response.revision;
|
|
795
|
-
this._run.state = response.state;
|
|
796
|
-
this._run.attempts = response.attempts;
|
|
797
|
-
} catch (error) {
|
|
798
|
-
if (isWorkflowRunRevisionConflictError(error)) {
|
|
799
|
-
throw new WorkflowRunRevisionConflictError3(this.run.id);
|
|
800
|
-
}
|
|
801
|
-
throw error;
|
|
802
|
-
}
|
|
803
|
-
}
|
|
804
|
-
async transitionTaskState(request) {
|
|
805
|
-
try {
|
|
806
|
-
const { taskInfo } = await this.api.workflowRun.transitionTaskStateV1({
|
|
807
|
-
...request,
|
|
808
|
-
id: this.run.id,
|
|
809
|
-
expectedWorkflowRunRevision: this.run.revision
|
|
810
|
-
});
|
|
811
|
-
return taskInfo;
|
|
812
|
-
} catch (error) {
|
|
813
|
-
if (isWorkflowRunRevisionConflictError(error)) {
|
|
814
|
-
throw new WorkflowRunRevisionConflictError3(this.run.id);
|
|
815
|
-
}
|
|
816
|
-
throw error;
|
|
817
|
-
}
|
|
818
|
-
}
|
|
819
|
-
assertExecutionAllowed() {
|
|
820
|
-
const status = this.run.state.status;
|
|
821
|
-
if (status !== "queued" && status !== "running") {
|
|
822
|
-
throw new WorkflowRunNotExecutableError(this.run.id, status);
|
|
823
|
-
}
|
|
824
|
-
}
|
|
825
|
-
};
|
|
826
|
-
function isWorkflowRunRevisionConflictError(error) {
|
|
827
|
-
return error != null && typeof error === "object" && "code" in error && error.code === "WORKFLOW_RUN_REVISION_CONFLICT";
|
|
828
|
-
}
|
|
829
|
-
|
|
830
|
-
// ../workflow/run/handle-child.ts
|
|
831
|
-
import { INTERNAL as INTERNAL4 } from "@aikirun/types/symbols";
|
|
832
|
-
import {
|
|
833
|
-
isTerminalWorkflowRunStatus,
|
|
834
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError4,
|
|
835
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError3
|
|
836
|
-
} from "@aikirun/types/workflow-run";
|
|
837
|
-
async function childWorkflowRunHandle(client, run, parentRun, childWorkflowRunWaitQueues, logger, eventsDefinition) {
|
|
838
|
-
const handle = await workflowRunHandle(client, run, eventsDefinition, logger);
|
|
839
|
-
return {
|
|
840
|
-
run: handle.run,
|
|
841
|
-
events: handle.events,
|
|
842
|
-
refresh: handle.refresh.bind(handle),
|
|
843
|
-
waitForStatus: createStatusWaiter(handle, parentRun, childWorkflowRunWaitQueues, logger),
|
|
844
|
-
cancel: handle.cancel.bind(handle),
|
|
845
|
-
pause: handle.pause.bind(handle),
|
|
846
|
-
resume: handle.resume.bind(handle),
|
|
847
|
-
awake: handle.awake.bind(handle),
|
|
848
|
-
[INTERNAL4]: handle[INTERNAL4]
|
|
849
|
-
};
|
|
850
|
-
}
|
|
851
|
-
function createStatusWaiter(handle, parentRun, childWorkflowRunWaitQueues, logger) {
|
|
852
|
-
const nextIndexByStatus = {
|
|
853
|
-
cancelled: 0,
|
|
854
|
-
completed: 0,
|
|
855
|
-
failed: 0
|
|
856
|
-
};
|
|
857
|
-
async function waitForStatus(expectedStatus, options) {
|
|
858
|
-
const parentRunHandle = parentRun[INTERNAL4].handle;
|
|
859
|
-
const nextIndex = nextIndexByStatus[expectedStatus];
|
|
860
|
-
const { run } = handle;
|
|
861
|
-
const childWorkflowRunWaits = childWorkflowRunWaitQueues[expectedStatus].childWorkflowRunWaits;
|
|
862
|
-
const existingChildWorkflowRunWait = childWorkflowRunWaits[nextIndex];
|
|
863
|
-
if (existingChildWorkflowRunWait) {
|
|
864
|
-
nextIndexByStatus[expectedStatus] = nextIndex + 1;
|
|
865
|
-
if (existingChildWorkflowRunWait.status === "timeout") {
|
|
866
|
-
logger.debug("Timed out waiting for child workflow status", {
|
|
867
|
-
"aiki.childWorkflowExpectedStatus": expectedStatus
|
|
868
|
-
});
|
|
869
|
-
return {
|
|
870
|
-
success: false,
|
|
871
|
-
cause: "timeout"
|
|
872
|
-
};
|
|
873
|
-
}
|
|
874
|
-
const childWorkflowRunStatus = existingChildWorkflowRunWait.childWorkflowRunState.status;
|
|
875
|
-
if (childWorkflowRunStatus === expectedStatus) {
|
|
876
|
-
return {
|
|
877
|
-
success: true,
|
|
878
|
-
state: existingChildWorkflowRunWait.childWorkflowRunState
|
|
879
|
-
};
|
|
880
|
-
}
|
|
881
|
-
if (isTerminalWorkflowRunStatus(childWorkflowRunStatus)) {
|
|
882
|
-
logger.debug("Child workflow run reached termnial state", {
|
|
883
|
-
"aiki.childWorkflowTerminalStatus": childWorkflowRunStatus
|
|
884
|
-
});
|
|
885
|
-
return {
|
|
886
|
-
success: false,
|
|
887
|
-
cause: "run_terminated"
|
|
888
|
-
};
|
|
889
|
-
}
|
|
890
|
-
childWorkflowRunStatus;
|
|
891
|
-
}
|
|
892
|
-
const timeoutInMs = options?.timeout && toMilliseconds(options.timeout);
|
|
893
|
-
try {
|
|
894
|
-
await parentRunHandle[INTERNAL4].transitionState({
|
|
895
|
-
status: "awaiting_child_workflow",
|
|
896
|
-
childWorkflowRunId: run.id,
|
|
897
|
-
childWorkflowRunStatus: expectedStatus,
|
|
898
|
-
timeoutInMs
|
|
899
|
-
});
|
|
900
|
-
logger.info("Waiting for child Workflow", {
|
|
901
|
-
"aiki.childWorkflowExpectedStatus": expectedStatus,
|
|
902
|
-
...timeoutInMs !== void 0 ? { "aiki.timeoutInMs": timeoutInMs } : {}
|
|
903
|
-
});
|
|
904
|
-
} catch (error) {
|
|
905
|
-
if (error instanceof WorkflowRunRevisionConflictError4) {
|
|
906
|
-
throw new WorkflowRunSuspendedError3(parentRun.id);
|
|
907
|
-
}
|
|
908
|
-
throw error;
|
|
909
|
-
}
|
|
910
|
-
throw new WorkflowRunSuspendedError3(parentRun.id);
|
|
911
|
-
}
|
|
912
|
-
return waitForStatus;
|
|
913
|
-
}
|
|
914
|
-
|
|
915
|
-
// ../workflow/workflow-version.ts
|
|
916
|
-
var WorkflowVersionImpl = class {
|
|
917
|
-
constructor(name, versionId, params) {
|
|
918
|
-
this.name = name;
|
|
919
|
-
this.versionId = versionId;
|
|
920
|
-
this.params = params;
|
|
921
|
-
const eventsDefinition = this.params.events ?? {};
|
|
922
|
-
this.events = createEventMulticasters(this.name, this.versionId, eventsDefinition);
|
|
923
|
-
this[INTERNAL5] = {
|
|
924
|
-
eventsDefinition,
|
|
925
|
-
handler: this.handler.bind(this)
|
|
926
|
-
};
|
|
927
|
-
}
|
|
928
|
-
events;
|
|
929
|
-
[INTERNAL5];
|
|
930
|
-
with() {
|
|
931
|
-
const startOpts = this.params.opts ?? {};
|
|
932
|
-
const startOptsOverrider = objectOverrider(startOpts);
|
|
933
|
-
return new WorkflowBuilderImpl(this, startOptsOverrider());
|
|
934
|
-
}
|
|
935
|
-
async start(client, ...args) {
|
|
936
|
-
return this.startWithOpts(client, this.params.opts ?? {}, ...args);
|
|
937
|
-
}
|
|
938
|
-
async startWithOpts(client, startOpts, ...args) {
|
|
939
|
-
let input = args[0];
|
|
940
|
-
const schema = this.params.schema?.input;
|
|
941
|
-
if (schema) {
|
|
942
|
-
const schemaValidation = schema["~standard"].validate(input);
|
|
943
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
944
|
-
if (schemaValidationResult.issues) {
|
|
945
|
-
client.logger.error("Invalid workflow data", { "aiki.issues": schemaValidationResult.issues });
|
|
946
|
-
throw new SchemaValidationError2("Invalid workflow data", schemaValidationResult.issues);
|
|
947
|
-
}
|
|
948
|
-
input = schemaValidationResult.value;
|
|
949
|
-
}
|
|
950
|
-
const { id } = await client.api.workflowRun.createV1({
|
|
951
|
-
name: this.name,
|
|
952
|
-
versionId: this.versionId,
|
|
953
|
-
input,
|
|
954
|
-
options: startOpts
|
|
955
|
-
});
|
|
956
|
-
client.logger.info("Created workflow", {
|
|
957
|
-
"aiki.workflowName": this.name,
|
|
958
|
-
"aiki.workflowVersionId": this.versionId,
|
|
959
|
-
"aiki.workflowRunId": id
|
|
960
|
-
});
|
|
961
|
-
return workflowRunHandle(client, id, this[INTERNAL5].eventsDefinition);
|
|
962
|
-
}
|
|
963
|
-
async startAsChild(parentRun, ...args) {
|
|
964
|
-
return this.startAsChildWithOpts(parentRun, this.params.opts ?? {}, ...args);
|
|
965
|
-
}
|
|
966
|
-
async startAsChildWithOpts(parentRun, startOpts, ...args) {
|
|
967
|
-
const parentRunHandle = parentRun[INTERNAL5].handle;
|
|
968
|
-
parentRunHandle[INTERNAL5].assertExecutionAllowed();
|
|
969
|
-
const { client } = parentRunHandle[INTERNAL5];
|
|
970
|
-
const inputRaw = args[0];
|
|
971
|
-
const input = await this.parse(parentRunHandle, this.params.schema?.input, inputRaw, parentRun.logger);
|
|
972
|
-
const inputHash = await hashInput(input);
|
|
973
|
-
const referenceId = startOpts.reference?.id;
|
|
974
|
-
const address = getWorkflowRunAddress(this.name, this.versionId, referenceId ?? inputHash);
|
|
975
|
-
const replayManifest = parentRun[INTERNAL5].replayManifest;
|
|
976
|
-
if (replayManifest.hasUnconsumedEntries()) {
|
|
977
|
-
const existingRunInfo = replayManifest.consumeNextChildWorkflowRun(address);
|
|
978
|
-
if (existingRunInfo) {
|
|
979
|
-
const { run: existingRun } = await client.api.workflowRun.getByIdV1({ id: existingRunInfo.id });
|
|
980
|
-
if (existingRun.state.status === "completed") {
|
|
981
|
-
await this.parse(parentRunHandle, this.params.schema?.output, existingRun.state.output, parentRun.logger);
|
|
982
|
-
}
|
|
983
|
-
const logger2 = parentRun.logger.child({
|
|
984
|
-
"aiki.childWorkflowName": existingRun.name,
|
|
985
|
-
"aiki.childWorkflowVersionId": existingRun.versionId,
|
|
986
|
-
"aiki.childWorkflowRunId": existingRun.id
|
|
987
|
-
});
|
|
988
|
-
return childWorkflowRunHandle(
|
|
989
|
-
client,
|
|
990
|
-
existingRun,
|
|
991
|
-
parentRun,
|
|
992
|
-
existingRunInfo.childWorkflowRunWaitQueues,
|
|
993
|
-
logger2,
|
|
994
|
-
this[INTERNAL5].eventsDefinition
|
|
995
|
-
);
|
|
996
|
-
}
|
|
997
|
-
await this.throwNonDeterminismError(parentRun, parentRunHandle, inputHash, referenceId, replayManifest);
|
|
998
|
-
}
|
|
999
|
-
const shard = parentRun.options.shard;
|
|
1000
|
-
const { id: newRunId } = await client.api.workflowRun.createV1({
|
|
1001
|
-
name: this.name,
|
|
1002
|
-
versionId: this.versionId,
|
|
1003
|
-
input,
|
|
1004
|
-
parentWorkflowRunId: parentRun.id,
|
|
1005
|
-
options: shard === void 0 ? startOpts : { ...startOpts, shard }
|
|
1006
|
-
});
|
|
1007
|
-
const { run: newRun } = await client.api.workflowRun.getByIdV1({ id: newRunId });
|
|
1008
|
-
const logger = parentRun.logger.child({
|
|
1009
|
-
"aiki.childWorkflowName": newRun.name,
|
|
1010
|
-
"aiki.childWorkflowVersionId": newRun.versionId,
|
|
1011
|
-
"aiki.childWorkflowRunId": newRun.id
|
|
1012
|
-
});
|
|
1013
|
-
logger.info("Created child workflow");
|
|
1014
|
-
return childWorkflowRunHandle(
|
|
1015
|
-
client,
|
|
1016
|
-
newRun,
|
|
1017
|
-
parentRun,
|
|
1018
|
-
{
|
|
1019
|
-
cancelled: { childWorkflowRunWaits: [] },
|
|
1020
|
-
completed: { childWorkflowRunWaits: [] },
|
|
1021
|
-
failed: { childWorkflowRunWaits: [] }
|
|
1022
|
-
},
|
|
1023
|
-
logger,
|
|
1024
|
-
this[INTERNAL5].eventsDefinition
|
|
1025
|
-
);
|
|
1026
|
-
}
|
|
1027
|
-
async throwNonDeterminismError(parentRun, parentRunHandle, inputHash, referenceId, manifest) {
|
|
1028
|
-
const unconsumedManifestEntries = manifest.getUnconsumedEntries();
|
|
1029
|
-
const logMeta = {
|
|
1030
|
-
"aiki.workflowName": this.name,
|
|
1031
|
-
"aiki.inputHash": inputHash,
|
|
1032
|
-
"aiki.unconsumedManifestEntries": unconsumedManifestEntries
|
|
1033
|
-
};
|
|
1034
|
-
if (referenceId !== void 0) {
|
|
1035
|
-
logMeta["aiki.referenceId"] = referenceId;
|
|
1036
|
-
}
|
|
1037
|
-
parentRun.logger.error("Replay divergence", logMeta);
|
|
1038
|
-
const error = new NonDeterminismError2(parentRun.id, parentRunHandle.run.attempts, unconsumedManifestEntries);
|
|
1039
|
-
await parentRunHandle[INTERNAL5].transitionState({
|
|
1040
|
-
status: "failed",
|
|
1041
|
-
cause: "self",
|
|
1042
|
-
error: createSerializableError(error)
|
|
1043
|
-
});
|
|
1044
|
-
throw error;
|
|
1045
|
-
}
|
|
1046
|
-
async getHandleById(client, runId) {
|
|
1047
|
-
return workflowRunHandle(client, runId, this[INTERNAL5].eventsDefinition);
|
|
1048
|
-
}
|
|
1049
|
-
async getHandleByReferenceId(client, referenceId) {
|
|
1050
|
-
const { run } = await client.api.workflowRun.getByReferenceIdV1({
|
|
1051
|
-
name: this.name,
|
|
1052
|
-
versionId: this.versionId,
|
|
1053
|
-
referenceId
|
|
1054
|
-
});
|
|
1055
|
-
return workflowRunHandle(client, run, this[INTERNAL5].eventsDefinition);
|
|
1056
|
-
}
|
|
1057
|
-
async handler(run, input, context) {
|
|
1058
|
-
const { logger } = run;
|
|
1059
|
-
const { handle } = run[INTERNAL5];
|
|
1060
|
-
handle[INTERNAL5].assertExecutionAllowed();
|
|
1061
|
-
const retryStrategy = this.params.opts?.retry ?? { type: "never" };
|
|
1062
|
-
const state = handle.run.state;
|
|
1063
|
-
if (state.status === "queued" && state.reason === "retry") {
|
|
1064
|
-
await this.assertRetryAllowed(handle, retryStrategy, logger);
|
|
1065
|
-
}
|
|
1066
|
-
logger.info("Starting workflow");
|
|
1067
|
-
await handle[INTERNAL5].transitionState({ status: "running" });
|
|
1068
|
-
const output = await this.tryExecuteWorkflow(input, run, context, retryStrategy);
|
|
1069
|
-
await handle[INTERNAL5].transitionState({ status: "completed", output });
|
|
1070
|
-
logger.info("Workflow complete");
|
|
1071
|
-
}
|
|
1072
|
-
async tryExecuteWorkflow(input, run, context, retryStrategy) {
|
|
1073
|
-
const { handle } = run[INTERNAL5];
|
|
1074
|
-
while (true) {
|
|
1075
|
-
try {
|
|
1076
|
-
const outputRaw = await this.params.handler(run, input, context);
|
|
1077
|
-
const output = await this.parse(handle, this.params.schema?.output, outputRaw, run.logger);
|
|
1078
|
-
return output;
|
|
1079
|
-
} catch (error) {
|
|
1080
|
-
if (error instanceof WorkflowRunSuspendedError4 || error instanceof WorkflowRunFailedError3 || error instanceof WorkflowRunRevisionConflictError5 || error instanceof NonDeterminismError2) {
|
|
1081
|
-
throw error;
|
|
1082
|
-
}
|
|
1083
|
-
const attempts = handle.run.attempts;
|
|
1084
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
1085
|
-
if (!retryParams.retriesLeft) {
|
|
1086
|
-
const failedState = this.createFailedState(error);
|
|
1087
|
-
await handle[INTERNAL5].transitionState(failedState);
|
|
1088
|
-
const logMeta2 = {};
|
|
1089
|
-
for (const [key, value] of Object.entries(failedState)) {
|
|
1090
|
-
logMeta2[`aiki.${key}`] = value;
|
|
1091
|
-
}
|
|
1092
|
-
run.logger.error("Workflow failed", {
|
|
1093
|
-
"aiki.attempts": attempts,
|
|
1094
|
-
...logMeta2
|
|
1095
|
-
});
|
|
1096
|
-
throw new WorkflowRunFailedError3(run.id, attempts);
|
|
1097
|
-
}
|
|
1098
|
-
const awaitingRetryState = this.createAwaitingRetryState(error, retryParams.delayMs);
|
|
1099
|
-
await handle[INTERNAL5].transitionState(awaitingRetryState);
|
|
1100
|
-
const logMeta = {};
|
|
1101
|
-
for (const [key, value] of Object.entries(awaitingRetryState)) {
|
|
1102
|
-
logMeta[`aiki.${key}`] = value;
|
|
1103
|
-
}
|
|
1104
|
-
run.logger.info("Workflow awaiting retry", {
|
|
1105
|
-
"aiki.attempts": attempts,
|
|
1106
|
-
...logMeta
|
|
1107
|
-
});
|
|
1108
|
-
throw new WorkflowRunSuspendedError4(run.id);
|
|
1109
|
-
}
|
|
1110
|
-
}
|
|
1111
|
-
}
|
|
1112
|
-
async assertRetryAllowed(handle, retryStrategy, logger) {
|
|
1113
|
-
const { id, attempts } = handle.run;
|
|
1114
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
1115
|
-
if (!retryParams.retriesLeft) {
|
|
1116
|
-
logger.error("Workflow retry not allowed", { "aiki.attempts": attempts });
|
|
1117
|
-
const error = new WorkflowRunFailedError3(id, attempts);
|
|
1118
|
-
await handle[INTERNAL5].transitionState({
|
|
1119
|
-
status: "failed",
|
|
1120
|
-
cause: "self",
|
|
1121
|
-
error: createSerializableError(error)
|
|
1122
|
-
});
|
|
1123
|
-
throw error;
|
|
1124
|
-
}
|
|
1125
|
-
}
|
|
1126
|
-
async parse(handle, schema, data, logger) {
|
|
1127
|
-
if (!schema) {
|
|
1128
|
-
return data;
|
|
1129
|
-
}
|
|
1130
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
1131
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
1132
|
-
if (!schemaValidationResult.issues) {
|
|
1133
|
-
return schemaValidationResult.value;
|
|
1134
|
-
}
|
|
1135
|
-
logger.error("Invalid workflow data", { "aiki.issues": schemaValidationResult.issues });
|
|
1136
|
-
await handle[INTERNAL5].transitionState({
|
|
1137
|
-
status: "failed",
|
|
1138
|
-
cause: "self",
|
|
1139
|
-
error: {
|
|
1140
|
-
name: "SchemaValidationError",
|
|
1141
|
-
message: JSON.stringify(schemaValidationResult.issues)
|
|
1142
|
-
}
|
|
1143
|
-
});
|
|
1144
|
-
throw new WorkflowRunFailedError3(handle.run.id, handle.run.attempts);
|
|
1145
|
-
}
|
|
1146
|
-
createFailedState(error) {
|
|
1147
|
-
if (error instanceof TaskFailedError2) {
|
|
1148
|
-
return {
|
|
1149
|
-
status: "failed",
|
|
1150
|
-
cause: "task",
|
|
1151
|
-
taskId: error.taskId
|
|
1152
|
-
};
|
|
1153
|
-
}
|
|
1154
|
-
return {
|
|
1155
|
-
status: "failed",
|
|
1156
|
-
cause: "self",
|
|
1157
|
-
error: createSerializableError(error)
|
|
1158
|
-
};
|
|
1159
|
-
}
|
|
1160
|
-
createAwaitingRetryState(error, nextAttemptInMs) {
|
|
1161
|
-
if (error instanceof TaskFailedError2) {
|
|
1162
|
-
return {
|
|
1163
|
-
status: "awaiting_retry",
|
|
1164
|
-
cause: "task",
|
|
1165
|
-
nextAttemptInMs,
|
|
1166
|
-
taskId: error.taskId
|
|
1167
|
-
};
|
|
1168
|
-
}
|
|
1169
|
-
return {
|
|
1170
|
-
status: "awaiting_retry",
|
|
1171
|
-
cause: "self",
|
|
1172
|
-
nextAttemptInMs,
|
|
1173
|
-
error: createSerializableError(error)
|
|
1174
|
-
};
|
|
1175
|
-
}
|
|
1176
|
-
};
|
|
1177
|
-
var WorkflowBuilderImpl = class _WorkflowBuilderImpl {
|
|
1178
|
-
constructor(workflow2, startOptsBuilder) {
|
|
1179
|
-
this.workflow = workflow2;
|
|
1180
|
-
this.startOptsBuilder = startOptsBuilder;
|
|
1181
|
-
}
|
|
1182
|
-
opt(path, value) {
|
|
1183
|
-
return new _WorkflowBuilderImpl(this.workflow, this.startOptsBuilder.with(path, value));
|
|
1184
|
-
}
|
|
1185
|
-
start(client, ...args) {
|
|
1186
|
-
return this.workflow.startWithOpts(client, this.startOptsBuilder.build(), ...args);
|
|
1187
|
-
}
|
|
1188
|
-
startAsChild(parentRun, ...args) {
|
|
1189
|
-
return this.workflow.startAsChildWithOpts(parentRun, this.startOptsBuilder.build(), ...args);
|
|
1190
|
-
}
|
|
1191
|
-
};
|
|
1192
|
-
|
|
1193
|
-
// ../workflow/workflow.ts
|
|
1194
|
-
function workflow(params) {
|
|
1195
|
-
return new WorkflowImpl(params);
|
|
1196
|
-
}
|
|
1197
|
-
var WorkflowImpl = class {
|
|
1198
|
-
name;
|
|
1199
|
-
[INTERNAL6];
|
|
1200
|
-
workflowVersions = /* @__PURE__ */ new Map();
|
|
1201
|
-
constructor(params) {
|
|
1202
|
-
this.name = params.name;
|
|
1203
|
-
this[INTERNAL6] = {
|
|
1204
|
-
getAllVersions: this.getAllVersions.bind(this),
|
|
1205
|
-
getVersion: this.getVersion.bind(this)
|
|
1206
|
-
};
|
|
1207
|
-
}
|
|
1208
|
-
v(versionId, params) {
|
|
1209
|
-
if (this.workflowVersions.has(versionId)) {
|
|
1210
|
-
throw new Error(`Workflow "${this.name}:${versionId}" already exists`);
|
|
1211
|
-
}
|
|
1212
|
-
const workflowVersion = new WorkflowVersionImpl(this.name, versionId, params);
|
|
1213
|
-
this.workflowVersions.set(
|
|
1214
|
-
versionId,
|
|
1215
|
-
workflowVersion
|
|
1216
|
-
);
|
|
1217
|
-
return workflowVersion;
|
|
1218
|
-
}
|
|
1219
|
-
getAllVersions() {
|
|
1220
|
-
return Array.from(this.workflowVersions.values());
|
|
1221
|
-
}
|
|
1222
|
-
getVersion(versionId) {
|
|
1223
|
-
return this.workflowVersions.get(versionId);
|
|
1224
|
-
}
|
|
1225
|
-
};
|
|
1226
|
-
|
|
1227
|
-
// ../workflow/system/cancel-child-runs.ts
|
|
1228
|
-
var createCancelChildRunsV1 = (api) => {
|
|
1229
|
-
const listNonTerminalChildRuns = task({
|
|
1230
|
-
name: "aiki:list-non-terminal-child-runs",
|
|
1231
|
-
async handler(parentRunId) {
|
|
1232
|
-
const { runs } = await api.workflowRun.listChildRunsV1({
|
|
1233
|
-
parentRunId,
|
|
1234
|
-
status: NON_TERMINAL_WORKFLOW_RUN_STATUSES
|
|
1235
|
-
});
|
|
1236
|
-
return runs.map((r) => r.id);
|
|
1237
|
-
}
|
|
1238
|
-
});
|
|
1239
|
-
const cancelRuns = task({
|
|
1240
|
-
name: "aiki:cancel-runs",
|
|
1241
|
-
async handler(runIds) {
|
|
1242
|
-
const { cancelledIds } = await api.workflowRun.cancelByIdsV1({ ids: runIds });
|
|
1243
|
-
return cancelledIds;
|
|
1244
|
-
}
|
|
1245
|
-
});
|
|
1246
|
-
return workflow({ name: "aiki:cancel-child-runs" }).v("1.0.0", {
|
|
1247
|
-
async handler(run, parentRunId) {
|
|
1248
|
-
const childRunIds = await listNonTerminalChildRuns.start(run, parentRunId);
|
|
1249
|
-
if (!isNonEmptyArray(childRunIds)) {
|
|
1250
|
-
return;
|
|
1251
|
-
}
|
|
1252
|
-
await cancelRuns.start(run, childRunIds);
|
|
1253
|
-
}
|
|
1254
|
-
});
|
|
1255
|
-
};
|
|
1256
|
-
|
|
1257
|
-
// ../workflow/system/index.ts
|
|
1258
|
-
function getSystemWorkflows(api) {
|
|
1259
|
-
return [createCancelChildRunsV1(api)];
|
|
1260
|
-
}
|
|
1261
|
-
|
|
1262
|
-
// worker.ts
|
|
1263
62
|
import { ulid } from "ulidx";
|
|
1264
63
|
function worker(params) {
|
|
1265
64
|
return new WorkerImpl(params);
|
|
@@ -1267,76 +66,61 @@ function worker(params) {
|
|
|
1267
66
|
var WorkerImpl = class {
|
|
1268
67
|
constructor(params) {
|
|
1269
68
|
this.params = params;
|
|
1270
|
-
this.name = params.name;
|
|
1271
69
|
}
|
|
1272
|
-
name;
|
|
1273
70
|
with() {
|
|
1274
|
-
const
|
|
1275
|
-
const
|
|
1276
|
-
return new WorkerBuilderImpl(this,
|
|
71
|
+
const spawnOptions = this.params.options ?? {};
|
|
72
|
+
const spawnOptionsOverrider = objectOverrider(spawnOptions);
|
|
73
|
+
return new WorkerBuilderImpl(this, spawnOptionsOverrider());
|
|
1277
74
|
}
|
|
1278
75
|
async spawn(client) {
|
|
1279
|
-
return this.
|
|
76
|
+
return this.spawnWithOptions(client, this.params.options ?? {});
|
|
1280
77
|
}
|
|
1281
|
-
async
|
|
1282
|
-
const handle = new WorkerHandleImpl(client, this.params,
|
|
78
|
+
async spawnWithOptions(client, spawnOptions) {
|
|
79
|
+
const handle = new WorkerHandleImpl(client, this.params, spawnOptions);
|
|
1283
80
|
await handle._start();
|
|
1284
81
|
return handle;
|
|
1285
82
|
}
|
|
1286
83
|
};
|
|
1287
84
|
var WorkerHandleImpl = class {
|
|
1288
|
-
constructor(client, params,
|
|
85
|
+
constructor(client, params, spawnOptions) {
|
|
1289
86
|
this.client = client;
|
|
1290
87
|
this.params = params;
|
|
1291
|
-
this.
|
|
88
|
+
this.spawnOptions = spawnOptions;
|
|
1292
89
|
this.id = ulid();
|
|
1293
|
-
this.
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
spinThresholdMs: this.spawnOpts.workflowRun?.spinThresholdMs ?? 10
|
|
90
|
+
this.workflowRunOptions = {
|
|
91
|
+
heartbeatIntervalMs: this.spawnOptions.workflowRun?.heartbeatIntervalMs ?? 3e4,
|
|
92
|
+
spinThresholdMs: this.spawnOptions.workflowRun?.spinThresholdMs ?? 10
|
|
1297
93
|
};
|
|
1298
94
|
this.registry = workflowRegistry().addMany(getSystemWorkflows(client.api)).addMany(this.params.workflows);
|
|
1299
|
-
const reference = this.
|
|
95
|
+
const reference = this.spawnOptions.reference;
|
|
1300
96
|
this.logger = client.logger.child({
|
|
1301
97
|
"aiki.component": "worker",
|
|
1302
98
|
"aiki.workerId": this.id,
|
|
1303
|
-
"aiki.workerName": this.name,
|
|
1304
99
|
...reference && { "aiki.workerReferenceId": reference.id }
|
|
1305
100
|
});
|
|
1306
101
|
}
|
|
1307
102
|
id;
|
|
1308
|
-
|
|
1309
|
-
workflowRunOpts;
|
|
103
|
+
workflowRunOptions;
|
|
1310
104
|
registry;
|
|
1311
105
|
logger;
|
|
1312
106
|
abortController;
|
|
1313
|
-
|
|
1314
|
-
|
|
107
|
+
subscriber;
|
|
108
|
+
fallbackSubscriber;
|
|
1315
109
|
pollPromise;
|
|
1316
110
|
activeWorkflowRunsById = /* @__PURE__ */ new Map();
|
|
1317
111
|
async _start() {
|
|
1318
|
-
const
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
);
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
const fallbackSubscriberStrategyBuilder = this.client[INTERNAL7].subscriber.create(
|
|
1331
|
-
{ type: "db" },
|
|
1332
|
-
workflows,
|
|
1333
|
-
this.spawnOpts.shards
|
|
1334
|
-
);
|
|
1335
|
-
this.fallbackSubscriberStrategy = await fallbackSubscriberStrategyBuilder.init(this.id, {
|
|
1336
|
-
onError: (error) => this.handleSubscriberError(error),
|
|
1337
|
-
onStop: () => this.stop()
|
|
1338
|
-
});
|
|
1339
|
-
}
|
|
112
|
+
const subscriberContext = {
|
|
113
|
+
workerId: this.id,
|
|
114
|
+
workflows: this.registry.getAll(),
|
|
115
|
+
shards: this.spawnOptions.shards,
|
|
116
|
+
logger: this.logger
|
|
117
|
+
};
|
|
118
|
+
const createSubscriber = this.params.subscriber ?? dbSubscriber({ api: this.client.api });
|
|
119
|
+
const subscriber = createSubscriber(subscriberContext);
|
|
120
|
+
this.subscriber = subscriber instanceof Promise ? await subscriber : subscriber;
|
|
121
|
+
const createFallbackSubscriber = dbSubscriber({ api: this.client.api });
|
|
122
|
+
const fallbackSubscriber = createFallbackSubscriber(subscriberContext);
|
|
123
|
+
this.fallbackSubscriber = fallbackSubscriber instanceof Promise ? await fallbackSubscriber : fallbackSubscriber;
|
|
1340
124
|
this.abortController = new AbortController();
|
|
1341
125
|
const abortSignal = this.abortController.signal;
|
|
1342
126
|
this.pollPromise = this.poll(abortSignal).catch((error) => {
|
|
@@ -1351,11 +135,13 @@ var WorkerHandleImpl = class {
|
|
|
1351
135
|
this.logger.info("Worker stopping");
|
|
1352
136
|
this.abortController?.abort();
|
|
1353
137
|
await this.pollPromise;
|
|
138
|
+
await this.subscriber?.close?.();
|
|
139
|
+
await this.fallbackSubscriber?.close?.();
|
|
1354
140
|
const activeWorkflowRuns = Array.from(this.activeWorkflowRunsById.values());
|
|
1355
141
|
if (activeWorkflowRuns.length === 0) {
|
|
1356
142
|
return;
|
|
1357
143
|
}
|
|
1358
|
-
const timeoutMs = this.
|
|
144
|
+
const timeoutMs = this.spawnOptions.gracefulShutdownTimeoutMs ?? 5e3;
|
|
1359
145
|
if (timeoutMs > 0) {
|
|
1360
146
|
await Promise.race([Promise.allSettled(activeWorkflowRuns.map((w) => w.executionPromise)), delay(timeoutMs)]);
|
|
1361
147
|
}
|
|
@@ -1369,60 +155,59 @@ var WorkerHandleImpl = class {
|
|
|
1369
155
|
this.activeWorkflowRunsById.clear();
|
|
1370
156
|
}
|
|
1371
157
|
async poll(abortSignal) {
|
|
1372
|
-
if (!this.
|
|
1373
|
-
throw new Error("Subscriber
|
|
158
|
+
if (!this.subscriber) {
|
|
159
|
+
throw new Error("Subscriber not initialized");
|
|
1374
160
|
}
|
|
1375
161
|
this.logger.info("Worker started", {
|
|
1376
162
|
"aiki.registeredWorkflows": this.params.workflows.map((w) => `${w.name}:${w.versionId}`)
|
|
1377
163
|
});
|
|
1378
|
-
const maxConcurrentWorkflowRuns = this.
|
|
1379
|
-
let
|
|
164
|
+
const maxConcurrentWorkflowRuns = this.spawnOptions.maxConcurrentWorkflowRuns ?? 1;
|
|
165
|
+
let activeSubscriber = this.subscriber;
|
|
166
|
+
let nextDelayMs = activeSubscriber.getNextDelay({ type: "polled", foundWork: false });
|
|
1380
167
|
let subscriberFailedAttempts = 0;
|
|
1381
168
|
while (!abortSignal.aborted) {
|
|
1382
169
|
await delay(nextDelayMs, { abortSignal });
|
|
1383
170
|
const availableCapacity = maxConcurrentWorkflowRuns - this.activeWorkflowRunsById.size;
|
|
1384
171
|
if (availableCapacity <= 0) {
|
|
1385
|
-
nextDelayMs =
|
|
172
|
+
nextDelayMs = activeSubscriber.getNextDelay({ type: "at_capacity" });
|
|
1386
173
|
continue;
|
|
1387
174
|
}
|
|
1388
175
|
const nextBatchResponse = await this.fetchNextWorkflowRunBatch(availableCapacity, subscriberFailedAttempts);
|
|
1389
176
|
if (!nextBatchResponse.success) {
|
|
1390
177
|
subscriberFailedAttempts++;
|
|
1391
|
-
nextDelayMs =
|
|
178
|
+
nextDelayMs = activeSubscriber.getNextDelay({
|
|
1392
179
|
type: "retry",
|
|
1393
180
|
attemptNumber: subscriberFailedAttempts
|
|
1394
181
|
});
|
|
1395
182
|
continue;
|
|
1396
183
|
}
|
|
1397
184
|
subscriberFailedAttempts = 0;
|
|
185
|
+
activeSubscriber = nextBatchResponse.subscriber;
|
|
1398
186
|
if (!isNonEmptyArray(nextBatchResponse.batch)) {
|
|
1399
|
-
nextDelayMs =
|
|
187
|
+
nextDelayMs = activeSubscriber.getNextDelay({ type: "polled", foundWork: false });
|
|
1400
188
|
continue;
|
|
1401
189
|
}
|
|
1402
190
|
await this.enqueueWorkflowRunBatch(nextBatchResponse.batch, nextBatchResponse.subscriber, abortSignal);
|
|
1403
|
-
nextDelayMs =
|
|
191
|
+
nextDelayMs = activeSubscriber.getNextDelay({ type: "polled", foundWork: true });
|
|
1404
192
|
}
|
|
1405
193
|
}
|
|
1406
194
|
async fetchNextWorkflowRunBatch(size, subscriberFailedAttempts) {
|
|
1407
|
-
if (!this.
|
|
1408
|
-
|
|
1409
|
-
success: false,
|
|
1410
|
-
error: new Error("Subscriber strategy not initialized")
|
|
1411
|
-
};
|
|
195
|
+
if (!this.subscriber) {
|
|
196
|
+
throw new Error("Subscriber not initialized");
|
|
1412
197
|
}
|
|
1413
198
|
try {
|
|
1414
|
-
const batch = await this.
|
|
1415
|
-
return { success: true, batch, subscriber: this.
|
|
199
|
+
const batch = await this.subscriber.getNextBatch(size);
|
|
200
|
+
return { success: true, batch, subscriber: this.subscriber };
|
|
1416
201
|
} catch (error) {
|
|
1417
202
|
this.logger.error("Error getting next workflow runs batch", {
|
|
1418
203
|
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
1419
204
|
});
|
|
1420
|
-
if (this.
|
|
205
|
+
if (this.fallbackSubscriber && subscriberFailedAttempts >= 2) {
|
|
1421
206
|
try {
|
|
1422
|
-
const batch = await this.
|
|
1423
|
-
return { success: true, batch, subscriber: this.
|
|
207
|
+
const batch = await this.fallbackSubscriber.getNextBatch(size);
|
|
208
|
+
return { success: true, batch, subscriber: this.fallbackSubscriber };
|
|
1424
209
|
} catch (fallbackError) {
|
|
1425
|
-
this.logger.error("Fallback subscriber
|
|
210
|
+
this.logger.error("Fallback subscriber also failed to get next workflow runs batch", {
|
|
1426
211
|
"aiki.error": fallbackError instanceof Error ? fallbackError.message : String(fallbackError)
|
|
1427
212
|
});
|
|
1428
213
|
}
|
|
@@ -1439,8 +224,15 @@ var WorkerHandleImpl = class {
|
|
|
1439
224
|
});
|
|
1440
225
|
continue;
|
|
1441
226
|
}
|
|
1442
|
-
|
|
1443
|
-
|
|
227
|
+
let workflowRun;
|
|
228
|
+
try {
|
|
229
|
+
const response = await this.client.api.workflowRun.getByIdV1({ id: workflowRunId });
|
|
230
|
+
workflowRun = response.run;
|
|
231
|
+
} catch (error) {
|
|
232
|
+
this.logger.warn("Failed to fetch workflow run", {
|
|
233
|
+
"aiki.workflowRunId": workflowRunId,
|
|
234
|
+
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
235
|
+
});
|
|
1444
236
|
if (subscriber.acknowledge) {
|
|
1445
237
|
await subscriber.acknowledge(workflowRunId).catch(() => {
|
|
1446
238
|
});
|
|
@@ -1475,94 +267,49 @@ var WorkerHandleImpl = class {
|
|
|
1475
267
|
}
|
|
1476
268
|
async executeWorkflow(workflowRun, workflowVersion, subscriber) {
|
|
1477
269
|
const logger = this.logger.child({
|
|
1478
|
-
"aiki.component": "workflow-execution",
|
|
1479
270
|
"aiki.workflowName": workflowRun.name,
|
|
1480
271
|
"aiki.workflowVersionId": workflowRun.versionId,
|
|
1481
272
|
"aiki.workflowRunId": workflowRun.id
|
|
1482
273
|
});
|
|
1483
|
-
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
name: workflowRun.name,
|
|
1505
|
-
versionId: workflowRun.versionId,
|
|
1506
|
-
options: workflowRun.options ?? {},
|
|
1507
|
-
logger,
|
|
1508
|
-
sleep: createSleeper(handle, logger),
|
|
1509
|
-
events: createEventWaiters(handle, eventsDefinition, logger),
|
|
1510
|
-
[INTERNAL7]: {
|
|
1511
|
-
handle,
|
|
1512
|
-
replayManifest: createReplayManifest(workflowRun),
|
|
1513
|
-
options: { spinThresholdMs: this.workflowRunOpts.spinThresholdMs }
|
|
1514
|
-
}
|
|
1515
|
-
},
|
|
1516
|
-
workflowRun.input,
|
|
1517
|
-
appContext
|
|
1518
|
-
);
|
|
1519
|
-
shouldAcknowledge = true;
|
|
1520
|
-
} catch (error) {
|
|
1521
|
-
if (error instanceof WorkflowRunNotExecutableError2 || error instanceof WorkflowRunSuspendedError5 || error instanceof WorkflowRunFailedError4 || error instanceof WorkflowRunRevisionConflictError6 || error instanceof NonDeterminismError3) {
|
|
1522
|
-
shouldAcknowledge = true;
|
|
1523
|
-
} else {
|
|
1524
|
-
logger.error("Unexpected error during workflow execution", {
|
|
1525
|
-
"aiki.error": error instanceof Error ? error.message : String(error),
|
|
1526
|
-
"aiki.stack": error instanceof Error ? error.stack : void 0
|
|
1527
|
-
});
|
|
1528
|
-
shouldAcknowledge = false;
|
|
1529
|
-
}
|
|
1530
|
-
} finally {
|
|
1531
|
-
if (heartbeatInterval) clearInterval(heartbeatInterval);
|
|
1532
|
-
if (subscriber.acknowledge) {
|
|
1533
|
-
if (shouldAcknowledge) {
|
|
1534
|
-
try {
|
|
1535
|
-
await subscriber.acknowledge(workflowRun.id);
|
|
1536
|
-
} catch (error) {
|
|
1537
|
-
logger.error("Failed to acknowledge message, it may be reprocessed", {
|
|
1538
|
-
"aiki.errorType": "MESSAGE_ACK_FAILED",
|
|
1539
|
-
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
1540
|
-
});
|
|
1541
|
-
}
|
|
1542
|
-
} else {
|
|
1543
|
-
logger.debug("Message left in PEL for retry");
|
|
274
|
+
const heartbeat = subscriber.heartbeat;
|
|
275
|
+
const success = await executeWorkflowRun({
|
|
276
|
+
client: this.client,
|
|
277
|
+
workflowRun,
|
|
278
|
+
workflowVersion,
|
|
279
|
+
logger,
|
|
280
|
+
options: {
|
|
281
|
+
spinThresholdMs: this.workflowRunOptions.spinThresholdMs,
|
|
282
|
+
heartbeatIntervalMs: this.workflowRunOptions.heartbeatIntervalMs
|
|
283
|
+
},
|
|
284
|
+
heartbeat: heartbeat ? () => heartbeat(workflowRun.id) : void 0
|
|
285
|
+
});
|
|
286
|
+
if (subscriber.acknowledge) {
|
|
287
|
+
if (success) {
|
|
288
|
+
try {
|
|
289
|
+
await subscriber.acknowledge(workflowRun.id);
|
|
290
|
+
} catch (error) {
|
|
291
|
+
logger.error("Failed to acknowledge message, it may be reprocessed", {
|
|
292
|
+
"aiki.errorType": "MESSAGE_ACK_FAILED",
|
|
293
|
+
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
294
|
+
});
|
|
1544
295
|
}
|
|
296
|
+
} else {
|
|
297
|
+
logger.debug("Message left pending for retry");
|
|
1545
298
|
}
|
|
1546
|
-
this.activeWorkflowRunsById.delete(workflowRun.id);
|
|
1547
299
|
}
|
|
1548
|
-
|
|
1549
|
-
handleSubscriberError(error) {
|
|
1550
|
-
this.logger.warn("Subscriber error", {
|
|
1551
|
-
"aiki.error": error.message,
|
|
1552
|
-
"aiki.stack": error.stack
|
|
1553
|
-
});
|
|
300
|
+
this.activeWorkflowRunsById.delete(workflowRun.id);
|
|
1554
301
|
}
|
|
1555
302
|
};
|
|
1556
303
|
var WorkerBuilderImpl = class _WorkerBuilderImpl {
|
|
1557
|
-
constructor(worker2,
|
|
304
|
+
constructor(worker2, spawnOptionsBuilder) {
|
|
1558
305
|
this.worker = worker2;
|
|
1559
|
-
this.
|
|
306
|
+
this.spawnOptionsBuilder = spawnOptionsBuilder;
|
|
1560
307
|
}
|
|
1561
308
|
opt(path, value) {
|
|
1562
|
-
return new _WorkerBuilderImpl(this.worker, this.
|
|
309
|
+
return new _WorkerBuilderImpl(this.worker, this.spawnOptionsBuilder.with(path, value));
|
|
1563
310
|
}
|
|
1564
311
|
spawn(client) {
|
|
1565
|
-
return this.worker.
|
|
312
|
+
return this.worker.spawnWithOptions(client, this.spawnOptionsBuilder.build());
|
|
1566
313
|
}
|
|
1567
314
|
};
|
|
1568
315
|
export {
|