@aikirun/worker 0.23.1 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -4
- package/dist/index.d.ts +10 -27
- package/dist/index.js +132 -1286
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -52,125 +52,7 @@ var objectOverrider = (defaultObj) => (obj) => {
|
|
|
52
52
|
return createBuilder([]);
|
|
53
53
|
};
|
|
54
54
|
|
|
55
|
-
// worker.ts
|
|
56
|
-
import { INTERNAL as INTERNAL7 } from "@aikirun/types/symbols";
|
|
57
|
-
import {
|
|
58
|
-
NonDeterminismError as NonDeterminismError3,
|
|
59
|
-
WorkflowRunFailedError as WorkflowRunFailedError4,
|
|
60
|
-
WorkflowRunNotExecutableError as WorkflowRunNotExecutableError2,
|
|
61
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError6,
|
|
62
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError5
|
|
63
|
-
} from "@aikirun/types/workflow-run";
|
|
64
|
-
import {
|
|
65
|
-
createEventWaiters,
|
|
66
|
-
createReplayManifest,
|
|
67
|
-
createSleeper,
|
|
68
|
-
workflowRegistry,
|
|
69
|
-
workflowRunHandle as workflowRunHandle2
|
|
70
|
-
} from "@aikirun/workflow";
|
|
71
|
-
|
|
72
|
-
// ../workflow/system/cancel-child-runs.ts
|
|
73
|
-
import { NON_TERMINAL_WORKFLOW_RUN_STATUSES } from "@aikirun/types/workflow-run";
|
|
74
|
-
|
|
75
|
-
// ../../lib/address/index.ts
|
|
76
|
-
function getTaskAddress(name, inputHash) {
|
|
77
|
-
return `${name}:${inputHash}`;
|
|
78
|
-
}
|
|
79
|
-
function getWorkflowRunAddress(name, versionId, referenceId) {
|
|
80
|
-
return `${name}:${versionId}:${referenceId}`;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
// ../../lib/crypto/hash.ts
|
|
84
|
-
import { createHash } from "crypto";
|
|
85
|
-
|
|
86
|
-
// ../../lib/json/stable-stringify.ts
|
|
87
|
-
function stableStringify(value) {
|
|
88
|
-
return stringifyValue(value);
|
|
89
|
-
}
|
|
90
|
-
function stringifyValue(value) {
|
|
91
|
-
if (value === null || value === void 0) {
|
|
92
|
-
return "null";
|
|
93
|
-
}
|
|
94
|
-
if (typeof value !== "object") {
|
|
95
|
-
return JSON.stringify(value);
|
|
96
|
-
}
|
|
97
|
-
if (Array.isArray(value)) {
|
|
98
|
-
return `[${value.map(stringifyValue).join(",")}]`;
|
|
99
|
-
}
|
|
100
|
-
const keys = Object.keys(value).sort();
|
|
101
|
-
const pairs = [];
|
|
102
|
-
for (const key of keys) {
|
|
103
|
-
const keyValue = value[key];
|
|
104
|
-
if (keyValue !== void 0) {
|
|
105
|
-
pairs.push(`${JSON.stringify(key)}:${stringifyValue(keyValue)}`);
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
return `{${pairs.join(",")}}`;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
// ../../lib/crypto/hash.ts
|
|
112
|
-
async function sha256(input) {
|
|
113
|
-
const data = new TextEncoder().encode(input);
|
|
114
|
-
const hashBuffer = await crypto.subtle.digest("SHA-256", data);
|
|
115
|
-
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
|
116
|
-
return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
117
|
-
}
|
|
118
|
-
async function hashInput(input) {
|
|
119
|
-
return sha256(stableStringify({ input }));
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// ../../lib/error/serializable.ts
|
|
123
|
-
function createSerializableError(error) {
|
|
124
|
-
return error instanceof Error ? {
|
|
125
|
-
message: error.message,
|
|
126
|
-
name: error.name,
|
|
127
|
-
stack: error.stack,
|
|
128
|
-
cause: error.cause ? createSerializableError(error.cause) : void 0
|
|
129
|
-
} : {
|
|
130
|
-
message: String(error),
|
|
131
|
-
name: "UnknownError"
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
|
-
|
|
135
55
|
// ../../lib/retry/strategy.ts
|
|
136
|
-
function withRetry(fn, strategy, options) {
|
|
137
|
-
return {
|
|
138
|
-
run: async (...args) => {
|
|
139
|
-
let attempts = 0;
|
|
140
|
-
while (true) {
|
|
141
|
-
if (options?.abortSignal?.aborted) {
|
|
142
|
-
return {
|
|
143
|
-
state: "aborted",
|
|
144
|
-
reason: options.abortSignal.reason
|
|
145
|
-
};
|
|
146
|
-
}
|
|
147
|
-
attempts++;
|
|
148
|
-
let result;
|
|
149
|
-
try {
|
|
150
|
-
result = await fn(...args);
|
|
151
|
-
if (options?.shouldRetryOnResult === void 0 || !await options.shouldRetryOnResult(result)) {
|
|
152
|
-
return {
|
|
153
|
-
state: "completed",
|
|
154
|
-
result,
|
|
155
|
-
attempts
|
|
156
|
-
};
|
|
157
|
-
}
|
|
158
|
-
} catch (err) {
|
|
159
|
-
if (options?.shouldNotRetryOnError !== void 0 && await options.shouldNotRetryOnError(err)) {
|
|
160
|
-
throw err;
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
const retryParams = getRetryParams(attempts, strategy);
|
|
164
|
-
if (!retryParams.retriesLeft) {
|
|
165
|
-
return {
|
|
166
|
-
state: "timeout"
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
await delay(retryParams.delayMs, { abortSignal: options?.abortSignal });
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
56
|
function getRetryParams(attempts, strategy) {
|
|
175
57
|
const strategyType = strategy.type;
|
|
176
58
|
switch (strategyType) {
|
|
@@ -218,1049 +100,65 @@ function getRetryParams(attempts, strategy) {
|
|
|
218
100
|
}
|
|
219
101
|
}
|
|
220
102
|
|
|
221
|
-
// ../
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
const startOptsOverrider = objectOverrider(startOpts);
|
|
242
|
-
return new TaskBuilderImpl(this, startOptsOverrider());
|
|
243
|
-
}
|
|
244
|
-
async start(run, ...args) {
|
|
245
|
-
return this.startWithOpts(run, this.params.opts ?? {}, ...args);
|
|
246
|
-
}
|
|
247
|
-
async startWithOpts(run, startOpts, ...args) {
|
|
248
|
-
const handle = run[INTERNAL].handle;
|
|
249
|
-
handle[INTERNAL].assertExecutionAllowed();
|
|
250
|
-
const inputRaw = args[0];
|
|
251
|
-
const input = await this.parse(handle, this.params.schema?.input, inputRaw, run.logger);
|
|
252
|
-
const inputHash = await hashInput(input);
|
|
253
|
-
const address = getTaskAddress(this.name, inputHash);
|
|
254
|
-
const replayManifest = run[INTERNAL].replayManifest;
|
|
255
|
-
if (replayManifest.hasUnconsumedEntries()) {
|
|
256
|
-
const existingTaskInfo = replayManifest.consumeNextTask(address);
|
|
257
|
-
if (existingTaskInfo) {
|
|
258
|
-
return this.getExistingTaskResult(run, handle, startOpts, input, existingTaskInfo);
|
|
259
|
-
}
|
|
260
|
-
await this.throwNonDeterminismError(run, handle, inputHash, replayManifest);
|
|
261
|
-
}
|
|
262
|
-
const attempts = 1;
|
|
263
|
-
const retryStrategy = startOpts.retry ?? { type: "never" };
|
|
264
|
-
const taskInfo = await handle[INTERNAL].transitionTaskState({
|
|
265
|
-
type: "create",
|
|
266
|
-
taskName: this.name,
|
|
267
|
-
options: startOpts,
|
|
268
|
-
taskState: { status: "running", attempts, input }
|
|
269
|
-
});
|
|
270
|
-
const logger = run.logger.child({
|
|
271
|
-
"aiki.component": "task-execution",
|
|
272
|
-
"aiki.taskName": this.name,
|
|
273
|
-
"aiki.taskId": taskInfo.id
|
|
274
|
-
});
|
|
275
|
-
logger.info("Task started", { "aiki.attempts": attempts });
|
|
276
|
-
const { output, lastAttempt } = await this.tryExecuteTask(
|
|
277
|
-
handle,
|
|
278
|
-
input,
|
|
279
|
-
taskInfo.id,
|
|
280
|
-
retryStrategy,
|
|
281
|
-
attempts,
|
|
282
|
-
run[INTERNAL].options.spinThresholdMs,
|
|
283
|
-
logger
|
|
284
|
-
);
|
|
285
|
-
await handle[INTERNAL].transitionTaskState({
|
|
286
|
-
taskId: taskInfo.id,
|
|
287
|
-
taskState: { status: "completed", attempts: lastAttempt, output }
|
|
288
|
-
});
|
|
289
|
-
logger.info("Task complete", { "aiki.attempts": lastAttempt });
|
|
290
|
-
return output;
|
|
291
|
-
}
|
|
292
|
-
async getExistingTaskResult(run, handle, startOpts, input, existingTaskInfo) {
|
|
293
|
-
const existingTaskState = existingTaskInfo.state;
|
|
294
|
-
if (existingTaskState.status === "completed") {
|
|
295
|
-
return this.parse(handle, this.params.schema?.output, existingTaskState.output, run.logger);
|
|
296
|
-
}
|
|
297
|
-
if (existingTaskState.status === "failed") {
|
|
298
|
-
throw new TaskFailedError(
|
|
299
|
-
existingTaskInfo.id,
|
|
300
|
-
existingTaskState.attempts,
|
|
301
|
-
existingTaskState.error.message
|
|
302
|
-
);
|
|
303
|
-
}
|
|
304
|
-
existingTaskState.status;
|
|
305
|
-
const attempts = existingTaskState.attempts;
|
|
306
|
-
const retryStrategy = startOpts.retry ?? { type: "never" };
|
|
307
|
-
this.assertRetryAllowed(existingTaskInfo.id, attempts, retryStrategy, run.logger);
|
|
308
|
-
run.logger.debug("Retrying task", {
|
|
309
|
-
"aiki.taskName": this.name,
|
|
310
|
-
"aiki.taskId": existingTaskInfo.id,
|
|
311
|
-
"aiki.attempts": attempts,
|
|
312
|
-
"aiki.taskStatus": existingTaskState.status
|
|
313
|
-
});
|
|
314
|
-
return this.retryAndExecute(run, handle, input, existingTaskInfo.id, startOpts, retryStrategy, attempts);
|
|
315
|
-
}
|
|
316
|
-
async throwNonDeterminismError(run, handle, inputHash, manifest) {
|
|
317
|
-
const unconsumedManifestEntries = manifest.getUnconsumedEntries();
|
|
318
|
-
run.logger.error("Replay divergence", {
|
|
319
|
-
"aiki.taskName": this.name,
|
|
320
|
-
"aiki.inputHash": inputHash,
|
|
321
|
-
"aiki.unconsumedManifestEntries": unconsumedManifestEntries
|
|
322
|
-
});
|
|
323
|
-
const error = new NonDeterminismError(run.id, handle.run.attempts, unconsumedManifestEntries);
|
|
324
|
-
await handle[INTERNAL].transitionState({
|
|
325
|
-
status: "failed",
|
|
326
|
-
cause: "self",
|
|
327
|
-
error: createSerializableError(error)
|
|
328
|
-
});
|
|
329
|
-
throw error;
|
|
330
|
-
}
|
|
331
|
-
async retryAndExecute(run, handle, input, taskId, startOpts, retryStrategy, previousAttempts) {
|
|
332
|
-
const attempts = previousAttempts + 1;
|
|
333
|
-
const taskInfo = await handle[INTERNAL].transitionTaskState({
|
|
334
|
-
type: "retry",
|
|
335
|
-
taskId,
|
|
336
|
-
options: startOpts,
|
|
337
|
-
taskState: { status: "running", attempts, input }
|
|
338
|
-
});
|
|
339
|
-
const logger = run.logger.child({
|
|
340
|
-
"aiki.component": "task-execution",
|
|
341
|
-
"aiki.taskName": this.name,
|
|
342
|
-
"aiki.taskId": taskInfo.id
|
|
343
|
-
});
|
|
344
|
-
logger.info("Task started", { "aiki.attempts": attempts });
|
|
345
|
-
const { output, lastAttempt } = await this.tryExecuteTask(
|
|
346
|
-
handle,
|
|
347
|
-
input,
|
|
348
|
-
taskInfo.id,
|
|
349
|
-
retryStrategy,
|
|
350
|
-
attempts,
|
|
351
|
-
run[INTERNAL].options.spinThresholdMs,
|
|
352
|
-
logger
|
|
353
|
-
);
|
|
354
|
-
await handle[INTERNAL].transitionTaskState({
|
|
355
|
-
taskId: taskInfo.id,
|
|
356
|
-
taskState: { status: "completed", attempts: lastAttempt, output }
|
|
357
|
-
});
|
|
358
|
-
logger.info("Task complete", { "aiki.attempts": lastAttempt });
|
|
359
|
-
return output;
|
|
360
|
-
}
|
|
361
|
-
async tryExecuteTask(handle, input, taskId, retryStrategy, currentAttempt, spinThresholdMs, logger) {
|
|
362
|
-
let attempts = currentAttempt;
|
|
363
|
-
while (true) {
|
|
364
|
-
try {
|
|
365
|
-
const outputRaw = await this.params.handler(input);
|
|
366
|
-
const output = await this.parse(handle, this.params.schema?.output, outputRaw, logger);
|
|
367
|
-
return { output, lastAttempt: attempts };
|
|
368
|
-
} catch (error) {
|
|
369
|
-
if (error instanceof WorkflowRunSuspendedError || error instanceof WorkflowRunFailedError || error instanceof WorkflowRunRevisionConflictError) {
|
|
370
|
-
throw error;
|
|
371
|
-
}
|
|
372
|
-
const serializableError = createSerializableError(error);
|
|
373
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
374
|
-
if (!retryParams.retriesLeft) {
|
|
375
|
-
logger.error("Task failed", {
|
|
376
|
-
"aiki.attempts": attempts,
|
|
377
|
-
"aiki.reason": serializableError.message
|
|
378
|
-
});
|
|
379
|
-
await handle[INTERNAL].transitionTaskState({
|
|
380
|
-
taskId,
|
|
381
|
-
taskState: { status: "failed", attempts, error: serializableError }
|
|
382
|
-
});
|
|
383
|
-
throw new TaskFailedError(taskId, attempts, serializableError.message);
|
|
384
|
-
}
|
|
385
|
-
logger.debug("Task failed. It will be retried", {
|
|
386
|
-
"aiki.attempts": attempts,
|
|
387
|
-
"aiki.nextAttemptInMs": retryParams.delayMs,
|
|
388
|
-
"aiki.reason": serializableError.message
|
|
103
|
+
// ../subscriber/db/db-subscriber.ts
|
|
104
|
+
function dbSubscriber(params) {
|
|
105
|
+
const { api, options } = params;
|
|
106
|
+
const intervalMs = options?.intervalMs ?? 1e3;
|
|
107
|
+
const maxRetryIntervalMs = options?.maxRetryIntervalMs ?? 3e4;
|
|
108
|
+
const atCapacityIntervalMs = options?.atCapacityIntervalMs ?? 500;
|
|
109
|
+
const claimMinIdleTimeMs = options?.claimMinIdleTimeMs ?? 9e4;
|
|
110
|
+
const getNextDelay = (delayParams) => {
|
|
111
|
+
switch (delayParams.type) {
|
|
112
|
+
case "polled":
|
|
113
|
+
case "heartbeat":
|
|
114
|
+
return intervalMs;
|
|
115
|
+
case "at_capacity":
|
|
116
|
+
return atCapacityIntervalMs;
|
|
117
|
+
case "retry": {
|
|
118
|
+
const retryParams = getRetryParams(delayParams.attemptNumber, {
|
|
119
|
+
type: "jittered",
|
|
120
|
+
maxAttempts: Number.POSITIVE_INFINITY,
|
|
121
|
+
baseDelayMs: intervalMs,
|
|
122
|
+
maxDelayMs: maxRetryIntervalMs
|
|
389
123
|
});
|
|
390
|
-
if (retryParams.
|
|
391
|
-
|
|
392
|
-
attempts++;
|
|
393
|
-
continue;
|
|
124
|
+
if (!retryParams.retriesLeft) {
|
|
125
|
+
return maxRetryIntervalMs;
|
|
394
126
|
}
|
|
395
|
-
|
|
396
|
-
taskId,
|
|
397
|
-
taskState: {
|
|
398
|
-
status: "awaiting_retry",
|
|
399
|
-
attempts,
|
|
400
|
-
error: serializableError,
|
|
401
|
-
nextAttemptInMs: retryParams.delayMs
|
|
402
|
-
}
|
|
403
|
-
});
|
|
404
|
-
throw new WorkflowRunSuspendedError(handle.run.id);
|
|
127
|
+
return retryParams.delayMs;
|
|
405
128
|
}
|
|
129
|
+
default:
|
|
130
|
+
return delayParams;
|
|
406
131
|
}
|
|
407
|
-
}
|
|
408
|
-
assertRetryAllowed(taskId, attempts, retryStrategy, logger) {
|
|
409
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
410
|
-
if (!retryParams.retriesLeft) {
|
|
411
|
-
logger.error("Task retry not allowed", {
|
|
412
|
-
"aiki.taskName": this.name,
|
|
413
|
-
"aiki.taskId": taskId,
|
|
414
|
-
"aiki.attempts": attempts
|
|
415
|
-
});
|
|
416
|
-
throw new TaskFailedError(taskId, attempts, "Task retry not allowed");
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
async parse(handle, schema, data, logger) {
|
|
420
|
-
if (!schema) {
|
|
421
|
-
return data;
|
|
422
|
-
}
|
|
423
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
424
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
425
|
-
if (!schemaValidationResult.issues) {
|
|
426
|
-
return schemaValidationResult.value;
|
|
427
|
-
}
|
|
428
|
-
logger.error("Invalid task data", { "aiki.issues": schemaValidationResult.issues });
|
|
429
|
-
await handle[INTERNAL].transitionState({
|
|
430
|
-
status: "failed",
|
|
431
|
-
cause: "self",
|
|
432
|
-
error: {
|
|
433
|
-
name: "SchemaValidationError",
|
|
434
|
-
message: JSON.stringify(schemaValidationResult.issues)
|
|
435
|
-
}
|
|
436
|
-
});
|
|
437
|
-
throw new WorkflowRunFailedError(handle.run.id, handle.run.attempts);
|
|
438
|
-
}
|
|
439
|
-
};
|
|
440
|
-
var TaskBuilderImpl = class _TaskBuilderImpl {
|
|
441
|
-
constructor(task2, startOptsBuilder) {
|
|
442
|
-
this.task = task2;
|
|
443
|
-
this.startOptsBuilder = startOptsBuilder;
|
|
444
|
-
}
|
|
445
|
-
opt(path, value) {
|
|
446
|
-
return new _TaskBuilderImpl(this.task, this.startOptsBuilder.with(path, value));
|
|
447
|
-
}
|
|
448
|
-
start(run, ...args) {
|
|
449
|
-
return this.task.startWithOpts(run, this.startOptsBuilder.build(), ...args);
|
|
450
|
-
}
|
|
451
|
-
};
|
|
452
|
-
|
|
453
|
-
// ../workflow/workflow.ts
|
|
454
|
-
import { INTERNAL as INTERNAL6 } from "@aikirun/types/symbols";
|
|
455
|
-
|
|
456
|
-
// ../workflow/workflow-version.ts
|
|
457
|
-
import { INTERNAL as INTERNAL5 } from "@aikirun/types/symbols";
|
|
458
|
-
import { TaskFailedError as TaskFailedError2 } from "@aikirun/types/task";
|
|
459
|
-
import { SchemaValidationError as SchemaValidationError2 } from "@aikirun/types/validator";
|
|
460
|
-
import {
|
|
461
|
-
NonDeterminismError as NonDeterminismError2,
|
|
462
|
-
WorkflowRunFailedError as WorkflowRunFailedError3,
|
|
463
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError5,
|
|
464
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError4
|
|
465
|
-
} from "@aikirun/types/workflow-run";
|
|
466
|
-
|
|
467
|
-
// ../../lib/duration/convert.ts
|
|
468
|
-
var MS_PER_SECOND = 1e3;
|
|
469
|
-
var MS_PER_MINUTE = 60 * MS_PER_SECOND;
|
|
470
|
-
var MS_PER_HOUR = 60 * MS_PER_MINUTE;
|
|
471
|
-
var MS_PER_DAY = 24 * MS_PER_HOUR;
|
|
472
|
-
function toMilliseconds(duration) {
|
|
473
|
-
if (typeof duration === "number") {
|
|
474
|
-
assertIsPositiveNumber(duration);
|
|
475
|
-
return duration;
|
|
476
|
-
}
|
|
477
|
-
let totalMs = 0;
|
|
478
|
-
if (duration.days !== void 0) {
|
|
479
|
-
assertIsPositiveNumber(duration.days, "days");
|
|
480
|
-
totalMs += duration.days * MS_PER_DAY;
|
|
481
|
-
}
|
|
482
|
-
if (duration.hours !== void 0) {
|
|
483
|
-
assertIsPositiveNumber(duration.hours, "hours");
|
|
484
|
-
totalMs += duration.hours * MS_PER_HOUR;
|
|
485
|
-
}
|
|
486
|
-
if (duration.minutes !== void 0) {
|
|
487
|
-
assertIsPositiveNumber(duration.minutes, "minutes");
|
|
488
|
-
totalMs += duration.minutes * MS_PER_MINUTE;
|
|
489
|
-
}
|
|
490
|
-
if (duration.seconds !== void 0) {
|
|
491
|
-
assertIsPositiveNumber(duration.seconds, "seconds");
|
|
492
|
-
totalMs += duration.seconds * MS_PER_SECOND;
|
|
493
|
-
}
|
|
494
|
-
if (duration.milliseconds !== void 0) {
|
|
495
|
-
assertIsPositiveNumber(duration.milliseconds, "milliseconds");
|
|
496
|
-
totalMs += duration.milliseconds;
|
|
497
|
-
}
|
|
498
|
-
return totalMs;
|
|
499
|
-
}
|
|
500
|
-
function assertIsPositiveNumber(value, field) {
|
|
501
|
-
if (!Number.isFinite(value)) {
|
|
502
|
-
throw new Error(
|
|
503
|
-
field !== void 0 ? `'${field}' duration must be finite. Received: ${value}` : `Duration must be finite. Received: ${value}`
|
|
504
|
-
);
|
|
505
|
-
}
|
|
506
|
-
if (value < 0) {
|
|
507
|
-
throw new Error(
|
|
508
|
-
field !== void 0 ? `'${field}' duration must be non-negative. Received: ${value}` : `Duration must be non-negative. Received: ${value}`
|
|
509
|
-
);
|
|
510
|
-
}
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
// ../workflow/run/event.ts
|
|
514
|
-
import { INTERNAL as INTERNAL2 } from "@aikirun/types/symbols";
|
|
515
|
-
import { SchemaValidationError } from "@aikirun/types/validator";
|
|
516
|
-
import {
|
|
517
|
-
WorkflowRunFailedError as WorkflowRunFailedError2,
|
|
518
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError2,
|
|
519
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError2
|
|
520
|
-
} from "@aikirun/types/workflow-run";
|
|
521
|
-
function createEventSenders(api, workflowRunId, eventsDefinition, logger) {
|
|
522
|
-
const senders = {};
|
|
523
|
-
for (const [eventName, eventDefinition] of Object.entries(eventsDefinition)) {
|
|
524
|
-
const sender = createEventSender(
|
|
525
|
-
api,
|
|
526
|
-
workflowRunId,
|
|
527
|
-
eventName,
|
|
528
|
-
eventDefinition.schema,
|
|
529
|
-
logger.child({ "aiki.eventName": eventName })
|
|
530
|
-
);
|
|
531
|
-
senders[eventName] = sender;
|
|
532
|
-
}
|
|
533
|
-
return senders;
|
|
534
|
-
}
|
|
535
|
-
function createEventSender(api, workflowRunId, eventName, schema, logger, options) {
|
|
536
|
-
const optsOverrider = objectOverrider(options ?? {});
|
|
537
|
-
const createBuilder = (optsBuilder) => ({
|
|
538
|
-
opt: (path, value) => createBuilder(optsBuilder.with(path, value)),
|
|
539
|
-
send: (...args) => createEventSender(api, workflowRunId, eventName, schema, logger, optsBuilder.build()).send(...args)
|
|
540
|
-
});
|
|
541
|
-
async function send(...args) {
|
|
542
|
-
let data = args[0];
|
|
543
|
-
if (schema) {
|
|
544
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
545
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
546
|
-
if (schemaValidationResult.issues) {
|
|
547
|
-
logger.error("Invalid event data", { "aiki.issues": schemaValidationResult.issues });
|
|
548
|
-
throw new SchemaValidationError("Invalid event data", schemaValidationResult.issues);
|
|
549
|
-
}
|
|
550
|
-
data = schemaValidationResult.value;
|
|
551
|
-
}
|
|
552
|
-
await api.workflowRun.sendEventV1({
|
|
553
|
-
id: workflowRunId,
|
|
554
|
-
eventName,
|
|
555
|
-
data,
|
|
556
|
-
options
|
|
557
|
-
});
|
|
558
|
-
logger.info("Sent event to workflow", {
|
|
559
|
-
...options?.reference ? { "aiki.referenceId": options.reference.id } : {}
|
|
560
|
-
});
|
|
561
|
-
}
|
|
562
|
-
return {
|
|
563
|
-
with: () => createBuilder(optsOverrider()),
|
|
564
|
-
send
|
|
565
132
|
};
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
const sender = createEventMulticaster(
|
|
571
|
-
workflowName,
|
|
572
|
-
workflowVersionId,
|
|
573
|
-
eventName,
|
|
574
|
-
eventDefinition.schema
|
|
133
|
+
return (context) => {
|
|
134
|
+
const { workerId, workflows, shards } = context;
|
|
135
|
+
const workflowFilters = !isNonEmptyArray(shards) ? workflows.map((workflow) => ({ name: workflow.name, versionId: workflow.versionId })) : workflows.flatMap(
|
|
136
|
+
(workflow) => shards.map((shard) => ({ name: workflow.name, versionId: workflow.versionId, shard }))
|
|
575
137
|
);
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
send: (client, runId, ...args) => createEventMulticaster(workflowName, workflowVersionId, eventName, schema, optsBuilder.build()).send(
|
|
585
|
-
client,
|
|
586
|
-
runId,
|
|
587
|
-
...args
|
|
588
|
-
),
|
|
589
|
-
sendByReferenceId: (client, referenceId, ...args) => createEventMulticaster(workflowName, workflowVersionId, eventName, schema, optsBuilder.build()).sendByReferenceId(
|
|
590
|
-
client,
|
|
591
|
-
referenceId,
|
|
592
|
-
...args
|
|
593
|
-
)
|
|
594
|
-
});
|
|
595
|
-
async function send(client, runId, ...args) {
|
|
596
|
-
let data = args[0];
|
|
597
|
-
if (schema) {
|
|
598
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
599
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
600
|
-
if (schemaValidationResult.issues) {
|
|
601
|
-
client.logger.error("Invalid event data", {
|
|
602
|
-
"aiki.workflowName": workflowName,
|
|
603
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
604
|
-
"aiki.eventName": eventName,
|
|
605
|
-
"aiki.issues": schemaValidationResult.issues
|
|
606
|
-
});
|
|
607
|
-
throw new SchemaValidationError("Invalid event data", schemaValidationResult.issues);
|
|
608
|
-
}
|
|
609
|
-
data = schemaValidationResult.value;
|
|
610
|
-
}
|
|
611
|
-
const runIds = Array.isArray(runId) ? runId : [runId];
|
|
612
|
-
if (!isNonEmptyArray(runIds)) {
|
|
613
|
-
return;
|
|
614
|
-
}
|
|
615
|
-
await client.api.workflowRun.multicastEventV1({
|
|
616
|
-
ids: runIds,
|
|
617
|
-
eventName,
|
|
618
|
-
data,
|
|
619
|
-
options
|
|
620
|
-
});
|
|
621
|
-
client.logger.info("Multicasted event to workflows", {
|
|
622
|
-
"aiki.workflowName": workflowName,
|
|
623
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
624
|
-
"aiki.workflowRunIds": runIds,
|
|
625
|
-
"aiki.eventName": eventName,
|
|
626
|
-
...options?.reference ? { "aiki.eventReferenceId": options.reference.id } : {}
|
|
627
|
-
});
|
|
628
|
-
}
|
|
629
|
-
async function sendByReferenceId(client, referenceId, ...args) {
|
|
630
|
-
let data = args[0];
|
|
631
|
-
if (schema) {
|
|
632
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
633
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
634
|
-
if (schemaValidationResult.issues) {
|
|
635
|
-
client.logger.error("Invalid event data", {
|
|
636
|
-
"aiki.workflowName": workflowName,
|
|
637
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
638
|
-
"aiki.eventName": eventName,
|
|
639
|
-
"aiki.issues": schemaValidationResult.issues
|
|
640
|
-
});
|
|
641
|
-
throw new SchemaValidationError("Invalid event data", schemaValidationResult.issues);
|
|
642
|
-
}
|
|
643
|
-
data = schemaValidationResult.value;
|
|
644
|
-
}
|
|
645
|
-
const referenceIds = Array.isArray(referenceId) ? referenceId : [referenceId];
|
|
646
|
-
if (!isNonEmptyArray(referenceIds)) {
|
|
647
|
-
return;
|
|
648
|
-
}
|
|
649
|
-
await client.api.workflowRun.multicastEventByReferenceV1({
|
|
650
|
-
references: referenceIds.map((referenceId2) => ({
|
|
651
|
-
name: workflowName,
|
|
652
|
-
versionId: workflowVersionId,
|
|
653
|
-
referenceId: referenceId2
|
|
654
|
-
})),
|
|
655
|
-
eventName,
|
|
656
|
-
data,
|
|
657
|
-
options
|
|
658
|
-
});
|
|
659
|
-
client.logger.info("Multicasted event by reference", {
|
|
660
|
-
"aiki.workflowName": workflowName,
|
|
661
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
662
|
-
"aiki.referenceIds": referenceIds,
|
|
663
|
-
"aiki.eventName": eventName,
|
|
664
|
-
...options?.reference ? { "aiki.eventReferenceId": options.reference.id } : {}
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
|
-
return {
|
|
668
|
-
with: () => createBuilder(optsOverrider()),
|
|
669
|
-
send,
|
|
670
|
-
sendByReferenceId
|
|
671
|
-
};
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
// ../workflow/run/handle.ts
|
|
675
|
-
import { INTERNAL as INTERNAL3 } from "@aikirun/types/symbols";
|
|
676
|
-
import {
|
|
677
|
-
WorkflowRunNotExecutableError,
|
|
678
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError3
|
|
679
|
-
} from "@aikirun/types/workflow-run";
|
|
680
|
-
async function workflowRunHandle(client, runOrId, eventsDefinition, logger) {
|
|
681
|
-
const run = typeof runOrId !== "string" ? runOrId : (await client.api.workflowRun.getByIdV1({ id: runOrId })).run;
|
|
682
|
-
return new WorkflowRunHandleImpl(
|
|
683
|
-
client,
|
|
684
|
-
run,
|
|
685
|
-
eventsDefinition ?? {},
|
|
686
|
-
logger ?? client.logger.child({
|
|
687
|
-
"aiki.workflowName": run.name,
|
|
688
|
-
"aiki.workflowVersionId": run.versionId,
|
|
689
|
-
"aiki.workflowRunId": run.id
|
|
690
|
-
})
|
|
691
|
-
);
|
|
692
|
-
}
|
|
693
|
-
var WorkflowRunHandleImpl = class {
|
|
694
|
-
constructor(client, _run, eventsDefinition, logger) {
|
|
695
|
-
this._run = _run;
|
|
696
|
-
this.logger = logger;
|
|
697
|
-
this.api = client.api;
|
|
698
|
-
this.events = createEventSenders(client.api, this._run.id, eventsDefinition, this.logger);
|
|
699
|
-
this[INTERNAL3] = {
|
|
700
|
-
client,
|
|
701
|
-
transitionState: this.transitionState.bind(this),
|
|
702
|
-
transitionTaskState: this.transitionTaskState.bind(this),
|
|
703
|
-
assertExecutionAllowed: this.assertExecutionAllowed.bind(this)
|
|
704
|
-
};
|
|
705
|
-
}
|
|
706
|
-
api;
|
|
707
|
-
events;
|
|
708
|
-
[INTERNAL3];
|
|
709
|
-
get run() {
|
|
710
|
-
return this._run;
|
|
711
|
-
}
|
|
712
|
-
async refresh() {
|
|
713
|
-
const { run: currentRun } = await this.api.workflowRun.getByIdV1({ id: this.run.id });
|
|
714
|
-
this._run = currentRun;
|
|
715
|
-
}
|
|
716
|
-
async waitForStatus(status, options) {
|
|
717
|
-
return this.waitForStatusByPolling(status, options);
|
|
718
|
-
}
|
|
719
|
-
async waitForStatusByPolling(expectedStatus, options) {
|
|
720
|
-
if (options?.abortSignal?.aborted) {
|
|
721
|
-
return {
|
|
722
|
-
success: false,
|
|
723
|
-
cause: "aborted"
|
|
724
|
-
};
|
|
725
|
-
}
|
|
726
|
-
const delayMs = options?.interval ? toMilliseconds(options.interval) : 1e3;
|
|
727
|
-
const maxAttempts = options?.timeout ? Math.ceil(toMilliseconds(options.timeout) / delayMs) : Number.POSITIVE_INFINITY;
|
|
728
|
-
const retryStrategy = { type: "fixed", maxAttempts, delayMs };
|
|
729
|
-
let afterStateTransitionId = this._run.stateTransitionId;
|
|
730
|
-
const hasTerminated = async () => {
|
|
731
|
-
const { terminated, latestStateTransitionId } = await this.api.workflowRun.hasTerminatedV1({
|
|
732
|
-
id: this._run.id,
|
|
733
|
-
afterStateTransitionId
|
|
734
|
-
});
|
|
735
|
-
afterStateTransitionId = latestStateTransitionId;
|
|
736
|
-
return terminated;
|
|
737
|
-
};
|
|
738
|
-
const shouldRetryOnResult = async (terminated) => !terminated;
|
|
739
|
-
const maybeResult = options?.abortSignal ? await withRetry(hasTerminated, retryStrategy, {
|
|
740
|
-
abortSignal: options.abortSignal,
|
|
741
|
-
shouldRetryOnResult
|
|
742
|
-
}).run() : await withRetry(hasTerminated, retryStrategy, { shouldRetryOnResult }).run();
|
|
743
|
-
if (maybeResult.state === "timeout") {
|
|
744
|
-
if (!Number.isFinite(maxAttempts)) {
|
|
745
|
-
throw new Error("Something's wrong, this should've never timed out");
|
|
746
|
-
}
|
|
747
|
-
return { success: false, cause: "timeout" };
|
|
748
|
-
}
|
|
749
|
-
if (maybeResult.state === "aborted") {
|
|
750
|
-
return { success: false, cause: "aborted" };
|
|
751
|
-
}
|
|
752
|
-
maybeResult.state;
|
|
753
|
-
await this.refresh();
|
|
754
|
-
if (this._run.state.status === expectedStatus) {
|
|
755
|
-
return {
|
|
756
|
-
success: true,
|
|
757
|
-
state: this._run.state
|
|
758
|
-
};
|
|
759
|
-
}
|
|
760
|
-
return { success: false, cause: "run_terminated" };
|
|
761
|
-
}
|
|
762
|
-
async cancel(reason) {
|
|
763
|
-
await this.transitionState({ status: "cancelled", reason });
|
|
764
|
-
this.logger.info("Workflow cancelled");
|
|
765
|
-
}
|
|
766
|
-
async pause() {
|
|
767
|
-
await this.transitionState({ status: "paused" });
|
|
768
|
-
this.logger.info("Workflow paused");
|
|
769
|
-
}
|
|
770
|
-
async resume() {
|
|
771
|
-
await this.transitionState({ status: "scheduled", scheduledInMs: 0, reason: "resume" });
|
|
772
|
-
this.logger.info("Workflow resumed");
|
|
773
|
-
}
|
|
774
|
-
async awake() {
|
|
775
|
-
await this.transitionState({ status: "scheduled", scheduledInMs: 0, reason: "awake_early" });
|
|
776
|
-
this.logger.info("Workflow awoken");
|
|
777
|
-
}
|
|
778
|
-
async transitionState(targetState) {
|
|
779
|
-
try {
|
|
780
|
-
let response;
|
|
781
|
-
if (targetState.status === "scheduled" && (targetState.reason === "new" || targetState.reason === "resume" || targetState.reason === "awake_early") || targetState.status === "paused" || targetState.status === "cancelled") {
|
|
782
|
-
response = await this.api.workflowRun.transitionStateV1({
|
|
783
|
-
type: "pessimistic",
|
|
784
|
-
id: this.run.id,
|
|
785
|
-
state: targetState
|
|
786
|
-
});
|
|
787
|
-
} else {
|
|
788
|
-
response = await this.api.workflowRun.transitionStateV1({
|
|
789
|
-
type: "optimistic",
|
|
790
|
-
id: this.run.id,
|
|
791
|
-
state: targetState,
|
|
792
|
-
expectedRevision: this.run.revision
|
|
793
|
-
});
|
|
794
|
-
}
|
|
795
|
-
this._run.revision = response.revision;
|
|
796
|
-
this._run.state = response.state;
|
|
797
|
-
this._run.attempts = response.attempts;
|
|
798
|
-
} catch (error) {
|
|
799
|
-
if (isWorkflowRunRevisionConflictError(error)) {
|
|
800
|
-
throw new WorkflowRunRevisionConflictError3(this.run.id);
|
|
801
|
-
}
|
|
802
|
-
throw error;
|
|
803
|
-
}
|
|
804
|
-
}
|
|
805
|
-
async transitionTaskState(request) {
|
|
806
|
-
try {
|
|
807
|
-
const { taskInfo } = await this.api.workflowRun.transitionTaskStateV1({
|
|
808
|
-
...request,
|
|
809
|
-
id: this.run.id,
|
|
810
|
-
expectedWorkflowRunRevision: this.run.revision
|
|
811
|
-
});
|
|
812
|
-
return taskInfo;
|
|
813
|
-
} catch (error) {
|
|
814
|
-
if (isWorkflowRunRevisionConflictError(error)) {
|
|
815
|
-
throw new WorkflowRunRevisionConflictError3(this.run.id);
|
|
816
|
-
}
|
|
817
|
-
throw error;
|
|
818
|
-
}
|
|
819
|
-
}
|
|
820
|
-
assertExecutionAllowed() {
|
|
821
|
-
const status = this.run.state.status;
|
|
822
|
-
if (status !== "queued" && status !== "running") {
|
|
823
|
-
throw new WorkflowRunNotExecutableError(this.run.id, status);
|
|
824
|
-
}
|
|
825
|
-
}
|
|
826
|
-
};
|
|
827
|
-
function isWorkflowRunRevisionConflictError(error) {
|
|
828
|
-
return error != null && typeof error === "object" && "code" in error && error.code === "WORKFLOW_RUN_REVISION_CONFLICT";
|
|
829
|
-
}
|
|
830
|
-
|
|
831
|
-
// ../workflow/run/handle-child.ts
|
|
832
|
-
import { INTERNAL as INTERNAL4 } from "@aikirun/types/symbols";
|
|
833
|
-
import {
|
|
834
|
-
isTerminalWorkflowRunStatus,
|
|
835
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError4,
|
|
836
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError3
|
|
837
|
-
} from "@aikirun/types/workflow-run";
|
|
838
|
-
async function childWorkflowRunHandle(client, run, parentRun, childWorkflowRunWaitQueues, logger, eventsDefinition) {
|
|
839
|
-
const handle = await workflowRunHandle(client, run, eventsDefinition, logger);
|
|
840
|
-
return {
|
|
841
|
-
run: handle.run,
|
|
842
|
-
events: handle.events,
|
|
843
|
-
refresh: handle.refresh.bind(handle),
|
|
844
|
-
waitForStatus: createStatusWaiter(handle, parentRun, childWorkflowRunWaitQueues, logger),
|
|
845
|
-
cancel: handle.cancel.bind(handle),
|
|
846
|
-
pause: handle.pause.bind(handle),
|
|
847
|
-
resume: handle.resume.bind(handle),
|
|
848
|
-
awake: handle.awake.bind(handle),
|
|
849
|
-
[INTERNAL4]: handle[INTERNAL4]
|
|
850
|
-
};
|
|
851
|
-
}
|
|
852
|
-
function createStatusWaiter(handle, parentRun, childWorkflowRunWaitQueues, logger) {
|
|
853
|
-
const nextIndexByStatus = {
|
|
854
|
-
cancelled: 0,
|
|
855
|
-
completed: 0,
|
|
856
|
-
failed: 0
|
|
857
|
-
};
|
|
858
|
-
async function waitForStatus(expectedStatus, options) {
|
|
859
|
-
const parentRunHandle = parentRun[INTERNAL4].handle;
|
|
860
|
-
const nextIndex = nextIndexByStatus[expectedStatus];
|
|
861
|
-
const { run } = handle;
|
|
862
|
-
const childWorkflowRunWaits = childWorkflowRunWaitQueues[expectedStatus].childWorkflowRunWaits;
|
|
863
|
-
const existingChildWorkflowRunWait = childWorkflowRunWaits[nextIndex];
|
|
864
|
-
if (existingChildWorkflowRunWait) {
|
|
865
|
-
nextIndexByStatus[expectedStatus] = nextIndex + 1;
|
|
866
|
-
if (existingChildWorkflowRunWait.status === "timeout") {
|
|
867
|
-
logger.debug("Timed out waiting for child workflow status", {
|
|
868
|
-
"aiki.childWorkflowExpectedStatus": expectedStatus
|
|
869
|
-
});
|
|
870
|
-
return {
|
|
871
|
-
success: false,
|
|
872
|
-
cause: "timeout"
|
|
873
|
-
};
|
|
874
|
-
}
|
|
875
|
-
const childWorkflowRunStatus = existingChildWorkflowRunWait.childWorkflowRunState.status;
|
|
876
|
-
if (childWorkflowRunStatus === expectedStatus) {
|
|
877
|
-
return {
|
|
878
|
-
success: true,
|
|
879
|
-
state: existingChildWorkflowRunWait.childWorkflowRunState
|
|
880
|
-
};
|
|
881
|
-
}
|
|
882
|
-
if (isTerminalWorkflowRunStatus(childWorkflowRunStatus)) {
|
|
883
|
-
logger.debug("Child workflow run reached termnial state", {
|
|
884
|
-
"aiki.childWorkflowTerminalStatus": childWorkflowRunStatus
|
|
885
|
-
});
|
|
886
|
-
return {
|
|
887
|
-
success: false,
|
|
888
|
-
cause: "run_terminated"
|
|
889
|
-
};
|
|
890
|
-
}
|
|
891
|
-
childWorkflowRunStatus;
|
|
892
|
-
}
|
|
893
|
-
const timeoutInMs = options?.timeout && toMilliseconds(options.timeout);
|
|
894
|
-
try {
|
|
895
|
-
await parentRunHandle[INTERNAL4].transitionState({
|
|
896
|
-
status: "awaiting_child_workflow",
|
|
897
|
-
childWorkflowRunId: run.id,
|
|
898
|
-
childWorkflowRunStatus: expectedStatus,
|
|
899
|
-
timeoutInMs
|
|
900
|
-
});
|
|
901
|
-
logger.info("Waiting for child Workflow", {
|
|
902
|
-
"aiki.childWorkflowExpectedStatus": expectedStatus,
|
|
903
|
-
...timeoutInMs !== void 0 ? { "aiki.timeoutInMs": timeoutInMs } : {}
|
|
904
|
-
});
|
|
905
|
-
} catch (error) {
|
|
906
|
-
if (error instanceof WorkflowRunRevisionConflictError4) {
|
|
907
|
-
throw new WorkflowRunSuspendedError3(parentRun.id);
|
|
908
|
-
}
|
|
909
|
-
throw error;
|
|
910
|
-
}
|
|
911
|
-
throw new WorkflowRunSuspendedError3(parentRun.id);
|
|
912
|
-
}
|
|
913
|
-
return waitForStatus;
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
// ../workflow/workflow-version.ts
|
|
917
|
-
var WorkflowVersionImpl = class {
|
|
918
|
-
constructor(name, versionId, params) {
|
|
919
|
-
this.name = name;
|
|
920
|
-
this.versionId = versionId;
|
|
921
|
-
this.params = params;
|
|
922
|
-
const eventsDefinition = this.params.events ?? {};
|
|
923
|
-
this.events = createEventMulticasters(this.name, this.versionId, eventsDefinition);
|
|
924
|
-
this[INTERNAL5] = {
|
|
925
|
-
eventsDefinition,
|
|
926
|
-
handler: this.handler.bind(this)
|
|
927
|
-
};
|
|
928
|
-
}
|
|
929
|
-
events;
|
|
930
|
-
[INTERNAL5];
|
|
931
|
-
with() {
|
|
932
|
-
const startOpts = this.params.opts ?? {};
|
|
933
|
-
const startOptsOverrider = objectOverrider(startOpts);
|
|
934
|
-
return new WorkflowBuilderImpl(this, startOptsOverrider());
|
|
935
|
-
}
|
|
936
|
-
async start(client, ...args) {
|
|
937
|
-
return this.startWithOpts(client, this.params.opts ?? {}, ...args);
|
|
938
|
-
}
|
|
939
|
-
async startWithOpts(client, startOpts, ...args) {
|
|
940
|
-
let input = args[0];
|
|
941
|
-
const schema = this.params.schema?.input;
|
|
942
|
-
if (schema) {
|
|
943
|
-
const schemaValidation = schema["~standard"].validate(input);
|
|
944
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
945
|
-
if (schemaValidationResult.issues) {
|
|
946
|
-
client.logger.error("Invalid workflow data", { "aiki.issues": schemaValidationResult.issues });
|
|
947
|
-
throw new SchemaValidationError2("Invalid workflow data", schemaValidationResult.issues);
|
|
948
|
-
}
|
|
949
|
-
input = schemaValidationResult.value;
|
|
950
|
-
}
|
|
951
|
-
const { id } = await client.api.workflowRun.createV1({
|
|
952
|
-
name: this.name,
|
|
953
|
-
versionId: this.versionId,
|
|
954
|
-
input,
|
|
955
|
-
options: startOpts
|
|
956
|
-
});
|
|
957
|
-
client.logger.info("Created workflow", {
|
|
958
|
-
"aiki.workflowName": this.name,
|
|
959
|
-
"aiki.workflowVersionId": this.versionId,
|
|
960
|
-
"aiki.workflowRunId": id
|
|
961
|
-
});
|
|
962
|
-
return workflowRunHandle(client, id, this[INTERNAL5].eventsDefinition);
|
|
963
|
-
}
|
|
964
|
-
async startAsChild(parentRun, ...args) {
|
|
965
|
-
return this.startAsChildWithOpts(parentRun, this.params.opts ?? {}, ...args);
|
|
966
|
-
}
|
|
967
|
-
async startAsChildWithOpts(parentRun, startOpts, ...args) {
|
|
968
|
-
const parentRunHandle = parentRun[INTERNAL5].handle;
|
|
969
|
-
parentRunHandle[INTERNAL5].assertExecutionAllowed();
|
|
970
|
-
const { client } = parentRunHandle[INTERNAL5];
|
|
971
|
-
const inputRaw = args[0];
|
|
972
|
-
const input = await this.parse(parentRunHandle, this.params.schema?.input, inputRaw, parentRun.logger);
|
|
973
|
-
const inputHash = await hashInput(input);
|
|
974
|
-
const referenceId = startOpts.reference?.id;
|
|
975
|
-
const address = getWorkflowRunAddress(this.name, this.versionId, referenceId ?? inputHash);
|
|
976
|
-
const replayManifest = parentRun[INTERNAL5].replayManifest;
|
|
977
|
-
if (replayManifest.hasUnconsumedEntries()) {
|
|
978
|
-
const existingRunInfo = replayManifest.consumeNextChildWorkflowRun(address);
|
|
979
|
-
if (existingRunInfo) {
|
|
980
|
-
const { run: existingRun } = await client.api.workflowRun.getByIdV1({ id: existingRunInfo.id });
|
|
981
|
-
if (existingRun.state.status === "completed") {
|
|
982
|
-
await this.parse(parentRunHandle, this.params.schema?.output, existingRun.state.output, parentRun.logger);
|
|
983
|
-
}
|
|
984
|
-
const logger2 = parentRun.logger.child({
|
|
985
|
-
"aiki.childWorkflowName": existingRun.name,
|
|
986
|
-
"aiki.childWorkflowVersionId": existingRun.versionId,
|
|
987
|
-
"aiki.childWorkflowRunId": existingRun.id
|
|
138
|
+
return {
|
|
139
|
+
getNextDelay,
|
|
140
|
+
async getNextBatch(size) {
|
|
141
|
+
const response = await api.workflowRun.claimReadyV1({
|
|
142
|
+
workerId,
|
|
143
|
+
workflows: workflowFilters,
|
|
144
|
+
limit: size,
|
|
145
|
+
claimMinIdleTimeMs
|
|
988
146
|
});
|
|
989
|
-
return
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
parentRun,
|
|
993
|
-
existingRunInfo.childWorkflowRunWaitQueues,
|
|
994
|
-
logger2,
|
|
995
|
-
this[INTERNAL5].eventsDefinition
|
|
996
|
-
);
|
|
997
|
-
}
|
|
998
|
-
await this.throwNonDeterminismError(parentRun, parentRunHandle, inputHash, referenceId, replayManifest);
|
|
999
|
-
}
|
|
1000
|
-
const shard = parentRun.options.shard;
|
|
1001
|
-
const { id: newRunId } = await client.api.workflowRun.createV1({
|
|
1002
|
-
name: this.name,
|
|
1003
|
-
versionId: this.versionId,
|
|
1004
|
-
input,
|
|
1005
|
-
parentWorkflowRunId: parentRun.id,
|
|
1006
|
-
options: shard === void 0 ? startOpts : { ...startOpts, shard }
|
|
1007
|
-
});
|
|
1008
|
-
const { run: newRun } = await client.api.workflowRun.getByIdV1({ id: newRunId });
|
|
1009
|
-
const logger = parentRun.logger.child({
|
|
1010
|
-
"aiki.childWorkflowName": newRun.name,
|
|
1011
|
-
"aiki.childWorkflowVersionId": newRun.versionId,
|
|
1012
|
-
"aiki.childWorkflowRunId": newRun.id
|
|
1013
|
-
});
|
|
1014
|
-
logger.info("Created child workflow");
|
|
1015
|
-
return childWorkflowRunHandle(
|
|
1016
|
-
client,
|
|
1017
|
-
newRun,
|
|
1018
|
-
parentRun,
|
|
1019
|
-
{
|
|
1020
|
-
cancelled: { childWorkflowRunWaits: [] },
|
|
1021
|
-
completed: { childWorkflowRunWaits: [] },
|
|
1022
|
-
failed: { childWorkflowRunWaits: [] }
|
|
147
|
+
return response.runs.map((run) => ({
|
|
148
|
+
data: { workflowRunId: run.id }
|
|
149
|
+
}));
|
|
1023
150
|
},
|
|
1024
|
-
|
|
1025
|
-
this[INTERNAL5].eventsDefinition
|
|
1026
|
-
);
|
|
1027
|
-
}
|
|
1028
|
-
async throwNonDeterminismError(parentRun, parentRunHandle, inputHash, referenceId, manifest) {
|
|
1029
|
-
const unconsumedManifestEntries = manifest.getUnconsumedEntries();
|
|
1030
|
-
const logMeta = {
|
|
1031
|
-
"aiki.workflowName": this.name,
|
|
1032
|
-
"aiki.inputHash": inputHash,
|
|
1033
|
-
"aiki.unconsumedManifestEntries": unconsumedManifestEntries
|
|
1034
|
-
};
|
|
1035
|
-
if (referenceId !== void 0) {
|
|
1036
|
-
logMeta["aiki.referenceId"] = referenceId;
|
|
1037
|
-
}
|
|
1038
|
-
parentRun.logger.error("Replay divergence", logMeta);
|
|
1039
|
-
const error = new NonDeterminismError2(parentRun.id, parentRunHandle.run.attempts, unconsumedManifestEntries);
|
|
1040
|
-
await parentRunHandle[INTERNAL5].transitionState({
|
|
1041
|
-
status: "failed",
|
|
1042
|
-
cause: "self",
|
|
1043
|
-
error: createSerializableError(error)
|
|
1044
|
-
});
|
|
1045
|
-
throw error;
|
|
1046
|
-
}
|
|
1047
|
-
async getHandleById(client, runId) {
|
|
1048
|
-
return workflowRunHandle(client, runId, this[INTERNAL5].eventsDefinition);
|
|
1049
|
-
}
|
|
1050
|
-
async getHandleByReferenceId(client, referenceId) {
|
|
1051
|
-
const { run } = await client.api.workflowRun.getByReferenceIdV1({
|
|
1052
|
-
name: this.name,
|
|
1053
|
-
versionId: this.versionId,
|
|
1054
|
-
referenceId
|
|
1055
|
-
});
|
|
1056
|
-
return workflowRunHandle(client, run, this[INTERNAL5].eventsDefinition);
|
|
1057
|
-
}
|
|
1058
|
-
async handler(run, input, context) {
|
|
1059
|
-
const { logger } = run;
|
|
1060
|
-
const { handle } = run[INTERNAL5];
|
|
1061
|
-
handle[INTERNAL5].assertExecutionAllowed();
|
|
1062
|
-
const retryStrategy = this.params.opts?.retry ?? { type: "never" };
|
|
1063
|
-
const state = handle.run.state;
|
|
1064
|
-
if (state.status === "queued" && state.reason === "retry") {
|
|
1065
|
-
await this.assertRetryAllowed(handle, retryStrategy, logger);
|
|
1066
|
-
}
|
|
1067
|
-
logger.info("Starting workflow");
|
|
1068
|
-
await handle[INTERNAL5].transitionState({ status: "running" });
|
|
1069
|
-
const output = await this.tryExecuteWorkflow(input, run, context, retryStrategy);
|
|
1070
|
-
await handle[INTERNAL5].transitionState({ status: "completed", output });
|
|
1071
|
-
logger.info("Workflow complete");
|
|
1072
|
-
}
|
|
1073
|
-
async tryExecuteWorkflow(input, run, context, retryStrategy) {
|
|
1074
|
-
const { handle } = run[INTERNAL5];
|
|
1075
|
-
while (true) {
|
|
1076
|
-
try {
|
|
1077
|
-
const outputRaw = await this.params.handler(run, input, context);
|
|
1078
|
-
const output = await this.parse(handle, this.params.schema?.output, outputRaw, run.logger);
|
|
1079
|
-
return output;
|
|
1080
|
-
} catch (error) {
|
|
1081
|
-
if (error instanceof WorkflowRunSuspendedError4 || error instanceof WorkflowRunFailedError3 || error instanceof WorkflowRunRevisionConflictError5 || error instanceof NonDeterminismError2) {
|
|
1082
|
-
throw error;
|
|
1083
|
-
}
|
|
1084
|
-
const attempts = handle.run.attempts;
|
|
1085
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
1086
|
-
if (!retryParams.retriesLeft) {
|
|
1087
|
-
const failedState = this.createFailedState(error);
|
|
1088
|
-
await handle[INTERNAL5].transitionState(failedState);
|
|
1089
|
-
const logMeta2 = {};
|
|
1090
|
-
for (const [key, value] of Object.entries(failedState)) {
|
|
1091
|
-
logMeta2[`aiki.${key}`] = value;
|
|
1092
|
-
}
|
|
1093
|
-
run.logger.error("Workflow failed", {
|
|
1094
|
-
"aiki.attempts": attempts,
|
|
1095
|
-
...logMeta2
|
|
1096
|
-
});
|
|
1097
|
-
throw new WorkflowRunFailedError3(run.id, attempts);
|
|
1098
|
-
}
|
|
1099
|
-
const awaitingRetryState = this.createAwaitingRetryState(error, retryParams.delayMs);
|
|
1100
|
-
await handle[INTERNAL5].transitionState(awaitingRetryState);
|
|
1101
|
-
const logMeta = {};
|
|
1102
|
-
for (const [key, value] of Object.entries(awaitingRetryState)) {
|
|
1103
|
-
logMeta[`aiki.${key}`] = value;
|
|
1104
|
-
}
|
|
1105
|
-
run.logger.info("Workflow awaiting retry", {
|
|
1106
|
-
"aiki.attempts": attempts,
|
|
1107
|
-
...logMeta
|
|
1108
|
-
});
|
|
1109
|
-
throw new WorkflowRunSuspendedError4(run.id);
|
|
1110
|
-
}
|
|
1111
|
-
}
|
|
1112
|
-
}
|
|
1113
|
-
async assertRetryAllowed(handle, retryStrategy, logger) {
|
|
1114
|
-
const { id, attempts } = handle.run;
|
|
1115
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
1116
|
-
if (!retryParams.retriesLeft) {
|
|
1117
|
-
logger.error("Workflow retry not allowed", { "aiki.attempts": attempts });
|
|
1118
|
-
const error = new WorkflowRunFailedError3(id, attempts);
|
|
1119
|
-
await handle[INTERNAL5].transitionState({
|
|
1120
|
-
status: "failed",
|
|
1121
|
-
cause: "self",
|
|
1122
|
-
error: createSerializableError(error)
|
|
1123
|
-
});
|
|
1124
|
-
throw error;
|
|
1125
|
-
}
|
|
1126
|
-
}
|
|
1127
|
-
async parse(handle, schema, data, logger) {
|
|
1128
|
-
if (!schema) {
|
|
1129
|
-
return data;
|
|
1130
|
-
}
|
|
1131
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
1132
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
1133
|
-
if (!schemaValidationResult.issues) {
|
|
1134
|
-
return schemaValidationResult.value;
|
|
1135
|
-
}
|
|
1136
|
-
logger.error("Invalid workflow data", { "aiki.issues": schemaValidationResult.issues });
|
|
1137
|
-
await handle[INTERNAL5].transitionState({
|
|
1138
|
-
status: "failed",
|
|
1139
|
-
cause: "self",
|
|
1140
|
-
error: {
|
|
1141
|
-
name: "SchemaValidationError",
|
|
1142
|
-
message: JSON.stringify(schemaValidationResult.issues)
|
|
1143
|
-
}
|
|
1144
|
-
});
|
|
1145
|
-
throw new WorkflowRunFailedError3(handle.run.id, handle.run.attempts);
|
|
1146
|
-
}
|
|
1147
|
-
createFailedState(error) {
|
|
1148
|
-
if (error instanceof TaskFailedError2) {
|
|
1149
|
-
return {
|
|
1150
|
-
status: "failed",
|
|
1151
|
-
cause: "task",
|
|
1152
|
-
taskId: error.taskId
|
|
1153
|
-
};
|
|
1154
|
-
}
|
|
1155
|
-
return {
|
|
1156
|
-
status: "failed",
|
|
1157
|
-
cause: "self",
|
|
1158
|
-
error: createSerializableError(error)
|
|
1159
|
-
};
|
|
1160
|
-
}
|
|
1161
|
-
createAwaitingRetryState(error, nextAttemptInMs) {
|
|
1162
|
-
if (error instanceof TaskFailedError2) {
|
|
1163
|
-
return {
|
|
1164
|
-
status: "awaiting_retry",
|
|
1165
|
-
cause: "task",
|
|
1166
|
-
nextAttemptInMs,
|
|
1167
|
-
taskId: error.taskId
|
|
1168
|
-
};
|
|
1169
|
-
}
|
|
1170
|
-
return {
|
|
1171
|
-
status: "awaiting_retry",
|
|
1172
|
-
cause: "self",
|
|
1173
|
-
nextAttemptInMs,
|
|
1174
|
-
error: createSerializableError(error)
|
|
1175
|
-
};
|
|
1176
|
-
}
|
|
1177
|
-
};
|
|
1178
|
-
var WorkflowBuilderImpl = class _WorkflowBuilderImpl {
|
|
1179
|
-
constructor(workflow2, startOptsBuilder) {
|
|
1180
|
-
this.workflow = workflow2;
|
|
1181
|
-
this.startOptsBuilder = startOptsBuilder;
|
|
1182
|
-
}
|
|
1183
|
-
opt(path, value) {
|
|
1184
|
-
return new _WorkflowBuilderImpl(this.workflow, this.startOptsBuilder.with(path, value));
|
|
1185
|
-
}
|
|
1186
|
-
start(client, ...args) {
|
|
1187
|
-
return this.workflow.startWithOpts(client, this.startOptsBuilder.build(), ...args);
|
|
1188
|
-
}
|
|
1189
|
-
startAsChild(parentRun, ...args) {
|
|
1190
|
-
return this.workflow.startAsChildWithOpts(parentRun, this.startOptsBuilder.build(), ...args);
|
|
1191
|
-
}
|
|
1192
|
-
};
|
|
1193
|
-
|
|
1194
|
-
// ../workflow/workflow.ts
|
|
1195
|
-
function workflow(params) {
|
|
1196
|
-
return new WorkflowImpl(params);
|
|
1197
|
-
}
|
|
1198
|
-
var WorkflowImpl = class {
|
|
1199
|
-
name;
|
|
1200
|
-
[INTERNAL6];
|
|
1201
|
-
workflowVersions = /* @__PURE__ */ new Map();
|
|
1202
|
-
constructor(params) {
|
|
1203
|
-
this.name = params.name;
|
|
1204
|
-
this[INTERNAL6] = {
|
|
1205
|
-
getAllVersions: this.getAllVersions.bind(this),
|
|
1206
|
-
getVersion: this.getVersion.bind(this)
|
|
151
|
+
heartbeat: (workflowRunId) => api.workflowRun.heartbeatV1({ id: workflowRunId })
|
|
1207
152
|
};
|
|
1208
|
-
}
|
|
1209
|
-
v(versionId, params) {
|
|
1210
|
-
if (this.workflowVersions.has(versionId)) {
|
|
1211
|
-
throw new Error(`Workflow "${this.name}:${versionId}" already exists`);
|
|
1212
|
-
}
|
|
1213
|
-
const workflowVersion = new WorkflowVersionImpl(this.name, versionId, params);
|
|
1214
|
-
this.workflowVersions.set(
|
|
1215
|
-
versionId,
|
|
1216
|
-
workflowVersion
|
|
1217
|
-
);
|
|
1218
|
-
return workflowVersion;
|
|
1219
|
-
}
|
|
1220
|
-
getAllVersions() {
|
|
1221
|
-
return Array.from(this.workflowVersions.values());
|
|
1222
|
-
}
|
|
1223
|
-
getVersion(versionId) {
|
|
1224
|
-
return this.workflowVersions.get(versionId);
|
|
1225
|
-
}
|
|
1226
|
-
};
|
|
1227
|
-
|
|
1228
|
-
// ../workflow/system/cancel-child-runs.ts
|
|
1229
|
-
var createCancelChildRunsV1 = (api) => {
|
|
1230
|
-
const listNonTerminalChildRuns = task({
|
|
1231
|
-
name: "aiki:list-non-terminal-child-runs",
|
|
1232
|
-
async handler(parentRunId) {
|
|
1233
|
-
const { runs } = await api.workflowRun.listChildRunsV1({
|
|
1234
|
-
parentRunId,
|
|
1235
|
-
status: NON_TERMINAL_WORKFLOW_RUN_STATUSES
|
|
1236
|
-
});
|
|
1237
|
-
return runs.map((r) => r.id);
|
|
1238
|
-
}
|
|
1239
|
-
});
|
|
1240
|
-
const cancelRuns = task({
|
|
1241
|
-
name: "aiki:cancel-runs",
|
|
1242
|
-
async handler(runIds) {
|
|
1243
|
-
const { cancelledIds } = await api.workflowRun.cancelByIdsV1({ ids: runIds });
|
|
1244
|
-
return cancelledIds;
|
|
1245
|
-
}
|
|
1246
|
-
});
|
|
1247
|
-
return workflow({ name: "aiki:cancel-child-runs" }).v("1.0.0", {
|
|
1248
|
-
async handler(run, parentRunId) {
|
|
1249
|
-
const childRunIds = await listNonTerminalChildRuns.start(run, parentRunId);
|
|
1250
|
-
if (!isNonEmptyArray(childRunIds)) {
|
|
1251
|
-
return;
|
|
1252
|
-
}
|
|
1253
|
-
await cancelRuns.start(run, childRunIds);
|
|
1254
|
-
}
|
|
1255
|
-
});
|
|
1256
|
-
};
|
|
1257
|
-
|
|
1258
|
-
// ../workflow/system/index.ts
|
|
1259
|
-
function getSystemWorkflows(api) {
|
|
1260
|
-
return [createCancelChildRunsV1(api)];
|
|
153
|
+
};
|
|
1261
154
|
}
|
|
1262
155
|
|
|
1263
156
|
// worker.ts
|
|
157
|
+
import {
|
|
158
|
+
executeWorkflowRun,
|
|
159
|
+
getSystemWorkflows,
|
|
160
|
+
workflowRegistry
|
|
161
|
+
} from "@aikirun/workflow";
|
|
1264
162
|
import { ulid } from "ulidx";
|
|
1265
163
|
function worker(params) {
|
|
1266
164
|
return new WorkerImpl(params);
|
|
@@ -1268,76 +166,61 @@ function worker(params) {
|
|
|
1268
166
|
var WorkerImpl = class {
|
|
1269
167
|
constructor(params) {
|
|
1270
168
|
this.params = params;
|
|
1271
|
-
this.name = params.name;
|
|
1272
169
|
}
|
|
1273
|
-
name;
|
|
1274
170
|
with() {
|
|
1275
|
-
const
|
|
1276
|
-
const
|
|
1277
|
-
return new WorkerBuilderImpl(this,
|
|
171
|
+
const spawnOptions = this.params.options ?? {};
|
|
172
|
+
const spawnOptionsOverrider = objectOverrider(spawnOptions);
|
|
173
|
+
return new WorkerBuilderImpl(this, spawnOptionsOverrider());
|
|
1278
174
|
}
|
|
1279
175
|
async spawn(client) {
|
|
1280
|
-
return this.
|
|
176
|
+
return this.spawnWithOptions(client, this.params.options ?? {});
|
|
1281
177
|
}
|
|
1282
|
-
async
|
|
1283
|
-
const handle = new WorkerHandleImpl(client, this.params,
|
|
178
|
+
async spawnWithOptions(client, spawnOptions) {
|
|
179
|
+
const handle = new WorkerHandleImpl(client, this.params, spawnOptions);
|
|
1284
180
|
await handle._start();
|
|
1285
181
|
return handle;
|
|
1286
182
|
}
|
|
1287
183
|
};
|
|
1288
184
|
var WorkerHandleImpl = class {
|
|
1289
|
-
constructor(client, params,
|
|
185
|
+
constructor(client, params, spawnOptions) {
|
|
1290
186
|
this.client = client;
|
|
1291
187
|
this.params = params;
|
|
1292
|
-
this.
|
|
188
|
+
this.spawnOptions = spawnOptions;
|
|
1293
189
|
this.id = ulid();
|
|
1294
|
-
this.
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
spinThresholdMs: this.spawnOpts.workflowRun?.spinThresholdMs ?? 10
|
|
190
|
+
this.workflowRunOptions = {
|
|
191
|
+
heartbeatIntervalMs: this.spawnOptions.workflowRun?.heartbeatIntervalMs ?? 3e4,
|
|
192
|
+
spinThresholdMs: this.spawnOptions.workflowRun?.spinThresholdMs ?? 10
|
|
1298
193
|
};
|
|
1299
194
|
this.registry = workflowRegistry().addMany(getSystemWorkflows(client.api)).addMany(this.params.workflows);
|
|
1300
|
-
const reference = this.
|
|
195
|
+
const reference = this.spawnOptions.reference;
|
|
1301
196
|
this.logger = client.logger.child({
|
|
1302
197
|
"aiki.component": "worker",
|
|
1303
198
|
"aiki.workerId": this.id,
|
|
1304
|
-
"aiki.workerName": this.name,
|
|
1305
199
|
...reference && { "aiki.workerReferenceId": reference.id }
|
|
1306
200
|
});
|
|
1307
201
|
}
|
|
1308
202
|
id;
|
|
1309
|
-
|
|
1310
|
-
workflowRunOpts;
|
|
203
|
+
workflowRunOptions;
|
|
1311
204
|
registry;
|
|
1312
205
|
logger;
|
|
1313
206
|
abortController;
|
|
1314
|
-
|
|
1315
|
-
|
|
207
|
+
subscriber;
|
|
208
|
+
fallbackSubscriber;
|
|
1316
209
|
pollPromise;
|
|
1317
210
|
activeWorkflowRunsById = /* @__PURE__ */ new Map();
|
|
1318
211
|
async _start() {
|
|
1319
|
-
const
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
);
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
const fallbackSubscriberStrategyBuilder = this.client[INTERNAL7].subscriber.create(
|
|
1332
|
-
{ type: "db" },
|
|
1333
|
-
workflows,
|
|
1334
|
-
this.spawnOpts.shards
|
|
1335
|
-
);
|
|
1336
|
-
this.fallbackSubscriberStrategy = await fallbackSubscriberStrategyBuilder.init(this.id, {
|
|
1337
|
-
onError: (error) => this.handleSubscriberError(error),
|
|
1338
|
-
onStop: () => this.stop()
|
|
1339
|
-
});
|
|
1340
|
-
}
|
|
212
|
+
const subscriberContext = {
|
|
213
|
+
workerId: this.id,
|
|
214
|
+
workflows: this.registry.getAll(),
|
|
215
|
+
shards: this.spawnOptions.shards,
|
|
216
|
+
logger: this.logger
|
|
217
|
+
};
|
|
218
|
+
const createSubscriber = this.params.subscriber ?? dbSubscriber({ api: this.client.api });
|
|
219
|
+
const subscriber = createSubscriber(subscriberContext);
|
|
220
|
+
this.subscriber = subscriber instanceof Promise ? await subscriber : subscriber;
|
|
221
|
+
const createFallbackSubscriber = dbSubscriber({ api: this.client.api });
|
|
222
|
+
const fallbackSubscriber = createFallbackSubscriber(subscriberContext);
|
|
223
|
+
this.fallbackSubscriber = fallbackSubscriber instanceof Promise ? await fallbackSubscriber : fallbackSubscriber;
|
|
1341
224
|
this.abortController = new AbortController();
|
|
1342
225
|
const abortSignal = this.abortController.signal;
|
|
1343
226
|
this.pollPromise = this.poll(abortSignal).catch((error) => {
|
|
@@ -1352,11 +235,13 @@ var WorkerHandleImpl = class {
|
|
|
1352
235
|
this.logger.info("Worker stopping");
|
|
1353
236
|
this.abortController?.abort();
|
|
1354
237
|
await this.pollPromise;
|
|
238
|
+
await this.subscriber?.close?.();
|
|
239
|
+
await this.fallbackSubscriber?.close?.();
|
|
1355
240
|
const activeWorkflowRuns = Array.from(this.activeWorkflowRunsById.values());
|
|
1356
241
|
if (activeWorkflowRuns.length === 0) {
|
|
1357
242
|
return;
|
|
1358
243
|
}
|
|
1359
|
-
const timeoutMs = this.
|
|
244
|
+
const timeoutMs = this.spawnOptions.gracefulShutdownTimeoutMs ?? 5e3;
|
|
1360
245
|
if (timeoutMs > 0) {
|
|
1361
246
|
await Promise.race([Promise.allSettled(activeWorkflowRuns.map((w) => w.executionPromise)), delay(timeoutMs)]);
|
|
1362
247
|
}
|
|
@@ -1370,60 +255,59 @@ var WorkerHandleImpl = class {
|
|
|
1370
255
|
this.activeWorkflowRunsById.clear();
|
|
1371
256
|
}
|
|
1372
257
|
async poll(abortSignal) {
|
|
1373
|
-
if (!this.
|
|
1374
|
-
throw new Error("Subscriber
|
|
258
|
+
if (!this.subscriber) {
|
|
259
|
+
throw new Error("Subscriber not initialized");
|
|
1375
260
|
}
|
|
1376
261
|
this.logger.info("Worker started", {
|
|
1377
262
|
"aiki.registeredWorkflows": this.params.workflows.map((w) => `${w.name}:${w.versionId}`)
|
|
1378
263
|
});
|
|
1379
|
-
const maxConcurrentWorkflowRuns = this.
|
|
1380
|
-
let
|
|
264
|
+
const maxConcurrentWorkflowRuns = this.spawnOptions.maxConcurrentWorkflowRuns ?? 1;
|
|
265
|
+
let activeSubscriber = this.subscriber;
|
|
266
|
+
let nextDelayMs = activeSubscriber.getNextDelay({ type: "polled", foundWork: false });
|
|
1381
267
|
let subscriberFailedAttempts = 0;
|
|
1382
268
|
while (!abortSignal.aborted) {
|
|
1383
269
|
await delay(nextDelayMs, { abortSignal });
|
|
1384
270
|
const availableCapacity = maxConcurrentWorkflowRuns - this.activeWorkflowRunsById.size;
|
|
1385
271
|
if (availableCapacity <= 0) {
|
|
1386
|
-
nextDelayMs =
|
|
272
|
+
nextDelayMs = activeSubscriber.getNextDelay({ type: "at_capacity" });
|
|
1387
273
|
continue;
|
|
1388
274
|
}
|
|
1389
275
|
const nextBatchResponse = await this.fetchNextWorkflowRunBatch(availableCapacity, subscriberFailedAttempts);
|
|
1390
276
|
if (!nextBatchResponse.success) {
|
|
1391
277
|
subscriberFailedAttempts++;
|
|
1392
|
-
nextDelayMs =
|
|
278
|
+
nextDelayMs = activeSubscriber.getNextDelay({
|
|
1393
279
|
type: "retry",
|
|
1394
280
|
attemptNumber: subscriberFailedAttempts
|
|
1395
281
|
});
|
|
1396
282
|
continue;
|
|
1397
283
|
}
|
|
1398
284
|
subscriberFailedAttempts = 0;
|
|
285
|
+
activeSubscriber = nextBatchResponse.subscriber;
|
|
1399
286
|
if (!isNonEmptyArray(nextBatchResponse.batch)) {
|
|
1400
|
-
nextDelayMs =
|
|
287
|
+
nextDelayMs = activeSubscriber.getNextDelay({ type: "polled", foundWork: false });
|
|
1401
288
|
continue;
|
|
1402
289
|
}
|
|
1403
290
|
await this.enqueueWorkflowRunBatch(nextBatchResponse.batch, nextBatchResponse.subscriber, abortSignal);
|
|
1404
|
-
nextDelayMs =
|
|
291
|
+
nextDelayMs = activeSubscriber.getNextDelay({ type: "polled", foundWork: true });
|
|
1405
292
|
}
|
|
1406
293
|
}
|
|
1407
294
|
async fetchNextWorkflowRunBatch(size, subscriberFailedAttempts) {
|
|
1408
|
-
if (!this.
|
|
1409
|
-
|
|
1410
|
-
success: false,
|
|
1411
|
-
error: new Error("Subscriber strategy not initialized")
|
|
1412
|
-
};
|
|
295
|
+
if (!this.subscriber) {
|
|
296
|
+
throw new Error("Subscriber not initialized");
|
|
1413
297
|
}
|
|
1414
298
|
try {
|
|
1415
|
-
const batch = await this.
|
|
1416
|
-
return { success: true, batch, subscriber: this.
|
|
299
|
+
const batch = await this.subscriber.getNextBatch(size);
|
|
300
|
+
return { success: true, batch, subscriber: this.subscriber };
|
|
1417
301
|
} catch (error) {
|
|
1418
302
|
this.logger.error("Error getting next workflow runs batch", {
|
|
1419
303
|
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
1420
304
|
});
|
|
1421
|
-
if (this.
|
|
305
|
+
if (this.fallbackSubscriber && subscriberFailedAttempts >= 2) {
|
|
1422
306
|
try {
|
|
1423
|
-
const batch = await this.
|
|
1424
|
-
return { success: true, batch, subscriber: this.
|
|
307
|
+
const batch = await this.fallbackSubscriber.getNextBatch(size);
|
|
308
|
+
return { success: true, batch, subscriber: this.fallbackSubscriber };
|
|
1425
309
|
} catch (fallbackError) {
|
|
1426
|
-
this.logger.error("Fallback subscriber
|
|
310
|
+
this.logger.error("Fallback subscriber also failed to get next workflow runs batch", {
|
|
1427
311
|
"aiki.error": fallbackError instanceof Error ? fallbackError.message : String(fallbackError)
|
|
1428
312
|
});
|
|
1429
313
|
}
|
|
@@ -1440,8 +324,15 @@ var WorkerHandleImpl = class {
|
|
|
1440
324
|
});
|
|
1441
325
|
continue;
|
|
1442
326
|
}
|
|
1443
|
-
|
|
1444
|
-
|
|
327
|
+
let workflowRun;
|
|
328
|
+
try {
|
|
329
|
+
const response = await this.client.api.workflowRun.getByIdV1({ id: workflowRunId });
|
|
330
|
+
workflowRun = response.run;
|
|
331
|
+
} catch (error) {
|
|
332
|
+
this.logger.warn("Failed to fetch workflow run", {
|
|
333
|
+
"aiki.workflowRunId": workflowRunId,
|
|
334
|
+
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
335
|
+
});
|
|
1445
336
|
if (subscriber.acknowledge) {
|
|
1446
337
|
await subscriber.acknowledge(workflowRunId).catch(() => {
|
|
1447
338
|
});
|
|
@@ -1476,94 +367,49 @@ var WorkerHandleImpl = class {
|
|
|
1476
367
|
}
|
|
1477
368
|
async executeWorkflow(workflowRun, workflowVersion, subscriber) {
|
|
1478
369
|
const logger = this.logger.child({
|
|
1479
|
-
"aiki.component": "workflow-execution",
|
|
1480
370
|
"aiki.workflowName": workflowRun.name,
|
|
1481
371
|
"aiki.workflowVersionId": workflowRun.versionId,
|
|
1482
372
|
"aiki.workflowRunId": workflowRun.id
|
|
1483
373
|
});
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
name: workflowRun.name,
|
|
1506
|
-
versionId: workflowRun.versionId,
|
|
1507
|
-
options: workflowRun.options ?? {},
|
|
1508
|
-
logger,
|
|
1509
|
-
sleep: createSleeper(handle, logger),
|
|
1510
|
-
events: createEventWaiters(handle, eventsDefinition, logger),
|
|
1511
|
-
[INTERNAL7]: {
|
|
1512
|
-
handle,
|
|
1513
|
-
replayManifest: createReplayManifest(workflowRun),
|
|
1514
|
-
options: { spinThresholdMs: this.workflowRunOpts.spinThresholdMs }
|
|
1515
|
-
}
|
|
1516
|
-
},
|
|
1517
|
-
workflowRun.input,
|
|
1518
|
-
appContext
|
|
1519
|
-
);
|
|
1520
|
-
shouldAcknowledge = true;
|
|
1521
|
-
} catch (error) {
|
|
1522
|
-
if (error instanceof WorkflowRunNotExecutableError2 || error instanceof WorkflowRunSuspendedError5 || error instanceof WorkflowRunFailedError4 || error instanceof WorkflowRunRevisionConflictError6 || error instanceof NonDeterminismError3) {
|
|
1523
|
-
shouldAcknowledge = true;
|
|
1524
|
-
} else {
|
|
1525
|
-
logger.error("Unexpected error during workflow execution", {
|
|
1526
|
-
"aiki.error": error instanceof Error ? error.message : String(error),
|
|
1527
|
-
"aiki.stack": error instanceof Error ? error.stack : void 0
|
|
1528
|
-
});
|
|
1529
|
-
shouldAcknowledge = false;
|
|
1530
|
-
}
|
|
1531
|
-
} finally {
|
|
1532
|
-
if (heartbeatInterval) clearInterval(heartbeatInterval);
|
|
1533
|
-
if (subscriber.acknowledge) {
|
|
1534
|
-
if (shouldAcknowledge) {
|
|
1535
|
-
try {
|
|
1536
|
-
await subscriber.acknowledge(workflowRun.id);
|
|
1537
|
-
} catch (error) {
|
|
1538
|
-
logger.error("Failed to acknowledge message, it may be reprocessed", {
|
|
1539
|
-
"aiki.errorType": "MESSAGE_ACK_FAILED",
|
|
1540
|
-
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
1541
|
-
});
|
|
1542
|
-
}
|
|
1543
|
-
} else {
|
|
1544
|
-
logger.debug("Message left in PEL for retry");
|
|
374
|
+
const heartbeat = subscriber.heartbeat;
|
|
375
|
+
const success = await executeWorkflowRun({
|
|
376
|
+
client: this.client,
|
|
377
|
+
workflowRun,
|
|
378
|
+
workflowVersion,
|
|
379
|
+
logger,
|
|
380
|
+
options: {
|
|
381
|
+
spinThresholdMs: this.workflowRunOptions.spinThresholdMs,
|
|
382
|
+
heartbeatIntervalMs: this.workflowRunOptions.heartbeatIntervalMs
|
|
383
|
+
},
|
|
384
|
+
heartbeat: heartbeat ? () => heartbeat(workflowRun.id) : void 0
|
|
385
|
+
});
|
|
386
|
+
if (subscriber.acknowledge) {
|
|
387
|
+
if (success) {
|
|
388
|
+
try {
|
|
389
|
+
await subscriber.acknowledge(workflowRun.id);
|
|
390
|
+
} catch (error) {
|
|
391
|
+
logger.error("Failed to acknowledge message, it may be reprocessed", {
|
|
392
|
+
"aiki.errorType": "MESSAGE_ACK_FAILED",
|
|
393
|
+
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
394
|
+
});
|
|
1545
395
|
}
|
|
396
|
+
} else {
|
|
397
|
+
logger.debug("Message left pending for retry");
|
|
1546
398
|
}
|
|
1547
|
-
this.activeWorkflowRunsById.delete(workflowRun.id);
|
|
1548
399
|
}
|
|
1549
|
-
|
|
1550
|
-
handleSubscriberError(error) {
|
|
1551
|
-
this.logger.warn("Subscriber error", {
|
|
1552
|
-
"aiki.error": error.message,
|
|
1553
|
-
"aiki.stack": error.stack
|
|
1554
|
-
});
|
|
400
|
+
this.activeWorkflowRunsById.delete(workflowRun.id);
|
|
1555
401
|
}
|
|
1556
402
|
};
|
|
1557
403
|
var WorkerBuilderImpl = class _WorkerBuilderImpl {
|
|
1558
|
-
constructor(worker2,
|
|
404
|
+
constructor(worker2, spawnOptionsBuilder) {
|
|
1559
405
|
this.worker = worker2;
|
|
1560
|
-
this.
|
|
406
|
+
this.spawnOptionsBuilder = spawnOptionsBuilder;
|
|
1561
407
|
}
|
|
1562
408
|
opt(path, value) {
|
|
1563
|
-
return new _WorkerBuilderImpl(this.worker, this.
|
|
409
|
+
return new _WorkerBuilderImpl(this.worker, this.spawnOptionsBuilder.with(path, value));
|
|
1564
410
|
}
|
|
1565
411
|
spawn(client) {
|
|
1566
|
-
return this.worker.
|
|
412
|
+
return this.worker.spawnWithOptions(client, this.spawnOptionsBuilder.build());
|
|
1567
413
|
}
|
|
1568
414
|
};
|
|
1569
415
|
export {
|