@aikirun/worker 0.23.1 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -4
- package/dist/index.d.ts +10 -27
- package/dist/index.js +89 -1343
- package/package.json +5 -4
package/dist/index.js
CHANGED
|
@@ -53,1214 +53,12 @@ var objectOverrider = (defaultObj) => (obj) => {
|
|
|
53
53
|
};
|
|
54
54
|
|
|
55
55
|
// worker.ts
|
|
56
|
-
import {
|
|
56
|
+
import { dbSubscriber } from "@aikirun/subscriber-db";
|
|
57
57
|
import {
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError6,
|
|
62
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError5
|
|
63
|
-
} from "@aikirun/types/workflow-run";
|
|
64
|
-
import {
|
|
65
|
-
createEventWaiters,
|
|
66
|
-
createReplayManifest,
|
|
67
|
-
createSleeper,
|
|
68
|
-
workflowRegistry,
|
|
69
|
-
workflowRunHandle as workflowRunHandle2
|
|
58
|
+
executeWorkflowRun,
|
|
59
|
+
getSystemWorkflows,
|
|
60
|
+
workflowRegistry
|
|
70
61
|
} from "@aikirun/workflow";
|
|
71
|
-
|
|
72
|
-
// ../workflow/system/cancel-child-runs.ts
|
|
73
|
-
import { NON_TERMINAL_WORKFLOW_RUN_STATUSES } from "@aikirun/types/workflow-run";
|
|
74
|
-
|
|
75
|
-
// ../../lib/address/index.ts
|
|
76
|
-
function getTaskAddress(name, inputHash) {
|
|
77
|
-
return `${name}:${inputHash}`;
|
|
78
|
-
}
|
|
79
|
-
function getWorkflowRunAddress(name, versionId, referenceId) {
|
|
80
|
-
return `${name}:${versionId}:${referenceId}`;
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
// ../../lib/crypto/hash.ts
|
|
84
|
-
import { createHash } from "crypto";
|
|
85
|
-
|
|
86
|
-
// ../../lib/json/stable-stringify.ts
|
|
87
|
-
function stableStringify(value) {
|
|
88
|
-
return stringifyValue(value);
|
|
89
|
-
}
|
|
90
|
-
function stringifyValue(value) {
|
|
91
|
-
if (value === null || value === void 0) {
|
|
92
|
-
return "null";
|
|
93
|
-
}
|
|
94
|
-
if (typeof value !== "object") {
|
|
95
|
-
return JSON.stringify(value);
|
|
96
|
-
}
|
|
97
|
-
if (Array.isArray(value)) {
|
|
98
|
-
return `[${value.map(stringifyValue).join(",")}]`;
|
|
99
|
-
}
|
|
100
|
-
const keys = Object.keys(value).sort();
|
|
101
|
-
const pairs = [];
|
|
102
|
-
for (const key of keys) {
|
|
103
|
-
const keyValue = value[key];
|
|
104
|
-
if (keyValue !== void 0) {
|
|
105
|
-
pairs.push(`${JSON.stringify(key)}:${stringifyValue(keyValue)}`);
|
|
106
|
-
}
|
|
107
|
-
}
|
|
108
|
-
return `{${pairs.join(",")}}`;
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
// ../../lib/crypto/hash.ts
|
|
112
|
-
async function sha256(input) {
|
|
113
|
-
const data = new TextEncoder().encode(input);
|
|
114
|
-
const hashBuffer = await crypto.subtle.digest("SHA-256", data);
|
|
115
|
-
const hashArray = Array.from(new Uint8Array(hashBuffer));
|
|
116
|
-
return hashArray.map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
117
|
-
}
|
|
118
|
-
async function hashInput(input) {
|
|
119
|
-
return sha256(stableStringify({ input }));
|
|
120
|
-
}
|
|
121
|
-
|
|
122
|
-
// ../../lib/error/serializable.ts
|
|
123
|
-
function createSerializableError(error) {
|
|
124
|
-
return error instanceof Error ? {
|
|
125
|
-
message: error.message,
|
|
126
|
-
name: error.name,
|
|
127
|
-
stack: error.stack,
|
|
128
|
-
cause: error.cause ? createSerializableError(error.cause) : void 0
|
|
129
|
-
} : {
|
|
130
|
-
message: String(error),
|
|
131
|
-
name: "UnknownError"
|
|
132
|
-
};
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
// ../../lib/retry/strategy.ts
|
|
136
|
-
function withRetry(fn, strategy, options) {
|
|
137
|
-
return {
|
|
138
|
-
run: async (...args) => {
|
|
139
|
-
let attempts = 0;
|
|
140
|
-
while (true) {
|
|
141
|
-
if (options?.abortSignal?.aborted) {
|
|
142
|
-
return {
|
|
143
|
-
state: "aborted",
|
|
144
|
-
reason: options.abortSignal.reason
|
|
145
|
-
};
|
|
146
|
-
}
|
|
147
|
-
attempts++;
|
|
148
|
-
let result;
|
|
149
|
-
try {
|
|
150
|
-
result = await fn(...args);
|
|
151
|
-
if (options?.shouldRetryOnResult === void 0 || !await options.shouldRetryOnResult(result)) {
|
|
152
|
-
return {
|
|
153
|
-
state: "completed",
|
|
154
|
-
result,
|
|
155
|
-
attempts
|
|
156
|
-
};
|
|
157
|
-
}
|
|
158
|
-
} catch (err) {
|
|
159
|
-
if (options?.shouldNotRetryOnError !== void 0 && await options.shouldNotRetryOnError(err)) {
|
|
160
|
-
throw err;
|
|
161
|
-
}
|
|
162
|
-
}
|
|
163
|
-
const retryParams = getRetryParams(attempts, strategy);
|
|
164
|
-
if (!retryParams.retriesLeft) {
|
|
165
|
-
return {
|
|
166
|
-
state: "timeout"
|
|
167
|
-
};
|
|
168
|
-
}
|
|
169
|
-
await delay(retryParams.delayMs, { abortSignal: options?.abortSignal });
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
};
|
|
173
|
-
}
|
|
174
|
-
function getRetryParams(attempts, strategy) {
|
|
175
|
-
const strategyType = strategy.type;
|
|
176
|
-
switch (strategyType) {
|
|
177
|
-
case "never":
|
|
178
|
-
return {
|
|
179
|
-
retriesLeft: false
|
|
180
|
-
};
|
|
181
|
-
case "fixed":
|
|
182
|
-
if (attempts >= strategy.maxAttempts) {
|
|
183
|
-
return {
|
|
184
|
-
retriesLeft: false
|
|
185
|
-
};
|
|
186
|
-
}
|
|
187
|
-
return {
|
|
188
|
-
retriesLeft: true,
|
|
189
|
-
delayMs: strategy.delayMs
|
|
190
|
-
};
|
|
191
|
-
case "exponential": {
|
|
192
|
-
if (attempts >= strategy.maxAttempts) {
|
|
193
|
-
return {
|
|
194
|
-
retriesLeft: false
|
|
195
|
-
};
|
|
196
|
-
}
|
|
197
|
-
const delayMs = strategy.baseDelayMs * (strategy.factor ?? 2) ** (attempts - 1);
|
|
198
|
-
return {
|
|
199
|
-
retriesLeft: true,
|
|
200
|
-
delayMs: Math.min(delayMs, strategy.maxDelayMs ?? Number.POSITIVE_INFINITY)
|
|
201
|
-
};
|
|
202
|
-
}
|
|
203
|
-
case "jittered": {
|
|
204
|
-
if (attempts >= strategy.maxAttempts) {
|
|
205
|
-
return {
|
|
206
|
-
retriesLeft: false
|
|
207
|
-
};
|
|
208
|
-
}
|
|
209
|
-
const base = strategy.baseDelayMs * (strategy.jitterFactor ?? 2) ** (attempts - 1);
|
|
210
|
-
const delayMs = Math.random() * base;
|
|
211
|
-
return {
|
|
212
|
-
retriesLeft: true,
|
|
213
|
-
delayMs: Math.min(delayMs, strategy.maxDelayMs ?? Number.POSITIVE_INFINITY)
|
|
214
|
-
};
|
|
215
|
-
}
|
|
216
|
-
default:
|
|
217
|
-
return strategyType;
|
|
218
|
-
}
|
|
219
|
-
}
|
|
220
|
-
|
|
221
|
-
// ../task/task.ts
|
|
222
|
-
import { INTERNAL } from "@aikirun/types/symbols";
|
|
223
|
-
import { TaskFailedError } from "@aikirun/types/task";
|
|
224
|
-
import {
|
|
225
|
-
NonDeterminismError,
|
|
226
|
-
WorkflowRunFailedError,
|
|
227
|
-
WorkflowRunRevisionConflictError,
|
|
228
|
-
WorkflowRunSuspendedError
|
|
229
|
-
} from "@aikirun/types/workflow-run";
|
|
230
|
-
function task(params) {
|
|
231
|
-
return new TaskImpl(params);
|
|
232
|
-
}
|
|
233
|
-
var TaskImpl = class {
|
|
234
|
-
constructor(params) {
|
|
235
|
-
this.params = params;
|
|
236
|
-
this.name = params.name;
|
|
237
|
-
}
|
|
238
|
-
name;
|
|
239
|
-
with() {
|
|
240
|
-
const startOpts = this.params.opts ?? {};
|
|
241
|
-
const startOptsOverrider = objectOverrider(startOpts);
|
|
242
|
-
return new TaskBuilderImpl(this, startOptsOverrider());
|
|
243
|
-
}
|
|
244
|
-
async start(run, ...args) {
|
|
245
|
-
return this.startWithOpts(run, this.params.opts ?? {}, ...args);
|
|
246
|
-
}
|
|
247
|
-
async startWithOpts(run, startOpts, ...args) {
|
|
248
|
-
const handle = run[INTERNAL].handle;
|
|
249
|
-
handle[INTERNAL].assertExecutionAllowed();
|
|
250
|
-
const inputRaw = args[0];
|
|
251
|
-
const input = await this.parse(handle, this.params.schema?.input, inputRaw, run.logger);
|
|
252
|
-
const inputHash = await hashInput(input);
|
|
253
|
-
const address = getTaskAddress(this.name, inputHash);
|
|
254
|
-
const replayManifest = run[INTERNAL].replayManifest;
|
|
255
|
-
if (replayManifest.hasUnconsumedEntries()) {
|
|
256
|
-
const existingTaskInfo = replayManifest.consumeNextTask(address);
|
|
257
|
-
if (existingTaskInfo) {
|
|
258
|
-
return this.getExistingTaskResult(run, handle, startOpts, input, existingTaskInfo);
|
|
259
|
-
}
|
|
260
|
-
await this.throwNonDeterminismError(run, handle, inputHash, replayManifest);
|
|
261
|
-
}
|
|
262
|
-
const attempts = 1;
|
|
263
|
-
const retryStrategy = startOpts.retry ?? { type: "never" };
|
|
264
|
-
const taskInfo = await handle[INTERNAL].transitionTaskState({
|
|
265
|
-
type: "create",
|
|
266
|
-
taskName: this.name,
|
|
267
|
-
options: startOpts,
|
|
268
|
-
taskState: { status: "running", attempts, input }
|
|
269
|
-
});
|
|
270
|
-
const logger = run.logger.child({
|
|
271
|
-
"aiki.component": "task-execution",
|
|
272
|
-
"aiki.taskName": this.name,
|
|
273
|
-
"aiki.taskId": taskInfo.id
|
|
274
|
-
});
|
|
275
|
-
logger.info("Task started", { "aiki.attempts": attempts });
|
|
276
|
-
const { output, lastAttempt } = await this.tryExecuteTask(
|
|
277
|
-
handle,
|
|
278
|
-
input,
|
|
279
|
-
taskInfo.id,
|
|
280
|
-
retryStrategy,
|
|
281
|
-
attempts,
|
|
282
|
-
run[INTERNAL].options.spinThresholdMs,
|
|
283
|
-
logger
|
|
284
|
-
);
|
|
285
|
-
await handle[INTERNAL].transitionTaskState({
|
|
286
|
-
taskId: taskInfo.id,
|
|
287
|
-
taskState: { status: "completed", attempts: lastAttempt, output }
|
|
288
|
-
});
|
|
289
|
-
logger.info("Task complete", { "aiki.attempts": lastAttempt });
|
|
290
|
-
return output;
|
|
291
|
-
}
|
|
292
|
-
async getExistingTaskResult(run, handle, startOpts, input, existingTaskInfo) {
|
|
293
|
-
const existingTaskState = existingTaskInfo.state;
|
|
294
|
-
if (existingTaskState.status === "completed") {
|
|
295
|
-
return this.parse(handle, this.params.schema?.output, existingTaskState.output, run.logger);
|
|
296
|
-
}
|
|
297
|
-
if (existingTaskState.status === "failed") {
|
|
298
|
-
throw new TaskFailedError(
|
|
299
|
-
existingTaskInfo.id,
|
|
300
|
-
existingTaskState.attempts,
|
|
301
|
-
existingTaskState.error.message
|
|
302
|
-
);
|
|
303
|
-
}
|
|
304
|
-
existingTaskState.status;
|
|
305
|
-
const attempts = existingTaskState.attempts;
|
|
306
|
-
const retryStrategy = startOpts.retry ?? { type: "never" };
|
|
307
|
-
this.assertRetryAllowed(existingTaskInfo.id, attempts, retryStrategy, run.logger);
|
|
308
|
-
run.logger.debug("Retrying task", {
|
|
309
|
-
"aiki.taskName": this.name,
|
|
310
|
-
"aiki.taskId": existingTaskInfo.id,
|
|
311
|
-
"aiki.attempts": attempts,
|
|
312
|
-
"aiki.taskStatus": existingTaskState.status
|
|
313
|
-
});
|
|
314
|
-
return this.retryAndExecute(run, handle, input, existingTaskInfo.id, startOpts, retryStrategy, attempts);
|
|
315
|
-
}
|
|
316
|
-
async throwNonDeterminismError(run, handle, inputHash, manifest) {
|
|
317
|
-
const unconsumedManifestEntries = manifest.getUnconsumedEntries();
|
|
318
|
-
run.logger.error("Replay divergence", {
|
|
319
|
-
"aiki.taskName": this.name,
|
|
320
|
-
"aiki.inputHash": inputHash,
|
|
321
|
-
"aiki.unconsumedManifestEntries": unconsumedManifestEntries
|
|
322
|
-
});
|
|
323
|
-
const error = new NonDeterminismError(run.id, handle.run.attempts, unconsumedManifestEntries);
|
|
324
|
-
await handle[INTERNAL].transitionState({
|
|
325
|
-
status: "failed",
|
|
326
|
-
cause: "self",
|
|
327
|
-
error: createSerializableError(error)
|
|
328
|
-
});
|
|
329
|
-
throw error;
|
|
330
|
-
}
|
|
331
|
-
async retryAndExecute(run, handle, input, taskId, startOpts, retryStrategy, previousAttempts) {
|
|
332
|
-
const attempts = previousAttempts + 1;
|
|
333
|
-
const taskInfo = await handle[INTERNAL].transitionTaskState({
|
|
334
|
-
type: "retry",
|
|
335
|
-
taskId,
|
|
336
|
-
options: startOpts,
|
|
337
|
-
taskState: { status: "running", attempts, input }
|
|
338
|
-
});
|
|
339
|
-
const logger = run.logger.child({
|
|
340
|
-
"aiki.component": "task-execution",
|
|
341
|
-
"aiki.taskName": this.name,
|
|
342
|
-
"aiki.taskId": taskInfo.id
|
|
343
|
-
});
|
|
344
|
-
logger.info("Task started", { "aiki.attempts": attempts });
|
|
345
|
-
const { output, lastAttempt } = await this.tryExecuteTask(
|
|
346
|
-
handle,
|
|
347
|
-
input,
|
|
348
|
-
taskInfo.id,
|
|
349
|
-
retryStrategy,
|
|
350
|
-
attempts,
|
|
351
|
-
run[INTERNAL].options.spinThresholdMs,
|
|
352
|
-
logger
|
|
353
|
-
);
|
|
354
|
-
await handle[INTERNAL].transitionTaskState({
|
|
355
|
-
taskId: taskInfo.id,
|
|
356
|
-
taskState: { status: "completed", attempts: lastAttempt, output }
|
|
357
|
-
});
|
|
358
|
-
logger.info("Task complete", { "aiki.attempts": lastAttempt });
|
|
359
|
-
return output;
|
|
360
|
-
}
|
|
361
|
-
async tryExecuteTask(handle, input, taskId, retryStrategy, currentAttempt, spinThresholdMs, logger) {
|
|
362
|
-
let attempts = currentAttempt;
|
|
363
|
-
while (true) {
|
|
364
|
-
try {
|
|
365
|
-
const outputRaw = await this.params.handler(input);
|
|
366
|
-
const output = await this.parse(handle, this.params.schema?.output, outputRaw, logger);
|
|
367
|
-
return { output, lastAttempt: attempts };
|
|
368
|
-
} catch (error) {
|
|
369
|
-
if (error instanceof WorkflowRunSuspendedError || error instanceof WorkflowRunFailedError || error instanceof WorkflowRunRevisionConflictError) {
|
|
370
|
-
throw error;
|
|
371
|
-
}
|
|
372
|
-
const serializableError = createSerializableError(error);
|
|
373
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
374
|
-
if (!retryParams.retriesLeft) {
|
|
375
|
-
logger.error("Task failed", {
|
|
376
|
-
"aiki.attempts": attempts,
|
|
377
|
-
"aiki.reason": serializableError.message
|
|
378
|
-
});
|
|
379
|
-
await handle[INTERNAL].transitionTaskState({
|
|
380
|
-
taskId,
|
|
381
|
-
taskState: { status: "failed", attempts, error: serializableError }
|
|
382
|
-
});
|
|
383
|
-
throw new TaskFailedError(taskId, attempts, serializableError.message);
|
|
384
|
-
}
|
|
385
|
-
logger.debug("Task failed. It will be retried", {
|
|
386
|
-
"aiki.attempts": attempts,
|
|
387
|
-
"aiki.nextAttemptInMs": retryParams.delayMs,
|
|
388
|
-
"aiki.reason": serializableError.message
|
|
389
|
-
});
|
|
390
|
-
if (retryParams.delayMs <= spinThresholdMs) {
|
|
391
|
-
await delay(retryParams.delayMs);
|
|
392
|
-
attempts++;
|
|
393
|
-
continue;
|
|
394
|
-
}
|
|
395
|
-
await handle[INTERNAL].transitionTaskState({
|
|
396
|
-
taskId,
|
|
397
|
-
taskState: {
|
|
398
|
-
status: "awaiting_retry",
|
|
399
|
-
attempts,
|
|
400
|
-
error: serializableError,
|
|
401
|
-
nextAttemptInMs: retryParams.delayMs
|
|
402
|
-
}
|
|
403
|
-
});
|
|
404
|
-
throw new WorkflowRunSuspendedError(handle.run.id);
|
|
405
|
-
}
|
|
406
|
-
}
|
|
407
|
-
}
|
|
408
|
-
assertRetryAllowed(taskId, attempts, retryStrategy, logger) {
|
|
409
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
410
|
-
if (!retryParams.retriesLeft) {
|
|
411
|
-
logger.error("Task retry not allowed", {
|
|
412
|
-
"aiki.taskName": this.name,
|
|
413
|
-
"aiki.taskId": taskId,
|
|
414
|
-
"aiki.attempts": attempts
|
|
415
|
-
});
|
|
416
|
-
throw new TaskFailedError(taskId, attempts, "Task retry not allowed");
|
|
417
|
-
}
|
|
418
|
-
}
|
|
419
|
-
async parse(handle, schema, data, logger) {
|
|
420
|
-
if (!schema) {
|
|
421
|
-
return data;
|
|
422
|
-
}
|
|
423
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
424
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
425
|
-
if (!schemaValidationResult.issues) {
|
|
426
|
-
return schemaValidationResult.value;
|
|
427
|
-
}
|
|
428
|
-
logger.error("Invalid task data", { "aiki.issues": schemaValidationResult.issues });
|
|
429
|
-
await handle[INTERNAL].transitionState({
|
|
430
|
-
status: "failed",
|
|
431
|
-
cause: "self",
|
|
432
|
-
error: {
|
|
433
|
-
name: "SchemaValidationError",
|
|
434
|
-
message: JSON.stringify(schemaValidationResult.issues)
|
|
435
|
-
}
|
|
436
|
-
});
|
|
437
|
-
throw new WorkflowRunFailedError(handle.run.id, handle.run.attempts);
|
|
438
|
-
}
|
|
439
|
-
};
|
|
440
|
-
var TaskBuilderImpl = class _TaskBuilderImpl {
|
|
441
|
-
constructor(task2, startOptsBuilder) {
|
|
442
|
-
this.task = task2;
|
|
443
|
-
this.startOptsBuilder = startOptsBuilder;
|
|
444
|
-
}
|
|
445
|
-
opt(path, value) {
|
|
446
|
-
return new _TaskBuilderImpl(this.task, this.startOptsBuilder.with(path, value));
|
|
447
|
-
}
|
|
448
|
-
start(run, ...args) {
|
|
449
|
-
return this.task.startWithOpts(run, this.startOptsBuilder.build(), ...args);
|
|
450
|
-
}
|
|
451
|
-
};
|
|
452
|
-
|
|
453
|
-
// ../workflow/workflow.ts
|
|
454
|
-
import { INTERNAL as INTERNAL6 } from "@aikirun/types/symbols";
|
|
455
|
-
|
|
456
|
-
// ../workflow/workflow-version.ts
|
|
457
|
-
import { INTERNAL as INTERNAL5 } from "@aikirun/types/symbols";
|
|
458
|
-
import { TaskFailedError as TaskFailedError2 } from "@aikirun/types/task";
|
|
459
|
-
import { SchemaValidationError as SchemaValidationError2 } from "@aikirun/types/validator";
|
|
460
|
-
import {
|
|
461
|
-
NonDeterminismError as NonDeterminismError2,
|
|
462
|
-
WorkflowRunFailedError as WorkflowRunFailedError3,
|
|
463
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError5,
|
|
464
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError4
|
|
465
|
-
} from "@aikirun/types/workflow-run";
|
|
466
|
-
|
|
467
|
-
// ../../lib/duration/convert.ts
|
|
468
|
-
var MS_PER_SECOND = 1e3;
|
|
469
|
-
var MS_PER_MINUTE = 60 * MS_PER_SECOND;
|
|
470
|
-
var MS_PER_HOUR = 60 * MS_PER_MINUTE;
|
|
471
|
-
var MS_PER_DAY = 24 * MS_PER_HOUR;
|
|
472
|
-
function toMilliseconds(duration) {
|
|
473
|
-
if (typeof duration === "number") {
|
|
474
|
-
assertIsPositiveNumber(duration);
|
|
475
|
-
return duration;
|
|
476
|
-
}
|
|
477
|
-
let totalMs = 0;
|
|
478
|
-
if (duration.days !== void 0) {
|
|
479
|
-
assertIsPositiveNumber(duration.days, "days");
|
|
480
|
-
totalMs += duration.days * MS_PER_DAY;
|
|
481
|
-
}
|
|
482
|
-
if (duration.hours !== void 0) {
|
|
483
|
-
assertIsPositiveNumber(duration.hours, "hours");
|
|
484
|
-
totalMs += duration.hours * MS_PER_HOUR;
|
|
485
|
-
}
|
|
486
|
-
if (duration.minutes !== void 0) {
|
|
487
|
-
assertIsPositiveNumber(duration.minutes, "minutes");
|
|
488
|
-
totalMs += duration.minutes * MS_PER_MINUTE;
|
|
489
|
-
}
|
|
490
|
-
if (duration.seconds !== void 0) {
|
|
491
|
-
assertIsPositiveNumber(duration.seconds, "seconds");
|
|
492
|
-
totalMs += duration.seconds * MS_PER_SECOND;
|
|
493
|
-
}
|
|
494
|
-
if (duration.milliseconds !== void 0) {
|
|
495
|
-
assertIsPositiveNumber(duration.milliseconds, "milliseconds");
|
|
496
|
-
totalMs += duration.milliseconds;
|
|
497
|
-
}
|
|
498
|
-
return totalMs;
|
|
499
|
-
}
|
|
500
|
-
function assertIsPositiveNumber(value, field) {
|
|
501
|
-
if (!Number.isFinite(value)) {
|
|
502
|
-
throw new Error(
|
|
503
|
-
field !== void 0 ? `'${field}' duration must be finite. Received: ${value}` : `Duration must be finite. Received: ${value}`
|
|
504
|
-
);
|
|
505
|
-
}
|
|
506
|
-
if (value < 0) {
|
|
507
|
-
throw new Error(
|
|
508
|
-
field !== void 0 ? `'${field}' duration must be non-negative. Received: ${value}` : `Duration must be non-negative. Received: ${value}`
|
|
509
|
-
);
|
|
510
|
-
}
|
|
511
|
-
}
|
|
512
|
-
|
|
513
|
-
// ../workflow/run/event.ts
|
|
514
|
-
import { INTERNAL as INTERNAL2 } from "@aikirun/types/symbols";
|
|
515
|
-
import { SchemaValidationError } from "@aikirun/types/validator";
|
|
516
|
-
import {
|
|
517
|
-
WorkflowRunFailedError as WorkflowRunFailedError2,
|
|
518
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError2,
|
|
519
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError2
|
|
520
|
-
} from "@aikirun/types/workflow-run";
|
|
521
|
-
function createEventSenders(api, workflowRunId, eventsDefinition, logger) {
|
|
522
|
-
const senders = {};
|
|
523
|
-
for (const [eventName, eventDefinition] of Object.entries(eventsDefinition)) {
|
|
524
|
-
const sender = createEventSender(
|
|
525
|
-
api,
|
|
526
|
-
workflowRunId,
|
|
527
|
-
eventName,
|
|
528
|
-
eventDefinition.schema,
|
|
529
|
-
logger.child({ "aiki.eventName": eventName })
|
|
530
|
-
);
|
|
531
|
-
senders[eventName] = sender;
|
|
532
|
-
}
|
|
533
|
-
return senders;
|
|
534
|
-
}
|
|
535
|
-
function createEventSender(api, workflowRunId, eventName, schema, logger, options) {
|
|
536
|
-
const optsOverrider = objectOverrider(options ?? {});
|
|
537
|
-
const createBuilder = (optsBuilder) => ({
|
|
538
|
-
opt: (path, value) => createBuilder(optsBuilder.with(path, value)),
|
|
539
|
-
send: (...args) => createEventSender(api, workflowRunId, eventName, schema, logger, optsBuilder.build()).send(...args)
|
|
540
|
-
});
|
|
541
|
-
async function send(...args) {
|
|
542
|
-
let data = args[0];
|
|
543
|
-
if (schema) {
|
|
544
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
545
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
546
|
-
if (schemaValidationResult.issues) {
|
|
547
|
-
logger.error("Invalid event data", { "aiki.issues": schemaValidationResult.issues });
|
|
548
|
-
throw new SchemaValidationError("Invalid event data", schemaValidationResult.issues);
|
|
549
|
-
}
|
|
550
|
-
data = schemaValidationResult.value;
|
|
551
|
-
}
|
|
552
|
-
await api.workflowRun.sendEventV1({
|
|
553
|
-
id: workflowRunId,
|
|
554
|
-
eventName,
|
|
555
|
-
data,
|
|
556
|
-
options
|
|
557
|
-
});
|
|
558
|
-
logger.info("Sent event to workflow", {
|
|
559
|
-
...options?.reference ? { "aiki.referenceId": options.reference.id } : {}
|
|
560
|
-
});
|
|
561
|
-
}
|
|
562
|
-
return {
|
|
563
|
-
with: () => createBuilder(optsOverrider()),
|
|
564
|
-
send
|
|
565
|
-
};
|
|
566
|
-
}
|
|
567
|
-
function createEventMulticasters(workflowName, workflowVersionId, eventsDefinition) {
|
|
568
|
-
const senders = {};
|
|
569
|
-
for (const [eventName, eventDefinition] of Object.entries(eventsDefinition)) {
|
|
570
|
-
const sender = createEventMulticaster(
|
|
571
|
-
workflowName,
|
|
572
|
-
workflowVersionId,
|
|
573
|
-
eventName,
|
|
574
|
-
eventDefinition.schema
|
|
575
|
-
);
|
|
576
|
-
senders[eventName] = sender;
|
|
577
|
-
}
|
|
578
|
-
return senders;
|
|
579
|
-
}
|
|
580
|
-
function createEventMulticaster(workflowName, workflowVersionId, eventName, schema, options) {
|
|
581
|
-
const optsOverrider = objectOverrider(options ?? {});
|
|
582
|
-
const createBuilder = (optsBuilder) => ({
|
|
583
|
-
opt: (path, value) => createBuilder(optsBuilder.with(path, value)),
|
|
584
|
-
send: (client, runId, ...args) => createEventMulticaster(workflowName, workflowVersionId, eventName, schema, optsBuilder.build()).send(
|
|
585
|
-
client,
|
|
586
|
-
runId,
|
|
587
|
-
...args
|
|
588
|
-
),
|
|
589
|
-
sendByReferenceId: (client, referenceId, ...args) => createEventMulticaster(workflowName, workflowVersionId, eventName, schema, optsBuilder.build()).sendByReferenceId(
|
|
590
|
-
client,
|
|
591
|
-
referenceId,
|
|
592
|
-
...args
|
|
593
|
-
)
|
|
594
|
-
});
|
|
595
|
-
async function send(client, runId, ...args) {
|
|
596
|
-
let data = args[0];
|
|
597
|
-
if (schema) {
|
|
598
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
599
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
600
|
-
if (schemaValidationResult.issues) {
|
|
601
|
-
client.logger.error("Invalid event data", {
|
|
602
|
-
"aiki.workflowName": workflowName,
|
|
603
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
604
|
-
"aiki.eventName": eventName,
|
|
605
|
-
"aiki.issues": schemaValidationResult.issues
|
|
606
|
-
});
|
|
607
|
-
throw new SchemaValidationError("Invalid event data", schemaValidationResult.issues);
|
|
608
|
-
}
|
|
609
|
-
data = schemaValidationResult.value;
|
|
610
|
-
}
|
|
611
|
-
const runIds = Array.isArray(runId) ? runId : [runId];
|
|
612
|
-
if (!isNonEmptyArray(runIds)) {
|
|
613
|
-
return;
|
|
614
|
-
}
|
|
615
|
-
await client.api.workflowRun.multicastEventV1({
|
|
616
|
-
ids: runIds,
|
|
617
|
-
eventName,
|
|
618
|
-
data,
|
|
619
|
-
options
|
|
620
|
-
});
|
|
621
|
-
client.logger.info("Multicasted event to workflows", {
|
|
622
|
-
"aiki.workflowName": workflowName,
|
|
623
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
624
|
-
"aiki.workflowRunIds": runIds,
|
|
625
|
-
"aiki.eventName": eventName,
|
|
626
|
-
...options?.reference ? { "aiki.eventReferenceId": options.reference.id } : {}
|
|
627
|
-
});
|
|
628
|
-
}
|
|
629
|
-
async function sendByReferenceId(client, referenceId, ...args) {
|
|
630
|
-
let data = args[0];
|
|
631
|
-
if (schema) {
|
|
632
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
633
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
634
|
-
if (schemaValidationResult.issues) {
|
|
635
|
-
client.logger.error("Invalid event data", {
|
|
636
|
-
"aiki.workflowName": workflowName,
|
|
637
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
638
|
-
"aiki.eventName": eventName,
|
|
639
|
-
"aiki.issues": schemaValidationResult.issues
|
|
640
|
-
});
|
|
641
|
-
throw new SchemaValidationError("Invalid event data", schemaValidationResult.issues);
|
|
642
|
-
}
|
|
643
|
-
data = schemaValidationResult.value;
|
|
644
|
-
}
|
|
645
|
-
const referenceIds = Array.isArray(referenceId) ? referenceId : [referenceId];
|
|
646
|
-
if (!isNonEmptyArray(referenceIds)) {
|
|
647
|
-
return;
|
|
648
|
-
}
|
|
649
|
-
await client.api.workflowRun.multicastEventByReferenceV1({
|
|
650
|
-
references: referenceIds.map((referenceId2) => ({
|
|
651
|
-
name: workflowName,
|
|
652
|
-
versionId: workflowVersionId,
|
|
653
|
-
referenceId: referenceId2
|
|
654
|
-
})),
|
|
655
|
-
eventName,
|
|
656
|
-
data,
|
|
657
|
-
options
|
|
658
|
-
});
|
|
659
|
-
client.logger.info("Multicasted event by reference", {
|
|
660
|
-
"aiki.workflowName": workflowName,
|
|
661
|
-
"aiki.workflowVersionId": workflowVersionId,
|
|
662
|
-
"aiki.referenceIds": referenceIds,
|
|
663
|
-
"aiki.eventName": eventName,
|
|
664
|
-
...options?.reference ? { "aiki.eventReferenceId": options.reference.id } : {}
|
|
665
|
-
});
|
|
666
|
-
}
|
|
667
|
-
return {
|
|
668
|
-
with: () => createBuilder(optsOverrider()),
|
|
669
|
-
send,
|
|
670
|
-
sendByReferenceId
|
|
671
|
-
};
|
|
672
|
-
}
|
|
673
|
-
|
|
674
|
-
// ../workflow/run/handle.ts
|
|
675
|
-
import { INTERNAL as INTERNAL3 } from "@aikirun/types/symbols";
|
|
676
|
-
import {
|
|
677
|
-
WorkflowRunNotExecutableError,
|
|
678
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError3
|
|
679
|
-
} from "@aikirun/types/workflow-run";
|
|
680
|
-
async function workflowRunHandle(client, runOrId, eventsDefinition, logger) {
|
|
681
|
-
const run = typeof runOrId !== "string" ? runOrId : (await client.api.workflowRun.getByIdV1({ id: runOrId })).run;
|
|
682
|
-
return new WorkflowRunHandleImpl(
|
|
683
|
-
client,
|
|
684
|
-
run,
|
|
685
|
-
eventsDefinition ?? {},
|
|
686
|
-
logger ?? client.logger.child({
|
|
687
|
-
"aiki.workflowName": run.name,
|
|
688
|
-
"aiki.workflowVersionId": run.versionId,
|
|
689
|
-
"aiki.workflowRunId": run.id
|
|
690
|
-
})
|
|
691
|
-
);
|
|
692
|
-
}
|
|
693
|
-
var WorkflowRunHandleImpl = class {
|
|
694
|
-
constructor(client, _run, eventsDefinition, logger) {
|
|
695
|
-
this._run = _run;
|
|
696
|
-
this.logger = logger;
|
|
697
|
-
this.api = client.api;
|
|
698
|
-
this.events = createEventSenders(client.api, this._run.id, eventsDefinition, this.logger);
|
|
699
|
-
this[INTERNAL3] = {
|
|
700
|
-
client,
|
|
701
|
-
transitionState: this.transitionState.bind(this),
|
|
702
|
-
transitionTaskState: this.transitionTaskState.bind(this),
|
|
703
|
-
assertExecutionAllowed: this.assertExecutionAllowed.bind(this)
|
|
704
|
-
};
|
|
705
|
-
}
|
|
706
|
-
api;
|
|
707
|
-
events;
|
|
708
|
-
[INTERNAL3];
|
|
709
|
-
get run() {
|
|
710
|
-
return this._run;
|
|
711
|
-
}
|
|
712
|
-
async refresh() {
|
|
713
|
-
const { run: currentRun } = await this.api.workflowRun.getByIdV1({ id: this.run.id });
|
|
714
|
-
this._run = currentRun;
|
|
715
|
-
}
|
|
716
|
-
async waitForStatus(status, options) {
|
|
717
|
-
return this.waitForStatusByPolling(status, options);
|
|
718
|
-
}
|
|
719
|
-
async waitForStatusByPolling(expectedStatus, options) {
|
|
720
|
-
if (options?.abortSignal?.aborted) {
|
|
721
|
-
return {
|
|
722
|
-
success: false,
|
|
723
|
-
cause: "aborted"
|
|
724
|
-
};
|
|
725
|
-
}
|
|
726
|
-
const delayMs = options?.interval ? toMilliseconds(options.interval) : 1e3;
|
|
727
|
-
const maxAttempts = options?.timeout ? Math.ceil(toMilliseconds(options.timeout) / delayMs) : Number.POSITIVE_INFINITY;
|
|
728
|
-
const retryStrategy = { type: "fixed", maxAttempts, delayMs };
|
|
729
|
-
let afterStateTransitionId = this._run.stateTransitionId;
|
|
730
|
-
const hasTerminated = async () => {
|
|
731
|
-
const { terminated, latestStateTransitionId } = await this.api.workflowRun.hasTerminatedV1({
|
|
732
|
-
id: this._run.id,
|
|
733
|
-
afterStateTransitionId
|
|
734
|
-
});
|
|
735
|
-
afterStateTransitionId = latestStateTransitionId;
|
|
736
|
-
return terminated;
|
|
737
|
-
};
|
|
738
|
-
const shouldRetryOnResult = async (terminated) => !terminated;
|
|
739
|
-
const maybeResult = options?.abortSignal ? await withRetry(hasTerminated, retryStrategy, {
|
|
740
|
-
abortSignal: options.abortSignal,
|
|
741
|
-
shouldRetryOnResult
|
|
742
|
-
}).run() : await withRetry(hasTerminated, retryStrategy, { shouldRetryOnResult }).run();
|
|
743
|
-
if (maybeResult.state === "timeout") {
|
|
744
|
-
if (!Number.isFinite(maxAttempts)) {
|
|
745
|
-
throw new Error("Something's wrong, this should've never timed out");
|
|
746
|
-
}
|
|
747
|
-
return { success: false, cause: "timeout" };
|
|
748
|
-
}
|
|
749
|
-
if (maybeResult.state === "aborted") {
|
|
750
|
-
return { success: false, cause: "aborted" };
|
|
751
|
-
}
|
|
752
|
-
maybeResult.state;
|
|
753
|
-
await this.refresh();
|
|
754
|
-
if (this._run.state.status === expectedStatus) {
|
|
755
|
-
return {
|
|
756
|
-
success: true,
|
|
757
|
-
state: this._run.state
|
|
758
|
-
};
|
|
759
|
-
}
|
|
760
|
-
return { success: false, cause: "run_terminated" };
|
|
761
|
-
}
|
|
762
|
-
async cancel(reason) {
|
|
763
|
-
await this.transitionState({ status: "cancelled", reason });
|
|
764
|
-
this.logger.info("Workflow cancelled");
|
|
765
|
-
}
|
|
766
|
-
async pause() {
|
|
767
|
-
await this.transitionState({ status: "paused" });
|
|
768
|
-
this.logger.info("Workflow paused");
|
|
769
|
-
}
|
|
770
|
-
async resume() {
|
|
771
|
-
await this.transitionState({ status: "scheduled", scheduledInMs: 0, reason: "resume" });
|
|
772
|
-
this.logger.info("Workflow resumed");
|
|
773
|
-
}
|
|
774
|
-
async awake() {
|
|
775
|
-
await this.transitionState({ status: "scheduled", scheduledInMs: 0, reason: "awake_early" });
|
|
776
|
-
this.logger.info("Workflow awoken");
|
|
777
|
-
}
|
|
778
|
-
async transitionState(targetState) {
|
|
779
|
-
try {
|
|
780
|
-
let response;
|
|
781
|
-
if (targetState.status === "scheduled" && (targetState.reason === "new" || targetState.reason === "resume" || targetState.reason === "awake_early") || targetState.status === "paused" || targetState.status === "cancelled") {
|
|
782
|
-
response = await this.api.workflowRun.transitionStateV1({
|
|
783
|
-
type: "pessimistic",
|
|
784
|
-
id: this.run.id,
|
|
785
|
-
state: targetState
|
|
786
|
-
});
|
|
787
|
-
} else {
|
|
788
|
-
response = await this.api.workflowRun.transitionStateV1({
|
|
789
|
-
type: "optimistic",
|
|
790
|
-
id: this.run.id,
|
|
791
|
-
state: targetState,
|
|
792
|
-
expectedRevision: this.run.revision
|
|
793
|
-
});
|
|
794
|
-
}
|
|
795
|
-
this._run.revision = response.revision;
|
|
796
|
-
this._run.state = response.state;
|
|
797
|
-
this._run.attempts = response.attempts;
|
|
798
|
-
} catch (error) {
|
|
799
|
-
if (isWorkflowRunRevisionConflictError(error)) {
|
|
800
|
-
throw new WorkflowRunRevisionConflictError3(this.run.id);
|
|
801
|
-
}
|
|
802
|
-
throw error;
|
|
803
|
-
}
|
|
804
|
-
}
|
|
805
|
-
async transitionTaskState(request) {
|
|
806
|
-
try {
|
|
807
|
-
const { taskInfo } = await this.api.workflowRun.transitionTaskStateV1({
|
|
808
|
-
...request,
|
|
809
|
-
id: this.run.id,
|
|
810
|
-
expectedWorkflowRunRevision: this.run.revision
|
|
811
|
-
});
|
|
812
|
-
return taskInfo;
|
|
813
|
-
} catch (error) {
|
|
814
|
-
if (isWorkflowRunRevisionConflictError(error)) {
|
|
815
|
-
throw new WorkflowRunRevisionConflictError3(this.run.id);
|
|
816
|
-
}
|
|
817
|
-
throw error;
|
|
818
|
-
}
|
|
819
|
-
}
|
|
820
|
-
assertExecutionAllowed() {
|
|
821
|
-
const status = this.run.state.status;
|
|
822
|
-
if (status !== "queued" && status !== "running") {
|
|
823
|
-
throw new WorkflowRunNotExecutableError(this.run.id, status);
|
|
824
|
-
}
|
|
825
|
-
}
|
|
826
|
-
};
|
|
827
|
-
function isWorkflowRunRevisionConflictError(error) {
|
|
828
|
-
return error != null && typeof error === "object" && "code" in error && error.code === "WORKFLOW_RUN_REVISION_CONFLICT";
|
|
829
|
-
}
|
|
830
|
-
|
|
831
|
-
// ../workflow/run/handle-child.ts
|
|
832
|
-
import { INTERNAL as INTERNAL4 } from "@aikirun/types/symbols";
|
|
833
|
-
import {
|
|
834
|
-
isTerminalWorkflowRunStatus,
|
|
835
|
-
WorkflowRunRevisionConflictError as WorkflowRunRevisionConflictError4,
|
|
836
|
-
WorkflowRunSuspendedError as WorkflowRunSuspendedError3
|
|
837
|
-
} from "@aikirun/types/workflow-run";
|
|
838
|
-
async function childWorkflowRunHandle(client, run, parentRun, childWorkflowRunWaitQueues, logger, eventsDefinition) {
|
|
839
|
-
const handle = await workflowRunHandle(client, run, eventsDefinition, logger);
|
|
840
|
-
return {
|
|
841
|
-
run: handle.run,
|
|
842
|
-
events: handle.events,
|
|
843
|
-
refresh: handle.refresh.bind(handle),
|
|
844
|
-
waitForStatus: createStatusWaiter(handle, parentRun, childWorkflowRunWaitQueues, logger),
|
|
845
|
-
cancel: handle.cancel.bind(handle),
|
|
846
|
-
pause: handle.pause.bind(handle),
|
|
847
|
-
resume: handle.resume.bind(handle),
|
|
848
|
-
awake: handle.awake.bind(handle),
|
|
849
|
-
[INTERNAL4]: handle[INTERNAL4]
|
|
850
|
-
};
|
|
851
|
-
}
|
|
852
|
-
function createStatusWaiter(handle, parentRun, childWorkflowRunWaitQueues, logger) {
|
|
853
|
-
const nextIndexByStatus = {
|
|
854
|
-
cancelled: 0,
|
|
855
|
-
completed: 0,
|
|
856
|
-
failed: 0
|
|
857
|
-
};
|
|
858
|
-
async function waitForStatus(expectedStatus, options) {
|
|
859
|
-
const parentRunHandle = parentRun[INTERNAL4].handle;
|
|
860
|
-
const nextIndex = nextIndexByStatus[expectedStatus];
|
|
861
|
-
const { run } = handle;
|
|
862
|
-
const childWorkflowRunWaits = childWorkflowRunWaitQueues[expectedStatus].childWorkflowRunWaits;
|
|
863
|
-
const existingChildWorkflowRunWait = childWorkflowRunWaits[nextIndex];
|
|
864
|
-
if (existingChildWorkflowRunWait) {
|
|
865
|
-
nextIndexByStatus[expectedStatus] = nextIndex + 1;
|
|
866
|
-
if (existingChildWorkflowRunWait.status === "timeout") {
|
|
867
|
-
logger.debug("Timed out waiting for child workflow status", {
|
|
868
|
-
"aiki.childWorkflowExpectedStatus": expectedStatus
|
|
869
|
-
});
|
|
870
|
-
return {
|
|
871
|
-
success: false,
|
|
872
|
-
cause: "timeout"
|
|
873
|
-
};
|
|
874
|
-
}
|
|
875
|
-
const childWorkflowRunStatus = existingChildWorkflowRunWait.childWorkflowRunState.status;
|
|
876
|
-
if (childWorkflowRunStatus === expectedStatus) {
|
|
877
|
-
return {
|
|
878
|
-
success: true,
|
|
879
|
-
state: existingChildWorkflowRunWait.childWorkflowRunState
|
|
880
|
-
};
|
|
881
|
-
}
|
|
882
|
-
if (isTerminalWorkflowRunStatus(childWorkflowRunStatus)) {
|
|
883
|
-
logger.debug("Child workflow run reached termnial state", {
|
|
884
|
-
"aiki.childWorkflowTerminalStatus": childWorkflowRunStatus
|
|
885
|
-
});
|
|
886
|
-
return {
|
|
887
|
-
success: false,
|
|
888
|
-
cause: "run_terminated"
|
|
889
|
-
};
|
|
890
|
-
}
|
|
891
|
-
childWorkflowRunStatus;
|
|
892
|
-
}
|
|
893
|
-
const timeoutInMs = options?.timeout && toMilliseconds(options.timeout);
|
|
894
|
-
try {
|
|
895
|
-
await parentRunHandle[INTERNAL4].transitionState({
|
|
896
|
-
status: "awaiting_child_workflow",
|
|
897
|
-
childWorkflowRunId: run.id,
|
|
898
|
-
childWorkflowRunStatus: expectedStatus,
|
|
899
|
-
timeoutInMs
|
|
900
|
-
});
|
|
901
|
-
logger.info("Waiting for child Workflow", {
|
|
902
|
-
"aiki.childWorkflowExpectedStatus": expectedStatus,
|
|
903
|
-
...timeoutInMs !== void 0 ? { "aiki.timeoutInMs": timeoutInMs } : {}
|
|
904
|
-
});
|
|
905
|
-
} catch (error) {
|
|
906
|
-
if (error instanceof WorkflowRunRevisionConflictError4) {
|
|
907
|
-
throw new WorkflowRunSuspendedError3(parentRun.id);
|
|
908
|
-
}
|
|
909
|
-
throw error;
|
|
910
|
-
}
|
|
911
|
-
throw new WorkflowRunSuspendedError3(parentRun.id);
|
|
912
|
-
}
|
|
913
|
-
return waitForStatus;
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
// ../workflow/workflow-version.ts
|
|
917
|
-
var WorkflowVersionImpl = class {
|
|
918
|
-
constructor(name, versionId, params) {
|
|
919
|
-
this.name = name;
|
|
920
|
-
this.versionId = versionId;
|
|
921
|
-
this.params = params;
|
|
922
|
-
const eventsDefinition = this.params.events ?? {};
|
|
923
|
-
this.events = createEventMulticasters(this.name, this.versionId, eventsDefinition);
|
|
924
|
-
this[INTERNAL5] = {
|
|
925
|
-
eventsDefinition,
|
|
926
|
-
handler: this.handler.bind(this)
|
|
927
|
-
};
|
|
928
|
-
}
|
|
929
|
-
events;
|
|
930
|
-
[INTERNAL5];
|
|
931
|
-
with() {
|
|
932
|
-
const startOpts = this.params.opts ?? {};
|
|
933
|
-
const startOptsOverrider = objectOverrider(startOpts);
|
|
934
|
-
return new WorkflowBuilderImpl(this, startOptsOverrider());
|
|
935
|
-
}
|
|
936
|
-
async start(client, ...args) {
|
|
937
|
-
return this.startWithOpts(client, this.params.opts ?? {}, ...args);
|
|
938
|
-
}
|
|
939
|
-
async startWithOpts(client, startOpts, ...args) {
|
|
940
|
-
let input = args[0];
|
|
941
|
-
const schema = this.params.schema?.input;
|
|
942
|
-
if (schema) {
|
|
943
|
-
const schemaValidation = schema["~standard"].validate(input);
|
|
944
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
945
|
-
if (schemaValidationResult.issues) {
|
|
946
|
-
client.logger.error("Invalid workflow data", { "aiki.issues": schemaValidationResult.issues });
|
|
947
|
-
throw new SchemaValidationError2("Invalid workflow data", schemaValidationResult.issues);
|
|
948
|
-
}
|
|
949
|
-
input = schemaValidationResult.value;
|
|
950
|
-
}
|
|
951
|
-
const { id } = await client.api.workflowRun.createV1({
|
|
952
|
-
name: this.name,
|
|
953
|
-
versionId: this.versionId,
|
|
954
|
-
input,
|
|
955
|
-
options: startOpts
|
|
956
|
-
});
|
|
957
|
-
client.logger.info("Created workflow", {
|
|
958
|
-
"aiki.workflowName": this.name,
|
|
959
|
-
"aiki.workflowVersionId": this.versionId,
|
|
960
|
-
"aiki.workflowRunId": id
|
|
961
|
-
});
|
|
962
|
-
return workflowRunHandle(client, id, this[INTERNAL5].eventsDefinition);
|
|
963
|
-
}
|
|
964
|
-
async startAsChild(parentRun, ...args) {
|
|
965
|
-
return this.startAsChildWithOpts(parentRun, this.params.opts ?? {}, ...args);
|
|
966
|
-
}
|
|
967
|
-
async startAsChildWithOpts(parentRun, startOpts, ...args) {
|
|
968
|
-
const parentRunHandle = parentRun[INTERNAL5].handle;
|
|
969
|
-
parentRunHandle[INTERNAL5].assertExecutionAllowed();
|
|
970
|
-
const { client } = parentRunHandle[INTERNAL5];
|
|
971
|
-
const inputRaw = args[0];
|
|
972
|
-
const input = await this.parse(parentRunHandle, this.params.schema?.input, inputRaw, parentRun.logger);
|
|
973
|
-
const inputHash = await hashInput(input);
|
|
974
|
-
const referenceId = startOpts.reference?.id;
|
|
975
|
-
const address = getWorkflowRunAddress(this.name, this.versionId, referenceId ?? inputHash);
|
|
976
|
-
const replayManifest = parentRun[INTERNAL5].replayManifest;
|
|
977
|
-
if (replayManifest.hasUnconsumedEntries()) {
|
|
978
|
-
const existingRunInfo = replayManifest.consumeNextChildWorkflowRun(address);
|
|
979
|
-
if (existingRunInfo) {
|
|
980
|
-
const { run: existingRun } = await client.api.workflowRun.getByIdV1({ id: existingRunInfo.id });
|
|
981
|
-
if (existingRun.state.status === "completed") {
|
|
982
|
-
await this.parse(parentRunHandle, this.params.schema?.output, existingRun.state.output, parentRun.logger);
|
|
983
|
-
}
|
|
984
|
-
const logger2 = parentRun.logger.child({
|
|
985
|
-
"aiki.childWorkflowName": existingRun.name,
|
|
986
|
-
"aiki.childWorkflowVersionId": existingRun.versionId,
|
|
987
|
-
"aiki.childWorkflowRunId": existingRun.id
|
|
988
|
-
});
|
|
989
|
-
return childWorkflowRunHandle(
|
|
990
|
-
client,
|
|
991
|
-
existingRun,
|
|
992
|
-
parentRun,
|
|
993
|
-
existingRunInfo.childWorkflowRunWaitQueues,
|
|
994
|
-
logger2,
|
|
995
|
-
this[INTERNAL5].eventsDefinition
|
|
996
|
-
);
|
|
997
|
-
}
|
|
998
|
-
await this.throwNonDeterminismError(parentRun, parentRunHandle, inputHash, referenceId, replayManifest);
|
|
999
|
-
}
|
|
1000
|
-
const shard = parentRun.options.shard;
|
|
1001
|
-
const { id: newRunId } = await client.api.workflowRun.createV1({
|
|
1002
|
-
name: this.name,
|
|
1003
|
-
versionId: this.versionId,
|
|
1004
|
-
input,
|
|
1005
|
-
parentWorkflowRunId: parentRun.id,
|
|
1006
|
-
options: shard === void 0 ? startOpts : { ...startOpts, shard }
|
|
1007
|
-
});
|
|
1008
|
-
const { run: newRun } = await client.api.workflowRun.getByIdV1({ id: newRunId });
|
|
1009
|
-
const logger = parentRun.logger.child({
|
|
1010
|
-
"aiki.childWorkflowName": newRun.name,
|
|
1011
|
-
"aiki.childWorkflowVersionId": newRun.versionId,
|
|
1012
|
-
"aiki.childWorkflowRunId": newRun.id
|
|
1013
|
-
});
|
|
1014
|
-
logger.info("Created child workflow");
|
|
1015
|
-
return childWorkflowRunHandle(
|
|
1016
|
-
client,
|
|
1017
|
-
newRun,
|
|
1018
|
-
parentRun,
|
|
1019
|
-
{
|
|
1020
|
-
cancelled: { childWorkflowRunWaits: [] },
|
|
1021
|
-
completed: { childWorkflowRunWaits: [] },
|
|
1022
|
-
failed: { childWorkflowRunWaits: [] }
|
|
1023
|
-
},
|
|
1024
|
-
logger,
|
|
1025
|
-
this[INTERNAL5].eventsDefinition
|
|
1026
|
-
);
|
|
1027
|
-
}
|
|
1028
|
-
async throwNonDeterminismError(parentRun, parentRunHandle, inputHash, referenceId, manifest) {
|
|
1029
|
-
const unconsumedManifestEntries = manifest.getUnconsumedEntries();
|
|
1030
|
-
const logMeta = {
|
|
1031
|
-
"aiki.workflowName": this.name,
|
|
1032
|
-
"aiki.inputHash": inputHash,
|
|
1033
|
-
"aiki.unconsumedManifestEntries": unconsumedManifestEntries
|
|
1034
|
-
};
|
|
1035
|
-
if (referenceId !== void 0) {
|
|
1036
|
-
logMeta["aiki.referenceId"] = referenceId;
|
|
1037
|
-
}
|
|
1038
|
-
parentRun.logger.error("Replay divergence", logMeta);
|
|
1039
|
-
const error = new NonDeterminismError2(parentRun.id, parentRunHandle.run.attempts, unconsumedManifestEntries);
|
|
1040
|
-
await parentRunHandle[INTERNAL5].transitionState({
|
|
1041
|
-
status: "failed",
|
|
1042
|
-
cause: "self",
|
|
1043
|
-
error: createSerializableError(error)
|
|
1044
|
-
});
|
|
1045
|
-
throw error;
|
|
1046
|
-
}
|
|
1047
|
-
async getHandleById(client, runId) {
|
|
1048
|
-
return workflowRunHandle(client, runId, this[INTERNAL5].eventsDefinition);
|
|
1049
|
-
}
|
|
1050
|
-
async getHandleByReferenceId(client, referenceId) {
|
|
1051
|
-
const { run } = await client.api.workflowRun.getByReferenceIdV1({
|
|
1052
|
-
name: this.name,
|
|
1053
|
-
versionId: this.versionId,
|
|
1054
|
-
referenceId
|
|
1055
|
-
});
|
|
1056
|
-
return workflowRunHandle(client, run, this[INTERNAL5].eventsDefinition);
|
|
1057
|
-
}
|
|
1058
|
-
async handler(run, input, context) {
|
|
1059
|
-
const { logger } = run;
|
|
1060
|
-
const { handle } = run[INTERNAL5];
|
|
1061
|
-
handle[INTERNAL5].assertExecutionAllowed();
|
|
1062
|
-
const retryStrategy = this.params.opts?.retry ?? { type: "never" };
|
|
1063
|
-
const state = handle.run.state;
|
|
1064
|
-
if (state.status === "queued" && state.reason === "retry") {
|
|
1065
|
-
await this.assertRetryAllowed(handle, retryStrategy, logger);
|
|
1066
|
-
}
|
|
1067
|
-
logger.info("Starting workflow");
|
|
1068
|
-
await handle[INTERNAL5].transitionState({ status: "running" });
|
|
1069
|
-
const output = await this.tryExecuteWorkflow(input, run, context, retryStrategy);
|
|
1070
|
-
await handle[INTERNAL5].transitionState({ status: "completed", output });
|
|
1071
|
-
logger.info("Workflow complete");
|
|
1072
|
-
}
|
|
1073
|
-
async tryExecuteWorkflow(input, run, context, retryStrategy) {
|
|
1074
|
-
const { handle } = run[INTERNAL5];
|
|
1075
|
-
while (true) {
|
|
1076
|
-
try {
|
|
1077
|
-
const outputRaw = await this.params.handler(run, input, context);
|
|
1078
|
-
const output = await this.parse(handle, this.params.schema?.output, outputRaw, run.logger);
|
|
1079
|
-
return output;
|
|
1080
|
-
} catch (error) {
|
|
1081
|
-
if (error instanceof WorkflowRunSuspendedError4 || error instanceof WorkflowRunFailedError3 || error instanceof WorkflowRunRevisionConflictError5 || error instanceof NonDeterminismError2) {
|
|
1082
|
-
throw error;
|
|
1083
|
-
}
|
|
1084
|
-
const attempts = handle.run.attempts;
|
|
1085
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
1086
|
-
if (!retryParams.retriesLeft) {
|
|
1087
|
-
const failedState = this.createFailedState(error);
|
|
1088
|
-
await handle[INTERNAL5].transitionState(failedState);
|
|
1089
|
-
const logMeta2 = {};
|
|
1090
|
-
for (const [key, value] of Object.entries(failedState)) {
|
|
1091
|
-
logMeta2[`aiki.${key}`] = value;
|
|
1092
|
-
}
|
|
1093
|
-
run.logger.error("Workflow failed", {
|
|
1094
|
-
"aiki.attempts": attempts,
|
|
1095
|
-
...logMeta2
|
|
1096
|
-
});
|
|
1097
|
-
throw new WorkflowRunFailedError3(run.id, attempts);
|
|
1098
|
-
}
|
|
1099
|
-
const awaitingRetryState = this.createAwaitingRetryState(error, retryParams.delayMs);
|
|
1100
|
-
await handle[INTERNAL5].transitionState(awaitingRetryState);
|
|
1101
|
-
const logMeta = {};
|
|
1102
|
-
for (const [key, value] of Object.entries(awaitingRetryState)) {
|
|
1103
|
-
logMeta[`aiki.${key}`] = value;
|
|
1104
|
-
}
|
|
1105
|
-
run.logger.info("Workflow awaiting retry", {
|
|
1106
|
-
"aiki.attempts": attempts,
|
|
1107
|
-
...logMeta
|
|
1108
|
-
});
|
|
1109
|
-
throw new WorkflowRunSuspendedError4(run.id);
|
|
1110
|
-
}
|
|
1111
|
-
}
|
|
1112
|
-
}
|
|
1113
|
-
async assertRetryAllowed(handle, retryStrategy, logger) {
|
|
1114
|
-
const { id, attempts } = handle.run;
|
|
1115
|
-
const retryParams = getRetryParams(attempts, retryStrategy);
|
|
1116
|
-
if (!retryParams.retriesLeft) {
|
|
1117
|
-
logger.error("Workflow retry not allowed", { "aiki.attempts": attempts });
|
|
1118
|
-
const error = new WorkflowRunFailedError3(id, attempts);
|
|
1119
|
-
await handle[INTERNAL5].transitionState({
|
|
1120
|
-
status: "failed",
|
|
1121
|
-
cause: "self",
|
|
1122
|
-
error: createSerializableError(error)
|
|
1123
|
-
});
|
|
1124
|
-
throw error;
|
|
1125
|
-
}
|
|
1126
|
-
}
|
|
1127
|
-
async parse(handle, schema, data, logger) {
|
|
1128
|
-
if (!schema) {
|
|
1129
|
-
return data;
|
|
1130
|
-
}
|
|
1131
|
-
const schemaValidation = schema["~standard"].validate(data);
|
|
1132
|
-
const schemaValidationResult = schemaValidation instanceof Promise ? await schemaValidation : schemaValidation;
|
|
1133
|
-
if (!schemaValidationResult.issues) {
|
|
1134
|
-
return schemaValidationResult.value;
|
|
1135
|
-
}
|
|
1136
|
-
logger.error("Invalid workflow data", { "aiki.issues": schemaValidationResult.issues });
|
|
1137
|
-
await handle[INTERNAL5].transitionState({
|
|
1138
|
-
status: "failed",
|
|
1139
|
-
cause: "self",
|
|
1140
|
-
error: {
|
|
1141
|
-
name: "SchemaValidationError",
|
|
1142
|
-
message: JSON.stringify(schemaValidationResult.issues)
|
|
1143
|
-
}
|
|
1144
|
-
});
|
|
1145
|
-
throw new WorkflowRunFailedError3(handle.run.id, handle.run.attempts);
|
|
1146
|
-
}
|
|
1147
|
-
createFailedState(error) {
|
|
1148
|
-
if (error instanceof TaskFailedError2) {
|
|
1149
|
-
return {
|
|
1150
|
-
status: "failed",
|
|
1151
|
-
cause: "task",
|
|
1152
|
-
taskId: error.taskId
|
|
1153
|
-
};
|
|
1154
|
-
}
|
|
1155
|
-
return {
|
|
1156
|
-
status: "failed",
|
|
1157
|
-
cause: "self",
|
|
1158
|
-
error: createSerializableError(error)
|
|
1159
|
-
};
|
|
1160
|
-
}
|
|
1161
|
-
createAwaitingRetryState(error, nextAttemptInMs) {
|
|
1162
|
-
if (error instanceof TaskFailedError2) {
|
|
1163
|
-
return {
|
|
1164
|
-
status: "awaiting_retry",
|
|
1165
|
-
cause: "task",
|
|
1166
|
-
nextAttemptInMs,
|
|
1167
|
-
taskId: error.taskId
|
|
1168
|
-
};
|
|
1169
|
-
}
|
|
1170
|
-
return {
|
|
1171
|
-
status: "awaiting_retry",
|
|
1172
|
-
cause: "self",
|
|
1173
|
-
nextAttemptInMs,
|
|
1174
|
-
error: createSerializableError(error)
|
|
1175
|
-
};
|
|
1176
|
-
}
|
|
1177
|
-
};
|
|
1178
|
-
var WorkflowBuilderImpl = class _WorkflowBuilderImpl {
|
|
1179
|
-
constructor(workflow2, startOptsBuilder) {
|
|
1180
|
-
this.workflow = workflow2;
|
|
1181
|
-
this.startOptsBuilder = startOptsBuilder;
|
|
1182
|
-
}
|
|
1183
|
-
opt(path, value) {
|
|
1184
|
-
return new _WorkflowBuilderImpl(this.workflow, this.startOptsBuilder.with(path, value));
|
|
1185
|
-
}
|
|
1186
|
-
start(client, ...args) {
|
|
1187
|
-
return this.workflow.startWithOpts(client, this.startOptsBuilder.build(), ...args);
|
|
1188
|
-
}
|
|
1189
|
-
startAsChild(parentRun, ...args) {
|
|
1190
|
-
return this.workflow.startAsChildWithOpts(parentRun, this.startOptsBuilder.build(), ...args);
|
|
1191
|
-
}
|
|
1192
|
-
};
|
|
1193
|
-
|
|
1194
|
-
// ../workflow/workflow.ts
|
|
1195
|
-
function workflow(params) {
|
|
1196
|
-
return new WorkflowImpl(params);
|
|
1197
|
-
}
|
|
1198
|
-
var WorkflowImpl = class {
|
|
1199
|
-
name;
|
|
1200
|
-
[INTERNAL6];
|
|
1201
|
-
workflowVersions = /* @__PURE__ */ new Map();
|
|
1202
|
-
constructor(params) {
|
|
1203
|
-
this.name = params.name;
|
|
1204
|
-
this[INTERNAL6] = {
|
|
1205
|
-
getAllVersions: this.getAllVersions.bind(this),
|
|
1206
|
-
getVersion: this.getVersion.bind(this)
|
|
1207
|
-
};
|
|
1208
|
-
}
|
|
1209
|
-
v(versionId, params) {
|
|
1210
|
-
if (this.workflowVersions.has(versionId)) {
|
|
1211
|
-
throw new Error(`Workflow "${this.name}:${versionId}" already exists`);
|
|
1212
|
-
}
|
|
1213
|
-
const workflowVersion = new WorkflowVersionImpl(this.name, versionId, params);
|
|
1214
|
-
this.workflowVersions.set(
|
|
1215
|
-
versionId,
|
|
1216
|
-
workflowVersion
|
|
1217
|
-
);
|
|
1218
|
-
return workflowVersion;
|
|
1219
|
-
}
|
|
1220
|
-
getAllVersions() {
|
|
1221
|
-
return Array.from(this.workflowVersions.values());
|
|
1222
|
-
}
|
|
1223
|
-
getVersion(versionId) {
|
|
1224
|
-
return this.workflowVersions.get(versionId);
|
|
1225
|
-
}
|
|
1226
|
-
};
|
|
1227
|
-
|
|
1228
|
-
// ../workflow/system/cancel-child-runs.ts
|
|
1229
|
-
var createCancelChildRunsV1 = (api) => {
|
|
1230
|
-
const listNonTerminalChildRuns = task({
|
|
1231
|
-
name: "aiki:list-non-terminal-child-runs",
|
|
1232
|
-
async handler(parentRunId) {
|
|
1233
|
-
const { runs } = await api.workflowRun.listChildRunsV1({
|
|
1234
|
-
parentRunId,
|
|
1235
|
-
status: NON_TERMINAL_WORKFLOW_RUN_STATUSES
|
|
1236
|
-
});
|
|
1237
|
-
return runs.map((r) => r.id);
|
|
1238
|
-
}
|
|
1239
|
-
});
|
|
1240
|
-
const cancelRuns = task({
|
|
1241
|
-
name: "aiki:cancel-runs",
|
|
1242
|
-
async handler(runIds) {
|
|
1243
|
-
const { cancelledIds } = await api.workflowRun.cancelByIdsV1({ ids: runIds });
|
|
1244
|
-
return cancelledIds;
|
|
1245
|
-
}
|
|
1246
|
-
});
|
|
1247
|
-
return workflow({ name: "aiki:cancel-child-runs" }).v("1.0.0", {
|
|
1248
|
-
async handler(run, parentRunId) {
|
|
1249
|
-
const childRunIds = await listNonTerminalChildRuns.start(run, parentRunId);
|
|
1250
|
-
if (!isNonEmptyArray(childRunIds)) {
|
|
1251
|
-
return;
|
|
1252
|
-
}
|
|
1253
|
-
await cancelRuns.start(run, childRunIds);
|
|
1254
|
-
}
|
|
1255
|
-
});
|
|
1256
|
-
};
|
|
1257
|
-
|
|
1258
|
-
// ../workflow/system/index.ts
|
|
1259
|
-
function getSystemWorkflows(api) {
|
|
1260
|
-
return [createCancelChildRunsV1(api)];
|
|
1261
|
-
}
|
|
1262
|
-
|
|
1263
|
-
// worker.ts
|
|
1264
62
|
import { ulid } from "ulidx";
|
|
1265
63
|
function worker(params) {
|
|
1266
64
|
return new WorkerImpl(params);
|
|
@@ -1268,76 +66,61 @@ function worker(params) {
|
|
|
1268
66
|
var WorkerImpl = class {
|
|
1269
67
|
constructor(params) {
|
|
1270
68
|
this.params = params;
|
|
1271
|
-
this.name = params.name;
|
|
1272
69
|
}
|
|
1273
|
-
name;
|
|
1274
70
|
with() {
|
|
1275
|
-
const
|
|
1276
|
-
const
|
|
1277
|
-
return new WorkerBuilderImpl(this,
|
|
71
|
+
const spawnOptions = this.params.options ?? {};
|
|
72
|
+
const spawnOptionsOverrider = objectOverrider(spawnOptions);
|
|
73
|
+
return new WorkerBuilderImpl(this, spawnOptionsOverrider());
|
|
1278
74
|
}
|
|
1279
75
|
async spawn(client) {
|
|
1280
|
-
return this.
|
|
76
|
+
return this.spawnWithOptions(client, this.params.options ?? {});
|
|
1281
77
|
}
|
|
1282
|
-
async
|
|
1283
|
-
const handle = new WorkerHandleImpl(client, this.params,
|
|
78
|
+
async spawnWithOptions(client, spawnOptions) {
|
|
79
|
+
const handle = new WorkerHandleImpl(client, this.params, spawnOptions);
|
|
1284
80
|
await handle._start();
|
|
1285
81
|
return handle;
|
|
1286
82
|
}
|
|
1287
83
|
};
|
|
1288
84
|
var WorkerHandleImpl = class {
|
|
1289
|
-
constructor(client, params,
|
|
85
|
+
constructor(client, params, spawnOptions) {
|
|
1290
86
|
this.client = client;
|
|
1291
87
|
this.params = params;
|
|
1292
|
-
this.
|
|
88
|
+
this.spawnOptions = spawnOptions;
|
|
1293
89
|
this.id = ulid();
|
|
1294
|
-
this.
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
spinThresholdMs: this.spawnOpts.workflowRun?.spinThresholdMs ?? 10
|
|
90
|
+
this.workflowRunOptions = {
|
|
91
|
+
heartbeatIntervalMs: this.spawnOptions.workflowRun?.heartbeatIntervalMs ?? 3e4,
|
|
92
|
+
spinThresholdMs: this.spawnOptions.workflowRun?.spinThresholdMs ?? 10
|
|
1298
93
|
};
|
|
1299
94
|
this.registry = workflowRegistry().addMany(getSystemWorkflows(client.api)).addMany(this.params.workflows);
|
|
1300
|
-
const reference = this.
|
|
95
|
+
const reference = this.spawnOptions.reference;
|
|
1301
96
|
this.logger = client.logger.child({
|
|
1302
97
|
"aiki.component": "worker",
|
|
1303
98
|
"aiki.workerId": this.id,
|
|
1304
|
-
"aiki.workerName": this.name,
|
|
1305
99
|
...reference && { "aiki.workerReferenceId": reference.id }
|
|
1306
100
|
});
|
|
1307
101
|
}
|
|
1308
102
|
id;
|
|
1309
|
-
|
|
1310
|
-
workflowRunOpts;
|
|
103
|
+
workflowRunOptions;
|
|
1311
104
|
registry;
|
|
1312
105
|
logger;
|
|
1313
106
|
abortController;
|
|
1314
|
-
|
|
1315
|
-
|
|
107
|
+
subscriber;
|
|
108
|
+
fallbackSubscriber;
|
|
1316
109
|
pollPromise;
|
|
1317
110
|
activeWorkflowRunsById = /* @__PURE__ */ new Map();
|
|
1318
111
|
async _start() {
|
|
1319
|
-
const
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
|
|
1323
|
-
|
|
1324
|
-
|
|
1325
|
-
);
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
const fallbackSubscriberStrategyBuilder = this.client[INTERNAL7].subscriber.create(
|
|
1332
|
-
{ type: "db" },
|
|
1333
|
-
workflows,
|
|
1334
|
-
this.spawnOpts.shards
|
|
1335
|
-
);
|
|
1336
|
-
this.fallbackSubscriberStrategy = await fallbackSubscriberStrategyBuilder.init(this.id, {
|
|
1337
|
-
onError: (error) => this.handleSubscriberError(error),
|
|
1338
|
-
onStop: () => this.stop()
|
|
1339
|
-
});
|
|
1340
|
-
}
|
|
112
|
+
const subscriberContext = {
|
|
113
|
+
workerId: this.id,
|
|
114
|
+
workflows: this.registry.getAll(),
|
|
115
|
+
shards: this.spawnOptions.shards,
|
|
116
|
+
logger: this.logger
|
|
117
|
+
};
|
|
118
|
+
const createSubscriber = this.params.subscriber ?? dbSubscriber({ api: this.client.api });
|
|
119
|
+
const subscriber = createSubscriber(subscriberContext);
|
|
120
|
+
this.subscriber = subscriber instanceof Promise ? await subscriber : subscriber;
|
|
121
|
+
const createFallbackSubscriber = dbSubscriber({ api: this.client.api });
|
|
122
|
+
const fallbackSubscriber = createFallbackSubscriber(subscriberContext);
|
|
123
|
+
this.fallbackSubscriber = fallbackSubscriber instanceof Promise ? await fallbackSubscriber : fallbackSubscriber;
|
|
1341
124
|
this.abortController = new AbortController();
|
|
1342
125
|
const abortSignal = this.abortController.signal;
|
|
1343
126
|
this.pollPromise = this.poll(abortSignal).catch((error) => {
|
|
@@ -1352,11 +135,13 @@ var WorkerHandleImpl = class {
|
|
|
1352
135
|
this.logger.info("Worker stopping");
|
|
1353
136
|
this.abortController?.abort();
|
|
1354
137
|
await this.pollPromise;
|
|
138
|
+
await this.subscriber?.close?.();
|
|
139
|
+
await this.fallbackSubscriber?.close?.();
|
|
1355
140
|
const activeWorkflowRuns = Array.from(this.activeWorkflowRunsById.values());
|
|
1356
141
|
if (activeWorkflowRuns.length === 0) {
|
|
1357
142
|
return;
|
|
1358
143
|
}
|
|
1359
|
-
const timeoutMs = this.
|
|
144
|
+
const timeoutMs = this.spawnOptions.gracefulShutdownTimeoutMs ?? 5e3;
|
|
1360
145
|
if (timeoutMs > 0) {
|
|
1361
146
|
await Promise.race([Promise.allSettled(activeWorkflowRuns.map((w) => w.executionPromise)), delay(timeoutMs)]);
|
|
1362
147
|
}
|
|
@@ -1370,60 +155,59 @@ var WorkerHandleImpl = class {
|
|
|
1370
155
|
this.activeWorkflowRunsById.clear();
|
|
1371
156
|
}
|
|
1372
157
|
async poll(abortSignal) {
|
|
1373
|
-
if (!this.
|
|
1374
|
-
throw new Error("Subscriber
|
|
158
|
+
if (!this.subscriber) {
|
|
159
|
+
throw new Error("Subscriber not initialized");
|
|
1375
160
|
}
|
|
1376
161
|
this.logger.info("Worker started", {
|
|
1377
162
|
"aiki.registeredWorkflows": this.params.workflows.map((w) => `${w.name}:${w.versionId}`)
|
|
1378
163
|
});
|
|
1379
|
-
const maxConcurrentWorkflowRuns = this.
|
|
1380
|
-
let
|
|
164
|
+
const maxConcurrentWorkflowRuns = this.spawnOptions.maxConcurrentWorkflowRuns ?? 1;
|
|
165
|
+
let activeSubscriber = this.subscriber;
|
|
166
|
+
let nextDelayMs = activeSubscriber.getNextDelay({ type: "polled", foundWork: false });
|
|
1381
167
|
let subscriberFailedAttempts = 0;
|
|
1382
168
|
while (!abortSignal.aborted) {
|
|
1383
169
|
await delay(nextDelayMs, { abortSignal });
|
|
1384
170
|
const availableCapacity = maxConcurrentWorkflowRuns - this.activeWorkflowRunsById.size;
|
|
1385
171
|
if (availableCapacity <= 0) {
|
|
1386
|
-
nextDelayMs =
|
|
172
|
+
nextDelayMs = activeSubscriber.getNextDelay({ type: "at_capacity" });
|
|
1387
173
|
continue;
|
|
1388
174
|
}
|
|
1389
175
|
const nextBatchResponse = await this.fetchNextWorkflowRunBatch(availableCapacity, subscriberFailedAttempts);
|
|
1390
176
|
if (!nextBatchResponse.success) {
|
|
1391
177
|
subscriberFailedAttempts++;
|
|
1392
|
-
nextDelayMs =
|
|
178
|
+
nextDelayMs = activeSubscriber.getNextDelay({
|
|
1393
179
|
type: "retry",
|
|
1394
180
|
attemptNumber: subscriberFailedAttempts
|
|
1395
181
|
});
|
|
1396
182
|
continue;
|
|
1397
183
|
}
|
|
1398
184
|
subscriberFailedAttempts = 0;
|
|
185
|
+
activeSubscriber = nextBatchResponse.subscriber;
|
|
1399
186
|
if (!isNonEmptyArray(nextBatchResponse.batch)) {
|
|
1400
|
-
nextDelayMs =
|
|
187
|
+
nextDelayMs = activeSubscriber.getNextDelay({ type: "polled", foundWork: false });
|
|
1401
188
|
continue;
|
|
1402
189
|
}
|
|
1403
190
|
await this.enqueueWorkflowRunBatch(nextBatchResponse.batch, nextBatchResponse.subscriber, abortSignal);
|
|
1404
|
-
nextDelayMs =
|
|
191
|
+
nextDelayMs = activeSubscriber.getNextDelay({ type: "polled", foundWork: true });
|
|
1405
192
|
}
|
|
1406
193
|
}
|
|
1407
194
|
async fetchNextWorkflowRunBatch(size, subscriberFailedAttempts) {
|
|
1408
|
-
if (!this.
|
|
1409
|
-
|
|
1410
|
-
success: false,
|
|
1411
|
-
error: new Error("Subscriber strategy not initialized")
|
|
1412
|
-
};
|
|
195
|
+
if (!this.subscriber) {
|
|
196
|
+
throw new Error("Subscriber not initialized");
|
|
1413
197
|
}
|
|
1414
198
|
try {
|
|
1415
|
-
const batch = await this.
|
|
1416
|
-
return { success: true, batch, subscriber: this.
|
|
199
|
+
const batch = await this.subscriber.getNextBatch(size);
|
|
200
|
+
return { success: true, batch, subscriber: this.subscriber };
|
|
1417
201
|
} catch (error) {
|
|
1418
202
|
this.logger.error("Error getting next workflow runs batch", {
|
|
1419
203
|
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
1420
204
|
});
|
|
1421
|
-
if (this.
|
|
205
|
+
if (this.fallbackSubscriber && subscriberFailedAttempts >= 2) {
|
|
1422
206
|
try {
|
|
1423
|
-
const batch = await this.
|
|
1424
|
-
return { success: true, batch, subscriber: this.
|
|
207
|
+
const batch = await this.fallbackSubscriber.getNextBatch(size);
|
|
208
|
+
return { success: true, batch, subscriber: this.fallbackSubscriber };
|
|
1425
209
|
} catch (fallbackError) {
|
|
1426
|
-
this.logger.error("Fallback subscriber
|
|
210
|
+
this.logger.error("Fallback subscriber also failed to get next workflow runs batch", {
|
|
1427
211
|
"aiki.error": fallbackError instanceof Error ? fallbackError.message : String(fallbackError)
|
|
1428
212
|
});
|
|
1429
213
|
}
|
|
@@ -1440,8 +224,15 @@ var WorkerHandleImpl = class {
|
|
|
1440
224
|
});
|
|
1441
225
|
continue;
|
|
1442
226
|
}
|
|
1443
|
-
|
|
1444
|
-
|
|
227
|
+
let workflowRun;
|
|
228
|
+
try {
|
|
229
|
+
const response = await this.client.api.workflowRun.getByIdV1({ id: workflowRunId });
|
|
230
|
+
workflowRun = response.run;
|
|
231
|
+
} catch (error) {
|
|
232
|
+
this.logger.warn("Failed to fetch workflow run", {
|
|
233
|
+
"aiki.workflowRunId": workflowRunId,
|
|
234
|
+
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
235
|
+
});
|
|
1445
236
|
if (subscriber.acknowledge) {
|
|
1446
237
|
await subscriber.acknowledge(workflowRunId).catch(() => {
|
|
1447
238
|
});
|
|
@@ -1476,94 +267,49 @@ var WorkerHandleImpl = class {
|
|
|
1476
267
|
}
|
|
1477
268
|
async executeWorkflow(workflowRun, workflowVersion, subscriber) {
|
|
1478
269
|
const logger = this.logger.child({
|
|
1479
|
-
"aiki.component": "workflow-execution",
|
|
1480
270
|
"aiki.workflowName": workflowRun.name,
|
|
1481
271
|
"aiki.workflowVersionId": workflowRun.versionId,
|
|
1482
272
|
"aiki.workflowRunId": workflowRun.id
|
|
1483
273
|
});
|
|
1484
|
-
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1497
|
-
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
name: workflowRun.name,
|
|
1506
|
-
versionId: workflowRun.versionId,
|
|
1507
|
-
options: workflowRun.options ?? {},
|
|
1508
|
-
logger,
|
|
1509
|
-
sleep: createSleeper(handle, logger),
|
|
1510
|
-
events: createEventWaiters(handle, eventsDefinition, logger),
|
|
1511
|
-
[INTERNAL7]: {
|
|
1512
|
-
handle,
|
|
1513
|
-
replayManifest: createReplayManifest(workflowRun),
|
|
1514
|
-
options: { spinThresholdMs: this.workflowRunOpts.spinThresholdMs }
|
|
1515
|
-
}
|
|
1516
|
-
},
|
|
1517
|
-
workflowRun.input,
|
|
1518
|
-
appContext
|
|
1519
|
-
);
|
|
1520
|
-
shouldAcknowledge = true;
|
|
1521
|
-
} catch (error) {
|
|
1522
|
-
if (error instanceof WorkflowRunNotExecutableError2 || error instanceof WorkflowRunSuspendedError5 || error instanceof WorkflowRunFailedError4 || error instanceof WorkflowRunRevisionConflictError6 || error instanceof NonDeterminismError3) {
|
|
1523
|
-
shouldAcknowledge = true;
|
|
1524
|
-
} else {
|
|
1525
|
-
logger.error("Unexpected error during workflow execution", {
|
|
1526
|
-
"aiki.error": error instanceof Error ? error.message : String(error),
|
|
1527
|
-
"aiki.stack": error instanceof Error ? error.stack : void 0
|
|
1528
|
-
});
|
|
1529
|
-
shouldAcknowledge = false;
|
|
1530
|
-
}
|
|
1531
|
-
} finally {
|
|
1532
|
-
if (heartbeatInterval) clearInterval(heartbeatInterval);
|
|
1533
|
-
if (subscriber.acknowledge) {
|
|
1534
|
-
if (shouldAcknowledge) {
|
|
1535
|
-
try {
|
|
1536
|
-
await subscriber.acknowledge(workflowRun.id);
|
|
1537
|
-
} catch (error) {
|
|
1538
|
-
logger.error("Failed to acknowledge message, it may be reprocessed", {
|
|
1539
|
-
"aiki.errorType": "MESSAGE_ACK_FAILED",
|
|
1540
|
-
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
1541
|
-
});
|
|
1542
|
-
}
|
|
1543
|
-
} else {
|
|
1544
|
-
logger.debug("Message left in PEL for retry");
|
|
274
|
+
const heartbeat = subscriber.heartbeat;
|
|
275
|
+
const success = await executeWorkflowRun({
|
|
276
|
+
client: this.client,
|
|
277
|
+
workflowRun,
|
|
278
|
+
workflowVersion,
|
|
279
|
+
logger,
|
|
280
|
+
options: {
|
|
281
|
+
spinThresholdMs: this.workflowRunOptions.spinThresholdMs,
|
|
282
|
+
heartbeatIntervalMs: this.workflowRunOptions.heartbeatIntervalMs
|
|
283
|
+
},
|
|
284
|
+
heartbeat: heartbeat ? () => heartbeat(workflowRun.id) : void 0
|
|
285
|
+
});
|
|
286
|
+
if (subscriber.acknowledge) {
|
|
287
|
+
if (success) {
|
|
288
|
+
try {
|
|
289
|
+
await subscriber.acknowledge(workflowRun.id);
|
|
290
|
+
} catch (error) {
|
|
291
|
+
logger.error("Failed to acknowledge message, it may be reprocessed", {
|
|
292
|
+
"aiki.errorType": "MESSAGE_ACK_FAILED",
|
|
293
|
+
"aiki.error": error instanceof Error ? error.message : String(error)
|
|
294
|
+
});
|
|
1545
295
|
}
|
|
296
|
+
} else {
|
|
297
|
+
logger.debug("Message left pending for retry");
|
|
1546
298
|
}
|
|
1547
|
-
this.activeWorkflowRunsById.delete(workflowRun.id);
|
|
1548
299
|
}
|
|
1549
|
-
|
|
1550
|
-
handleSubscriberError(error) {
|
|
1551
|
-
this.logger.warn("Subscriber error", {
|
|
1552
|
-
"aiki.error": error.message,
|
|
1553
|
-
"aiki.stack": error.stack
|
|
1554
|
-
});
|
|
300
|
+
this.activeWorkflowRunsById.delete(workflowRun.id);
|
|
1555
301
|
}
|
|
1556
302
|
};
|
|
1557
303
|
var WorkerBuilderImpl = class _WorkerBuilderImpl {
|
|
1558
|
-
constructor(worker2,
|
|
304
|
+
constructor(worker2, spawnOptionsBuilder) {
|
|
1559
305
|
this.worker = worker2;
|
|
1560
|
-
this.
|
|
306
|
+
this.spawnOptionsBuilder = spawnOptionsBuilder;
|
|
1561
307
|
}
|
|
1562
308
|
opt(path, value) {
|
|
1563
|
-
return new _WorkerBuilderImpl(this.worker, this.
|
|
309
|
+
return new _WorkerBuilderImpl(this.worker, this.spawnOptionsBuilder.with(path, value));
|
|
1564
310
|
}
|
|
1565
311
|
spawn(client) {
|
|
1566
|
-
return this.worker.
|
|
312
|
+
return this.worker.spawnWithOptions(client, this.spawnOptionsBuilder.build());
|
|
1567
313
|
}
|
|
1568
314
|
};
|
|
1569
315
|
export {
|