@coji/durably 0.13.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/{chunk-UCUP6NMJ.js → chunk-L42OCQEV.js} +3 -3
- package/dist/chunk-L42OCQEV.js.map +1 -0
- package/dist/{index-DWsJlgyh.d.ts → index-CXH4ozmK.d.ts} +104 -18
- package/dist/index.d.ts +26 -4
- package/dist/index.js +1250 -764
- package/dist/index.js.map +1 -1
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/docs/llms.md +149 -17
- package/package.json +5 -5
- package/dist/chunk-UCUP6NMJ.js.map +0 -1
package/dist/index.js
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import {
|
|
2
2
|
withLogPersistence
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-L42OCQEV.js";
|
|
4
4
|
|
|
5
5
|
// src/durably.ts
|
|
6
|
-
import { Kysely } from "kysely";
|
|
6
|
+
import { Kysely, sql as sql5 } from "kysely";
|
|
7
7
|
import { monotonicFactory as monotonicFactory2 } from "ulidx";
|
|
8
8
|
|
|
9
9
|
// src/errors.ts
|
|
@@ -19,197 +19,54 @@ var LeaseLostError = class extends Error {
|
|
|
19
19
|
this.name = "LeaseLostError";
|
|
20
20
|
}
|
|
21
21
|
};
|
|
22
|
+
var DurablyError = class extends Error {
|
|
23
|
+
statusCode;
|
|
24
|
+
constructor(message, statusCode) {
|
|
25
|
+
super(message);
|
|
26
|
+
this.name = "DurablyError";
|
|
27
|
+
this.statusCode = statusCode;
|
|
28
|
+
}
|
|
29
|
+
};
|
|
30
|
+
var NotFoundError = class extends DurablyError {
|
|
31
|
+
constructor(message) {
|
|
32
|
+
super(message, 404);
|
|
33
|
+
this.name = "NotFoundError";
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
var ValidationError = class extends DurablyError {
|
|
37
|
+
constructor(message) {
|
|
38
|
+
super(message, 400);
|
|
39
|
+
this.name = "ValidationError";
|
|
40
|
+
}
|
|
41
|
+
};
|
|
42
|
+
var ConflictError = class extends DurablyError {
|
|
43
|
+
constructor(message) {
|
|
44
|
+
super(message, 409);
|
|
45
|
+
this.name = "ConflictError";
|
|
46
|
+
}
|
|
47
|
+
};
|
|
22
48
|
function getErrorMessage(error) {
|
|
23
49
|
return error instanceof Error ? error.message : String(error);
|
|
24
50
|
}
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
var LEASE_LOST = "lease-lost";
|
|
28
|
-
function createStepContext(run, jobName, leaseGeneration, storage, eventEmitter) {
|
|
29
|
-
let stepIndex = run.currentStepIndex;
|
|
30
|
-
let currentStepName = null;
|
|
31
|
-
const controller = new AbortController();
|
|
32
|
-
function abortForLeaseLoss() {
|
|
33
|
-
if (!controller.signal.aborted) {
|
|
34
|
-
controller.abort(LEASE_LOST);
|
|
35
|
-
}
|
|
36
|
-
}
|
|
37
|
-
function throwIfAborted() {
|
|
38
|
-
if (!controller.signal.aborted) {
|
|
39
|
-
return;
|
|
40
|
-
}
|
|
41
|
-
if (controller.signal.reason === LEASE_LOST) {
|
|
42
|
-
throw new LeaseLostError(run.id);
|
|
43
|
-
}
|
|
44
|
-
throw new CancelledError(run.id);
|
|
45
|
-
}
|
|
46
|
-
const unsubscribe = eventEmitter.on("run:cancel", (event) => {
|
|
47
|
-
if (event.runId === run.id) {
|
|
48
|
-
controller.abort();
|
|
49
|
-
}
|
|
50
|
-
});
|
|
51
|
-
const step = {
|
|
52
|
-
get runId() {
|
|
53
|
-
return run.id;
|
|
54
|
-
},
|
|
55
|
-
get signal() {
|
|
56
|
-
return controller.signal;
|
|
57
|
-
},
|
|
58
|
-
isAborted() {
|
|
59
|
-
return controller.signal.aborted;
|
|
60
|
-
},
|
|
61
|
-
throwIfAborted() {
|
|
62
|
-
throwIfAborted();
|
|
63
|
-
},
|
|
64
|
-
async run(name, fn) {
|
|
65
|
-
throwIfAborted();
|
|
66
|
-
const currentRun = await storage.getRun(run.id);
|
|
67
|
-
if (currentRun?.status === "cancelled") {
|
|
68
|
-
controller.abort();
|
|
69
|
-
throwIfAborted();
|
|
70
|
-
}
|
|
71
|
-
if (currentRun && (currentRun.status === "leased" && currentRun.leaseGeneration !== leaseGeneration || currentRun.status === "completed" || currentRun.status === "failed")) {
|
|
72
|
-
abortForLeaseLoss();
|
|
73
|
-
throwIfAborted();
|
|
74
|
-
}
|
|
75
|
-
throwIfAborted();
|
|
76
|
-
const existingStep = await storage.getCompletedStep(run.id, name);
|
|
77
|
-
if (existingStep) {
|
|
78
|
-
stepIndex++;
|
|
79
|
-
return existingStep.output;
|
|
80
|
-
}
|
|
81
|
-
currentStepName = name;
|
|
82
|
-
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
83
|
-
const startTime = Date.now();
|
|
84
|
-
eventEmitter.emit({
|
|
85
|
-
type: "step:start",
|
|
86
|
-
runId: run.id,
|
|
87
|
-
jobName,
|
|
88
|
-
stepName: name,
|
|
89
|
-
stepIndex,
|
|
90
|
-
labels: run.labels
|
|
91
|
-
});
|
|
92
|
-
try {
|
|
93
|
-
const result = await fn(controller.signal);
|
|
94
|
-
throwIfAborted();
|
|
95
|
-
const savedStep = await storage.persistStep(run.id, leaseGeneration, {
|
|
96
|
-
name,
|
|
97
|
-
index: stepIndex,
|
|
98
|
-
status: "completed",
|
|
99
|
-
output: result,
|
|
100
|
-
startedAt
|
|
101
|
-
});
|
|
102
|
-
if (!savedStep) {
|
|
103
|
-
abortForLeaseLoss();
|
|
104
|
-
throwIfAborted();
|
|
105
|
-
}
|
|
106
|
-
stepIndex++;
|
|
107
|
-
eventEmitter.emit({
|
|
108
|
-
type: "step:complete",
|
|
109
|
-
runId: run.id,
|
|
110
|
-
jobName,
|
|
111
|
-
stepName: name,
|
|
112
|
-
stepIndex: stepIndex - 1,
|
|
113
|
-
output: result,
|
|
114
|
-
duration: Date.now() - startTime,
|
|
115
|
-
labels: run.labels
|
|
116
|
-
});
|
|
117
|
-
return result;
|
|
118
|
-
} catch (error) {
|
|
119
|
-
if (error instanceof LeaseLostError) {
|
|
120
|
-
throw error;
|
|
121
|
-
}
|
|
122
|
-
const isLeaseLost = controller.signal.aborted && controller.signal.reason === LEASE_LOST;
|
|
123
|
-
if (isLeaseLost) {
|
|
124
|
-
throw new LeaseLostError(run.id);
|
|
125
|
-
}
|
|
126
|
-
const isCancelled = controller.signal.aborted;
|
|
127
|
-
const errorMessage = getErrorMessage(error);
|
|
128
|
-
const savedStep = await storage.persistStep(run.id, leaseGeneration, {
|
|
129
|
-
name,
|
|
130
|
-
index: stepIndex,
|
|
131
|
-
status: isCancelled ? "cancelled" : "failed",
|
|
132
|
-
error: errorMessage,
|
|
133
|
-
startedAt
|
|
134
|
-
});
|
|
135
|
-
if (!savedStep) {
|
|
136
|
-
abortForLeaseLoss();
|
|
137
|
-
throw new LeaseLostError(run.id);
|
|
138
|
-
}
|
|
139
|
-
eventEmitter.emit({
|
|
140
|
-
...isCancelled ? { type: "step:cancel" } : { type: "step:fail", error: errorMessage },
|
|
141
|
-
runId: run.id,
|
|
142
|
-
jobName,
|
|
143
|
-
stepName: name,
|
|
144
|
-
stepIndex,
|
|
145
|
-
labels: run.labels
|
|
146
|
-
});
|
|
147
|
-
if (isCancelled) {
|
|
148
|
-
throwIfAborted();
|
|
149
|
-
}
|
|
150
|
-
throw error;
|
|
151
|
-
} finally {
|
|
152
|
-
currentStepName = null;
|
|
153
|
-
}
|
|
154
|
-
},
|
|
155
|
-
progress(current, total, message) {
|
|
156
|
-
const progressData = { current, total, message };
|
|
157
|
-
storage.updateProgress(run.id, leaseGeneration, progressData);
|
|
158
|
-
eventEmitter.emit({
|
|
159
|
-
type: "run:progress",
|
|
160
|
-
runId: run.id,
|
|
161
|
-
jobName,
|
|
162
|
-
progress: progressData,
|
|
163
|
-
labels: run.labels
|
|
164
|
-
});
|
|
165
|
-
},
|
|
166
|
-
log: {
|
|
167
|
-
info(message, data) {
|
|
168
|
-
eventEmitter.emit({
|
|
169
|
-
type: "log:write",
|
|
170
|
-
runId: run.id,
|
|
171
|
-
jobName,
|
|
172
|
-
labels: run.labels,
|
|
173
|
-
stepName: currentStepName,
|
|
174
|
-
level: "info",
|
|
175
|
-
message,
|
|
176
|
-
data
|
|
177
|
-
});
|
|
178
|
-
},
|
|
179
|
-
warn(message, data) {
|
|
180
|
-
eventEmitter.emit({
|
|
181
|
-
type: "log:write",
|
|
182
|
-
runId: run.id,
|
|
183
|
-
jobName,
|
|
184
|
-
labels: run.labels,
|
|
185
|
-
stepName: currentStepName,
|
|
186
|
-
level: "warn",
|
|
187
|
-
message,
|
|
188
|
-
data
|
|
189
|
-
});
|
|
190
|
-
},
|
|
191
|
-
error(message, data) {
|
|
192
|
-
eventEmitter.emit({
|
|
193
|
-
type: "log:write",
|
|
194
|
-
runId: run.id,
|
|
195
|
-
jobName,
|
|
196
|
-
labels: run.labels,
|
|
197
|
-
stepName: currentStepName,
|
|
198
|
-
level: "error",
|
|
199
|
-
message,
|
|
200
|
-
data
|
|
201
|
-
});
|
|
202
|
-
}
|
|
203
|
-
}
|
|
204
|
-
};
|
|
205
|
-
return {
|
|
206
|
-
step,
|
|
207
|
-
abortLeaseOwnership: abortForLeaseLoss,
|
|
208
|
-
dispose: unsubscribe
|
|
209
|
-
};
|
|
51
|
+
function toError(error) {
|
|
52
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
210
53
|
}
|
|
211
54
|
|
|
212
55
|
// src/events.ts
|
|
56
|
+
var DOMAIN_EVENT_TYPE_VALUES = [
|
|
57
|
+
"run:trigger",
|
|
58
|
+
"run:coalesced",
|
|
59
|
+
"run:complete",
|
|
60
|
+
"run:fail",
|
|
61
|
+
"run:cancel",
|
|
62
|
+
"run:delete"
|
|
63
|
+
];
|
|
64
|
+
var DOMAIN_EVENT_TYPES = new Set(
|
|
65
|
+
DOMAIN_EVENT_TYPE_VALUES
|
|
66
|
+
);
|
|
67
|
+
function isDomainEvent(event) {
|
|
68
|
+
return DOMAIN_EVENT_TYPES.has(event.type);
|
|
69
|
+
}
|
|
213
70
|
function createEventEmitter() {
|
|
214
71
|
const listeners = /* @__PURE__ */ new Map();
|
|
215
72
|
let sequence = 0;
|
|
@@ -239,16 +96,16 @@ function createEventEmitter() {
|
|
|
239
96
|
if (!typeListeners) {
|
|
240
97
|
return;
|
|
241
98
|
}
|
|
99
|
+
const reportError = (error) => errorHandler?.(toError(error), fullEvent);
|
|
242
100
|
for (const listener of typeListeners) {
|
|
243
101
|
try {
|
|
244
|
-
listener(fullEvent);
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
error instanceof Error ? error : new Error(String(error)),
|
|
249
|
-
fullEvent
|
|
250
|
-
);
|
|
102
|
+
const result = listener(fullEvent);
|
|
103
|
+
if (result != null && typeof result.then === "function") {
|
|
104
|
+
;
|
|
105
|
+
result.catch(reportError);
|
|
251
106
|
}
|
|
107
|
+
} catch (error) {
|
|
108
|
+
reportError(error);
|
|
252
109
|
}
|
|
253
110
|
}
|
|
254
111
|
}
|
|
@@ -257,13 +114,16 @@ function createEventEmitter() {
|
|
|
257
114
|
|
|
258
115
|
// src/job.ts
|
|
259
116
|
import { prettifyError } from "zod";
|
|
117
|
+
var DEFAULT_WAIT_POLLING_INTERVAL_MS = 1e3;
|
|
260
118
|
var noop = () => {
|
|
261
119
|
};
|
|
262
120
|
function validateJobInputOrThrow(schema, input, context) {
|
|
263
121
|
const result = schema.safeParse(input);
|
|
264
122
|
if (!result.success) {
|
|
265
123
|
const prefix = context ? `${context}: ` : "";
|
|
266
|
-
throw new
|
|
124
|
+
throw new ValidationError(
|
|
125
|
+
`${prefix}Invalid input: ${prettifyError(result.error)}`
|
|
126
|
+
);
|
|
267
127
|
}
|
|
268
128
|
return result.data;
|
|
269
129
|
}
|
|
@@ -281,7 +141,157 @@ function createJobRegistry() {
|
|
|
281
141
|
}
|
|
282
142
|
};
|
|
283
143
|
}
|
|
284
|
-
function
|
|
144
|
+
function waitForRunCompletion(runId, storage, eventEmitter, options, timeoutMessagePrefix = "waitForRun") {
|
|
145
|
+
return new Promise((resolve, reject) => {
|
|
146
|
+
let timeoutId;
|
|
147
|
+
let pollIntervalId;
|
|
148
|
+
let resolved = false;
|
|
149
|
+
let pollInFlight = false;
|
|
150
|
+
const unsubscribes = [];
|
|
151
|
+
const pollingMs = options?.pollingIntervalMs ?? DEFAULT_WAIT_POLLING_INTERVAL_MS;
|
|
152
|
+
if (!Number.isFinite(pollingMs) || pollingMs <= 0) {
|
|
153
|
+
throw new ValidationError(
|
|
154
|
+
"pollingIntervalMs must be a positive finite number"
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
const cleanup = () => {
|
|
158
|
+
if (resolved) return;
|
|
159
|
+
resolved = true;
|
|
160
|
+
for (const unsub of unsubscribes) unsub();
|
|
161
|
+
if (timeoutId !== void 0) {
|
|
162
|
+
clearTimeout(timeoutId);
|
|
163
|
+
timeoutId = void 0;
|
|
164
|
+
}
|
|
165
|
+
if (pollIntervalId !== void 0) {
|
|
166
|
+
clearInterval(pollIntervalId);
|
|
167
|
+
pollIntervalId = void 0;
|
|
168
|
+
}
|
|
169
|
+
};
|
|
170
|
+
const settleFromStorage = (run) => {
|
|
171
|
+
if (resolved) return;
|
|
172
|
+
if (!run) {
|
|
173
|
+
cleanup();
|
|
174
|
+
reject(new NotFoundError(`Run not found: ${runId}`));
|
|
175
|
+
return;
|
|
176
|
+
}
|
|
177
|
+
if (run.status === "completed") {
|
|
178
|
+
cleanup();
|
|
179
|
+
resolve(run);
|
|
180
|
+
return;
|
|
181
|
+
}
|
|
182
|
+
if (run.status === "failed") {
|
|
183
|
+
cleanup();
|
|
184
|
+
reject(new Error(run.error || "Run failed"));
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
if (run.status === "cancelled") {
|
|
188
|
+
cleanup();
|
|
189
|
+
reject(new CancelledError(runId));
|
|
190
|
+
return;
|
|
191
|
+
}
|
|
192
|
+
};
|
|
193
|
+
const poll = () => {
|
|
194
|
+
if (resolved || pollInFlight) return;
|
|
195
|
+
pollInFlight = true;
|
|
196
|
+
void storage.getRun(runId).then((run) => {
|
|
197
|
+
if (resolved) return;
|
|
198
|
+
settleFromStorage(run);
|
|
199
|
+
}).catch((err) => {
|
|
200
|
+
if (resolved) return;
|
|
201
|
+
cleanup();
|
|
202
|
+
reject(toError(err));
|
|
203
|
+
}).finally(() => {
|
|
204
|
+
pollInFlight = false;
|
|
205
|
+
});
|
|
206
|
+
};
|
|
207
|
+
unsubscribes.push(
|
|
208
|
+
eventEmitter.on("run:complete", (event) => {
|
|
209
|
+
if (event.runId !== runId || resolved) return;
|
|
210
|
+
cleanup();
|
|
211
|
+
storage.getRun(runId).then((run) => {
|
|
212
|
+
if (run) resolve(run);
|
|
213
|
+
else reject(new NotFoundError(`Run not found: ${runId}`));
|
|
214
|
+
}).catch((err) => reject(toError(err)));
|
|
215
|
+
})
|
|
216
|
+
);
|
|
217
|
+
unsubscribes.push(
|
|
218
|
+
eventEmitter.on("run:fail", (event) => {
|
|
219
|
+
if (event.runId !== runId || resolved) return;
|
|
220
|
+
cleanup();
|
|
221
|
+
reject(new Error(event.error));
|
|
222
|
+
})
|
|
223
|
+
);
|
|
224
|
+
unsubscribes.push(
|
|
225
|
+
eventEmitter.on("run:cancel", (event) => {
|
|
226
|
+
if (event.runId !== runId || resolved) return;
|
|
227
|
+
cleanup();
|
|
228
|
+
reject(new CancelledError(runId));
|
|
229
|
+
})
|
|
230
|
+
);
|
|
231
|
+
if (options?.onProgress) {
|
|
232
|
+
const onProgress = options.onProgress;
|
|
233
|
+
unsubscribes.push(
|
|
234
|
+
eventEmitter.on("run:progress", (event) => {
|
|
235
|
+
if (event.runId !== runId || resolved) return;
|
|
236
|
+
void Promise.resolve(onProgress(event.progress)).catch(noop);
|
|
237
|
+
})
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
if (options?.onLog) {
|
|
241
|
+
const onLog = options.onLog;
|
|
242
|
+
unsubscribes.push(
|
|
243
|
+
eventEmitter.on("log:write", (event) => {
|
|
244
|
+
if (event.runId !== runId || resolved) return;
|
|
245
|
+
const { level, message, data, stepName } = event;
|
|
246
|
+
void Promise.resolve(onLog({ level, message, data, stepName })).catch(
|
|
247
|
+
noop
|
|
248
|
+
);
|
|
249
|
+
})
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
storage.getRun(runId).then((currentRun) => {
|
|
253
|
+
if (resolved) return;
|
|
254
|
+
if (!currentRun) {
|
|
255
|
+
cleanup();
|
|
256
|
+
reject(new NotFoundError(`Run not found: ${runId}`));
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
if (currentRun.status === "completed") {
|
|
260
|
+
cleanup();
|
|
261
|
+
resolve(currentRun);
|
|
262
|
+
return;
|
|
263
|
+
}
|
|
264
|
+
if (currentRun.status === "failed") {
|
|
265
|
+
cleanup();
|
|
266
|
+
reject(new Error(currentRun.error || "Run failed"));
|
|
267
|
+
return;
|
|
268
|
+
}
|
|
269
|
+
if (currentRun.status === "cancelled") {
|
|
270
|
+
cleanup();
|
|
271
|
+
reject(new CancelledError(runId));
|
|
272
|
+
return;
|
|
273
|
+
}
|
|
274
|
+
pollIntervalId = setInterval(poll, pollingMs);
|
|
275
|
+
}).catch((error) => {
|
|
276
|
+
if (resolved) return;
|
|
277
|
+
cleanup();
|
|
278
|
+
reject(toError(error));
|
|
279
|
+
});
|
|
280
|
+
if (options?.timeout !== void 0) {
|
|
281
|
+
timeoutId = setTimeout(() => {
|
|
282
|
+
if (!resolved) {
|
|
283
|
+
cleanup();
|
|
284
|
+
reject(
|
|
285
|
+
new Error(
|
|
286
|
+
`${timeoutMessagePrefix} timeout after ${options.timeout}ms`
|
|
287
|
+
)
|
|
288
|
+
);
|
|
289
|
+
}
|
|
290
|
+
}, options.timeout);
|
|
291
|
+
}
|
|
292
|
+
});
|
|
293
|
+
}
|
|
294
|
+
function createJobHandle(jobDef, storage, eventEmitter, registry, labelsSchema, pollingIntervalMs) {
|
|
285
295
|
const existingJob = registry.get(jobDef.name);
|
|
286
296
|
if (existingJob) {
|
|
287
297
|
if (existingJob.jobDef === jobDef) {
|
|
@@ -293,113 +303,77 @@ function createJobHandle(jobDef, storage, eventEmitter, registry, labelsSchema)
|
|
|
293
303
|
}
|
|
294
304
|
const inputSchema = jobDef.input;
|
|
295
305
|
const outputSchema = jobDef.output;
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
306
|
+
function validateCoalesceOption(coalesce, concurrencyKey, context) {
|
|
307
|
+
if (coalesce === void 0) return;
|
|
308
|
+
const suffix = context ? ` ${context}` : "";
|
|
309
|
+
if (coalesce !== "skip") {
|
|
310
|
+
throw new ValidationError(
|
|
311
|
+
`Invalid coalesce value${suffix}: '${coalesce}'. Valid values: 'skip'`
|
|
312
|
+
);
|
|
313
|
+
}
|
|
314
|
+
if (!concurrencyKey) {
|
|
315
|
+
throw new ValidationError(`coalesce requires concurrencyKey${suffix}`);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
function emitDispositionEvent(disposition, run, input, labels) {
|
|
319
|
+
if (disposition === "created") {
|
|
310
320
|
eventEmitter.emit({
|
|
311
321
|
type: "run:trigger",
|
|
312
322
|
runId: run.id,
|
|
313
323
|
jobName: jobDef.name,
|
|
314
|
-
input
|
|
324
|
+
input,
|
|
315
325
|
labels: run.labels
|
|
316
326
|
});
|
|
317
|
-
|
|
327
|
+
} else if (disposition === "coalesced") {
|
|
328
|
+
eventEmitter.emit({
|
|
329
|
+
type: "run:coalesced",
|
|
330
|
+
runId: run.id,
|
|
331
|
+
jobName: jobDef.name,
|
|
332
|
+
labels: run.labels,
|
|
333
|
+
skippedInput: input,
|
|
334
|
+
skippedLabels: labels ?? {}
|
|
335
|
+
});
|
|
336
|
+
}
|
|
337
|
+
}
|
|
338
|
+
const handle = {
|
|
339
|
+
name: jobDef.name,
|
|
340
|
+
async trigger(input, options) {
|
|
341
|
+
validateCoalesceOption(options?.coalesce, options?.concurrencyKey);
|
|
342
|
+
const validatedInput = validateJobInputOrThrow(inputSchema, input);
|
|
343
|
+
const validatedLabels = labelsSchema && options?.labels ? validateJobInputOrThrow(labelsSchema, options.labels, "labels") : options?.labels;
|
|
344
|
+
const { run, disposition } = await storage.enqueue({
|
|
345
|
+
jobName: jobDef.name,
|
|
346
|
+
input: validatedInput,
|
|
347
|
+
idempotencyKey: options?.idempotencyKey,
|
|
348
|
+
concurrencyKey: options?.concurrencyKey,
|
|
349
|
+
labels: validatedLabels,
|
|
350
|
+
coalesce: options?.coalesce
|
|
351
|
+
});
|
|
352
|
+
emitDispositionEvent(
|
|
353
|
+
disposition,
|
|
354
|
+
run,
|
|
355
|
+
validatedInput,
|
|
356
|
+
validatedLabels
|
|
357
|
+
);
|
|
358
|
+
return { ...run, disposition };
|
|
318
359
|
},
|
|
319
360
|
async triggerAndWait(input, options) {
|
|
320
361
|
const run = await this.trigger(input, options);
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
cleanup();
|
|
337
|
-
resolve({
|
|
338
|
-
id: run.id,
|
|
339
|
-
output: event.output
|
|
340
|
-
});
|
|
341
|
-
}
|
|
342
|
-
})
|
|
343
|
-
);
|
|
344
|
-
unsubscribes.push(
|
|
345
|
-
eventEmitter.on("run:fail", (event) => {
|
|
346
|
-
if (event.runId === run.id && !resolved) {
|
|
347
|
-
cleanup();
|
|
348
|
-
reject(new Error(event.error));
|
|
349
|
-
}
|
|
350
|
-
})
|
|
351
|
-
);
|
|
352
|
-
if (options?.onProgress) {
|
|
353
|
-
const onProgress = options.onProgress;
|
|
354
|
-
unsubscribes.push(
|
|
355
|
-
eventEmitter.on("run:progress", (event) => {
|
|
356
|
-
if (event.runId === run.id && !resolved) {
|
|
357
|
-
void Promise.resolve(onProgress(event.progress)).catch(noop);
|
|
358
|
-
}
|
|
359
|
-
})
|
|
360
|
-
);
|
|
361
|
-
}
|
|
362
|
-
if (options?.onLog) {
|
|
363
|
-
const onLog = options.onLog;
|
|
364
|
-
unsubscribes.push(
|
|
365
|
-
eventEmitter.on("log:write", (event) => {
|
|
366
|
-
if (event.runId === run.id && !resolved) {
|
|
367
|
-
const { level, message, data, stepName } = event;
|
|
368
|
-
void Promise.resolve(
|
|
369
|
-
onLog({ level, message, data, stepName })
|
|
370
|
-
).catch(noop);
|
|
371
|
-
}
|
|
372
|
-
})
|
|
373
|
-
);
|
|
374
|
-
}
|
|
375
|
-
storage.getRun(run.id).then((currentRun) => {
|
|
376
|
-
if (resolved || !currentRun) return;
|
|
377
|
-
if (currentRun.status === "completed") {
|
|
378
|
-
cleanup();
|
|
379
|
-
resolve({
|
|
380
|
-
id: run.id,
|
|
381
|
-
output: currentRun.output
|
|
382
|
-
});
|
|
383
|
-
} else if (currentRun.status === "failed") {
|
|
384
|
-
cleanup();
|
|
385
|
-
reject(new Error(currentRun.error || "Run failed"));
|
|
386
|
-
}
|
|
387
|
-
}).catch((error) => {
|
|
388
|
-
if (resolved) return;
|
|
389
|
-
cleanup();
|
|
390
|
-
reject(error instanceof Error ? error : new Error(String(error)));
|
|
391
|
-
});
|
|
392
|
-
if (options?.timeout !== void 0) {
|
|
393
|
-
timeoutId = setTimeout(() => {
|
|
394
|
-
if (!resolved) {
|
|
395
|
-
cleanup();
|
|
396
|
-
reject(
|
|
397
|
-
new Error(`triggerAndWait timeout after ${options.timeout}ms`)
|
|
398
|
-
);
|
|
399
|
-
}
|
|
400
|
-
}, options.timeout);
|
|
401
|
-
}
|
|
402
|
-
});
|
|
362
|
+
const completedRun = await waitForRunCompletion(
|
|
363
|
+
run.id,
|
|
364
|
+
storage,
|
|
365
|
+
eventEmitter,
|
|
366
|
+
{
|
|
367
|
+
...options,
|
|
368
|
+
pollingIntervalMs: options?.pollingIntervalMs ?? pollingIntervalMs
|
|
369
|
+
},
|
|
370
|
+
"triggerAndWait"
|
|
371
|
+
);
|
|
372
|
+
return {
|
|
373
|
+
id: run.id,
|
|
374
|
+
output: completedRun.output,
|
|
375
|
+
disposition: run.disposition
|
|
376
|
+
};
|
|
403
377
|
},
|
|
404
378
|
async batchTrigger(inputs) {
|
|
405
379
|
if (inputs.length === 0) {
|
|
@@ -413,42 +387,51 @@ function createJobHandle(jobDef, storage, eventEmitter, registry, labelsSchema)
|
|
|
413
387
|
});
|
|
414
388
|
const validated = [];
|
|
415
389
|
for (let i = 0; i < normalized.length; i++) {
|
|
390
|
+
const opts = normalized[i].options;
|
|
391
|
+
validateCoalesceOption(
|
|
392
|
+
opts?.coalesce,
|
|
393
|
+
opts?.concurrencyKey,
|
|
394
|
+
`at index ${i}`
|
|
395
|
+
);
|
|
416
396
|
const validatedInput = validateJobInputOrThrow(
|
|
417
397
|
inputSchema,
|
|
418
398
|
normalized[i].input,
|
|
419
399
|
`at index ${i}`
|
|
420
400
|
);
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
);
|
|
427
|
-
}
|
|
401
|
+
const validatedLabels = labelsSchema && opts?.labels ? validateJobInputOrThrow(
|
|
402
|
+
labelsSchema,
|
|
403
|
+
opts.labels,
|
|
404
|
+
`labels at index ${i}`
|
|
405
|
+
) : opts?.labels;
|
|
428
406
|
validated.push({
|
|
429
407
|
input: validatedInput,
|
|
430
|
-
options:
|
|
408
|
+
options: opts ? { ...opts, labels: validatedLabels } : opts
|
|
431
409
|
});
|
|
432
410
|
}
|
|
433
|
-
const
|
|
411
|
+
const results = await storage.enqueueMany(
|
|
434
412
|
validated.map((v) => ({
|
|
435
413
|
jobName: jobDef.name,
|
|
436
414
|
input: v.input,
|
|
437
415
|
idempotencyKey: v.options?.idempotencyKey,
|
|
438
416
|
concurrencyKey: v.options?.concurrencyKey,
|
|
439
|
-
labels: v.options?.labels
|
|
417
|
+
labels: v.options?.labels,
|
|
418
|
+
coalesce: v.options?.coalesce
|
|
440
419
|
}))
|
|
441
420
|
);
|
|
442
|
-
for (let i = 0; i <
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
});
|
|
421
|
+
for (let i = 0; i < results.length; i++) {
|
|
422
|
+
emitDispositionEvent(
|
|
423
|
+
results[i].disposition,
|
|
424
|
+
results[i].run,
|
|
425
|
+
validated[i].input,
|
|
426
|
+
validated[i].options?.labels
|
|
427
|
+
);
|
|
450
428
|
}
|
|
451
|
-
return
|
|
429
|
+
return results.map(
|
|
430
|
+
(r) => ({
|
|
431
|
+
...r.run,
|
|
432
|
+
disposition: r.disposition
|
|
433
|
+
})
|
|
434
|
+
);
|
|
452
435
|
},
|
|
453
436
|
async getRun(id) {
|
|
454
437
|
const run = await storage.getRun(id);
|
|
@@ -487,6 +470,10 @@ var migrations = [
|
|
|
487
470
|
"current_step_index",
|
|
488
471
|
"integer",
|
|
489
472
|
(col) => col.notNull().defaultTo(0)
|
|
473
|
+
).addColumn(
|
|
474
|
+
"completed_step_count",
|
|
475
|
+
"integer",
|
|
476
|
+
(col) => col.notNull().defaultTo(0)
|
|
490
477
|
).addColumn("progress", "text").addColumn("output", "text").addColumn("error", "text").addColumn("lease_owner", "text").addColumn("lease_expires_at", "text").addColumn(
|
|
491
478
|
"lease_generation",
|
|
492
479
|
"integer",
|
|
@@ -510,6 +497,11 @@ var migrations = [
|
|
|
510
497
|
await db.schema.createTable("durably_logs").ifNotExists().addColumn("id", "text", (col) => col.primaryKey()).addColumn("run_id", "text", (col) => col.notNull()).addColumn("step_name", "text").addColumn("level", "text", (col) => col.notNull()).addColumn("message", "text", (col) => col.notNull()).addColumn("data", "text").addColumn("created_at", "text", (col) => col.notNull()).execute();
|
|
511
498
|
await db.schema.createIndex("idx_durably_logs_run_created").ifNotExists().on("durably_logs").columns(["run_id", "created_at"]).execute();
|
|
512
499
|
await db.schema.createTable("durably_schema_versions").ifNotExists().addColumn("version", "integer", (col) => col.primaryKey()).addColumn("applied_at", "text", (col) => col.notNull()).execute();
|
|
500
|
+
await sql`
|
|
501
|
+
CREATE UNIQUE INDEX IF NOT EXISTS idx_durably_runs_pending_concurrency
|
|
502
|
+
ON durably_runs (job_name, concurrency_key)
|
|
503
|
+
WHERE status = 'pending' AND concurrency_key IS NOT NULL
|
|
504
|
+
`.execute(db);
|
|
513
505
|
}
|
|
514
506
|
}
|
|
515
507
|
];
|
|
@@ -536,34 +528,365 @@ async function runMigrations(db) {
|
|
|
536
528
|
}
|
|
537
529
|
}
|
|
538
530
|
|
|
539
|
-
// src/
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
531
|
+
// src/context.ts
|
|
532
|
+
var LEASE_LOST = "lease-lost";
|
|
533
|
+
function createStepContext(run, jobName, leaseGeneration, storage, eventEmitter) {
|
|
534
|
+
let stepIndex = run.currentStepIndex;
|
|
535
|
+
let currentStepName = null;
|
|
536
|
+
const controller = new AbortController();
|
|
537
|
+
function abortForLeaseLoss() {
|
|
538
|
+
if (!controller.signal.aborted) {
|
|
539
|
+
controller.abort(LEASE_LOST);
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
function throwIfAborted() {
|
|
543
|
+
if (!controller.signal.aborted) {
|
|
544
|
+
return;
|
|
545
|
+
}
|
|
546
|
+
if (controller.signal.reason === LEASE_LOST) {
|
|
547
|
+
throw new LeaseLostError(run.id);
|
|
548
|
+
}
|
|
549
|
+
throw new CancelledError(run.id);
|
|
550
|
+
}
|
|
551
|
+
async function throwForRefusedStep(stepName, stepIndex2) {
|
|
552
|
+
const latestRun = await storage.getRun(run.id);
|
|
553
|
+
if (latestRun?.status === "cancelled") {
|
|
554
|
+
eventEmitter.emit({
|
|
555
|
+
type: "step:cancel",
|
|
556
|
+
runId: run.id,
|
|
557
|
+
jobName,
|
|
558
|
+
stepName,
|
|
559
|
+
stepIndex: stepIndex2,
|
|
560
|
+
labels: run.labels
|
|
561
|
+
});
|
|
562
|
+
throw new CancelledError(run.id);
|
|
563
|
+
}
|
|
564
|
+
abortForLeaseLoss();
|
|
565
|
+
throw new LeaseLostError(run.id);
|
|
566
|
+
}
|
|
567
|
+
const unsubscribe = eventEmitter.on("run:cancel", (event) => {
|
|
568
|
+
if (event.runId === run.id) {
|
|
569
|
+
controller.abort();
|
|
570
|
+
}
|
|
571
|
+
});
|
|
572
|
+
const step = {
|
|
573
|
+
get runId() {
|
|
574
|
+
return run.id;
|
|
575
|
+
},
|
|
576
|
+
get signal() {
|
|
577
|
+
return controller.signal;
|
|
578
|
+
},
|
|
579
|
+
isAborted() {
|
|
580
|
+
return controller.signal.aborted;
|
|
581
|
+
},
|
|
582
|
+
throwIfAborted() {
|
|
583
|
+
throwIfAborted();
|
|
584
|
+
},
|
|
585
|
+
async run(name, fn) {
|
|
586
|
+
throwIfAborted();
|
|
587
|
+
const currentRun = await storage.getRun(run.id);
|
|
588
|
+
if (currentRun?.status === "cancelled") {
|
|
589
|
+
controller.abort();
|
|
590
|
+
throwIfAborted();
|
|
591
|
+
}
|
|
592
|
+
if (currentRun && (currentRun.status === "leased" && currentRun.leaseGeneration !== leaseGeneration || currentRun.status === "completed" || currentRun.status === "failed")) {
|
|
593
|
+
abortForLeaseLoss();
|
|
594
|
+
throwIfAborted();
|
|
595
|
+
}
|
|
596
|
+
throwIfAborted();
|
|
597
|
+
const existingStep = await storage.getCompletedStep(run.id, name);
|
|
598
|
+
if (existingStep) {
|
|
599
|
+
stepIndex++;
|
|
600
|
+
return existingStep.output;
|
|
601
|
+
}
|
|
602
|
+
currentStepName = name;
|
|
603
|
+
const startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
604
|
+
const startTime = Date.now();
|
|
605
|
+
eventEmitter.emit({
|
|
606
|
+
type: "step:start",
|
|
607
|
+
runId: run.id,
|
|
608
|
+
jobName,
|
|
609
|
+
stepName: name,
|
|
610
|
+
stepIndex,
|
|
611
|
+
labels: run.labels
|
|
612
|
+
});
|
|
613
|
+
try {
|
|
614
|
+
const result = await fn(controller.signal);
|
|
615
|
+
throwIfAborted();
|
|
616
|
+
const savedStep = await storage.persistStep(run.id, leaseGeneration, {
|
|
617
|
+
name,
|
|
618
|
+
index: stepIndex,
|
|
619
|
+
status: "completed",
|
|
620
|
+
output: result,
|
|
621
|
+
startedAt
|
|
622
|
+
});
|
|
623
|
+
if (!savedStep) {
|
|
624
|
+
await throwForRefusedStep(name, stepIndex);
|
|
625
|
+
}
|
|
626
|
+
stepIndex++;
|
|
627
|
+
eventEmitter.emit({
|
|
628
|
+
type: "step:complete",
|
|
629
|
+
runId: run.id,
|
|
630
|
+
jobName,
|
|
631
|
+
stepName: name,
|
|
632
|
+
stepIndex: stepIndex - 1,
|
|
633
|
+
output: result,
|
|
634
|
+
duration: Date.now() - startTime,
|
|
635
|
+
labels: run.labels
|
|
636
|
+
});
|
|
637
|
+
return result;
|
|
638
|
+
} catch (error) {
|
|
639
|
+
if (error instanceof LeaseLostError) {
|
|
640
|
+
throw error;
|
|
641
|
+
}
|
|
642
|
+
const isLeaseLost = controller.signal.aborted && controller.signal.reason === LEASE_LOST;
|
|
643
|
+
if (isLeaseLost) {
|
|
644
|
+
throw new LeaseLostError(run.id);
|
|
645
|
+
}
|
|
646
|
+
const isCancelled = controller.signal.aborted;
|
|
647
|
+
const errorMessage = getErrorMessage(error);
|
|
648
|
+
const savedStep = await storage.persistStep(run.id, leaseGeneration, {
|
|
649
|
+
name,
|
|
650
|
+
index: stepIndex,
|
|
651
|
+
status: isCancelled ? "cancelled" : "failed",
|
|
652
|
+
error: errorMessage,
|
|
653
|
+
startedAt
|
|
654
|
+
});
|
|
655
|
+
if (!savedStep) {
|
|
656
|
+
await throwForRefusedStep(name, stepIndex);
|
|
657
|
+
}
|
|
658
|
+
eventEmitter.emit({
|
|
659
|
+
type: "step:fail",
|
|
660
|
+
error: errorMessage,
|
|
661
|
+
runId: run.id,
|
|
662
|
+
jobName,
|
|
663
|
+
stepName: name,
|
|
664
|
+
stepIndex,
|
|
665
|
+
labels: run.labels
|
|
666
|
+
});
|
|
667
|
+
throw error;
|
|
668
|
+
} finally {
|
|
669
|
+
currentStepName = null;
|
|
670
|
+
}
|
|
671
|
+
},
|
|
672
|
+
progress(current, total, message) {
|
|
673
|
+
const progressData = { current, total, message };
|
|
674
|
+
storage.updateProgress(run.id, leaseGeneration, progressData);
|
|
675
|
+
eventEmitter.emit({
|
|
676
|
+
type: "run:progress",
|
|
677
|
+
runId: run.id,
|
|
678
|
+
jobName,
|
|
679
|
+
progress: progressData,
|
|
680
|
+
labels: run.labels
|
|
681
|
+
});
|
|
682
|
+
},
|
|
683
|
+
log: {
|
|
684
|
+
info(message, data) {
|
|
685
|
+
eventEmitter.emit({
|
|
686
|
+
type: "log:write",
|
|
687
|
+
runId: run.id,
|
|
688
|
+
jobName,
|
|
689
|
+
labels: run.labels,
|
|
690
|
+
stepName: currentStepName,
|
|
691
|
+
level: "info",
|
|
692
|
+
message,
|
|
693
|
+
data
|
|
694
|
+
});
|
|
695
|
+
},
|
|
696
|
+
warn(message, data) {
|
|
697
|
+
eventEmitter.emit({
|
|
698
|
+
type: "log:write",
|
|
699
|
+
runId: run.id,
|
|
700
|
+
jobName,
|
|
701
|
+
labels: run.labels,
|
|
702
|
+
stepName: currentStepName,
|
|
703
|
+
level: "warn",
|
|
704
|
+
message,
|
|
705
|
+
data
|
|
706
|
+
});
|
|
707
|
+
},
|
|
708
|
+
error(message, data) {
|
|
709
|
+
eventEmitter.emit({
|
|
710
|
+
type: "log:write",
|
|
711
|
+
runId: run.id,
|
|
712
|
+
jobName,
|
|
713
|
+
labels: run.labels,
|
|
714
|
+
stepName: currentStepName,
|
|
715
|
+
level: "error",
|
|
716
|
+
message,
|
|
717
|
+
data
|
|
718
|
+
});
|
|
719
|
+
}
|
|
720
|
+
}
|
|
721
|
+
};
|
|
722
|
+
return {
|
|
723
|
+
step,
|
|
724
|
+
abortLeaseOwnership: abortForLeaseLoss,
|
|
725
|
+
dispose: unsubscribe
|
|
726
|
+
};
|
|
555
727
|
}
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
728
|
+
|
|
729
|
+
// src/runtime.ts
|
|
730
|
+
function isoNow(clock) {
|
|
731
|
+
return new Date(clock.now()).toISOString();
|
|
732
|
+
}
|
|
733
|
+
async function executeRun(run, job, config, environment) {
|
|
734
|
+
const { storage, eventEmitter, clock } = environment;
|
|
735
|
+
const { step, abortLeaseOwnership, dispose } = createStepContext(
|
|
736
|
+
run,
|
|
737
|
+
run.jobName,
|
|
738
|
+
run.leaseGeneration,
|
|
739
|
+
storage,
|
|
740
|
+
eventEmitter
|
|
741
|
+
);
|
|
742
|
+
let leaseDeadlineTimer = null;
|
|
743
|
+
const scheduleLeaseDeadline = (leaseExpiresAt) => {
|
|
744
|
+
if (leaseDeadlineTimer) {
|
|
745
|
+
clock.clearTimeout(leaseDeadlineTimer);
|
|
746
|
+
leaseDeadlineTimer = null;
|
|
747
|
+
}
|
|
748
|
+
if (!leaseExpiresAt) {
|
|
749
|
+
return;
|
|
750
|
+
}
|
|
751
|
+
const delay = Math.max(0, Date.parse(leaseExpiresAt) - clock.now());
|
|
752
|
+
leaseDeadlineTimer = clock.setTimeout(() => {
|
|
753
|
+
abortLeaseOwnership();
|
|
754
|
+
}, delay);
|
|
755
|
+
};
|
|
756
|
+
scheduleLeaseDeadline(run.leaseExpiresAt);
|
|
757
|
+
const leaseTimer = clock.setInterval(() => {
|
|
758
|
+
const now = isoNow(clock);
|
|
759
|
+
storage.renewLease(run.id, run.leaseGeneration, now, config.leaseMs).then((renewed) => {
|
|
760
|
+
if (!renewed) {
|
|
761
|
+
abortLeaseOwnership();
|
|
762
|
+
eventEmitter.emit({
|
|
763
|
+
type: "worker:error",
|
|
764
|
+
error: `Lease renewal lost ownership for run ${run.id}`,
|
|
765
|
+
context: "lease-renewal",
|
|
766
|
+
runId: run.id
|
|
767
|
+
});
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
const renewedLeaseExpiresAt = new Date(
|
|
771
|
+
Date.parse(now) + config.leaseMs
|
|
772
|
+
).toISOString();
|
|
773
|
+
scheduleLeaseDeadline(renewedLeaseExpiresAt);
|
|
774
|
+
eventEmitter.emit({
|
|
775
|
+
type: "run:lease-renewed",
|
|
776
|
+
runId: run.id,
|
|
777
|
+
jobName: run.jobName,
|
|
778
|
+
leaseOwner: run.leaseOwner ?? "",
|
|
779
|
+
leaseExpiresAt: renewedLeaseExpiresAt,
|
|
780
|
+
labels: run.labels
|
|
781
|
+
});
|
|
782
|
+
}).catch((error) => {
|
|
783
|
+
eventEmitter.emit({
|
|
784
|
+
type: "worker:error",
|
|
785
|
+
error: getErrorMessage(error),
|
|
786
|
+
context: "lease-renewal",
|
|
787
|
+
runId: run.id
|
|
788
|
+
});
|
|
789
|
+
});
|
|
790
|
+
}, config.leaseRenewIntervalMs);
|
|
791
|
+
const started = clock.now();
|
|
792
|
+
let reachedTerminalState = false;
|
|
793
|
+
try {
|
|
794
|
+
eventEmitter.emit({
|
|
795
|
+
type: "run:leased",
|
|
796
|
+
runId: run.id,
|
|
797
|
+
jobName: run.jobName,
|
|
798
|
+
input: run.input,
|
|
799
|
+
leaseOwner: run.leaseOwner ?? "",
|
|
800
|
+
leaseExpiresAt: run.leaseExpiresAt ?? isoNow(clock),
|
|
801
|
+
labels: run.labels
|
|
802
|
+
});
|
|
803
|
+
const output = await job.fn(step, run.input);
|
|
804
|
+
if (job.outputSchema) {
|
|
805
|
+
const parseResult = job.outputSchema.safeParse(output);
|
|
806
|
+
if (!parseResult.success) {
|
|
807
|
+
throw new Error(`Invalid output: ${parseResult.error.message}`);
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
const completedAt = isoNow(clock);
|
|
811
|
+
const completed = await storage.completeRun(
|
|
812
|
+
run.id,
|
|
813
|
+
run.leaseGeneration,
|
|
814
|
+
output,
|
|
815
|
+
completedAt
|
|
816
|
+
);
|
|
817
|
+
if (completed) {
|
|
818
|
+
reachedTerminalState = true;
|
|
819
|
+
eventEmitter.emit({
|
|
820
|
+
type: "run:complete",
|
|
821
|
+
runId: run.id,
|
|
822
|
+
jobName: run.jobName,
|
|
823
|
+
output,
|
|
824
|
+
duration: clock.now() - started,
|
|
825
|
+
labels: run.labels
|
|
826
|
+
});
|
|
827
|
+
return { kind: "completed" };
|
|
828
|
+
}
|
|
829
|
+
eventEmitter.emit({
|
|
830
|
+
type: "worker:error",
|
|
831
|
+
error: `Lease lost before completing run ${run.id}`,
|
|
832
|
+
context: "run-completion"
|
|
833
|
+
});
|
|
834
|
+
return { kind: "lease-lost" };
|
|
835
|
+
} catch (error) {
|
|
836
|
+
if (error instanceof LeaseLostError) {
|
|
837
|
+
return { kind: "lease-lost" };
|
|
838
|
+
}
|
|
839
|
+
if (error instanceof CancelledError) {
|
|
840
|
+
return { kind: "cancelled" };
|
|
841
|
+
}
|
|
842
|
+
const errorMessage = getErrorMessage(error);
|
|
843
|
+
const completedAt = isoNow(clock);
|
|
844
|
+
const failed = await storage.failRun(
|
|
845
|
+
run.id,
|
|
846
|
+
run.leaseGeneration,
|
|
847
|
+
errorMessage,
|
|
848
|
+
completedAt
|
|
849
|
+
);
|
|
850
|
+
if (failed) {
|
|
851
|
+
reachedTerminalState = true;
|
|
852
|
+
const steps = await storage.getSteps(run.id);
|
|
853
|
+
const failedStep = steps.find((entry) => entry.status === "failed");
|
|
854
|
+
eventEmitter.emit({
|
|
855
|
+
type: "run:fail",
|
|
856
|
+
runId: run.id,
|
|
857
|
+
jobName: run.jobName,
|
|
858
|
+
error: errorMessage,
|
|
859
|
+
failedStepName: failedStep?.name ?? "unknown",
|
|
860
|
+
labels: run.labels
|
|
861
|
+
});
|
|
862
|
+
return { kind: "failed" };
|
|
863
|
+
}
|
|
864
|
+
eventEmitter.emit({
|
|
865
|
+
type: "worker:error",
|
|
866
|
+
error: `Lease lost before recording failure for run ${run.id}`,
|
|
867
|
+
context: "run-failure"
|
|
868
|
+
});
|
|
869
|
+
return { kind: "lease-lost" };
|
|
870
|
+
} finally {
|
|
871
|
+
clock.clearInterval(leaseTimer);
|
|
872
|
+
if (leaseDeadlineTimer) {
|
|
873
|
+
clock.clearTimeout(leaseDeadlineTimer);
|
|
874
|
+
}
|
|
875
|
+
dispose();
|
|
876
|
+
if (!config.preserveSteps && reachedTerminalState) {
|
|
877
|
+
await storage.deleteSteps(run.id);
|
|
564
878
|
}
|
|
565
879
|
}
|
|
566
880
|
}
|
|
881
|
+
|
|
882
|
+
// src/storage.ts
|
|
883
|
+
import { sql as sql4 } from "kysely";
|
|
884
|
+
import { monotonicFactory } from "ulidx";
|
|
885
|
+
|
|
886
|
+
// src/claim-postgres.ts
|
|
887
|
+
import { sql as sql2 } from "kysely";
|
|
888
|
+
|
|
889
|
+
// src/transformers.ts
|
|
567
890
|
function rowToRun(row) {
|
|
568
891
|
return {
|
|
569
892
|
id: row.id,
|
|
@@ -573,7 +896,7 @@ function rowToRun(row) {
|
|
|
573
896
|
idempotencyKey: row.idempotency_key,
|
|
574
897
|
concurrencyKey: row.concurrency_key,
|
|
575
898
|
currentStepIndex: row.current_step_index,
|
|
576
|
-
|
|
899
|
+
completedStepCount: row.completed_step_count,
|
|
577
900
|
progress: row.progress ? JSON.parse(row.progress) : null,
|
|
578
901
|
output: row.output ? JSON.parse(row.output) : null,
|
|
579
902
|
error: row.error,
|
|
@@ -600,15 +923,135 @@ function rowToStep(row) {
|
|
|
600
923
|
completedAt: row.completed_at
|
|
601
924
|
};
|
|
602
925
|
}
|
|
603
|
-
function rowToLog(row) {
|
|
926
|
+
function rowToLog(row) {
|
|
927
|
+
return {
|
|
928
|
+
id: row.id,
|
|
929
|
+
runId: row.run_id,
|
|
930
|
+
stepName: row.step_name,
|
|
931
|
+
level: row.level,
|
|
932
|
+
message: row.message,
|
|
933
|
+
data: row.data ? JSON.parse(row.data) : null,
|
|
934
|
+
createdAt: row.created_at
|
|
935
|
+
};
|
|
936
|
+
}
|
|
937
|
+
var LABEL_KEY_PATTERN = /^[a-zA-Z0-9\-_./]+$/;
|
|
938
|
+
function validateLabels(labels) {
|
|
939
|
+
if (!labels) return;
|
|
940
|
+
for (const key of Object.keys(labels)) {
|
|
941
|
+
if (!LABEL_KEY_PATTERN.test(key)) {
|
|
942
|
+
throw new Error(
|
|
943
|
+
`Invalid label key "${key}": must contain only alphanumeric characters, dashes, underscores, dots, and slashes`
|
|
944
|
+
);
|
|
945
|
+
}
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
// src/claim-postgres.ts
|
|
950
|
+
async function claimNextPostgres(db, workerId, now, leaseExpiresAt, activeLeaseGuard) {
|
|
951
|
+
return await db.transaction().execute(async (trx) => {
|
|
952
|
+
const skipKeys = [];
|
|
953
|
+
for (; ; ) {
|
|
954
|
+
const concurrencyCondition = skipKeys.length > 0 ? sql2`
|
|
955
|
+
AND (
|
|
956
|
+
concurrency_key IS NULL
|
|
957
|
+
OR concurrency_key NOT IN (${sql2.join(skipKeys)})
|
|
958
|
+
)
|
|
959
|
+
` : sql2``;
|
|
960
|
+
const candidateResult = await sql2`
|
|
961
|
+
SELECT id, concurrency_key
|
|
962
|
+
FROM durably_runs
|
|
963
|
+
WHERE
|
|
964
|
+
(
|
|
965
|
+
status = 'pending'
|
|
966
|
+
OR (status = 'leased' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ${now})
|
|
967
|
+
)
|
|
968
|
+
AND ${activeLeaseGuard}
|
|
969
|
+
${concurrencyCondition}
|
|
970
|
+
ORDER BY created_at ASC, id ASC
|
|
971
|
+
FOR UPDATE SKIP LOCKED
|
|
972
|
+
LIMIT 1
|
|
973
|
+
`.execute(trx);
|
|
974
|
+
const candidate = candidateResult.rows[0];
|
|
975
|
+
if (!candidate) return null;
|
|
976
|
+
if (candidate.concurrency_key) {
|
|
977
|
+
await sql2`SELECT pg_advisory_xact_lock(hashtext(${candidate.concurrency_key}))`.execute(
|
|
978
|
+
trx
|
|
979
|
+
);
|
|
980
|
+
const conflict = await sql2`
|
|
981
|
+
SELECT 1 FROM durably_runs
|
|
982
|
+
WHERE concurrency_key = ${candidate.concurrency_key}
|
|
983
|
+
AND id <> ${candidate.id}
|
|
984
|
+
AND status = 'leased'
|
|
985
|
+
AND lease_expires_at IS NOT NULL
|
|
986
|
+
AND lease_expires_at > ${now}
|
|
987
|
+
LIMIT 1
|
|
988
|
+
`.execute(trx);
|
|
989
|
+
if (conflict.rows.length > 0) {
|
|
990
|
+
skipKeys.push(candidate.concurrency_key);
|
|
991
|
+
continue;
|
|
992
|
+
}
|
|
993
|
+
}
|
|
994
|
+
const result = await sql2`
|
|
995
|
+
UPDATE durably_runs
|
|
996
|
+
SET
|
|
997
|
+
status = 'leased',
|
|
998
|
+
lease_owner = ${workerId},
|
|
999
|
+
lease_expires_at = ${leaseExpiresAt},
|
|
1000
|
+
lease_generation = lease_generation + 1,
|
|
1001
|
+
started_at = COALESCE(started_at, ${now}),
|
|
1002
|
+
updated_at = ${now}
|
|
1003
|
+
WHERE id = ${candidate.id}
|
|
1004
|
+
RETURNING *
|
|
1005
|
+
`.execute(trx);
|
|
1006
|
+
const row = result.rows[0];
|
|
1007
|
+
if (!row) return null;
|
|
1008
|
+
return rowToRun(row);
|
|
1009
|
+
}
|
|
1010
|
+
});
|
|
1011
|
+
}
|
|
1012
|
+
|
|
1013
|
+
// src/claim-sqlite.ts
|
|
1014
|
+
import { sql as sql3 } from "kysely";
|
|
1015
|
+
async function claimNextSqlite(db, workerId, now, leaseExpiresAt, activeLeaseGuard) {
|
|
1016
|
+
const subquery = db.selectFrom("durably_runs").select("durably_runs.id").where(
|
|
1017
|
+
(eb) => eb.or([
|
|
1018
|
+
eb("status", "=", "pending"),
|
|
1019
|
+
eb.and([
|
|
1020
|
+
eb("status", "=", "leased"),
|
|
1021
|
+
eb("lease_expires_at", "is not", null),
|
|
1022
|
+
eb("lease_expires_at", "<=", now)
|
|
1023
|
+
])
|
|
1024
|
+
])
|
|
1025
|
+
).where(activeLeaseGuard).orderBy("created_at", "asc").orderBy("id", "asc").limit(1);
|
|
1026
|
+
const row = await db.updateTable("durably_runs").set({
|
|
1027
|
+
status: "leased",
|
|
1028
|
+
lease_owner: workerId,
|
|
1029
|
+
lease_expires_at: leaseExpiresAt,
|
|
1030
|
+
lease_generation: sql3`lease_generation + 1`,
|
|
1031
|
+
started_at: sql3`COALESCE(started_at, ${now})`,
|
|
1032
|
+
updated_at: now
|
|
1033
|
+
}).where("id", "=", (eb) => eb.selectFrom(subquery.as("sub")).select("id")).returningAll().executeTakeFirst();
|
|
1034
|
+
if (!row) return null;
|
|
1035
|
+
return rowToRun(row);
|
|
1036
|
+
}
|
|
1037
|
+
|
|
1038
|
+
// src/storage.ts
|
|
1039
|
+
var ulid = monotonicFactory();
|
|
1040
|
+
var TERMINAL_STATUSES = ["completed", "failed", "cancelled"];
|
|
1041
|
+
function toClientRun(run) {
|
|
1042
|
+
const {
|
|
1043
|
+
idempotencyKey,
|
|
1044
|
+
concurrencyKey,
|
|
1045
|
+
leaseOwner,
|
|
1046
|
+
leaseExpiresAt,
|
|
1047
|
+
leaseGeneration,
|
|
1048
|
+
updatedAt,
|
|
1049
|
+
...clientRun
|
|
1050
|
+
} = run;
|
|
604
1051
|
return {
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
level: row.level,
|
|
609
|
-
message: row.message,
|
|
610
|
-
data: row.data ? JSON.parse(row.data) : null,
|
|
611
|
-
createdAt: row.created_at
|
|
1052
|
+
...clientRun,
|
|
1053
|
+
isTerminal: TERMINAL_STATUSES.includes(run.status),
|
|
1054
|
+
isActive: run.status === "pending" || run.status === "leased"
|
|
612
1055
|
};
|
|
613
1056
|
}
|
|
614
1057
|
function createWriteMutex() {
|
|
@@ -628,6 +1071,29 @@ function createWriteMutex() {
|
|
|
628
1071
|
}
|
|
629
1072
|
};
|
|
630
1073
|
}
|
|
1074
|
+
function isUniqueViolation(err) {
|
|
1075
|
+
if (!(err instanceof Error)) return false;
|
|
1076
|
+
const pgCode = err.code;
|
|
1077
|
+
if (pgCode === "23505") return true;
|
|
1078
|
+
if (err.constraint) return true;
|
|
1079
|
+
if (/unique constraint/i.test(err.message)) return true;
|
|
1080
|
+
return false;
|
|
1081
|
+
}
|
|
1082
|
+
function parseUniqueViolation(err) {
|
|
1083
|
+
if (!(err instanceof Error)) return null;
|
|
1084
|
+
const msg = err.message;
|
|
1085
|
+
const pgConstraint = err.constraint;
|
|
1086
|
+
if (pgConstraint) {
|
|
1087
|
+
if (pgConstraint.includes("idempotency")) return "idempotency";
|
|
1088
|
+
if (pgConstraint.includes("pending_concurrency"))
|
|
1089
|
+
return "pending_concurrency";
|
|
1090
|
+
}
|
|
1091
|
+
if (/unique constraint/i.test(msg)) {
|
|
1092
|
+
if (msg.includes("idempotency_key")) return "idempotency";
|
|
1093
|
+
if (msg.includes("concurrency_key")) return "pending_concurrency";
|
|
1094
|
+
}
|
|
1095
|
+
return null;
|
|
1096
|
+
}
|
|
631
1097
|
function createKyselyStore(db, backend = "generic") {
|
|
632
1098
|
const withWriteLock = createWriteMutex();
|
|
633
1099
|
async function cascadeDeleteRuns(trx, ids) {
|
|
@@ -653,119 +1119,138 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
653
1119
|
}).where("id", "=", runId).where("status", "=", "leased").where("lease_generation", "=", leaseGeneration).executeTakeFirst();
|
|
654
1120
|
return Number(result.numUpdatedRows) > 0;
|
|
655
1121
|
}
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
1122
|
+
function findPendingByConcurrencyKey(queryDb, jobName, concurrencyKey) {
|
|
1123
|
+
return queryDb.selectFrom("durably_runs").selectAll().where("job_name", "=", jobName).where("concurrency_key", "=", concurrencyKey).where("status", "=", "pending").orderBy("created_at", "asc").orderBy("id", "asc").limit(1).executeTakeFirst();
|
|
1124
|
+
}
|
|
1125
|
+
async function enqueueInTx(trx, input, retried = false) {
|
|
1126
|
+
const queryDb = trx ?? db;
|
|
1127
|
+
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1128
|
+
if (input.idempotencyKey) {
|
|
1129
|
+
const existing = await queryDb.selectFrom("durably_runs").selectAll().where("job_name", "=", input.jobName).where("idempotency_key", "=", input.idempotencyKey).executeTakeFirst();
|
|
1130
|
+
if (existing) {
|
|
1131
|
+
return { run: rowToRun(existing), disposition: "idempotent" };
|
|
1132
|
+
}
|
|
1133
|
+
}
|
|
1134
|
+
validateLabels(input.labels);
|
|
1135
|
+
const id = ulid();
|
|
1136
|
+
const row = {
|
|
1137
|
+
id,
|
|
1138
|
+
job_name: input.jobName,
|
|
1139
|
+
input: JSON.stringify(input.input),
|
|
1140
|
+
status: "pending",
|
|
1141
|
+
idempotency_key: input.idempotencyKey ?? null,
|
|
1142
|
+
concurrency_key: input.concurrencyKey ?? null,
|
|
1143
|
+
current_step_index: 0,
|
|
1144
|
+
completed_step_count: 0,
|
|
1145
|
+
progress: null,
|
|
1146
|
+
output: null,
|
|
1147
|
+
error: null,
|
|
1148
|
+
labels: JSON.stringify(input.labels ?? {}),
|
|
1149
|
+
lease_owner: null,
|
|
1150
|
+
lease_expires_at: null,
|
|
1151
|
+
lease_generation: 0,
|
|
1152
|
+
started_at: null,
|
|
1153
|
+
completed_at: null,
|
|
1154
|
+
created_at: now,
|
|
1155
|
+
updated_at: now
|
|
1156
|
+
};
|
|
1157
|
+
const doInsert = async (insertDb) => {
|
|
1158
|
+
await sql4`SAVEPOINT sp_enqueue`.execute(insertDb);
|
|
1159
|
+
try {
|
|
1160
|
+
await insertDb.insertInto("durably_runs").values(row).execute();
|
|
1161
|
+
await insertLabelRows(insertDb, id, input.labels);
|
|
1162
|
+
await sql4`RELEASE SAVEPOINT sp_enqueue`.execute(insertDb);
|
|
1163
|
+
} catch (err) {
|
|
1164
|
+
await sql4`ROLLBACK TO SAVEPOINT sp_enqueue`.execute(insertDb);
|
|
1165
|
+
throw err;
|
|
1166
|
+
}
|
|
1167
|
+
};
|
|
1168
|
+
try {
|
|
1169
|
+
if (trx) {
|
|
1170
|
+
await doInsert(trx);
|
|
1171
|
+
} else {
|
|
1172
|
+
await db.transaction().execute(doInsert);
|
|
1173
|
+
}
|
|
1174
|
+
return { run: rowToRun(row), disposition: "created" };
|
|
1175
|
+
} catch (err) {
|
|
1176
|
+
if (!isUniqueViolation(err)) throw err;
|
|
1177
|
+
const violation = parseUniqueViolation(err);
|
|
659
1178
|
if (input.idempotencyKey) {
|
|
660
|
-
const
|
|
661
|
-
if (
|
|
662
|
-
return rowToRun(
|
|
1179
|
+
const idempotent = await queryDb.selectFrom("durably_runs").selectAll().where("job_name", "=", input.jobName).where("idempotency_key", "=", input.idempotencyKey).executeTakeFirst();
|
|
1180
|
+
if (idempotent) {
|
|
1181
|
+
return { run: rowToRun(idempotent), disposition: "idempotent" };
|
|
663
1182
|
}
|
|
664
1183
|
}
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
1184
|
+
if ((violation === "pending_concurrency" || violation === null) && input.concurrencyKey) {
|
|
1185
|
+
if (input.coalesce === "skip") {
|
|
1186
|
+
const pending = await findPendingByConcurrencyKey(
|
|
1187
|
+
queryDb,
|
|
1188
|
+
input.jobName,
|
|
1189
|
+
input.concurrencyKey
|
|
1190
|
+
);
|
|
1191
|
+
if (pending) {
|
|
1192
|
+
return { run: rowToRun(pending), disposition: "coalesced" };
|
|
1193
|
+
}
|
|
1194
|
+
if (!retried) {
|
|
1195
|
+
return enqueueInTx(trx, input, true);
|
|
1196
|
+
}
|
|
1197
|
+
const lastChance = await findPendingByConcurrencyKey(
|
|
1198
|
+
queryDb,
|
|
1199
|
+
input.jobName,
|
|
1200
|
+
input.concurrencyKey
|
|
1201
|
+
);
|
|
1202
|
+
if (lastChance) {
|
|
1203
|
+
return { run: rowToRun(lastChance), disposition: "coalesced" };
|
|
1204
|
+
}
|
|
1205
|
+
throw new ConflictError(
|
|
1206
|
+
`Conflict after retry for concurrency key "${input.concurrencyKey}" in job "${input.jobName}". Concurrent modification detected.`
|
|
1207
|
+
);
|
|
1208
|
+
}
|
|
1209
|
+
throw new ConflictError(
|
|
1210
|
+
`A pending run already exists for concurrency key "${input.concurrencyKey}" in job "${input.jobName}". Use coalesce: 'skip' to return the existing run instead.`
|
|
1211
|
+
);
|
|
1212
|
+
}
|
|
1213
|
+
throw err;
|
|
1214
|
+
}
|
|
1215
|
+
}
|
|
1216
|
+
const store = {
|
|
1217
|
+
async enqueue(input) {
|
|
1218
|
+
return enqueueInTx(null, input);
|
|
692
1219
|
},
|
|
693
1220
|
async enqueueMany(inputs) {
|
|
694
1221
|
if (inputs.length === 0) {
|
|
695
1222
|
return [];
|
|
696
1223
|
}
|
|
697
|
-
return
|
|
698
|
-
const
|
|
699
|
-
const runs = [];
|
|
1224
|
+
return db.transaction().execute(async (trx) => {
|
|
1225
|
+
const results = [];
|
|
700
1226
|
for (const input of inputs) {
|
|
701
|
-
|
|
702
|
-
}
|
|
703
|
-
const allLabelRows = [];
|
|
704
|
-
for (const input of inputs) {
|
|
705
|
-
if (input.idempotencyKey) {
|
|
706
|
-
const existing = await trx.selectFrom("durably_runs").selectAll().where("job_name", "=", input.jobName).where("idempotency_key", "=", input.idempotencyKey).executeTakeFirst();
|
|
707
|
-
if (existing) {
|
|
708
|
-
runs.push(existing);
|
|
709
|
-
continue;
|
|
710
|
-
}
|
|
711
|
-
}
|
|
712
|
-
const id = ulid();
|
|
713
|
-
if (input.labels) {
|
|
714
|
-
for (const [key, value] of Object.entries(input.labels)) {
|
|
715
|
-
allLabelRows.push({ run_id: id, key, value });
|
|
716
|
-
}
|
|
717
|
-
}
|
|
718
|
-
runs.push({
|
|
719
|
-
id,
|
|
720
|
-
job_name: input.jobName,
|
|
721
|
-
input: JSON.stringify(input.input),
|
|
722
|
-
status: "pending",
|
|
723
|
-
idempotency_key: input.idempotencyKey ?? null,
|
|
724
|
-
concurrency_key: input.concurrencyKey ?? null,
|
|
725
|
-
current_step_index: 0,
|
|
726
|
-
progress: null,
|
|
727
|
-
output: null,
|
|
728
|
-
error: null,
|
|
729
|
-
labels: JSON.stringify(input.labels ?? {}),
|
|
730
|
-
lease_owner: null,
|
|
731
|
-
lease_expires_at: null,
|
|
732
|
-
lease_generation: 0,
|
|
733
|
-
started_at: null,
|
|
734
|
-
completed_at: null,
|
|
735
|
-
created_at: now,
|
|
736
|
-
updated_at: now
|
|
737
|
-
});
|
|
738
|
-
}
|
|
739
|
-
const newRuns = runs.filter((r) => r.created_at === now);
|
|
740
|
-
if (newRuns.length > 0) {
|
|
741
|
-
await trx.insertInto("durably_runs").values(newRuns).execute();
|
|
742
|
-
if (allLabelRows.length > 0) {
|
|
743
|
-
await trx.insertInto("durably_run_labels").values(allLabelRows).execute();
|
|
744
|
-
}
|
|
1227
|
+
results.push(await enqueueInTx(trx, input));
|
|
745
1228
|
}
|
|
746
|
-
return
|
|
1229
|
+
return results;
|
|
747
1230
|
});
|
|
748
1231
|
},
|
|
749
1232
|
async getRun(runId) {
|
|
750
|
-
const row = await db.selectFrom("durably_runs").
|
|
751
|
-
(eb) => eb.fn.count("durably_steps.id").as("step_count")
|
|
752
|
-
).where("durably_runs.id", "=", runId).groupBy("durably_runs.id").executeTakeFirst();
|
|
1233
|
+
const row = await db.selectFrom("durably_runs").selectAll().where("id", "=", runId).executeTakeFirst();
|
|
753
1234
|
return row ? rowToRun(row) : null;
|
|
754
1235
|
},
|
|
755
1236
|
async getRuns(filter) {
|
|
756
|
-
let query = db.selectFrom("durably_runs").
|
|
757
|
-
(eb) => eb.fn.count("durably_steps.id").as("step_count")
|
|
758
|
-
).groupBy("durably_runs.id");
|
|
1237
|
+
let query = db.selectFrom("durably_runs").selectAll();
|
|
759
1238
|
if (filter?.status) {
|
|
760
|
-
|
|
1239
|
+
if (Array.isArray(filter.status)) {
|
|
1240
|
+
if (filter.status.length > 0) {
|
|
1241
|
+
query = query.where("status", "in", filter.status);
|
|
1242
|
+
}
|
|
1243
|
+
} else {
|
|
1244
|
+
query = query.where("status", "=", filter.status);
|
|
1245
|
+
}
|
|
761
1246
|
}
|
|
762
1247
|
if (filter?.jobName) {
|
|
763
1248
|
if (Array.isArray(filter.jobName)) {
|
|
764
1249
|
if (filter.jobName.length > 0) {
|
|
765
|
-
query = query.where("
|
|
1250
|
+
query = query.where("job_name", "in", filter.jobName);
|
|
766
1251
|
}
|
|
767
1252
|
} else {
|
|
768
|
-
query = query.where("
|
|
1253
|
+
query = query.where("job_name", "=", filter.jobName);
|
|
769
1254
|
}
|
|
770
1255
|
}
|
|
771
1256
|
if (filter?.labels) {
|
|
@@ -773,18 +1258,14 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
773
1258
|
validateLabels(labels);
|
|
774
1259
|
for (const [key, value] of Object.entries(labels)) {
|
|
775
1260
|
if (value === void 0) continue;
|
|
776
|
-
const jsonFallback = backend === "postgres" ? sql2`durably_runs.labels ->> ${key} = ${value}` : sql2`json_extract(durably_runs.labels, ${`$.${key}`}) = ${value}`;
|
|
777
1261
|
query = query.where(
|
|
778
|
-
(eb) => eb.
|
|
779
|
-
eb.
|
|
780
|
-
|
|
781
|
-
),
|
|
782
|
-
jsonFallback
|
|
783
|
-
])
|
|
1262
|
+
(eb) => eb.exists(
|
|
1263
|
+
eb.selectFrom("durably_run_labels").select(sql4.lit(1).as("one")).whereRef("durably_run_labels.run_id", "=", "durably_runs.id").where("durably_run_labels.key", "=", key).where("durably_run_labels.value", "=", value)
|
|
1264
|
+
)
|
|
784
1265
|
);
|
|
785
1266
|
}
|
|
786
1267
|
}
|
|
787
|
-
query = query.orderBy("
|
|
1268
|
+
query = query.orderBy("created_at", "desc");
|
|
788
1269
|
if (filter?.limit !== void 0) {
|
|
789
1270
|
query = query.limit(filter.limit);
|
|
790
1271
|
}
|
|
@@ -830,7 +1311,7 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
830
1311
|
},
|
|
831
1312
|
async claimNext(workerId, now, leaseMs) {
|
|
832
1313
|
const leaseExpiresAt = new Date(Date.parse(now) + leaseMs).toISOString();
|
|
833
|
-
const activeLeaseGuard =
|
|
1314
|
+
const activeLeaseGuard = sql4`
|
|
834
1315
|
(
|
|
835
1316
|
concurrency_key IS NULL
|
|
836
1317
|
OR NOT EXISTS (
|
|
@@ -844,92 +1325,7 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
844
1325
|
)
|
|
845
1326
|
)
|
|
846
1327
|
`;
|
|
847
|
-
|
|
848
|
-
return await db.transaction().execute(async (trx) => {
|
|
849
|
-
const skipKeys = [];
|
|
850
|
-
for (; ; ) {
|
|
851
|
-
const concurrencyCondition = skipKeys.length > 0 ? sql2`
|
|
852
|
-
AND (
|
|
853
|
-
concurrency_key IS NULL
|
|
854
|
-
OR concurrency_key NOT IN (${sql2.join(skipKeys)})
|
|
855
|
-
)
|
|
856
|
-
` : sql2``;
|
|
857
|
-
const candidateResult = await sql2`
|
|
858
|
-
SELECT id, concurrency_key
|
|
859
|
-
FROM durably_runs
|
|
860
|
-
WHERE
|
|
861
|
-
(
|
|
862
|
-
status = 'pending'
|
|
863
|
-
OR (status = 'leased' AND lease_expires_at IS NOT NULL AND lease_expires_at <= ${now})
|
|
864
|
-
)
|
|
865
|
-
AND ${activeLeaseGuard}
|
|
866
|
-
${concurrencyCondition}
|
|
867
|
-
ORDER BY created_at ASC, id ASC
|
|
868
|
-
FOR UPDATE SKIP LOCKED
|
|
869
|
-
LIMIT 1
|
|
870
|
-
`.execute(trx);
|
|
871
|
-
const candidate = candidateResult.rows[0];
|
|
872
|
-
if (!candidate) return null;
|
|
873
|
-
if (candidate.concurrency_key) {
|
|
874
|
-
await sql2`SELECT pg_advisory_xact_lock(hashtext(${candidate.concurrency_key}))`.execute(
|
|
875
|
-
trx
|
|
876
|
-
);
|
|
877
|
-
const conflict = await sql2`
|
|
878
|
-
SELECT 1 FROM durably_runs
|
|
879
|
-
WHERE concurrency_key = ${candidate.concurrency_key}
|
|
880
|
-
AND id <> ${candidate.id}
|
|
881
|
-
AND status = 'leased'
|
|
882
|
-
AND lease_expires_at IS NOT NULL
|
|
883
|
-
AND lease_expires_at > ${now}
|
|
884
|
-
LIMIT 1
|
|
885
|
-
`.execute(trx);
|
|
886
|
-
if (conflict.rows.length > 0) {
|
|
887
|
-
skipKeys.push(candidate.concurrency_key);
|
|
888
|
-
continue;
|
|
889
|
-
}
|
|
890
|
-
}
|
|
891
|
-
const result = await sql2`
|
|
892
|
-
UPDATE durably_runs
|
|
893
|
-
SET
|
|
894
|
-
status = 'leased',
|
|
895
|
-
lease_owner = ${workerId},
|
|
896
|
-
lease_expires_at = ${leaseExpiresAt},
|
|
897
|
-
lease_generation = lease_generation + 1,
|
|
898
|
-
started_at = COALESCE(started_at, ${now}),
|
|
899
|
-
updated_at = ${now}
|
|
900
|
-
WHERE id = ${candidate.id}
|
|
901
|
-
RETURNING *
|
|
902
|
-
`.execute(trx);
|
|
903
|
-
const row2 = result.rows[0];
|
|
904
|
-
if (!row2) return null;
|
|
905
|
-
return rowToRun({ ...row2, step_count: 0 });
|
|
906
|
-
}
|
|
907
|
-
});
|
|
908
|
-
}
|
|
909
|
-
let subquery = db.selectFrom("durably_runs").select("durably_runs.id").where(
|
|
910
|
-
(eb) => eb.or([
|
|
911
|
-
eb("status", "=", "pending"),
|
|
912
|
-
eb.and([
|
|
913
|
-
eb("status", "=", "leased"),
|
|
914
|
-
eb("lease_expires_at", "is not", null),
|
|
915
|
-
eb("lease_expires_at", "<=", now)
|
|
916
|
-
])
|
|
917
|
-
])
|
|
918
|
-
).where(activeLeaseGuard).orderBy("created_at", "asc").orderBy("id", "asc").limit(1);
|
|
919
|
-
const row = await db.updateTable("durably_runs").set({
|
|
920
|
-
status: "leased",
|
|
921
|
-
lease_owner: workerId,
|
|
922
|
-
lease_expires_at: leaseExpiresAt,
|
|
923
|
-
lease_generation: sql2`lease_generation + 1`,
|
|
924
|
-
started_at: sql2`COALESCE(started_at, ${now})`,
|
|
925
|
-
updated_at: now
|
|
926
|
-
}).where(
|
|
927
|
-
"id",
|
|
928
|
-
"=",
|
|
929
|
-
(eb) => eb.selectFrom(subquery.as("sub")).select("id")
|
|
930
|
-
).returningAll().executeTakeFirst();
|
|
931
|
-
if (!row) return null;
|
|
932
|
-
return rowToRun({ ...row, step_count: 0 });
|
|
1328
|
+
return backend === "postgres" ? claimNextPostgres(db, workerId, now, leaseExpiresAt, activeLeaseGuard) : claimNextSqlite(db, workerId, now, leaseExpiresAt, activeLeaseGuard);
|
|
933
1329
|
},
|
|
934
1330
|
async renewLease(runId, leaseGeneration, now, leaseMs) {
|
|
935
1331
|
const leaseExpiresAt = new Date(Date.parse(now) + leaseMs).toISOString();
|
|
@@ -940,13 +1336,54 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
940
1336
|
return Number(result.numUpdatedRows) > 0;
|
|
941
1337
|
},
|
|
942
1338
|
async releaseExpiredLeases(now) {
|
|
943
|
-
const
|
|
944
|
-
status: "
|
|
1339
|
+
const conflicting = await db.updateTable("durably_runs").set({
|
|
1340
|
+
status: "failed",
|
|
1341
|
+
error: "Lease expired; pending run already exists",
|
|
945
1342
|
lease_owner: null,
|
|
946
1343
|
lease_expires_at: null,
|
|
1344
|
+
completed_at: now,
|
|
947
1345
|
updated_at: now
|
|
948
|
-
}).where("status", "=", "leased").where("lease_expires_at", "is not", null).where("lease_expires_at", "<=", now).
|
|
949
|
-
|
|
1346
|
+
}).where("status", "=", "leased").where("lease_expires_at", "is not", null).where("lease_expires_at", "<=", now).where(
|
|
1347
|
+
({ exists, selectFrom }) => exists(
|
|
1348
|
+
selectFrom("durably_runs as other").select(sql4.lit(1).as("one")).whereRef("other.job_name", "=", "durably_runs.job_name").whereRef(
|
|
1349
|
+
"other.concurrency_key",
|
|
1350
|
+
"=",
|
|
1351
|
+
"durably_runs.concurrency_key"
|
|
1352
|
+
).where("other.status", "=", "pending").whereRef("other.id", "<>", "durably_runs.id")
|
|
1353
|
+
)
|
|
1354
|
+
).executeTakeFirst();
|
|
1355
|
+
let count = Number(conflicting.numUpdatedRows);
|
|
1356
|
+
const remaining = await db.selectFrom("durably_runs").select("id").where("status", "=", "leased").where("lease_expires_at", "is not", null).where("lease_expires_at", "<=", now).execute();
|
|
1357
|
+
if (remaining.length > 0) {
|
|
1358
|
+
await db.transaction().execute(async (trx) => {
|
|
1359
|
+
for (const row of remaining) {
|
|
1360
|
+
try {
|
|
1361
|
+
await sql4`SAVEPOINT sp_release`.execute(trx);
|
|
1362
|
+
const reset = await trx.updateTable("durably_runs").set({
|
|
1363
|
+
status: "pending",
|
|
1364
|
+
lease_owner: null,
|
|
1365
|
+
lease_expires_at: null,
|
|
1366
|
+
updated_at: now
|
|
1367
|
+
}).where("id", "=", row.id).where("status", "=", "leased").where("lease_expires_at", "<=", now).executeTakeFirst();
|
|
1368
|
+
await sql4`RELEASE SAVEPOINT sp_release`.execute(trx);
|
|
1369
|
+
count += Number(reset.numUpdatedRows);
|
|
1370
|
+
} catch (err) {
|
|
1371
|
+
await sql4`ROLLBACK TO SAVEPOINT sp_release`.execute(trx);
|
|
1372
|
+
if (!isUniqueViolation(err)) throw err;
|
|
1373
|
+
const failed = await trx.updateTable("durably_runs").set({
|
|
1374
|
+
status: "failed",
|
|
1375
|
+
error: "Lease expired; pending run already exists",
|
|
1376
|
+
lease_owner: null,
|
|
1377
|
+
lease_expires_at: null,
|
|
1378
|
+
completed_at: now,
|
|
1379
|
+
updated_at: now
|
|
1380
|
+
}).where("id", "=", row.id).where("status", "=", "leased").executeTakeFirst();
|
|
1381
|
+
count += Number(failed.numUpdatedRows);
|
|
1382
|
+
}
|
|
1383
|
+
}
|
|
1384
|
+
});
|
|
1385
|
+
}
|
|
1386
|
+
return count;
|
|
950
1387
|
},
|
|
951
1388
|
async completeRun(runId, leaseGeneration, output, completedAt) {
|
|
952
1389
|
return terminateRun(runId, leaseGeneration, completedAt, {
|
|
@@ -977,19 +1414,20 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
977
1414
|
const outputJson = input.output !== void 0 ? JSON.stringify(input.output) : null;
|
|
978
1415
|
const errorValue = input.error ?? null;
|
|
979
1416
|
return await db.transaction().execute(async (trx) => {
|
|
980
|
-
const insertResult = await
|
|
1417
|
+
const insertResult = await sql4`
|
|
981
1418
|
INSERT INTO durably_steps (id, run_id, name, "index", status, output, error, started_at, completed_at)
|
|
982
1419
|
SELECT ${id}, ${runId}, ${input.name}, ${input.index}, ${input.status},
|
|
983
1420
|
${outputJson}, ${errorValue}, ${input.startedAt}, ${completedAt}
|
|
984
1421
|
FROM durably_runs
|
|
985
|
-
WHERE id = ${runId} AND lease_generation = ${leaseGeneration}
|
|
1422
|
+
WHERE id = ${runId} AND status = 'leased' AND lease_generation = ${leaseGeneration}
|
|
986
1423
|
`.execute(trx);
|
|
987
1424
|
if (Number(insertResult.numAffectedRows) === 0) return null;
|
|
988
1425
|
if (input.status === "completed") {
|
|
989
1426
|
await trx.updateTable("durably_runs").set({
|
|
990
1427
|
current_step_index: input.index + 1,
|
|
1428
|
+
completed_step_count: sql4`completed_step_count + 1`,
|
|
991
1429
|
updated_at: completedAt
|
|
992
|
-
}).where("id", "=", runId).where("lease_generation", "=", leaseGeneration).execute();
|
|
1430
|
+
}).where("id", "=", runId).where("status", "=", "leased").where("lease_generation", "=", leaseGeneration).execute();
|
|
993
1431
|
}
|
|
994
1432
|
return {
|
|
995
1433
|
id,
|
|
@@ -1020,7 +1458,7 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
1020
1458
|
await db.updateTable("durably_runs").set({
|
|
1021
1459
|
progress: progress ? JSON.stringify(progress) : null,
|
|
1022
1460
|
updated_at: (/* @__PURE__ */ new Date()).toISOString()
|
|
1023
|
-
}).where("id", "=", runId).where("lease_generation", "=", leaseGeneration).execute();
|
|
1461
|
+
}).where("id", "=", runId).where("status", "=", "leased").where("lease_generation", "=", leaseGeneration).execute();
|
|
1024
1462
|
},
|
|
1025
1463
|
async createLog(input) {
|
|
1026
1464
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
@@ -1042,56 +1480,107 @@ function createKyselyStore(db, backend = "generic") {
|
|
|
1042
1480
|
return rows.map(rowToLog);
|
|
1043
1481
|
}
|
|
1044
1482
|
};
|
|
1045
|
-
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
const
|
|
1064
|
-
|
|
1483
|
+
if (backend !== "postgres") {
|
|
1484
|
+
const mutatingKeys = [
|
|
1485
|
+
"enqueue",
|
|
1486
|
+
"enqueueMany",
|
|
1487
|
+
"updateRun",
|
|
1488
|
+
"deleteRun",
|
|
1489
|
+
"purgeRuns",
|
|
1490
|
+
"claimNext",
|
|
1491
|
+
"renewLease",
|
|
1492
|
+
"releaseExpiredLeases",
|
|
1493
|
+
"completeRun",
|
|
1494
|
+
"failRun",
|
|
1495
|
+
"cancelRun",
|
|
1496
|
+
"persistStep",
|
|
1497
|
+
"deleteSteps",
|
|
1498
|
+
"updateProgress",
|
|
1499
|
+
"createLog"
|
|
1500
|
+
];
|
|
1501
|
+
for (const key of mutatingKeys) {
|
|
1502
|
+
const original = store[key];
|
|
1503
|
+
store[key] = (...args) => withWriteLock(() => original.apply(store, args));
|
|
1504
|
+
}
|
|
1065
1505
|
}
|
|
1066
1506
|
return store;
|
|
1067
1507
|
}
|
|
1068
1508
|
|
|
1069
1509
|
// src/worker.ts
|
|
1070
|
-
function createWorker(config, processOne) {
|
|
1510
|
+
function createWorker(config, processOne, onIdle) {
|
|
1511
|
+
const maxConcurrentRuns = config.maxConcurrentRuns;
|
|
1071
1512
|
let running = false;
|
|
1072
1513
|
let pollingTimeout = null;
|
|
1073
|
-
let
|
|
1074
|
-
let stopResolver = null;
|
|
1514
|
+
let activeCount = 0;
|
|
1075
1515
|
let activeWorkerId;
|
|
1076
|
-
|
|
1516
|
+
const activePromises = /* @__PURE__ */ new Set();
|
|
1517
|
+
let idleMaintenanceInFlight = null;
|
|
1518
|
+
function scheduleDelayedPoll() {
|
|
1077
1519
|
if (!running) {
|
|
1078
1520
|
return;
|
|
1079
1521
|
}
|
|
1522
|
+
if (pollingTimeout) {
|
|
1523
|
+
clearTimeout(pollingTimeout);
|
|
1524
|
+
pollingTimeout = null;
|
|
1525
|
+
}
|
|
1526
|
+
pollingTimeout = setTimeout(() => {
|
|
1527
|
+
pollingTimeout = null;
|
|
1528
|
+
if (running) {
|
|
1529
|
+
fillSlots();
|
|
1530
|
+
}
|
|
1531
|
+
}, config.pollingIntervalMs);
|
|
1532
|
+
}
|
|
1533
|
+
async function runIdleMaintenanceSafe() {
|
|
1534
|
+
if (!onIdle) {
|
|
1535
|
+
return;
|
|
1536
|
+
}
|
|
1537
|
+
const cycle = (async () => {
|
|
1538
|
+
try {
|
|
1539
|
+
await onIdle();
|
|
1540
|
+
} catch {
|
|
1541
|
+
}
|
|
1542
|
+
})();
|
|
1543
|
+
idleMaintenanceInFlight = cycle;
|
|
1080
1544
|
try {
|
|
1081
|
-
|
|
1082
|
-
await inFlight;
|
|
1545
|
+
await cycle;
|
|
1083
1546
|
} finally {
|
|
1084
|
-
|
|
1547
|
+
if (idleMaintenanceInFlight === cycle) {
|
|
1548
|
+
idleMaintenanceInFlight = null;
|
|
1549
|
+
}
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
async function processSlotCycle() {
|
|
1553
|
+
try {
|
|
1554
|
+
const didProcess = await processOne({ workerId: activeWorkerId });
|
|
1555
|
+
activeCount--;
|
|
1556
|
+
if (didProcess && running) {
|
|
1557
|
+
fillSlots();
|
|
1558
|
+
} else if (!didProcess && running) {
|
|
1559
|
+
if (activeCount === 0) {
|
|
1560
|
+
await runIdleMaintenanceSafe();
|
|
1561
|
+
}
|
|
1562
|
+
scheduleDelayedPoll();
|
|
1563
|
+
}
|
|
1564
|
+
} catch (err) {
|
|
1565
|
+
activeCount--;
|
|
1566
|
+
if (running) {
|
|
1567
|
+
fillSlots();
|
|
1568
|
+
}
|
|
1569
|
+
throw err;
|
|
1085
1570
|
}
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
}, config.pollingIntervalMs);
|
|
1571
|
+
}
|
|
1572
|
+
function fillSlots() {
|
|
1573
|
+
if (!running) {
|
|
1090
1574
|
return;
|
|
1091
1575
|
}
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1576
|
+
while (running && activeCount < maxConcurrentRuns) {
|
|
1577
|
+
activeCount++;
|
|
1578
|
+
const p = processSlotCycle();
|
|
1579
|
+
activePromises.add(p);
|
|
1580
|
+
void p.finally(() => {
|
|
1581
|
+
activePromises.delete(p);
|
|
1582
|
+
}).catch(() => {
|
|
1583
|
+
});
|
|
1095
1584
|
}
|
|
1096
1585
|
}
|
|
1097
1586
|
return {
|
|
@@ -1104,7 +1593,7 @@ function createWorker(config, processOne) {
|
|
|
1104
1593
|
}
|
|
1105
1594
|
activeWorkerId = options?.workerId;
|
|
1106
1595
|
running = true;
|
|
1107
|
-
|
|
1596
|
+
fillSlots();
|
|
1108
1597
|
},
|
|
1109
1598
|
async stop() {
|
|
1110
1599
|
if (!running) {
|
|
@@ -1115,11 +1604,14 @@ function createWorker(config, processOne) {
|
|
|
1115
1604
|
clearTimeout(pollingTimeout);
|
|
1116
1605
|
pollingTimeout = null;
|
|
1117
1606
|
}
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
1121
|
-
|
|
1607
|
+
const pending = [...activePromises];
|
|
1608
|
+
if (idleMaintenanceInFlight) {
|
|
1609
|
+
pending.push(idleMaintenanceInFlight);
|
|
1610
|
+
}
|
|
1611
|
+
if (pending.length === 0) {
|
|
1612
|
+
return;
|
|
1122
1613
|
}
|
|
1614
|
+
await Promise.allSettled(pending);
|
|
1123
1615
|
}
|
|
1124
1616
|
};
|
|
1125
1617
|
}
|
|
@@ -1127,10 +1619,20 @@ function createWorker(config, processOne) {
|
|
|
1127
1619
|
// src/durably.ts
|
|
1128
1620
|
var DEFAULTS = {
|
|
1129
1621
|
pollingIntervalMs: 1e3,
|
|
1622
|
+
maxConcurrentRuns: 1,
|
|
1130
1623
|
leaseRenewIntervalMs: 5e3,
|
|
1131
1624
|
leaseMs: 3e4,
|
|
1132
1625
|
preserveSteps: false
|
|
1133
1626
|
};
|
|
1627
|
+
var MAX_CONCURRENT_RUNS = 1e3;
|
|
1628
|
+
function validateMaxConcurrentRuns(value) {
|
|
1629
|
+
if (!Number.isSafeInteger(value) || value < 1 || value > MAX_CONCURRENT_RUNS) {
|
|
1630
|
+
throw new ValidationError(
|
|
1631
|
+
`maxConcurrentRuns must be between 1 and ${MAX_CONCURRENT_RUNS}`
|
|
1632
|
+
);
|
|
1633
|
+
}
|
|
1634
|
+
return value;
|
|
1635
|
+
}
|
|
1134
1636
|
function parseDuration(value) {
|
|
1135
1637
|
const match = value.match(/^(\d+)(d|h|m)$/);
|
|
1136
1638
|
if (!match) {
|
|
@@ -1148,6 +1650,14 @@ function parseDuration(value) {
|
|
|
1148
1650
|
return num * multipliers[unit];
|
|
1149
1651
|
}
|
|
1150
1652
|
var PURGE_INTERVAL_MS = 6e4;
|
|
1653
|
+
var CHECKPOINT_INTERVAL_MS = 6e4;
|
|
1654
|
+
var realClock = {
|
|
1655
|
+
now: () => Date.now(),
|
|
1656
|
+
setTimeout: (fn, ms) => globalThis.setTimeout(fn, ms),
|
|
1657
|
+
clearTimeout: (id) => globalThis.clearTimeout(id),
|
|
1658
|
+
setInterval: (fn, ms) => globalThis.setInterval(fn, ms),
|
|
1659
|
+
clearInterval: (id) => globalThis.clearInterval(id)
|
|
1660
|
+
};
|
|
1151
1661
|
var ulid2 = monotonicFactory2();
|
|
1152
1662
|
var BROWSER_SINGLETON_REGISTRY_KEY = "__durablyBrowserSingletonRegistry";
|
|
1153
1663
|
var BROWSER_LOCAL_DIALECT_KEY = "__durablyBrowserLocalKey";
|
|
@@ -1213,11 +1723,11 @@ function createDurablyInstance(state, jobs) {
|
|
|
1213
1723
|
async function getRunOrThrow(runId) {
|
|
1214
1724
|
const run = await storage.getRun(runId);
|
|
1215
1725
|
if (!run) {
|
|
1216
|
-
throw new
|
|
1726
|
+
throw new NotFoundError(`Run not found: ${runId}`);
|
|
1217
1727
|
}
|
|
1218
1728
|
return run;
|
|
1219
1729
|
}
|
|
1220
|
-
async function
|
|
1730
|
+
async function executeRun2(run, _workerId) {
|
|
1221
1731
|
const job = jobRegistry.get(run.jobName);
|
|
1222
1732
|
if (!job) {
|
|
1223
1733
|
await storage.failRun(
|
|
@@ -1228,146 +1738,16 @@ function createDurablyInstance(state, jobs) {
|
|
|
1228
1738
|
);
|
|
1229
1739
|
return;
|
|
1230
1740
|
}
|
|
1231
|
-
|
|
1741
|
+
await executeRun(
|
|
1232
1742
|
run,
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1743
|
+
job,
|
|
1744
|
+
{
|
|
1745
|
+
leaseMs: state.leaseMs,
|
|
1746
|
+
leaseRenewIntervalMs: state.leaseRenewIntervalMs,
|
|
1747
|
+
preserveSteps: state.preserveSteps
|
|
1748
|
+
},
|
|
1749
|
+
{ storage, eventEmitter, clock: realClock }
|
|
1237
1750
|
);
|
|
1238
|
-
let leaseDeadlineTimer = null;
|
|
1239
|
-
const scheduleLeaseDeadline = (leaseExpiresAt) => {
|
|
1240
|
-
if (leaseDeadlineTimer) {
|
|
1241
|
-
clearTimeout(leaseDeadlineTimer);
|
|
1242
|
-
leaseDeadlineTimer = null;
|
|
1243
|
-
}
|
|
1244
|
-
if (!leaseExpiresAt) {
|
|
1245
|
-
return;
|
|
1246
|
-
}
|
|
1247
|
-
const delay = Math.max(0, Date.parse(leaseExpiresAt) - Date.now());
|
|
1248
|
-
leaseDeadlineTimer = setTimeout(() => {
|
|
1249
|
-
abortLeaseOwnership();
|
|
1250
|
-
}, delay);
|
|
1251
|
-
};
|
|
1252
|
-
scheduleLeaseDeadline(run.leaseExpiresAt);
|
|
1253
|
-
const leaseTimer = setInterval(() => {
|
|
1254
|
-
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1255
|
-
storage.renewLease(run.id, run.leaseGeneration, now, state.leaseMs).then((renewed) => {
|
|
1256
|
-
if (!renewed) {
|
|
1257
|
-
abortLeaseOwnership();
|
|
1258
|
-
eventEmitter.emit({
|
|
1259
|
-
type: "worker:error",
|
|
1260
|
-
error: `Lease renewal lost ownership for run ${run.id}`,
|
|
1261
|
-
context: "lease-renewal",
|
|
1262
|
-
runId: run.id
|
|
1263
|
-
});
|
|
1264
|
-
return;
|
|
1265
|
-
}
|
|
1266
|
-
const renewedLeaseExpiresAt = new Date(
|
|
1267
|
-
Date.parse(now) + state.leaseMs
|
|
1268
|
-
).toISOString();
|
|
1269
|
-
scheduleLeaseDeadline(renewedLeaseExpiresAt);
|
|
1270
|
-
eventEmitter.emit({
|
|
1271
|
-
type: "run:lease-renewed",
|
|
1272
|
-
runId: run.id,
|
|
1273
|
-
jobName: run.jobName,
|
|
1274
|
-
leaseOwner: workerId,
|
|
1275
|
-
leaseExpiresAt: renewedLeaseExpiresAt,
|
|
1276
|
-
labels: run.labels
|
|
1277
|
-
});
|
|
1278
|
-
}).catch((error) => {
|
|
1279
|
-
eventEmitter.emit({
|
|
1280
|
-
type: "worker:error",
|
|
1281
|
-
error: getErrorMessage(error),
|
|
1282
|
-
context: "lease-renewal",
|
|
1283
|
-
runId: run.id
|
|
1284
|
-
});
|
|
1285
|
-
});
|
|
1286
|
-
}, state.leaseRenewIntervalMs);
|
|
1287
|
-
const started = Date.now();
|
|
1288
|
-
let reachedTerminalState = false;
|
|
1289
|
-
try {
|
|
1290
|
-
eventEmitter.emit({
|
|
1291
|
-
type: "run:leased",
|
|
1292
|
-
runId: run.id,
|
|
1293
|
-
jobName: run.jobName,
|
|
1294
|
-
input: run.input,
|
|
1295
|
-
leaseOwner: workerId,
|
|
1296
|
-
leaseExpiresAt: run.leaseExpiresAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
1297
|
-
labels: run.labels
|
|
1298
|
-
});
|
|
1299
|
-
const output = await job.fn(step, run.input);
|
|
1300
|
-
if (job.outputSchema) {
|
|
1301
|
-
const parseResult = job.outputSchema.safeParse(output);
|
|
1302
|
-
if (!parseResult.success) {
|
|
1303
|
-
throw new Error(`Invalid output: ${parseResult.error.message}`);
|
|
1304
|
-
}
|
|
1305
|
-
}
|
|
1306
|
-
const completedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1307
|
-
const completed = await storage.completeRun(
|
|
1308
|
-
run.id,
|
|
1309
|
-
run.leaseGeneration,
|
|
1310
|
-
output,
|
|
1311
|
-
completedAt
|
|
1312
|
-
);
|
|
1313
|
-
if (completed) {
|
|
1314
|
-
reachedTerminalState = true;
|
|
1315
|
-
eventEmitter.emit({
|
|
1316
|
-
type: "run:complete",
|
|
1317
|
-
runId: run.id,
|
|
1318
|
-
jobName: run.jobName,
|
|
1319
|
-
output,
|
|
1320
|
-
duration: Date.now() - started,
|
|
1321
|
-
labels: run.labels
|
|
1322
|
-
});
|
|
1323
|
-
} else {
|
|
1324
|
-
eventEmitter.emit({
|
|
1325
|
-
type: "worker:error",
|
|
1326
|
-
error: `Lease lost before completing run ${run.id}`,
|
|
1327
|
-
context: "run-completion"
|
|
1328
|
-
});
|
|
1329
|
-
}
|
|
1330
|
-
} catch (error) {
|
|
1331
|
-
if (error instanceof LeaseLostError || error instanceof CancelledError) {
|
|
1332
|
-
return;
|
|
1333
|
-
}
|
|
1334
|
-
const errorMessage = getErrorMessage(error);
|
|
1335
|
-
const completedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
1336
|
-
const failed = await storage.failRun(
|
|
1337
|
-
run.id,
|
|
1338
|
-
run.leaseGeneration,
|
|
1339
|
-
errorMessage,
|
|
1340
|
-
completedAt
|
|
1341
|
-
);
|
|
1342
|
-
if (failed) {
|
|
1343
|
-
reachedTerminalState = true;
|
|
1344
|
-
const steps = await storage.getSteps(run.id);
|
|
1345
|
-
const failedStep = steps.find((entry) => entry.status === "failed");
|
|
1346
|
-
eventEmitter.emit({
|
|
1347
|
-
type: "run:fail",
|
|
1348
|
-
runId: run.id,
|
|
1349
|
-
jobName: run.jobName,
|
|
1350
|
-
error: errorMessage,
|
|
1351
|
-
failedStepName: failedStep?.name ?? "unknown",
|
|
1352
|
-
labels: run.labels
|
|
1353
|
-
});
|
|
1354
|
-
} else {
|
|
1355
|
-
eventEmitter.emit({
|
|
1356
|
-
type: "worker:error",
|
|
1357
|
-
error: `Lease lost before recording failure for run ${run.id}`,
|
|
1358
|
-
context: "run-failure"
|
|
1359
|
-
});
|
|
1360
|
-
}
|
|
1361
|
-
} finally {
|
|
1362
|
-
clearInterval(leaseTimer);
|
|
1363
|
-
if (leaseDeadlineTimer) {
|
|
1364
|
-
clearTimeout(leaseDeadlineTimer);
|
|
1365
|
-
}
|
|
1366
|
-
dispose();
|
|
1367
|
-
if (!state.preserveSteps && reachedTerminalState) {
|
|
1368
|
-
await storage.deleteSteps(run.id);
|
|
1369
|
-
}
|
|
1370
|
-
}
|
|
1371
1751
|
}
|
|
1372
1752
|
const durably = {
|
|
1373
1753
|
db,
|
|
@@ -1391,7 +1771,8 @@ function createDurablyInstance(state, jobs) {
|
|
|
1391
1771
|
storage,
|
|
1392
1772
|
eventEmitter,
|
|
1393
1773
|
jobRegistry,
|
|
1394
|
-
state.labelsSchema
|
|
1774
|
+
state.labelsSchema,
|
|
1775
|
+
state.pollingIntervalMs
|
|
1395
1776
|
);
|
|
1396
1777
|
newHandles[key] = handle;
|
|
1397
1778
|
}
|
|
@@ -1403,6 +1784,19 @@ function createDurablyInstance(state, jobs) {
|
|
|
1403
1784
|
},
|
|
1404
1785
|
getRun: storage.getRun.bind(storage),
|
|
1405
1786
|
getRuns: storage.getRuns.bind(storage),
|
|
1787
|
+
async waitForRun(runId, options) {
|
|
1788
|
+
const run = await waitForRunCompletion(
|
|
1789
|
+
runId,
|
|
1790
|
+
storage,
|
|
1791
|
+
eventEmitter,
|
|
1792
|
+
{
|
|
1793
|
+
...options,
|
|
1794
|
+
pollingIntervalMs: options?.pollingIntervalMs ?? state.pollingIntervalMs
|
|
1795
|
+
},
|
|
1796
|
+
"waitForRun"
|
|
1797
|
+
);
|
|
1798
|
+
return run;
|
|
1799
|
+
},
|
|
1406
1800
|
use(plugin) {
|
|
1407
1801
|
plugin.install(durably);
|
|
1408
1802
|
},
|
|
@@ -1520,21 +1914,21 @@ function createDurablyInstance(state, jobs) {
|
|
|
1520
1914
|
async retrigger(runId) {
|
|
1521
1915
|
const run = await getRunOrThrow(runId);
|
|
1522
1916
|
if (run.status === "pending") {
|
|
1523
|
-
throw new
|
|
1917
|
+
throw new ConflictError(`Cannot retrigger pending run: ${runId}`);
|
|
1524
1918
|
}
|
|
1525
1919
|
if (run.status === "leased") {
|
|
1526
|
-
throw new
|
|
1920
|
+
throw new ConflictError(`Cannot retrigger leased run: ${runId}`);
|
|
1527
1921
|
}
|
|
1528
1922
|
const job = jobRegistry.get(run.jobName);
|
|
1529
1923
|
if (!job) {
|
|
1530
|
-
throw new
|
|
1924
|
+
throw new NotFoundError(`Unknown job: ${run.jobName}`);
|
|
1531
1925
|
}
|
|
1532
1926
|
const validatedInput = validateJobInputOrThrow(
|
|
1533
1927
|
job.inputSchema,
|
|
1534
1928
|
run.input,
|
|
1535
1929
|
`Cannot retrigger run ${runId}`
|
|
1536
1930
|
);
|
|
1537
|
-
const nextRun = await storage.enqueue({
|
|
1931
|
+
const { run: nextRun } = await storage.enqueue({
|
|
1538
1932
|
jobName: run.jobName,
|
|
1539
1933
|
input: validatedInput,
|
|
1540
1934
|
concurrencyKey: run.concurrencyKey ?? void 0,
|
|
@@ -1552,19 +1946,19 @@ function createDurablyInstance(state, jobs) {
|
|
|
1552
1946
|
async cancel(runId) {
|
|
1553
1947
|
const run = await getRunOrThrow(runId);
|
|
1554
1948
|
if (run.status === "completed") {
|
|
1555
|
-
throw new
|
|
1949
|
+
throw new ConflictError(`Cannot cancel completed run: ${runId}`);
|
|
1556
1950
|
}
|
|
1557
1951
|
if (run.status === "failed") {
|
|
1558
|
-
throw new
|
|
1952
|
+
throw new ConflictError(`Cannot cancel failed run: ${runId}`);
|
|
1559
1953
|
}
|
|
1560
1954
|
if (run.status === "cancelled") {
|
|
1561
|
-
throw new
|
|
1955
|
+
throw new ConflictError(`Cannot cancel already cancelled run: ${runId}`);
|
|
1562
1956
|
}
|
|
1563
1957
|
const wasPending = run.status === "pending";
|
|
1564
1958
|
const cancelled = await storage.cancelRun(runId, (/* @__PURE__ */ new Date()).toISOString());
|
|
1565
1959
|
if (!cancelled) {
|
|
1566
1960
|
const current = await getRunOrThrow(runId);
|
|
1567
|
-
throw new
|
|
1961
|
+
throw new ConflictError(
|
|
1568
1962
|
`Cannot cancel run ${runId}: status changed to ${current.status}`
|
|
1569
1963
|
);
|
|
1570
1964
|
}
|
|
@@ -1581,10 +1975,10 @@ function createDurablyInstance(state, jobs) {
|
|
|
1581
1975
|
async deleteRun(runId) {
|
|
1582
1976
|
const run = await getRunOrThrow(runId);
|
|
1583
1977
|
if (run.status === "pending") {
|
|
1584
|
-
throw new
|
|
1978
|
+
throw new ConflictError(`Cannot delete pending run: ${runId}`);
|
|
1585
1979
|
}
|
|
1586
1980
|
if (run.status === "leased") {
|
|
1587
|
-
throw new
|
|
1981
|
+
throw new ConflictError(`Cannot delete leased run: ${runId}`);
|
|
1588
1982
|
}
|
|
1589
1983
|
await storage.deleteRun(runId);
|
|
1590
1984
|
eventEmitter.emit({
|
|
@@ -1603,37 +1997,29 @@ function createDurablyInstance(state, jobs) {
|
|
|
1603
1997
|
async processOne(options) {
|
|
1604
1998
|
const workerId = options?.workerId ?? defaultWorkerId();
|
|
1605
1999
|
const now = (/* @__PURE__ */ new Date()).toISOString();
|
|
1606
|
-
await storage.releaseExpiredLeases(now);
|
|
1607
2000
|
const run = await storage.claimNext(workerId, now, state.leaseMs);
|
|
1608
2001
|
if (!run) {
|
|
1609
|
-
if (state.retainRunsMs !== null && Date.now() - state.lastPurgeAt >= PURGE_INTERVAL_MS) {
|
|
1610
|
-
const purgeNow = Date.now();
|
|
1611
|
-
state.lastPurgeAt = purgeNow;
|
|
1612
|
-
const cutoff = new Date(purgeNow - state.retainRunsMs).toISOString();
|
|
1613
|
-
storage.purgeRuns({ olderThan: cutoff, limit: 100 }).catch((error) => {
|
|
1614
|
-
eventEmitter.emit({
|
|
1615
|
-
type: "worker:error",
|
|
1616
|
-
error: getErrorMessage(error),
|
|
1617
|
-
context: "auto-purge"
|
|
1618
|
-
});
|
|
1619
|
-
});
|
|
1620
|
-
}
|
|
1621
2002
|
return false;
|
|
1622
2003
|
}
|
|
1623
|
-
await
|
|
2004
|
+
await executeRun2(run, workerId);
|
|
1624
2005
|
return true;
|
|
1625
2006
|
},
|
|
1626
2007
|
async processUntilIdle(options) {
|
|
1627
2008
|
const workerId = options?.workerId ?? defaultWorkerId();
|
|
1628
2009
|
const maxRuns = options?.maxRuns ?? Number.POSITIVE_INFINITY;
|
|
1629
2010
|
let processed = 0;
|
|
2011
|
+
let reachedIdle = false;
|
|
1630
2012
|
while (processed < maxRuns) {
|
|
1631
2013
|
const didProcess = await this.processOne({ workerId });
|
|
1632
2014
|
if (!didProcess) {
|
|
2015
|
+
reachedIdle = true;
|
|
1633
2016
|
break;
|
|
1634
2017
|
}
|
|
1635
2018
|
processed++;
|
|
1636
2019
|
}
|
|
2020
|
+
if (reachedIdle) {
|
|
2021
|
+
await state.runIdleMaintenance();
|
|
2022
|
+
}
|
|
1637
2023
|
return processed;
|
|
1638
2024
|
},
|
|
1639
2025
|
async migrate() {
|
|
@@ -1643,8 +2029,10 @@ function createDurablyInstance(state, jobs) {
|
|
|
1643
2029
|
if (state.migrating) {
|
|
1644
2030
|
return state.migrating;
|
|
1645
2031
|
}
|
|
1646
|
-
state.migrating = runMigrations(db).then(() => {
|
|
2032
|
+
state.migrating = runMigrations(db).then(async () => {
|
|
1647
2033
|
state.migrated = true;
|
|
2034
|
+
await state.probeWalCheckpoint?.();
|
|
2035
|
+
state.probeWalCheckpoint = null;
|
|
1648
2036
|
}).finally(() => {
|
|
1649
2037
|
state.migrating = null;
|
|
1650
2038
|
});
|
|
@@ -1658,8 +2046,10 @@ function createDurablyInstance(state, jobs) {
|
|
|
1658
2046
|
return durably;
|
|
1659
2047
|
}
|
|
1660
2048
|
function createDurably(options) {
|
|
2049
|
+
const maxConcurrentRuns = options.maxConcurrentRuns !== void 0 ? validateMaxConcurrentRuns(options.maxConcurrentRuns) : DEFAULTS.maxConcurrentRuns;
|
|
1661
2050
|
const config = {
|
|
1662
2051
|
pollingIntervalMs: options.pollingIntervalMs ?? DEFAULTS.pollingIntervalMs,
|
|
2052
|
+
maxConcurrentRuns,
|
|
1663
2053
|
leaseRenewIntervalMs: options.leaseRenewIntervalMs ?? DEFAULTS.leaseRenewIntervalMs,
|
|
1664
2054
|
leaseMs: options.leaseMs ?? DEFAULTS.leaseMs,
|
|
1665
2055
|
preserveSteps: options.preserveSteps ?? DEFAULTS.preserveSteps,
|
|
@@ -1681,15 +2071,60 @@ function createDurably(options) {
|
|
|
1681
2071
|
});
|
|
1682
2072
|
const eventEmitter = createEventEmitter();
|
|
1683
2073
|
const jobRegistry = createJobRegistry();
|
|
2074
|
+
let lastPurgeAt = 0;
|
|
2075
|
+
let lastCheckpointAt = 0;
|
|
2076
|
+
const runIdleMaintenance = async () => {
|
|
2077
|
+
try {
|
|
2078
|
+
const nowMs = Date.now();
|
|
2079
|
+
await storage.releaseExpiredLeases(new Date(nowMs).toISOString());
|
|
2080
|
+
if (config.retainRunsMs !== null) {
|
|
2081
|
+
if (nowMs - lastPurgeAt >= PURGE_INTERVAL_MS) {
|
|
2082
|
+
lastPurgeAt = nowMs;
|
|
2083
|
+
const cutoff = new Date(nowMs - config.retainRunsMs).toISOString();
|
|
2084
|
+
await storage.purgeRuns({ olderThan: cutoff, limit: 100 });
|
|
2085
|
+
}
|
|
2086
|
+
}
|
|
2087
|
+
if (state.walCheckpointSupported) {
|
|
2088
|
+
if (nowMs - lastCheckpointAt >= CHECKPOINT_INTERVAL_MS) {
|
|
2089
|
+
lastCheckpointAt = nowMs;
|
|
2090
|
+
try {
|
|
2091
|
+
const result = await sql5`PRAGMA wal_checkpoint(TRUNCATE)`.execute(
|
|
2092
|
+
db
|
|
2093
|
+
);
|
|
2094
|
+
const row = result.rows[0];
|
|
2095
|
+
if (row?.busy !== 0) {
|
|
2096
|
+
lastCheckpointAt = nowMs - CHECKPOINT_INTERVAL_MS / 2;
|
|
2097
|
+
}
|
|
2098
|
+
} catch (checkpointError) {
|
|
2099
|
+
eventEmitter.emit({
|
|
2100
|
+
type: "worker:error",
|
|
2101
|
+
error: getErrorMessage(checkpointError),
|
|
2102
|
+
context: "wal-checkpoint"
|
|
2103
|
+
});
|
|
2104
|
+
}
|
|
2105
|
+
}
|
|
2106
|
+
}
|
|
2107
|
+
} catch (error) {
|
|
2108
|
+
eventEmitter.emit({
|
|
2109
|
+
type: "worker:error",
|
|
2110
|
+
error: getErrorMessage(error),
|
|
2111
|
+
context: "idle-maintenance"
|
|
2112
|
+
});
|
|
2113
|
+
}
|
|
2114
|
+
};
|
|
1684
2115
|
let processOneImpl = null;
|
|
1685
2116
|
const worker = createWorker(
|
|
1686
|
-
{
|
|
2117
|
+
{
|
|
2118
|
+
pollingIntervalMs: config.pollingIntervalMs,
|
|
2119
|
+
maxConcurrentRuns: config.maxConcurrentRuns
|
|
2120
|
+
},
|
|
1687
2121
|
(runtimeOptions) => {
|
|
1688
2122
|
if (!processOneImpl) {
|
|
1689
2123
|
throw new Error("Durably runtime is not initialized");
|
|
1690
2124
|
}
|
|
1691
2125
|
return processOneImpl(runtimeOptions);
|
|
1692
|
-
}
|
|
2126
|
+
},
|
|
2127
|
+
runIdleMaintenance
|
|
1693
2128
|
);
|
|
1694
2129
|
const state = {
|
|
1695
2130
|
db,
|
|
@@ -1701,12 +2136,27 @@ function createDurably(options) {
|
|
|
1701
2136
|
preserveSteps: config.preserveSteps,
|
|
1702
2137
|
migrating: null,
|
|
1703
2138
|
migrated: false,
|
|
2139
|
+
walCheckpointSupported: false,
|
|
2140
|
+
probeWalCheckpoint: null,
|
|
1704
2141
|
leaseMs: config.leaseMs,
|
|
1705
2142
|
leaseRenewIntervalMs: config.leaseRenewIntervalMs,
|
|
2143
|
+
pollingIntervalMs: config.pollingIntervalMs,
|
|
1706
2144
|
retainRunsMs: config.retainRunsMs,
|
|
1707
|
-
|
|
1708
|
-
|
|
2145
|
+
releaseBrowserSingleton,
|
|
2146
|
+
runIdleMaintenance
|
|
1709
2147
|
};
|
|
2148
|
+
if (backend === "generic" && !isBrowserLikeEnvironment()) {
|
|
2149
|
+
state.probeWalCheckpoint = async () => {
|
|
2150
|
+
try {
|
|
2151
|
+
const result = await sql5`PRAGMA wal_checkpoint(PASSIVE)`.execute(db);
|
|
2152
|
+
const row = result.rows[0];
|
|
2153
|
+
if (row && row.log !== -1) {
|
|
2154
|
+
state.walCheckpointSupported = true;
|
|
2155
|
+
}
|
|
2156
|
+
} catch {
|
|
2157
|
+
}
|
|
2158
|
+
};
|
|
2159
|
+
}
|
|
1710
2160
|
const instance = createDurablyInstance(
|
|
1711
2161
|
state,
|
|
1712
2162
|
{}
|
|
@@ -1973,15 +2423,27 @@ function parseLabelsFromParams(searchParams) {
|
|
|
1973
2423
|
}
|
|
1974
2424
|
function parseRunFilter(url) {
|
|
1975
2425
|
const jobNames = url.searchParams.getAll("jobName");
|
|
1976
|
-
const
|
|
2426
|
+
const statusParams = url.searchParams.getAll("status");
|
|
1977
2427
|
const limitParam = url.searchParams.get("limit");
|
|
1978
2428
|
const offsetParam = url.searchParams.get("offset");
|
|
1979
2429
|
const labels = parseLabelsFromParams(url.searchParams);
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
|
|
2430
|
+
let status;
|
|
2431
|
+
if (statusParams.length > 0) {
|
|
2432
|
+
for (const s of statusParams) {
|
|
2433
|
+
if (s === "") {
|
|
2434
|
+
return errorResponse(
|
|
2435
|
+
`Invalid status: empty value. Must be one of: ${VALID_STATUSES.join(", ")}`,
|
|
2436
|
+
400
|
|
2437
|
+
);
|
|
2438
|
+
}
|
|
2439
|
+
if (!VALID_STATUSES_SET.has(s)) {
|
|
2440
|
+
return errorResponse(
|
|
2441
|
+
`Invalid status: ${s}. Must be one of: ${VALID_STATUSES.join(", ")}`,
|
|
2442
|
+
400
|
|
2443
|
+
);
|
|
2444
|
+
}
|
|
2445
|
+
}
|
|
2446
|
+
status = statusParams.length === 1 ? statusParams[0] : statusParams;
|
|
1985
2447
|
}
|
|
1986
2448
|
let limit;
|
|
1987
2449
|
if (limitParam) {
|
|
@@ -2002,7 +2464,7 @@ function parseRunFilter(url) {
|
|
|
2002
2464
|
}
|
|
2003
2465
|
return {
|
|
2004
2466
|
jobName: jobNames.length > 0 ? jobNames : void 0,
|
|
2005
|
-
status
|
|
2467
|
+
status,
|
|
2006
2468
|
labels,
|
|
2007
2469
|
limit,
|
|
2008
2470
|
offset
|
|
@@ -2035,6 +2497,12 @@ function createDurablyHandler(durably, options) {
|
|
|
2035
2497
|
return await fn();
|
|
2036
2498
|
} catch (error) {
|
|
2037
2499
|
if (error instanceof Response) throw error;
|
|
2500
|
+
if (error instanceof DurablyError) {
|
|
2501
|
+
return errorResponse(
|
|
2502
|
+
error.message,
|
|
2503
|
+
error.statusCode
|
|
2504
|
+
);
|
|
2505
|
+
}
|
|
2038
2506
|
return errorResponse(getErrorMessage(error), 500);
|
|
2039
2507
|
}
|
|
2040
2508
|
}
|
|
@@ -2063,15 +2531,16 @@ function createDurablyHandler(durably, options) {
|
|
|
2063
2531
|
if (auth?.onTrigger && ctx !== void 0) {
|
|
2064
2532
|
await auth.onTrigger(ctx, body);
|
|
2065
2533
|
}
|
|
2066
|
-
const run = await job.trigger(
|
|
2067
|
-
body.
|
|
2068
|
-
|
|
2069
|
-
|
|
2070
|
-
|
|
2071
|
-
|
|
2072
|
-
|
|
2073
|
-
|
|
2074
|
-
|
|
2534
|
+
const run = await job.trigger(body.input, {
|
|
2535
|
+
idempotencyKey: body.idempotencyKey,
|
|
2536
|
+
concurrencyKey: body.concurrencyKey,
|
|
2537
|
+
labels: body.labels,
|
|
2538
|
+
coalesce: body.coalesce
|
|
2539
|
+
});
|
|
2540
|
+
const response = {
|
|
2541
|
+
runId: run.id,
|
|
2542
|
+
disposition: run.disposition
|
|
2543
|
+
};
|
|
2075
2544
|
return jsonResponse(response);
|
|
2076
2545
|
});
|
|
2077
2546
|
}
|
|
@@ -2186,6 +2655,18 @@ function createDurablyHandler(durably, options) {
|
|
|
2186
2655
|
});
|
|
2187
2656
|
}
|
|
2188
2657
|
}),
|
|
2658
|
+
durably.on("run:coalesced", (event) => {
|
|
2659
|
+
if (matchesFilter(event.jobName, event.labels)) {
|
|
2660
|
+
ctrl.enqueue({
|
|
2661
|
+
type: "run:coalesced",
|
|
2662
|
+
runId: event.runId,
|
|
2663
|
+
jobName: event.jobName,
|
|
2664
|
+
labels: event.labels,
|
|
2665
|
+
skippedInput: event.skippedInput,
|
|
2666
|
+
skippedLabels: event.skippedLabels
|
|
2667
|
+
});
|
|
2668
|
+
}
|
|
2669
|
+
}),
|
|
2189
2670
|
durably.on("run:leased", (event) => {
|
|
2190
2671
|
if (matchesFilter(event.jobName, event.labels)) {
|
|
2191
2672
|
ctrl.enqueue({
|
|
@@ -2355,11 +2836,16 @@ function createDurablyHandler(durably, options) {
|
|
|
2355
2836
|
}
|
|
2356
2837
|
export {
|
|
2357
2838
|
CancelledError,
|
|
2839
|
+
ConflictError,
|
|
2840
|
+
DurablyError,
|
|
2358
2841
|
LeaseLostError,
|
|
2842
|
+
NotFoundError,
|
|
2843
|
+
ValidationError,
|
|
2359
2844
|
createDurably,
|
|
2360
2845
|
createDurablyHandler,
|
|
2361
2846
|
createKyselyStore,
|
|
2362
2847
|
defineJob,
|
|
2848
|
+
isDomainEvent,
|
|
2363
2849
|
toClientRun,
|
|
2364
2850
|
withLogPersistence
|
|
2365
2851
|
};
|