@aigne/afs-scheduler 1.12.0-beta.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +26 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.cjs +11 -0
- package/dist/_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.mjs +10 -0
- package/dist/cron-parser.cjs +202 -0
- package/dist/cron-parser.d.cts +65 -0
- package/dist/cron-parser.d.cts.map +1 -0
- package/dist/cron-parser.d.mts +65 -0
- package/dist/cron-parser.d.mts.map +1 -0
- package/dist/cron-parser.mjs +200 -0
- package/dist/cron-parser.mjs.map +1 -0
- package/dist/do-executor.cjs +59 -0
- package/dist/do-executor.d.cts +30 -0
- package/dist/do-executor.d.cts.map +1 -0
- package/dist/do-executor.d.mts +30 -0
- package/dist/do-executor.d.mts.map +1 -0
- package/dist/do-executor.mjs +59 -0
- package/dist/do-executor.mjs.map +1 -0
- package/dist/do-job-executor.cjs +53 -0
- package/dist/do-job-executor.d.cts +28 -0
- package/dist/do-job-executor.d.cts.map +1 -0
- package/dist/do-job-executor.d.mts +28 -0
- package/dist/do-job-executor.d.mts.map +1 -0
- package/dist/do-job-executor.mjs +53 -0
- package/dist/do-job-executor.mjs.map +1 -0
- package/dist/do-types.d.cts +66 -0
- package/dist/do-types.d.cts.map +1 -0
- package/dist/do-types.d.mts +66 -0
- package/dist/do-types.d.mts.map +1 -0
- package/dist/index.cjs +34 -0
- package/dist/index.d.cts +11 -0
- package/dist/index.d.mts +11 -0
- package/dist/index.mjs +11 -0
- package/dist/job-coordinator.cjs +216 -0
- package/dist/job-coordinator.d.cts +8 -0
- package/dist/job-coordinator.d.cts.map +1 -0
- package/dist/job-coordinator.d.mts +8 -0
- package/dist/job-coordinator.d.mts.map +1 -0
- package/dist/job-coordinator.mjs +216 -0
- package/dist/job-coordinator.mjs.map +1 -0
- package/dist/local-executor.cjs +25 -0
- package/dist/local-executor.d.cts +12 -0
- package/dist/local-executor.d.cts.map +1 -0
- package/dist/local-executor.d.mts +12 -0
- package/dist/local-executor.d.mts.map +1 -0
- package/dist/local-executor.mjs +25 -0
- package/dist/local-executor.mjs.map +1 -0
- package/dist/scheduler-provider.cjs +1236 -0
- package/dist/scheduler-provider.d.cts +95 -0
- package/dist/scheduler-provider.d.cts.map +1 -0
- package/dist/scheduler-provider.d.mts +95 -0
- package/dist/scheduler-provider.d.mts.map +1 -0
- package/dist/scheduler-provider.mjs +1237 -0
- package/dist/scheduler-provider.mjs.map +1 -0
- package/dist/task-runner.cjs +70 -0
- package/dist/task-runner.d.cts +25 -0
- package/dist/task-runner.d.cts.map +1 -0
- package/dist/task-runner.d.mts +25 -0
- package/dist/task-runner.d.mts.map +1 -0
- package/dist/task-runner.mjs +70 -0
- package/dist/task-runner.mjs.map +1 -0
- package/dist/task-store.cjs +186 -0
- package/dist/task-store.d.cts +60 -0
- package/dist/task-store.d.cts.map +1 -0
- package/dist/task-store.d.mts +60 -0
- package/dist/task-store.d.mts.map +1 -0
- package/dist/task-store.mjs +187 -0
- package/dist/task-store.mjs.map +1 -0
- package/dist/types.cjs +185 -0
- package/dist/types.d.cts +384 -0
- package/dist/types.d.cts.map +1 -0
- package/dist/types.d.mts +384 -0
- package/dist/types.d.mts.map +1 -0
- package/dist/types.mjs +176 -0
- package/dist/types.mjs.map +1 -0
- package/package.json +59 -0
|
@@ -0,0 +1,1237 @@
|
|
|
1
|
+
import { DOJobExecutor } from "./do-job-executor.mjs";
|
|
2
|
+
import { LocalExecutor } from "./local-executor.mjs";
|
|
3
|
+
import { TaskStore } from "./task-store.mjs";
|
|
4
|
+
import { CancelArgsSchema, CancelJobArgsSchema, ClaimArgsSchema, DispatchArgsSchema, HeartbeatArgsSchema, RetryArgsSchema, ScheduleArgsSchema, SchedulerSnapshotSchema, SubmitArgsSchema, SubmitJobArgsSchema, UnscheduleArgsSchema } from "./types.mjs";
|
|
5
|
+
import { __decorate } from "./_virtual/_@oxc-project_runtime@0.108.0/helpers/decorate.mjs";
|
|
6
|
+
import { v7 } from "@aigne/uuid";
|
|
7
|
+
import { AFSConflictError, makeNsLog } from "@aigne/afs";
|
|
8
|
+
import { AFSBaseProvider, Actions, Explain, List, Meta, PersistenceHelper, Read } from "@aigne/afs/provider";
|
|
9
|
+
import { joinURL } from "ufo";
|
|
10
|
+
import { z } from "zod/v4";
|
|
11
|
+
|
|
12
|
+
//#region src/scheduler-provider.ts
|
|
13
|
+
const log = makeNsLog("provider:scheduler");
|
|
14
|
+
const DEFAULT_SCHEDULED_TASK_TIMEOUT_MS = 0;
|
|
15
|
+
/** Default lease duration for claim-based tasks (#1067) when `claim` omits `leaseDuration`. */
|
|
16
|
+
const DEFAULT_LEASE_DURATION_MS = 300 * 1e3;
|
|
17
|
+
const ACTION_NAMES = [
|
|
18
|
+
"dispatch",
|
|
19
|
+
"cancel",
|
|
20
|
+
"retry",
|
|
21
|
+
"schedule",
|
|
22
|
+
"unschedule",
|
|
23
|
+
"heartbeat",
|
|
24
|
+
"submit-job",
|
|
25
|
+
"cancel-job",
|
|
26
|
+
"claim",
|
|
27
|
+
"submit"
|
|
28
|
+
];
|
|
29
|
+
var AFSScheduler = class extends AFSBaseProvider {
|
|
30
|
+
static manifest() {
|
|
31
|
+
return {
|
|
32
|
+
name: "scheduler",
|
|
33
|
+
description: "Task dispatch and execution management.\n- Dispatch tasks for background execution with optional proc budget integration\n- Cron-based scheduling with duplicate protection\n- Map-reduce job coordination with batch processing",
|
|
34
|
+
uriTemplate: "scheduler://",
|
|
35
|
+
category: "compute",
|
|
36
|
+
schema: z.object({
|
|
37
|
+
name: z.string().optional(),
|
|
38
|
+
description: z.string().optional(),
|
|
39
|
+
maxHistory: z.number().int().positive().optional(),
|
|
40
|
+
storagePath: z.string().optional(),
|
|
41
|
+
procPath: z.string().optional(),
|
|
42
|
+
budgetBlockedTTL: z.number().int().nonnegative().optional()
|
|
43
|
+
}),
|
|
44
|
+
tags: [
|
|
45
|
+
"scheduler",
|
|
46
|
+
"dispatch",
|
|
47
|
+
"cron",
|
|
48
|
+
"background"
|
|
49
|
+
],
|
|
50
|
+
capabilityTags: [
|
|
51
|
+
"read-write",
|
|
52
|
+
"auth:none",
|
|
53
|
+
"local"
|
|
54
|
+
],
|
|
55
|
+
security: {
|
|
56
|
+
riskLevel: "system",
|
|
57
|
+
resourceAccess: ["process-spawn"],
|
|
58
|
+
dataSensitivity: ["code"],
|
|
59
|
+
notes: ["Executes tasks by delegating to AFS exec, can spawn background processes"]
|
|
60
|
+
},
|
|
61
|
+
capabilities: { crossProvider: { afsAccess: true } }
|
|
62
|
+
};
|
|
63
|
+
}
|
|
64
|
+
name;
|
|
65
|
+
description;
|
|
66
|
+
accessMode = "readwrite";
|
|
67
|
+
/** @internal — exposed for conformance test pre-population */
|
|
68
|
+
store;
|
|
69
|
+
executor;
|
|
70
|
+
jobExecutor;
|
|
71
|
+
root;
|
|
72
|
+
options;
|
|
73
|
+
persistence;
|
|
74
|
+
constructor(options = {}) {
|
|
75
|
+
super();
|
|
76
|
+
this.name = options.name ?? "scheduler";
|
|
77
|
+
this.description = options.description ?? "Task dispatch and execution management";
|
|
78
|
+
this.options = options;
|
|
79
|
+
this.store = new TaskStore(options.maxHistory ?? 100, options.budgetBlockedTTL);
|
|
80
|
+
this.persistence = new PersistenceHelper({
|
|
81
|
+
storagePath: options.storagePath,
|
|
82
|
+
getSnapshot: () => this.store.snapshot(),
|
|
83
|
+
applySnapshot: (data, now) => this.store.restore(data, now),
|
|
84
|
+
snapshotVersion: 3,
|
|
85
|
+
validateSnapshot: (raw) => SchedulerSnapshotSchema.parse(raw)
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
onMount(root, _mountPath) {
|
|
89
|
+
this.root = root;
|
|
90
|
+
this.executor = this.options.executor ?? new LocalExecutor(() => this.root);
|
|
91
|
+
if (this.options.coordinatorNamespace) this.jobExecutor = new DOJobExecutor(this.options.coordinatorNamespace);
|
|
92
|
+
this.persistence.attachRoot(root);
|
|
93
|
+
}
|
|
94
|
+
/** Await all pending snapshot writes. Use before reading persisted state. */
|
|
95
|
+
async flush() {
|
|
96
|
+
await this.persistence.flush();
|
|
97
|
+
}
|
|
98
|
+
async read(path, options) {
|
|
99
|
+
await this.persistence.ensureReady();
|
|
100
|
+
return super.read(path, options);
|
|
101
|
+
}
|
|
102
|
+
async list(path, options) {
|
|
103
|
+
await this.persistence.ensureReady();
|
|
104
|
+
return super.list(path, options);
|
|
105
|
+
}
|
|
106
|
+
readRoot(ctx) {
|
|
107
|
+
return this.buildEntry(ctx.path, {
|
|
108
|
+
content: this.description,
|
|
109
|
+
meta: {
|
|
110
|
+
kind: "scheduler:root",
|
|
111
|
+
childrenCount: 4
|
|
112
|
+
}
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
readTasks(ctx) {
|
|
116
|
+
const tasks = this.store.listActive();
|
|
117
|
+
return this.buildEntry(ctx.path, {
|
|
118
|
+
content: `${tasks.length} active tasks`,
|
|
119
|
+
meta: {
|
|
120
|
+
kind: "scheduler:task-list",
|
|
121
|
+
childrenCount: tasks.length
|
|
122
|
+
}
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
readTask(ctx) {
|
|
126
|
+
const entry = this.store.getAny(ctx.params.id);
|
|
127
|
+
if (!entry) return void 0;
|
|
128
|
+
return this.buildEntry(ctx.path, {
|
|
129
|
+
id: entry.id,
|
|
130
|
+
content: entry,
|
|
131
|
+
meta: { kind: "scheduler:task" }
|
|
132
|
+
});
|
|
133
|
+
}
|
|
134
|
+
readTaskField(ctx) {
|
|
135
|
+
const entry = this.store.getAny(ctx.params.id);
|
|
136
|
+
if (!entry) return void 0;
|
|
137
|
+
return this.readEntryField(ctx.path, entry, ctx.params.field);
|
|
138
|
+
}
|
|
139
|
+
readHistory(ctx) {
|
|
140
|
+
const history = this.store.listHistory();
|
|
141
|
+
return this.buildEntry(ctx.path, {
|
|
142
|
+
content: `${history.length} history entries`,
|
|
143
|
+
meta: {
|
|
144
|
+
kind: "scheduler:history",
|
|
145
|
+
childrenCount: history.length
|
|
146
|
+
}
|
|
147
|
+
});
|
|
148
|
+
}
|
|
149
|
+
readHistoryTask(ctx) {
|
|
150
|
+
const entry = this.store.getFromHistory(ctx.params.id);
|
|
151
|
+
if (!entry) return void 0;
|
|
152
|
+
return this.buildEntry(ctx.path, {
|
|
153
|
+
id: entry.id,
|
|
154
|
+
content: entry,
|
|
155
|
+
meta: { kind: "scheduler:task" }
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
readHistoryTaskField(ctx) {
|
|
159
|
+
const entry = this.store.getFromHistory(ctx.params.id);
|
|
160
|
+
if (!entry) return void 0;
|
|
161
|
+
return this.readEntryField(ctx.path, entry, ctx.params.field);
|
|
162
|
+
}
|
|
163
|
+
readSchedules(ctx) {
|
|
164
|
+
const schedules = this.store.listSchedules();
|
|
165
|
+
return this.buildEntry(ctx.path, {
|
|
166
|
+
content: `${schedules.length} schedules`,
|
|
167
|
+
meta: {
|
|
168
|
+
kind: "scheduler:schedule-list",
|
|
169
|
+
childrenCount: schedules.length
|
|
170
|
+
}
|
|
171
|
+
});
|
|
172
|
+
}
|
|
173
|
+
readSchedule(ctx) {
|
|
174
|
+
const schedule = this.store.getSchedule(ctx.params.id);
|
|
175
|
+
if (!schedule) return void 0;
|
|
176
|
+
return this.buildEntry(ctx.path, {
|
|
177
|
+
id: schedule.id,
|
|
178
|
+
content: schedule,
|
|
179
|
+
meta: { kind: "scheduler:schedule" }
|
|
180
|
+
});
|
|
181
|
+
}
|
|
182
|
+
readJobs(ctx) {
|
|
183
|
+
const jobs = this.store.listJobs();
|
|
184
|
+
return this.buildEntry(ctx.path, {
|
|
185
|
+
content: `${jobs.length} jobs`,
|
|
186
|
+
meta: {
|
|
187
|
+
kind: "scheduler:job-list",
|
|
188
|
+
childrenCount: jobs.length
|
|
189
|
+
}
|
|
190
|
+
});
|
|
191
|
+
}
|
|
192
|
+
async readJob(ctx) {
|
|
193
|
+
if (this.jobExecutor) {
|
|
194
|
+
const job$1 = await this.jobExecutor.getStatus(ctx.params.id);
|
|
195
|
+
if (!job$1) return void 0;
|
|
196
|
+
return this.buildEntry(ctx.path, {
|
|
197
|
+
id: job$1.id,
|
|
198
|
+
content: job$1,
|
|
199
|
+
meta: { kind: "scheduler:job" }
|
|
200
|
+
});
|
|
201
|
+
}
|
|
202
|
+
const job = this.store.getJob(ctx.params.id);
|
|
203
|
+
if (!job) return void 0;
|
|
204
|
+
return this.buildEntry(ctx.path, {
|
|
205
|
+
id: job.id,
|
|
206
|
+
content: job,
|
|
207
|
+
meta: { kind: "scheduler:job" }
|
|
208
|
+
});
|
|
209
|
+
}
|
|
210
|
+
readCapabilities(ctx) {
|
|
211
|
+
return this.buildEntry(ctx.path, { content: { operations: this.getOperationsDeclaration() } });
|
|
212
|
+
}
|
|
213
|
+
listRoot(ctx) {
|
|
214
|
+
return { data: [
|
|
215
|
+
this.buildEntry(joinURL(ctx.path, "tasks"), { meta: {
|
|
216
|
+
kind: "scheduler:task-list",
|
|
217
|
+
childrenCount: this.store.listActive().length
|
|
218
|
+
} }),
|
|
219
|
+
this.buildEntry(joinURL(ctx.path, "history"), { meta: {
|
|
220
|
+
kind: "scheduler:history",
|
|
221
|
+
childrenCount: this.store.listHistory().length
|
|
222
|
+
} }),
|
|
223
|
+
this.buildEntry(joinURL(ctx.path, "schedules"), { meta: {
|
|
224
|
+
kind: "scheduler:schedule-list",
|
|
225
|
+
childrenCount: this.store.listSchedules().length
|
|
226
|
+
} }),
|
|
227
|
+
this.buildEntry(joinURL(ctx.path, "jobs"), { meta: {
|
|
228
|
+
kind: "scheduler:job-list",
|
|
229
|
+
childrenCount: this.store.listJobs().length
|
|
230
|
+
} })
|
|
231
|
+
] };
|
|
232
|
+
}
|
|
233
|
+
listTasks(ctx) {
|
|
234
|
+
return { data: this.store.listActive().map((t) => this.buildEntry(joinURL(ctx.path, t.id), {
|
|
235
|
+
id: t.id,
|
|
236
|
+
content: t,
|
|
237
|
+
meta: { kind: "scheduler:task" }
|
|
238
|
+
})) };
|
|
239
|
+
}
|
|
240
|
+
listHistory(ctx) {
|
|
241
|
+
return { data: this.store.listHistory().map((t) => this.buildEntry(joinURL(ctx.path, t.id), {
|
|
242
|
+
id: t.id,
|
|
243
|
+
content: t,
|
|
244
|
+
meta: { kind: "scheduler:task" }
|
|
245
|
+
})) };
|
|
246
|
+
}
|
|
247
|
+
listSchedules(ctx) {
|
|
248
|
+
return { data: this.store.listSchedules().map((s) => this.buildEntry(joinURL(ctx.path, s.id), {
|
|
249
|
+
id: s.id,
|
|
250
|
+
content: s,
|
|
251
|
+
meta: { kind: "scheduler:schedule" }
|
|
252
|
+
})) };
|
|
253
|
+
}
|
|
254
|
+
listJobsHandler(ctx) {
|
|
255
|
+
return { data: this.store.listJobs().map((j) => this.buildEntry(joinURL(ctx.path, j.id), {
|
|
256
|
+
id: j.id,
|
|
257
|
+
content: j,
|
|
258
|
+
meta: { kind: "scheduler:job" }
|
|
259
|
+
})) };
|
|
260
|
+
}
|
|
261
|
+
readRootMeta(ctx) {
|
|
262
|
+
const meta = {
|
|
263
|
+
kind: "scheduler:root",
|
|
264
|
+
childrenCount: 4
|
|
265
|
+
};
|
|
266
|
+
return this.buildEntry(ctx.path, {
|
|
267
|
+
content: meta,
|
|
268
|
+
meta
|
|
269
|
+
});
|
|
270
|
+
}
|
|
271
|
+
readTasksMeta(ctx) {
|
|
272
|
+
const meta = {
|
|
273
|
+
kind: "scheduler:task-list",
|
|
274
|
+
childrenCount: this.store.listActive().length
|
|
275
|
+
};
|
|
276
|
+
return this.buildEntry(ctx.path, {
|
|
277
|
+
content: meta,
|
|
278
|
+
meta
|
|
279
|
+
});
|
|
280
|
+
}
|
|
281
|
+
readHistoryMeta(ctx) {
|
|
282
|
+
const meta = {
|
|
283
|
+
kind: "scheduler:history",
|
|
284
|
+
childrenCount: this.store.listHistory().length
|
|
285
|
+
};
|
|
286
|
+
return this.buildEntry(ctx.path, {
|
|
287
|
+
content: meta,
|
|
288
|
+
meta
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
readSchedulesMeta(ctx) {
|
|
292
|
+
const meta = {
|
|
293
|
+
kind: "scheduler:schedule-list",
|
|
294
|
+
childrenCount: this.store.listSchedules().length
|
|
295
|
+
};
|
|
296
|
+
return this.buildEntry(ctx.path, {
|
|
297
|
+
content: meta,
|
|
298
|
+
meta
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
readJobsMeta(ctx) {
|
|
302
|
+
const meta = {
|
|
303
|
+
kind: "scheduler:job-list",
|
|
304
|
+
childrenCount: this.store.listJobs().length
|
|
305
|
+
};
|
|
306
|
+
return this.buildEntry(ctx.path, {
|
|
307
|
+
content: meta,
|
|
308
|
+
meta
|
|
309
|
+
});
|
|
310
|
+
}
|
|
311
|
+
explainRoot() {
|
|
312
|
+
return {
|
|
313
|
+
content: "Scheduler provider — dispatches tasks for background execution. Use dispatch to submit tasks, cancel to stop them, retry to re-run failed tasks. Read /tasks/{id}/status to check progress.",
|
|
314
|
+
format: "text"
|
|
315
|
+
};
|
|
316
|
+
}
|
|
317
|
+
explainPath(ctx) {
|
|
318
|
+
return {
|
|
319
|
+
content: `Scheduler path: /${ctx.params.path}`,
|
|
320
|
+
format: "text"
|
|
321
|
+
};
|
|
322
|
+
}
|
|
323
|
+
listActions(ctx) {
|
|
324
|
+
return { data: ACTION_NAMES.map((name) => this.buildEntry(joinURL(ctx.path, name), {
|
|
325
|
+
id: name,
|
|
326
|
+
meta: { kind: "afs:executable" }
|
|
327
|
+
})) };
|
|
328
|
+
}
|
|
329
|
+
async execDispatch(ctx, args) {
|
|
330
|
+
await this.persistence.ensureReady();
|
|
331
|
+
const parsed = DispatchArgsSchema.safeParse(args);
|
|
332
|
+
if (!parsed.success) return {
|
|
333
|
+
success: false,
|
|
334
|
+
error: {
|
|
335
|
+
code: "VALIDATION_ERROR",
|
|
336
|
+
message: parsed.error.message
|
|
337
|
+
}
|
|
338
|
+
};
|
|
339
|
+
let procId;
|
|
340
|
+
if (parsed.data.proc) {
|
|
341
|
+
const procPath = this.options.procPath ?? "/proc";
|
|
342
|
+
if (!this.root.exec) return {
|
|
343
|
+
success: false,
|
|
344
|
+
error: {
|
|
345
|
+
code: "PROC_REGISTRATION_FAILED",
|
|
346
|
+
message: "AFS root does not support exec"
|
|
347
|
+
}
|
|
348
|
+
};
|
|
349
|
+
try {
|
|
350
|
+
const procData = {
|
|
351
|
+
...parsed.data.proc,
|
|
352
|
+
parentId: parsed.data.proc.parentId ?? args.parentProcId
|
|
353
|
+
};
|
|
354
|
+
const regResult = await this.root.exec(joinURL(procPath, ".actions", "register"), procData, {});
|
|
355
|
+
if (!regResult.success) return {
|
|
356
|
+
success: false,
|
|
357
|
+
error: {
|
|
358
|
+
code: "PROC_REGISTRATION_FAILED",
|
|
359
|
+
message: regResult.error?.message ?? "Failed to register proc"
|
|
360
|
+
}
|
|
361
|
+
};
|
|
362
|
+
procId = regResult.data.procId;
|
|
363
|
+
} catch (err) {
|
|
364
|
+
return {
|
|
365
|
+
success: false,
|
|
366
|
+
error: {
|
|
367
|
+
code: "PROC_REGISTRATION_FAILED",
|
|
368
|
+
message: String(err)
|
|
369
|
+
}
|
|
370
|
+
};
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
const isClaimable = parsed.data.assignee !== void 0;
|
|
374
|
+
const entry = this.store.create(parsed.data.task, parsed.data.args, procId, parsed.data.estimate, {
|
|
375
|
+
ownerKey: this.getCallerKey(ctx),
|
|
376
|
+
claimable: isClaimable,
|
|
377
|
+
assignee: parsed.data.assignee
|
|
378
|
+
});
|
|
379
|
+
this.persistence.schedulePersist();
|
|
380
|
+
if (isClaimable) return {
|
|
381
|
+
success: true,
|
|
382
|
+
data: {
|
|
383
|
+
taskId: entry.id,
|
|
384
|
+
procId,
|
|
385
|
+
claimable: true
|
|
386
|
+
}
|
|
387
|
+
};
|
|
388
|
+
const taskPromise = this.executeTask(entry.id).catch((err) => {
|
|
389
|
+
log.error("[scheduler] executeTask failed:", err);
|
|
390
|
+
});
|
|
391
|
+
this.options.waitUntil?.(taskPromise);
|
|
392
|
+
return {
|
|
393
|
+
success: true,
|
|
394
|
+
data: {
|
|
395
|
+
taskId: entry.id,
|
|
396
|
+
procId
|
|
397
|
+
}
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
async execCancel(ctx, args) {
|
|
401
|
+
await this.persistence.ensureReady();
|
|
402
|
+
const parsed = CancelArgsSchema.safeParse(args);
|
|
403
|
+
if (!parsed.success) return {
|
|
404
|
+
success: false,
|
|
405
|
+
error: {
|
|
406
|
+
code: "VALIDATION_ERROR",
|
|
407
|
+
message: parsed.error.message
|
|
408
|
+
}
|
|
409
|
+
};
|
|
410
|
+
const { taskId } = parsed.data;
|
|
411
|
+
const entry = this.store.get(taskId);
|
|
412
|
+
if (!entry) return {
|
|
413
|
+
success: false,
|
|
414
|
+
error: {
|
|
415
|
+
code: "NOT_FOUND",
|
|
416
|
+
message: `Task ${taskId} not found`
|
|
417
|
+
}
|
|
418
|
+
};
|
|
419
|
+
const ownerError = this.checkTaskOwner(entry, ctx);
|
|
420
|
+
if (ownerError) return ownerError;
|
|
421
|
+
if (entry.status === "done" || entry.status === "failed" || entry.status === "timeout") return {
|
|
422
|
+
success: false,
|
|
423
|
+
error: {
|
|
424
|
+
code: "INVALID_STATE",
|
|
425
|
+
message: `Task ${taskId} is already ${entry.status}`
|
|
426
|
+
}
|
|
427
|
+
};
|
|
428
|
+
if (entry.status === "cancelled") return {
|
|
429
|
+
success: true,
|
|
430
|
+
data: {
|
|
431
|
+
cancelled: true,
|
|
432
|
+
alreadyCancelled: true
|
|
433
|
+
}
|
|
434
|
+
};
|
|
435
|
+
this.store.update(taskId, {
|
|
436
|
+
status: "cancelled",
|
|
437
|
+
ended: Date.now()
|
|
438
|
+
});
|
|
439
|
+
this.store.moveToHistory(taskId);
|
|
440
|
+
this.persistence.schedulePersist();
|
|
441
|
+
if (this.executor.cancel) try {
|
|
442
|
+
await this.executor.cancel(taskId);
|
|
443
|
+
} catch {}
|
|
444
|
+
if (entry.procId) await this.completeProc(entry.procId, "failed");
|
|
445
|
+
return {
|
|
446
|
+
success: true,
|
|
447
|
+
data: { cancelled: true }
|
|
448
|
+
};
|
|
449
|
+
}
|
|
450
|
+
async execRetry(ctx, args) {
|
|
451
|
+
await this.persistence.ensureReady();
|
|
452
|
+
const parsed = RetryArgsSchema.safeParse(args);
|
|
453
|
+
if (!parsed.success) return {
|
|
454
|
+
success: false,
|
|
455
|
+
error: {
|
|
456
|
+
code: "VALIDATION_ERROR",
|
|
457
|
+
message: parsed.error.message
|
|
458
|
+
}
|
|
459
|
+
};
|
|
460
|
+
const { taskId } = parsed.data;
|
|
461
|
+
let entry = this.store.get(taskId);
|
|
462
|
+
if (entry) {
|
|
463
|
+
const ownerError = this.checkTaskOwner(entry, ctx);
|
|
464
|
+
if (ownerError) return ownerError;
|
|
465
|
+
if (entry.status === "running" || entry.status === "pending") return {
|
|
466
|
+
success: false,
|
|
467
|
+
error: {
|
|
468
|
+
code: "INVALID_STATE",
|
|
469
|
+
message: `Task ${taskId} is ${entry.status}, cannot retry`
|
|
470
|
+
}
|
|
471
|
+
};
|
|
472
|
+
if (entry.status === "done") return {
|
|
473
|
+
success: false,
|
|
474
|
+
error: {
|
|
475
|
+
code: "INVALID_STATE",
|
|
476
|
+
message: `Task ${taskId} is done, cannot retry`
|
|
477
|
+
}
|
|
478
|
+
};
|
|
479
|
+
} else {
|
|
480
|
+
entry = this.store.restoreFromHistory(taskId);
|
|
481
|
+
if (!entry) return {
|
|
482
|
+
success: false,
|
|
483
|
+
error: {
|
|
484
|
+
code: "NOT_FOUND",
|
|
485
|
+
message: `Task ${taskId} not found`
|
|
486
|
+
}
|
|
487
|
+
};
|
|
488
|
+
const ownerError = this.checkTaskOwner(entry, ctx);
|
|
489
|
+
if (ownerError) {
|
|
490
|
+
this.store.moveToHistory(taskId);
|
|
491
|
+
return ownerError;
|
|
492
|
+
}
|
|
493
|
+
if (entry.status === "done") {
|
|
494
|
+
this.store.moveToHistory(taskId);
|
|
495
|
+
return {
|
|
496
|
+
success: false,
|
|
497
|
+
error: {
|
|
498
|
+
code: "INVALID_STATE",
|
|
499
|
+
message: `Task ${taskId} is done, cannot retry`
|
|
500
|
+
}
|
|
501
|
+
};
|
|
502
|
+
}
|
|
503
|
+
}
|
|
504
|
+
this.store.update(taskId, {
|
|
505
|
+
status: "pending",
|
|
506
|
+
result: void 0,
|
|
507
|
+
error: void 0,
|
|
508
|
+
started: void 0,
|
|
509
|
+
ended: void 0
|
|
510
|
+
});
|
|
511
|
+
this.persistence.schedulePersist();
|
|
512
|
+
const retryPromise = this.executeTask(taskId).catch((err) => {
|
|
513
|
+
log.error("[scheduler] executeTask failed:", err);
|
|
514
|
+
});
|
|
515
|
+
this.options.waitUntil?.(retryPromise);
|
|
516
|
+
return {
|
|
517
|
+
success: true,
|
|
518
|
+
data: { taskId }
|
|
519
|
+
};
|
|
520
|
+
}
|
|
521
|
+
async execSchedule(ctx, args) {
|
|
522
|
+
await this.persistence.ensureReady();
|
|
523
|
+
const parsed = ScheduleArgsSchema.safeParse(args);
|
|
524
|
+
if (!parsed.success) return {
|
|
525
|
+
success: false,
|
|
526
|
+
error: {
|
|
527
|
+
code: "VALIDATION_ERROR",
|
|
528
|
+
message: parsed.error.message
|
|
529
|
+
}
|
|
530
|
+
};
|
|
531
|
+
try {
|
|
532
|
+
const schedule = this.store.addSchedule({
|
|
533
|
+
name: parsed.data.name,
|
|
534
|
+
cron: parsed.data.cron,
|
|
535
|
+
taskPath: parsed.data.task,
|
|
536
|
+
args: parsed.data.args,
|
|
537
|
+
enabled: parsed.data.enabled,
|
|
538
|
+
ownerKey: this.getCallerKey(ctx)
|
|
539
|
+
});
|
|
540
|
+
this.persistence.schedulePersist();
|
|
541
|
+
return {
|
|
542
|
+
success: true,
|
|
543
|
+
data: { scheduleId: schedule.id }
|
|
544
|
+
};
|
|
545
|
+
} catch (err) {
|
|
546
|
+
return {
|
|
547
|
+
success: false,
|
|
548
|
+
error: {
|
|
549
|
+
code: "SCHEDULE_ERROR",
|
|
550
|
+
message: err instanceof Error ? err.message : String(err)
|
|
551
|
+
}
|
|
552
|
+
};
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
async execUnschedule(ctx, args) {
|
|
556
|
+
await this.persistence.ensureReady();
|
|
557
|
+
const parsed = UnscheduleArgsSchema.safeParse(args);
|
|
558
|
+
if (!parsed.success) return {
|
|
559
|
+
success: false,
|
|
560
|
+
error: {
|
|
561
|
+
code: "VALIDATION_ERROR",
|
|
562
|
+
message: parsed.error.message
|
|
563
|
+
}
|
|
564
|
+
};
|
|
565
|
+
let removed = false;
|
|
566
|
+
let scheduleOwnerError;
|
|
567
|
+
if (parsed.data.scheduleId) {
|
|
568
|
+
const schedule = this.store.getSchedule(parsed.data.scheduleId);
|
|
569
|
+
scheduleOwnerError = schedule ? this.checkScheduleOwner(schedule, ctx) : void 0;
|
|
570
|
+
if (!scheduleOwnerError) removed = this.store.removeSchedule(parsed.data.scheduleId);
|
|
571
|
+
} else if (parsed.data.name) {
|
|
572
|
+
const schedule = this.store.getScheduleByName(parsed.data.name);
|
|
573
|
+
if (schedule) {
|
|
574
|
+
scheduleOwnerError = this.checkScheduleOwner(schedule, ctx);
|
|
575
|
+
if (!scheduleOwnerError) removed = this.store.removeSchedule(schedule.id);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
if (scheduleOwnerError) return scheduleOwnerError;
|
|
579
|
+
if (!removed) return {
|
|
580
|
+
success: false,
|
|
581
|
+
error: {
|
|
582
|
+
code: "NOT_FOUND",
|
|
583
|
+
message: "Schedule not found"
|
|
584
|
+
}
|
|
585
|
+
};
|
|
586
|
+
this.persistence.schedulePersist();
|
|
587
|
+
return { success: true };
|
|
588
|
+
}
|
|
589
|
+
async execHeartbeat(_ctx, args) {
|
|
590
|
+
await this.persistence.ensureReady();
|
|
591
|
+
const parsed = HeartbeatArgsSchema.safeParse(args);
|
|
592
|
+
if (!parsed.success) return {
|
|
593
|
+
success: false,
|
|
594
|
+
error: {
|
|
595
|
+
code: "VALIDATION_ERROR",
|
|
596
|
+
message: parsed.error.message
|
|
597
|
+
}
|
|
598
|
+
};
|
|
599
|
+
const { taskId } = parsed.data;
|
|
600
|
+
const entry = this.store.get(taskId);
|
|
601
|
+
if (!entry) return {
|
|
602
|
+
success: false,
|
|
603
|
+
error: {
|
|
604
|
+
code: "NOT_FOUND",
|
|
605
|
+
message: `Task ${taskId} not found`
|
|
606
|
+
}
|
|
607
|
+
};
|
|
608
|
+
if (entry.status !== "running") return {
|
|
609
|
+
success: false,
|
|
610
|
+
error: {
|
|
611
|
+
code: "INVALID_STATE",
|
|
612
|
+
message: `Task ${taskId} is ${entry.status}, not running`
|
|
613
|
+
}
|
|
614
|
+
};
|
|
615
|
+
if (!entry.timeout || !entry.deadline) return {
|
|
616
|
+
success: false,
|
|
617
|
+
error: {
|
|
618
|
+
code: "NO_TIMEOUT",
|
|
619
|
+
message: `Task ${taskId} has no timeout configured`
|
|
620
|
+
}
|
|
621
|
+
};
|
|
622
|
+
const now = Date.now();
|
|
623
|
+
this.store.update(taskId, {
|
|
624
|
+
lastHeartbeat: now,
|
|
625
|
+
deadline: now + entry.timeout
|
|
626
|
+
});
|
|
627
|
+
this.persistence.schedulePersist();
|
|
628
|
+
return { success: true };
|
|
629
|
+
}
|
|
630
|
+
async execClaim(_ctx, args) {
|
|
631
|
+
await this.persistence.ensureReady();
|
|
632
|
+
const parsed = ClaimArgsSchema.safeParse(args);
|
|
633
|
+
if (!parsed.success) return {
|
|
634
|
+
success: false,
|
|
635
|
+
error: {
|
|
636
|
+
code: "VALIDATION_ERROR",
|
|
637
|
+
message: parsed.error.message
|
|
638
|
+
}
|
|
639
|
+
};
|
|
640
|
+
const { taskId, workerId, leaseDuration, ifMatch } = parsed.data;
|
|
641
|
+
const entry = this.store.get(taskId);
|
|
642
|
+
if (!entry) return {
|
|
643
|
+
success: false,
|
|
644
|
+
error: {
|
|
645
|
+
code: "NOT_FOUND",
|
|
646
|
+
message: `Task ${taskId} not found`
|
|
647
|
+
}
|
|
648
|
+
};
|
|
649
|
+
if (!entry.claimable) return {
|
|
650
|
+
success: false,
|
|
651
|
+
error: {
|
|
652
|
+
code: "NOT_CLAIMABLE",
|
|
653
|
+
message: `Task ${taskId} was not dispatched for claiming (missing assignee field)`
|
|
654
|
+
}
|
|
655
|
+
};
|
|
656
|
+
if (entry.assignee && entry.assignee !== workerId) return {
|
|
657
|
+
success: false,
|
|
658
|
+
error: {
|
|
659
|
+
code: "PERMISSION_DENIED",
|
|
660
|
+
message: `Task ${taskId} is reserved for a different assignee`
|
|
661
|
+
}
|
|
662
|
+
};
|
|
663
|
+
const now = Date.now();
|
|
664
|
+
if (ifMatch !== void 0 && ifMatch !== entry.version) throw new AFSConflictError(joinURL("/tasks", taskId), { cid: entry.version ?? null });
|
|
665
|
+
const leaseExpired = entry.claimedBy !== void 0 && entry.deadline !== void 0 && entry.deadline < now;
|
|
666
|
+
if (entry.claimedBy !== void 0 && entry.claimedBy !== workerId && !leaseExpired) return {
|
|
667
|
+
success: false,
|
|
668
|
+
error: {
|
|
669
|
+
code: "ALREADY_CLAIMED",
|
|
670
|
+
message: `Task ${taskId} is already claimed by another worker`
|
|
671
|
+
}
|
|
672
|
+
};
|
|
673
|
+
if (entry.status !== "pending" && !(entry.status === "running" && leaseExpired)) return {
|
|
674
|
+
success: false,
|
|
675
|
+
error: {
|
|
676
|
+
code: "INVALID_STATE",
|
|
677
|
+
message: `Task ${taskId} is ${entry.status}, cannot claim`
|
|
678
|
+
}
|
|
679
|
+
};
|
|
680
|
+
const lease = leaseDuration ?? this.options.defaultLeaseDuration ?? DEFAULT_LEASE_DURATION_MS;
|
|
681
|
+
const newVersion = v7();
|
|
682
|
+
this.store.update(taskId, {
|
|
683
|
+
status: "running",
|
|
684
|
+
claimedBy: workerId,
|
|
685
|
+
claimedAt: now,
|
|
686
|
+
started: entry.started ?? now,
|
|
687
|
+
attempts: entry.attempts + 1,
|
|
688
|
+
lastHeartbeat: now,
|
|
689
|
+
deadline: now + lease,
|
|
690
|
+
timeout: lease,
|
|
691
|
+
version: newVersion
|
|
692
|
+
});
|
|
693
|
+
this.persistence.schedulePersist();
|
|
694
|
+
return {
|
|
695
|
+
success: true,
|
|
696
|
+
data: {
|
|
697
|
+
taskId,
|
|
698
|
+
claimedBy: workerId,
|
|
699
|
+
leaseDeadline: now + lease,
|
|
700
|
+
version: newVersion
|
|
701
|
+
}
|
|
702
|
+
};
|
|
703
|
+
}
|
|
704
|
+
async execSubmit(ctx, args) {
|
|
705
|
+
await this.persistence.ensureReady();
|
|
706
|
+
const parsed = SubmitArgsSchema.safeParse(args);
|
|
707
|
+
if (!parsed.success) return {
|
|
708
|
+
success: false,
|
|
709
|
+
error: {
|
|
710
|
+
code: "VALIDATION_ERROR",
|
|
711
|
+
message: parsed.error.message
|
|
712
|
+
}
|
|
713
|
+
};
|
|
714
|
+
const { taskId, workerId, success, result, artifacts, verifyArgs } = parsed.data;
|
|
715
|
+
const verifyTaskPath = parsed.data.verifyTask ?? this.options.verifyTaskPath;
|
|
716
|
+
if (!verifyTaskPath) return {
|
|
717
|
+
success: false,
|
|
718
|
+
error: {
|
|
719
|
+
code: "VALIDATION_ERROR",
|
|
720
|
+
message: "verifyTask is required (no verifyTaskPath configured on the scheduler provider)"
|
|
721
|
+
}
|
|
722
|
+
};
|
|
723
|
+
const entry = this.store.get(taskId);
|
|
724
|
+
if (!entry) return {
|
|
725
|
+
success: false,
|
|
726
|
+
error: {
|
|
727
|
+
code: "NOT_FOUND",
|
|
728
|
+
message: `Task ${taskId} not found`
|
|
729
|
+
}
|
|
730
|
+
};
|
|
731
|
+
if (!entry.claimable) return {
|
|
732
|
+
success: false,
|
|
733
|
+
error: {
|
|
734
|
+
code: "NOT_CLAIMABLE",
|
|
735
|
+
message: `Task ${taskId} is not a claim-based task`
|
|
736
|
+
}
|
|
737
|
+
};
|
|
738
|
+
if (entry.claimedBy !== workerId) return {
|
|
739
|
+
success: false,
|
|
740
|
+
error: {
|
|
741
|
+
code: "PERMISSION_DENIED",
|
|
742
|
+
message: `Task ${taskId} is not claimed by ${workerId}`
|
|
743
|
+
}
|
|
744
|
+
};
|
|
745
|
+
if (entry.status !== "running") return {
|
|
746
|
+
success: false,
|
|
747
|
+
error: {
|
|
748
|
+
code: "INVALID_STATE",
|
|
749
|
+
message: `Task ${taskId} is ${entry.status}, cannot submit`
|
|
750
|
+
}
|
|
751
|
+
};
|
|
752
|
+
const finalStatus = success === false ? "failed" : "done";
|
|
753
|
+
const now = Date.now();
|
|
754
|
+
this.store.update(taskId, {
|
|
755
|
+
status: finalStatus,
|
|
756
|
+
result: {
|
|
757
|
+
success: success !== false,
|
|
758
|
+
data: result
|
|
759
|
+
},
|
|
760
|
+
ended: now,
|
|
761
|
+
version: v7()
|
|
762
|
+
});
|
|
763
|
+
this.store.moveToHistory(taskId);
|
|
764
|
+
if (entry.procId) await this.completeProc(entry.procId, finalStatus === "done" ? "done" : "failed");
|
|
765
|
+
const verifyTaskId = this.store.create(verifyTaskPath, {
|
|
766
|
+
...typeof verifyArgs === "object" && verifyArgs !== null ? verifyArgs : {},
|
|
767
|
+
originalTaskId: taskId,
|
|
768
|
+
workerId,
|
|
769
|
+
result,
|
|
770
|
+
artifacts
|
|
771
|
+
}, void 0, void 0, { ownerKey: this.getCallerKey(ctx) }).id;
|
|
772
|
+
const verifyPromise = this.executeTask(verifyTaskId).catch((err) => {
|
|
773
|
+
log.error("[scheduler] executeTask failed:", err);
|
|
774
|
+
});
|
|
775
|
+
this.options.waitUntil?.(verifyPromise);
|
|
776
|
+
this.persistence.schedulePersist();
|
|
777
|
+
return {
|
|
778
|
+
success: true,
|
|
779
|
+
data: {
|
|
780
|
+
taskId,
|
|
781
|
+
status: finalStatus,
|
|
782
|
+
verifyTaskId
|
|
783
|
+
}
|
|
784
|
+
};
|
|
785
|
+
}
|
|
786
|
+
async execSubmitJob(_ctx, args) {
|
|
787
|
+
await this.persistence.ensureReady();
|
|
788
|
+
const parsed = SubmitJobArgsSchema.safeParse(args);
|
|
789
|
+
if (!parsed.success) return {
|
|
790
|
+
success: false,
|
|
791
|
+
error: {
|
|
792
|
+
code: "VALIDATION_ERROR",
|
|
793
|
+
message: parsed.error.message
|
|
794
|
+
}
|
|
795
|
+
};
|
|
796
|
+
const batchSize = parsed.data.batchSize ?? 100;
|
|
797
|
+
const concurrency = parsed.data.concurrency ?? 10;
|
|
798
|
+
const jobId = v7();
|
|
799
|
+
if (this.jobExecutor) try {
|
|
800
|
+
return {
|
|
801
|
+
success: true,
|
|
802
|
+
data: { jobId: (await this.jobExecutor.submitJob({
|
|
803
|
+
jobId,
|
|
804
|
+
name: parsed.data.name,
|
|
805
|
+
mapTask: parsed.data.mapTask,
|
|
806
|
+
reduceTask: parsed.data.reduceTask,
|
|
807
|
+
totalItems: parsed.data.totalItems,
|
|
808
|
+
batchSize,
|
|
809
|
+
concurrency,
|
|
810
|
+
args: parsed.data.args,
|
|
811
|
+
timeout: parsed.data.timeout
|
|
812
|
+
})).jobId }
|
|
813
|
+
};
|
|
814
|
+
} catch (err) {
|
|
815
|
+
return {
|
|
816
|
+
success: false,
|
|
817
|
+
error: {
|
|
818
|
+
code: "COORDINATOR_ERROR",
|
|
819
|
+
message: String(err)
|
|
820
|
+
}
|
|
821
|
+
};
|
|
822
|
+
}
|
|
823
|
+
const totalBatches = Math.ceil(parsed.data.totalItems / batchSize);
|
|
824
|
+
const job = {
|
|
825
|
+
id: jobId,
|
|
826
|
+
name: parsed.data.name,
|
|
827
|
+
status: "mapping",
|
|
828
|
+
mapTask: parsed.data.mapTask,
|
|
829
|
+
reduceTask: parsed.data.reduceTask,
|
|
830
|
+
args: parsed.data.args,
|
|
831
|
+
totalItems: parsed.data.totalItems,
|
|
832
|
+
batchSize,
|
|
833
|
+
concurrency,
|
|
834
|
+
timeout: parsed.data.timeout,
|
|
835
|
+
totalBatches,
|
|
836
|
+
progress: {
|
|
837
|
+
pending: totalBatches,
|
|
838
|
+
running: 0,
|
|
839
|
+
done: 0,
|
|
840
|
+
failed: 0
|
|
841
|
+
},
|
|
842
|
+
mapTaskIds: new Array(totalBatches).fill(""),
|
|
843
|
+
mapResults: new Array(totalBatches).fill(null),
|
|
844
|
+
created: Date.now(),
|
|
845
|
+
started: Date.now()
|
|
846
|
+
};
|
|
847
|
+
this.store.addJob(job);
|
|
848
|
+
this.persistence.schedulePersist();
|
|
849
|
+
const firstWave = Math.min(concurrency, totalBatches);
|
|
850
|
+
for (let i = 0; i < firstWave; i++) this.dispatchMapTask(job, i);
|
|
851
|
+
return {
|
|
852
|
+
success: true,
|
|
853
|
+
data: { jobId: job.id }
|
|
854
|
+
};
|
|
855
|
+
}
|
|
856
|
+
async execCancelJob(_ctx, args) {
|
|
857
|
+
await this.persistence.ensureReady();
|
|
858
|
+
const parsed = CancelJobArgsSchema.safeParse(args);
|
|
859
|
+
if (!parsed.success) return {
|
|
860
|
+
success: false,
|
|
861
|
+
error: {
|
|
862
|
+
code: "VALIDATION_ERROR",
|
|
863
|
+
message: parsed.error.message
|
|
864
|
+
}
|
|
865
|
+
};
|
|
866
|
+
if (this.jobExecutor) try {
|
|
867
|
+
await this.jobExecutor.cancelJob(parsed.data.jobId);
|
|
868
|
+
return { success: true };
|
|
869
|
+
} catch (err) {
|
|
870
|
+
return {
|
|
871
|
+
success: false,
|
|
872
|
+
error: {
|
|
873
|
+
code: "COORDINATOR_ERROR",
|
|
874
|
+
message: String(err)
|
|
875
|
+
}
|
|
876
|
+
};
|
|
877
|
+
}
|
|
878
|
+
const job = this.store.getJob(parsed.data.jobId);
|
|
879
|
+
if (!job) return {
|
|
880
|
+
success: false,
|
|
881
|
+
error: {
|
|
882
|
+
code: "NOT_FOUND",
|
|
883
|
+
message: `Job ${parsed.data.jobId} not found`
|
|
884
|
+
}
|
|
885
|
+
};
|
|
886
|
+
let cancelledTasks = 0;
|
|
887
|
+
for (const task of this.store.listActive()) if (task.jobId === job.id && (task.status === "pending" || task.status === "running")) {
|
|
888
|
+
this.store.update(task.id, {
|
|
889
|
+
status: "cancelled",
|
|
890
|
+
ended: Date.now()
|
|
891
|
+
});
|
|
892
|
+
this.store.moveToHistory(task.id);
|
|
893
|
+
cancelledTasks++;
|
|
894
|
+
}
|
|
895
|
+
this.store.updateJob(job.id, {
|
|
896
|
+
status: "cancelled",
|
|
897
|
+
ended: Date.now()
|
|
898
|
+
});
|
|
899
|
+
this.persistence.schedulePersist();
|
|
900
|
+
return {
|
|
901
|
+
success: true,
|
|
902
|
+
data: { cancelledTasks }
|
|
903
|
+
};
|
|
904
|
+
}
|
|
905
|
+
/**
|
|
906
|
+
* Evaluate cron schedules and scan for expired deadlines.
|
|
907
|
+
* Call from Worker `scheduled()` handler or `setInterval` in non-Worker environments.
|
|
908
|
+
*/
|
|
909
|
+
async tick() {
|
|
910
|
+
await this.persistence.ensureReady();
|
|
911
|
+
const now = Date.now();
|
|
912
|
+
let dispatched = 0;
|
|
913
|
+
let timedOut = 0;
|
|
914
|
+
let scheduleAdvanced = false;
|
|
915
|
+
for (const task of this.store.listActive()) {
|
|
916
|
+
const isScheduledTask = Boolean(task.scheduleId);
|
|
917
|
+
if (task.status !== "running" && !(isScheduledTask && task.status === "pending")) continue;
|
|
918
|
+
if (task.claimable) continue;
|
|
919
|
+
const deadline = this.getEffectiveDeadline(task);
|
|
920
|
+
if (deadline == null) continue;
|
|
921
|
+
if (deadline > now) continue;
|
|
922
|
+
this.store.update(task.id, {
|
|
923
|
+
status: "timeout",
|
|
924
|
+
error: `Task timed out (deadline exceeded by ${now - deadline}ms)`,
|
|
925
|
+
ended: now
|
|
926
|
+
});
|
|
927
|
+
if (task.procId) await this.completeProc(task.procId, "failed");
|
|
928
|
+
this.store.moveToHistory(task.id);
|
|
929
|
+
timedOut++;
|
|
930
|
+
}
|
|
931
|
+
for (const schedule of this.store.listSchedules()) {
|
|
932
|
+
if (!schedule.enabled) continue;
|
|
933
|
+
if (!schedule.nextRun || schedule.nextRun > now) continue;
|
|
934
|
+
if (this.store.listActive().some((t) => t.scheduleId === schedule.id && (t.status === "pending" || t.status === "running"))) {
|
|
935
|
+
this.store.recordScheduleRun(schedule.id, now);
|
|
936
|
+
scheduleAdvanced = true;
|
|
937
|
+
continue;
|
|
938
|
+
}
|
|
939
|
+
const entry = this.store.create(schedule.taskPath, schedule.args, void 0, void 0, {
|
|
940
|
+
ownerKey: schedule.ownerKey,
|
|
941
|
+
scheduleId: schedule.id
|
|
942
|
+
});
|
|
943
|
+
const scheduledTaskTimeoutMs = this.getScheduledTaskTimeoutMs();
|
|
944
|
+
if (scheduledTaskTimeoutMs > 0) this.store.update(entry.id, {
|
|
945
|
+
deadline: now + scheduledTaskTimeoutMs,
|
|
946
|
+
timeout: scheduledTaskTimeoutMs
|
|
947
|
+
});
|
|
948
|
+
this.store.recordScheduleRun(schedule.id, now);
|
|
949
|
+
const taskPromise = this.executeTask(entry.id).catch((err) => {
|
|
950
|
+
log.error("[scheduler] executeTask failed:", err);
|
|
951
|
+
});
|
|
952
|
+
this.options.waitUntil?.(taskPromise);
|
|
953
|
+
dispatched++;
|
|
954
|
+
}
|
|
955
|
+
if (dispatched > 0 || timedOut > 0 || scheduleAdvanced) this.persistence.schedulePersist();
|
|
956
|
+
return {
|
|
957
|
+
dispatched,
|
|
958
|
+
timedOut
|
|
959
|
+
};
|
|
960
|
+
}
|
|
961
|
+
getScheduledTaskTimeoutMs() {
|
|
962
|
+
return this.options.scheduledTaskTimeoutMs ?? DEFAULT_SCHEDULED_TASK_TIMEOUT_MS;
|
|
963
|
+
}
|
|
964
|
+
getEffectiveDeadline(task) {
|
|
965
|
+
if (task.deadline != null) return task.deadline;
|
|
966
|
+
if (!task.scheduleId) return void 0;
|
|
967
|
+
const scheduledTaskTimeoutMs = this.getScheduledTaskTimeoutMs();
|
|
968
|
+
if (scheduledTaskTimeoutMs <= 0) return void 0;
|
|
969
|
+
return (task.started ?? task.created) + scheduledTaskTimeoutMs;
|
|
970
|
+
}
|
|
971
|
+
getCallerKey(ctx) {
|
|
972
|
+
const context = ctx.context;
|
|
973
|
+
const userId = typeof context?.userId === "string" ? context.userId : void 0;
|
|
974
|
+
const sessionId = typeof context?.sessionId === "string" ? context.sessionId : void 0;
|
|
975
|
+
return userId ?? sessionId;
|
|
976
|
+
}
|
|
977
|
+
checkTaskOwner(entry, ctx) {
|
|
978
|
+
if (!entry.ownerKey) return void 0;
|
|
979
|
+
if (entry.ownerKey === this.getCallerKey(ctx)) return void 0;
|
|
980
|
+
return {
|
|
981
|
+
success: false,
|
|
982
|
+
error: {
|
|
983
|
+
code: "PERMISSION_DENIED",
|
|
984
|
+
message: `Task ${entry.id} is owned by a different caller`
|
|
985
|
+
}
|
|
986
|
+
};
|
|
987
|
+
}
|
|
988
|
+
checkScheduleOwner(schedule, ctx) {
|
|
989
|
+
if (!schedule.ownerKey) return void 0;
|
|
990
|
+
if (schedule.ownerKey === this.getCallerKey(ctx)) return void 0;
|
|
991
|
+
return {
|
|
992
|
+
success: false,
|
|
993
|
+
error: {
|
|
994
|
+
code: "PERMISSION_DENIED",
|
|
995
|
+
message: `Schedule ${schedule.id} is owned by a different caller`
|
|
996
|
+
}
|
|
997
|
+
};
|
|
998
|
+
}
|
|
999
|
+
async executeTask(taskId) {
|
|
1000
|
+
const entry = this.store.get(taskId);
|
|
1001
|
+
if (!entry) return;
|
|
1002
|
+
if (entry.procId) {
|
|
1003
|
+
const budgetCheck = await this.checkTaskBudget(entry.procId, entry.estimate);
|
|
1004
|
+
if (!budgetCheck.allowed) {
|
|
1005
|
+
this.store.update(taskId, {
|
|
1006
|
+
status: "budget-blocked",
|
|
1007
|
+
error: `Budget check failed: ${budgetCheck.reason}`
|
|
1008
|
+
});
|
|
1009
|
+
this.persistence.schedulePersist();
|
|
1010
|
+
await this.persistence.flush();
|
|
1011
|
+
return;
|
|
1012
|
+
}
|
|
1013
|
+
}
|
|
1014
|
+
this.store.update(taskId, {
|
|
1015
|
+
status: "running",
|
|
1016
|
+
started: Date.now(),
|
|
1017
|
+
attempts: entry.attempts + 1
|
|
1018
|
+
});
|
|
1019
|
+
this.persistence.schedulePersist();
|
|
1020
|
+
const context = entry.procId ? { procId: entry.procId } : void 0;
|
|
1021
|
+
let taskArgs = entry.args;
|
|
1022
|
+
if (taskArgs && typeof taskArgs === "object" && !Array.isArray(taskArgs)) {
|
|
1023
|
+
const a = taskArgs;
|
|
1024
|
+
if (!a.job_dir) taskArgs = {
|
|
1025
|
+
...a,
|
|
1026
|
+
job_dir: `/data/jobs/${taskId}`
|
|
1027
|
+
};
|
|
1028
|
+
}
|
|
1029
|
+
try {
|
|
1030
|
+
const result = await this.executor.run(taskId, entry.taskPath, taskArgs, context);
|
|
1031
|
+
const current = this.store.get(taskId);
|
|
1032
|
+
if (!current || current.status !== "running") return;
|
|
1033
|
+
this.store.update(taskId, {
|
|
1034
|
+
status: "done",
|
|
1035
|
+
result,
|
|
1036
|
+
ended: Date.now()
|
|
1037
|
+
});
|
|
1038
|
+
} catch (err) {
|
|
1039
|
+
const current = this.store.get(taskId);
|
|
1040
|
+
if (!current || current.status !== "running") return;
|
|
1041
|
+
this.store.update(taskId, {
|
|
1042
|
+
status: "failed",
|
|
1043
|
+
error: String(err),
|
|
1044
|
+
ended: Date.now()
|
|
1045
|
+
});
|
|
1046
|
+
}
|
|
1047
|
+
const finalEntry = this.store.get(taskId);
|
|
1048
|
+
if (finalEntry?.procId) await this.completeProc(finalEntry.procId, finalEntry.status === "done" ? "done" : "failed");
|
|
1049
|
+
this.store.moveToHistory(taskId);
|
|
1050
|
+
const completedEntry = this.store.getFromHistory(taskId);
|
|
1051
|
+
if (completedEntry?.jobId) await this.onJobTaskComplete(completedEntry);
|
|
1052
|
+
this.persistence.schedulePersist();
|
|
1053
|
+
await this.persistence.flush();
|
|
1054
|
+
}
|
|
1055
|
+
async checkTaskBudget(procId, estimate) {
|
|
1056
|
+
const procPath = this.options.procPath ?? "/proc";
|
|
1057
|
+
try {
|
|
1058
|
+
if (!this.root.exec) return {
|
|
1059
|
+
allowed: false,
|
|
1060
|
+
reason: "AFS root does not support exec"
|
|
1061
|
+
};
|
|
1062
|
+
const args = { id: procId };
|
|
1063
|
+
if (estimate) args.estimate = estimate;
|
|
1064
|
+
const result = await this.root.exec(joinURL(procPath, ".actions", "check-budget"), args, {});
|
|
1065
|
+
if (!result.success) return {
|
|
1066
|
+
allowed: false,
|
|
1067
|
+
reason: result.error?.message ?? "Budget check failed"
|
|
1068
|
+
};
|
|
1069
|
+
const data = result.data;
|
|
1070
|
+
return {
|
|
1071
|
+
allowed: data.allowed,
|
|
1072
|
+
reason: data.reason
|
|
1073
|
+
};
|
|
1074
|
+
} catch (err) {
|
|
1075
|
+
return {
|
|
1076
|
+
allowed: false,
|
|
1077
|
+
reason: `Budget check error: ${String(err)}`
|
|
1078
|
+
};
|
|
1079
|
+
}
|
|
1080
|
+
}
|
|
1081
|
+
async completeProc(procId, status) {
|
|
1082
|
+
const procPath = this.options.procPath ?? "/proc";
|
|
1083
|
+
try {
|
|
1084
|
+
if (this.root.exec) await this.root.exec(joinURL(procPath, ".actions", "complete"), {
|
|
1085
|
+
id: procId,
|
|
1086
|
+
status
|
|
1087
|
+
}, {});
|
|
1088
|
+
} catch {}
|
|
1089
|
+
}
|
|
1090
|
+
dispatchMapTask(job, batchIndex) {
|
|
1091
|
+
const batchStart = batchIndex * job.batchSize;
|
|
1092
|
+
const batchEnd = Math.min(batchStart + job.batchSize, job.totalItems);
|
|
1093
|
+
const taskArgs = {
|
|
1094
|
+
...job.args ?? {},
|
|
1095
|
+
jobId: job.id,
|
|
1096
|
+
batchIndex,
|
|
1097
|
+
batchSize: job.batchSize,
|
|
1098
|
+
batchStart,
|
|
1099
|
+
batchEnd,
|
|
1100
|
+
totalItems: job.totalItems
|
|
1101
|
+
};
|
|
1102
|
+
const entry = this.store.create(job.mapTask, taskArgs, void 0, void 0, {
|
|
1103
|
+
jobId: job.id,
|
|
1104
|
+
batchIndex
|
|
1105
|
+
});
|
|
1106
|
+
job.mapTaskIds[batchIndex] = entry.id;
|
|
1107
|
+
job.progress.pending--;
|
|
1108
|
+
job.progress.running++;
|
|
1109
|
+
const taskPromise = this.executeTask(entry.id).catch((err) => {
|
|
1110
|
+
log.error("[scheduler] executeTask failed:", err);
|
|
1111
|
+
});
|
|
1112
|
+
this.options.waitUntil?.(taskPromise);
|
|
1113
|
+
}
|
|
1114
|
+
async onJobTaskComplete(task) {
|
|
1115
|
+
const job = this.store.getJob(task.jobId);
|
|
1116
|
+
if (!job || job.status === "cancelled") return;
|
|
1117
|
+
if (job.reduceTaskId === task.id) {
|
|
1118
|
+
if (task.status === "done" && task.result) this.store.updateJob(job.id, {
|
|
1119
|
+
status: "done",
|
|
1120
|
+
result: task.result.data,
|
|
1121
|
+
ended: Date.now()
|
|
1122
|
+
});
|
|
1123
|
+
else this.store.updateJob(job.id, {
|
|
1124
|
+
status: "failed",
|
|
1125
|
+
error: task.error ?? "Reduce task failed",
|
|
1126
|
+
ended: Date.now()
|
|
1127
|
+
});
|
|
1128
|
+
this.persistence.schedulePersist();
|
|
1129
|
+
return;
|
|
1130
|
+
}
|
|
1131
|
+
const batchIndex = task.batchIndex;
|
|
1132
|
+
const taskSucceeded = task.status === "done" && task.result?.success !== false;
|
|
1133
|
+
if (taskSucceeded && task.result) job.mapResults[batchIndex] = task.result.data;
|
|
1134
|
+
else job.mapResults[batchIndex] = null;
|
|
1135
|
+
job.progress.running = Math.max(0, job.progress.running - 1);
|
|
1136
|
+
if (taskSucceeded) job.progress.done++;
|
|
1137
|
+
else job.progress.failed++;
|
|
1138
|
+
const nextBatch = this.store.nextPendingBatch(job.id);
|
|
1139
|
+
if (nextBatch >= 0) this.dispatchMapTask(job, nextBatch);
|
|
1140
|
+
if (job.progress.done + job.progress.failed === job.totalBatches) if (job.reduceTask) {
|
|
1141
|
+
job.status = "reducing";
|
|
1142
|
+
const reduceArgs = {
|
|
1143
|
+
...job.args ?? {},
|
|
1144
|
+
jobId: job.id,
|
|
1145
|
+
mapResults: job.mapResults,
|
|
1146
|
+
totalBatches: job.totalBatches
|
|
1147
|
+
};
|
|
1148
|
+
const reduceEntry = this.store.create(job.reduceTask, reduceArgs, void 0, void 0, { jobId: job.id });
|
|
1149
|
+
job.reduceTaskId = reduceEntry.id;
|
|
1150
|
+
const reducePromise = this.executeTask(reduceEntry.id).catch((err) => {
|
|
1151
|
+
log.error("[scheduler] executeTask failed:", err);
|
|
1152
|
+
});
|
|
1153
|
+
this.options.waitUntil?.(reducePromise);
|
|
1154
|
+
} else this.store.updateJob(job.id, {
|
|
1155
|
+
status: "done",
|
|
1156
|
+
result: job.mapResults,
|
|
1157
|
+
ended: Date.now()
|
|
1158
|
+
});
|
|
1159
|
+
this.persistence.schedulePersist();
|
|
1160
|
+
}
|
|
1161
|
+
readEntryField(path, entry, field) {
|
|
1162
|
+
let content;
|
|
1163
|
+
switch (field) {
|
|
1164
|
+
case "status":
|
|
1165
|
+
content = entry.status;
|
|
1166
|
+
break;
|
|
1167
|
+
case "result":
|
|
1168
|
+
content = entry.result ?? null;
|
|
1169
|
+
break;
|
|
1170
|
+
case "error":
|
|
1171
|
+
content = entry.error ?? null;
|
|
1172
|
+
break;
|
|
1173
|
+
case "task":
|
|
1174
|
+
content = {
|
|
1175
|
+
taskPath: entry.taskPath,
|
|
1176
|
+
args: entry.args
|
|
1177
|
+
};
|
|
1178
|
+
break;
|
|
1179
|
+
case "created":
|
|
1180
|
+
content = entry.created;
|
|
1181
|
+
break;
|
|
1182
|
+
case "started":
|
|
1183
|
+
content = entry.started ?? null;
|
|
1184
|
+
break;
|
|
1185
|
+
case "ended":
|
|
1186
|
+
content = entry.ended ?? null;
|
|
1187
|
+
break;
|
|
1188
|
+
case "attempts":
|
|
1189
|
+
content = entry.attempts;
|
|
1190
|
+
break;
|
|
1191
|
+
default: return;
|
|
1192
|
+
}
|
|
1193
|
+
return this.buildEntry(path, {
|
|
1194
|
+
content,
|
|
1195
|
+
meta: { kind: `scheduler:task:${field}` }
|
|
1196
|
+
});
|
|
1197
|
+
}
|
|
1198
|
+
};
|
|
1199
|
+
__decorate([Read("/")], AFSScheduler.prototype, "readRoot", null);
|
|
1200
|
+
__decorate([Read("/tasks")], AFSScheduler.prototype, "readTasks", null);
|
|
1201
|
+
__decorate([Read("/tasks/:id")], AFSScheduler.prototype, "readTask", null);
|
|
1202
|
+
__decorate([Read("/tasks/:id/:field")], AFSScheduler.prototype, "readTaskField", null);
|
|
1203
|
+
__decorate([Read("/history")], AFSScheduler.prototype, "readHistory", null);
|
|
1204
|
+
__decorate([Read("/history/:id")], AFSScheduler.prototype, "readHistoryTask", null);
|
|
1205
|
+
__decorate([Read("/history/:id/:field")], AFSScheduler.prototype, "readHistoryTaskField", null);
|
|
1206
|
+
__decorate([Read("/schedules")], AFSScheduler.prototype, "readSchedules", null);
|
|
1207
|
+
__decorate([Read("/schedules/:id")], AFSScheduler.prototype, "readSchedule", null);
|
|
1208
|
+
__decorate([Read("/jobs")], AFSScheduler.prototype, "readJobs", null);
|
|
1209
|
+
__decorate([Read("/jobs/:id")], AFSScheduler.prototype, "readJob", null);
|
|
1210
|
+
__decorate([Read("/.meta/.capabilities")], AFSScheduler.prototype, "readCapabilities", null);
|
|
1211
|
+
__decorate([List("/")], AFSScheduler.prototype, "listRoot", null);
|
|
1212
|
+
__decorate([List("/tasks")], AFSScheduler.prototype, "listTasks", null);
|
|
1213
|
+
__decorate([List("/history")], AFSScheduler.prototype, "listHistory", null);
|
|
1214
|
+
__decorate([List("/schedules")], AFSScheduler.prototype, "listSchedules", null);
|
|
1215
|
+
__decorate([List("/jobs")], AFSScheduler.prototype, "listJobsHandler", null);
|
|
1216
|
+
__decorate([Meta("/")], AFSScheduler.prototype, "readRootMeta", null);
|
|
1217
|
+
__decorate([Meta("/tasks")], AFSScheduler.prototype, "readTasksMeta", null);
|
|
1218
|
+
__decorate([Meta("/history")], AFSScheduler.prototype, "readHistoryMeta", null);
|
|
1219
|
+
__decorate([Meta("/schedules")], AFSScheduler.prototype, "readSchedulesMeta", null);
|
|
1220
|
+
__decorate([Meta("/jobs")], AFSScheduler.prototype, "readJobsMeta", null);
|
|
1221
|
+
__decorate([Explain("/")], AFSScheduler.prototype, "explainRoot", null);
|
|
1222
|
+
__decorate([Explain("/:path+")], AFSScheduler.prototype, "explainPath", null);
|
|
1223
|
+
__decorate([Actions("/")], AFSScheduler.prototype, "listActions", null);
|
|
1224
|
+
__decorate([Actions.Exec("/", "dispatch", "Submit a task for execution", { effect: "write" })], AFSScheduler.prototype, "execDispatch", null);
|
|
1225
|
+
__decorate([Actions.Exec("/", "cancel", "Cancel a running task", { effect: "write" })], AFSScheduler.prototype, "execCancel", null);
|
|
1226
|
+
__decorate([Actions.Exec("/", "retry", "Retry a failed task", { effect: "write" })], AFSScheduler.prototype, "execRetry", null);
|
|
1227
|
+
__decorate([Actions.Exec("/", "schedule", "Create a cron schedule", { effect: "write" })], AFSScheduler.prototype, "execSchedule", null);
|
|
1228
|
+
__decorate([Actions.Exec("/", "unschedule", "Remove a cron schedule", { effect: "write" })], AFSScheduler.prototype, "execUnschedule", null);
|
|
1229
|
+
__decorate([Actions.Exec("/", "heartbeat", "Send heartbeat for a running task", { effect: "write" })], AFSScheduler.prototype, "execHeartbeat", null);
|
|
1230
|
+
__decorate([Actions.Exec("/", "claim", "Atomically claim an unclaimed (or lease-expired) task for external worker execution", { effect: "write" })], AFSScheduler.prototype, "execClaim", null);
|
|
1231
|
+
__decorate([Actions.Exec("/", "submit", "Submit worker completion + artifacts for a claimed task, enqueuing a verification task", { effect: "write" })], AFSScheduler.prototype, "execSubmit", null);
|
|
1232
|
+
__decorate([Actions.Exec("/", "submit-job", "Submit a map-reduce job", { effect: "write" })], AFSScheduler.prototype, "execSubmitJob", null);
|
|
1233
|
+
__decorate([Actions.Exec("/", "cancel-job", "Cancel a map-reduce job", { effect: "write" })], AFSScheduler.prototype, "execCancelJob", null);
|
|
1234
|
+
|
|
1235
|
+
//#endregion
|
|
1236
|
+
export { AFSScheduler };
|
|
1237
|
+
//# sourceMappingURL=scheduler-provider.mjs.map
|