@sentry/junior-scheduler 0.57.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/dist/cadence.d.ts +24 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.js +1655 -0
- package/dist/plugin.d.ts +4 -0
- package/dist/prompt.d.ts +7 -0
- package/dist/schedule-tools.d.ts +25 -0
- package/dist/store.d.ts +49 -0
- package/dist/types.d.ts +86 -0
- package/package.json +38 -0
- package/plugin.yaml +2 -0
- package/src/cadence.ts +501 -0
- package/src/index.ts +27 -0
- package/src/plugin.ts +312 -0
- package/src/prompt.ts +89 -0
- package/src/schedule-tools.ts +640 -0
- package/src/store.ts +819 -0
- package/src/types.ts +110 -0
package/src/store.ts
ADDED
|
@@ -0,0 +1,819 @@
|
|
|
1
|
+
import type { AgentPluginState } from "@sentry/junior-plugin-api";
|
|
2
|
+
import { getNextRunAtMs } from "./cadence";
|
|
3
|
+
import type { ScheduledRun, ScheduledTask } from "./types";
|
|
4
|
+
|
|
5
|
+
const SCHEDULER_KEY_PREFIX = "junior:scheduler";
|
|
6
|
+
const SCHEDULER_RECORD_TTL_MS = 5 * 365 * 24 * 60 * 60 * 1000;
|
|
7
|
+
const SCHEDULED_RUN_TTL_MS = 90 * 24 * 60 * 60 * 1000;
|
|
8
|
+
const CLAIM_TTL_MS = 6 * 60 * 60 * 1000;
|
|
9
|
+
const PENDING_CLAIM_STALE_MS = 60_000;
|
|
10
|
+
const MISSED_RUN_MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
|
11
|
+
const LOCK_TTL_MS = 10_000;
|
|
12
|
+
|
|
13
|
+
export interface SchedulerStore {
|
|
14
|
+
claimDueRun(args: { nowMs: number }): Promise<ScheduledRun | undefined>;
|
|
15
|
+
getRun(runId: string): Promise<ScheduledRun | undefined>;
|
|
16
|
+
getTask(taskId: string): Promise<ScheduledTask | undefined>;
|
|
17
|
+
listIncompleteRuns(): Promise<ScheduledRun[]>;
|
|
18
|
+
listTasksForTeam(teamId: string): Promise<ScheduledTask[]>;
|
|
19
|
+
markRunBlocked(args: {
|
|
20
|
+
completedAtMs: number;
|
|
21
|
+
errorMessage: string;
|
|
22
|
+
runId: string;
|
|
23
|
+
startedAtMs?: number;
|
|
24
|
+
}): Promise<ScheduledRun | undefined>;
|
|
25
|
+
markRunCompleted(args: {
|
|
26
|
+
completedAtMs: number;
|
|
27
|
+
resultMessageTs?: string;
|
|
28
|
+
runId: string;
|
|
29
|
+
startedAtMs: number;
|
|
30
|
+
}): Promise<ScheduledRun | undefined>;
|
|
31
|
+
markRunFailed(args: {
|
|
32
|
+
completedAtMs: number;
|
|
33
|
+
errorMessage: string;
|
|
34
|
+
startedAtMs?: number;
|
|
35
|
+
runId: string;
|
|
36
|
+
}): Promise<ScheduledRun | undefined>;
|
|
37
|
+
markRunSkipped(args: {
|
|
38
|
+
completedAtMs: number;
|
|
39
|
+
errorMessage: string;
|
|
40
|
+
runId: string;
|
|
41
|
+
}): Promise<ScheduledRun | undefined>;
|
|
42
|
+
markRunDispatched(args: {
|
|
43
|
+
claimedAtMs: number;
|
|
44
|
+
dispatchId: string;
|
|
45
|
+
nowMs: number;
|
|
46
|
+
runId: string;
|
|
47
|
+
}): Promise<ScheduledRun | undefined>;
|
|
48
|
+
saveTask(task: ScheduledTask): Promise<void>;
|
|
49
|
+
updateTaskAfterRun(args: {
|
|
50
|
+
errorMessage?: string;
|
|
51
|
+
nowMs: number;
|
|
52
|
+
run: ScheduledRun;
|
|
53
|
+
status: "blocked" | "completed" | "failed";
|
|
54
|
+
}): Promise<void>;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function taskKey(taskId: string): string {
|
|
58
|
+
return `${SCHEDULER_KEY_PREFIX}:task:${taskId}`;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function taskLockKey(taskId: string): string {
|
|
62
|
+
return `${taskKey(taskId)}:lock`;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
function runKey(runId: string): string {
|
|
66
|
+
return `${SCHEDULER_KEY_PREFIX}:run:${runId}`;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
function claimKey(taskId: string, scheduledForMs: number): string {
|
|
70
|
+
return `${SCHEDULER_KEY_PREFIX}:claim:${taskId}:${scheduledForMs}`;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function activeRunKey(taskId: string): string {
|
|
74
|
+
return `${SCHEDULER_KEY_PREFIX}:active:${taskId}`;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function globalTaskIndexKey(): string {
|
|
78
|
+
return `${SCHEDULER_KEY_PREFIX}:tasks`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function teamTaskIndexKey(teamId: string): string {
|
|
82
|
+
return `${SCHEDULER_KEY_PREFIX}:team:${teamId}:tasks`;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
function indexLockKey(indexKey: string): string {
|
|
86
|
+
return `${indexKey}:lock`;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function buildRunId(taskId: string, scheduledForMs: number): string {
|
|
90
|
+
return `${taskId}:${scheduledForMs}`;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
function unique(values: string[]): string[] {
|
|
94
|
+
return [...new Set(values.filter(Boolean))];
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
async function withLock<T>(
|
|
98
|
+
state: AgentPluginState,
|
|
99
|
+
key: string,
|
|
100
|
+
callback: () => Promise<T>,
|
|
101
|
+
): Promise<T> {
|
|
102
|
+
return await state.withLock(key, LOCK_TTL_MS, callback);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function addToIndex(
|
|
106
|
+
state: AgentPluginState,
|
|
107
|
+
key: string,
|
|
108
|
+
taskId: string,
|
|
109
|
+
): Promise<void> {
|
|
110
|
+
await withLock(state, indexLockKey(key), async () => {
|
|
111
|
+
const current = ((await state.get<string[]>(key)) ?? []).filter(
|
|
112
|
+
(value): value is string => typeof value === "string",
|
|
113
|
+
);
|
|
114
|
+
await state.set(key, unique([...current, taskId]), SCHEDULER_RECORD_TTL_MS);
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
async function removeFromIndex(
|
|
119
|
+
state: AgentPluginState,
|
|
120
|
+
key: string,
|
|
121
|
+
taskId: string,
|
|
122
|
+
): Promise<void> {
|
|
123
|
+
await withLock(state, indexLockKey(key), async () => {
|
|
124
|
+
const current = unique(
|
|
125
|
+
((await state.get<string[]>(key)) ?? []).filter(
|
|
126
|
+
(value): value is string => typeof value === "string",
|
|
127
|
+
),
|
|
128
|
+
);
|
|
129
|
+
const next = current.filter((value) => value !== taskId);
|
|
130
|
+
if (next.length === current.length) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
if (next.length === 0) {
|
|
134
|
+
await state.delete(key);
|
|
135
|
+
return;
|
|
136
|
+
}
|
|
137
|
+
await state.set(key, next, SCHEDULER_RECORD_TTL_MS);
|
|
138
|
+
});
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
async function getIndex(
|
|
142
|
+
state: AgentPluginState,
|
|
143
|
+
key: string,
|
|
144
|
+
): Promise<string[]> {
|
|
145
|
+
const values = (await state.get<string[]>(key)) ?? [];
|
|
146
|
+
return unique(
|
|
147
|
+
values.filter((value): value is string => typeof value === "string"),
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function clearActiveRun(
|
|
152
|
+
state: AgentPluginState,
|
|
153
|
+
taskId: string,
|
|
154
|
+
runId: string,
|
|
155
|
+
): Promise<void> {
|
|
156
|
+
await withLock(state, indexLockKey(activeRunKey(taskId)), async () => {
|
|
157
|
+
const current = await state.get<{ runId?: unknown }>(activeRunKey(taskId));
|
|
158
|
+
if (current?.runId === runId) {
|
|
159
|
+
await state.delete(activeRunKey(taskId));
|
|
160
|
+
}
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
async function clearStaleActiveRun(
|
|
165
|
+
state: AgentPluginState,
|
|
166
|
+
taskId: string,
|
|
167
|
+
nowMs: number,
|
|
168
|
+
): Promise<boolean> {
|
|
169
|
+
const active = await state.get<{
|
|
170
|
+
claimedAtMs?: unknown;
|
|
171
|
+
runId?: unknown;
|
|
172
|
+
scheduledForMs?: unknown;
|
|
173
|
+
}>(activeRunKey(taskId));
|
|
174
|
+
if (typeof active?.runId !== "string") {
|
|
175
|
+
await state.delete(activeRunKey(taskId));
|
|
176
|
+
return true;
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
const activeRun =
|
|
180
|
+
(await state.get<ScheduledRun>(runKey(active.runId))) ?? undefined;
|
|
181
|
+
if (!isStaleActiveRun(active, activeRun, nowMs)) {
|
|
182
|
+
return false;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
await clearActiveRun(state, taskId, active.runId);
|
|
186
|
+
if (typeof active.scheduledForMs === "number") {
|
|
187
|
+
await state.delete(claimKey(taskId, active.scheduledForMs));
|
|
188
|
+
}
|
|
189
|
+
return true;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function isFinishedRun(run: ScheduledRun): boolean {
|
|
193
|
+
return (
|
|
194
|
+
run.status === "completed" ||
|
|
195
|
+
run.status === "failed" ||
|
|
196
|
+
run.status === "blocked" ||
|
|
197
|
+
run.status === "skipped"
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
function isStaleActiveRun(
|
|
202
|
+
active: { claimedAtMs?: unknown },
|
|
203
|
+
run: ScheduledRun | undefined,
|
|
204
|
+
nowMs: number,
|
|
205
|
+
): boolean {
|
|
206
|
+
if (run) {
|
|
207
|
+
return isFinishedRun(run) || isStalePendingRun(run, nowMs);
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
return (
|
|
211
|
+
typeof active.claimedAtMs === "number" &&
|
|
212
|
+
active.claimedAtMs + PENDING_CLAIM_STALE_MS <= nowMs
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function isStalePendingRun(
|
|
217
|
+
run: ScheduledRun | undefined,
|
|
218
|
+
nowMs: number,
|
|
219
|
+
): boolean {
|
|
220
|
+
return (
|
|
221
|
+
run?.status === "pending" &&
|
|
222
|
+
run.claimedAtMs + PENDING_CLAIM_STALE_MS <= nowMs
|
|
223
|
+
);
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function isDueTask(
|
|
227
|
+
task: ScheduledTask,
|
|
228
|
+
nowMs: number,
|
|
229
|
+
): task is ScheduledTask & {
|
|
230
|
+
nextRunAtMs?: number;
|
|
231
|
+
runNowAtMs?: number;
|
|
232
|
+
} {
|
|
233
|
+
return (
|
|
234
|
+
task.status === "active" &&
|
|
235
|
+
((typeof task.runNowAtMs === "number" &&
|
|
236
|
+
Number.isFinite(task.runNowAtMs) &&
|
|
237
|
+
task.runNowAtMs <= nowMs) ||
|
|
238
|
+
(typeof task.nextRunAtMs === "number" &&
|
|
239
|
+
Number.isFinite(task.nextRunAtMs) &&
|
|
240
|
+
task.nextRunAtMs <= nowMs))
|
|
241
|
+
);
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
function getDueRunAtMs(task: ScheduledTask, nowMs: number): number | undefined {
|
|
245
|
+
if (
|
|
246
|
+
typeof task.runNowAtMs === "number" &&
|
|
247
|
+
Number.isFinite(task.runNowAtMs) &&
|
|
248
|
+
task.runNowAtMs <= nowMs
|
|
249
|
+
) {
|
|
250
|
+
return task.runNowAtMs;
|
|
251
|
+
}
|
|
252
|
+
if (
|
|
253
|
+
typeof task.nextRunAtMs === "number" &&
|
|
254
|
+
Number.isFinite(task.nextRunAtMs) &&
|
|
255
|
+
task.nextRunAtMs <= nowMs
|
|
256
|
+
) {
|
|
257
|
+
return task.nextRunAtMs;
|
|
258
|
+
}
|
|
259
|
+
return undefined;
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
function buildScheduledRun(args: {
|
|
263
|
+
claimedAtMs: number;
|
|
264
|
+
scheduledForMs: number;
|
|
265
|
+
task: ScheduledTask;
|
|
266
|
+
}): ScheduledRun {
|
|
267
|
+
const idempotencyKey = `${args.task.id}:${args.scheduledForMs}`;
|
|
268
|
+
return {
|
|
269
|
+
id: buildRunId(args.task.id, args.scheduledForMs),
|
|
270
|
+
attempt: 1,
|
|
271
|
+
claimedAtMs: args.claimedAtMs,
|
|
272
|
+
idempotencyKey,
|
|
273
|
+
scheduledForMs: args.scheduledForMs,
|
|
274
|
+
status: "pending",
|
|
275
|
+
taskId: args.task.id,
|
|
276
|
+
taskVersion: args.task.version,
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
function buildSkippedScheduledRun(args: {
|
|
281
|
+
completedAtMs: number;
|
|
282
|
+
errorMessage: string;
|
|
283
|
+
scheduledForMs: number;
|
|
284
|
+
task: ScheduledTask;
|
|
285
|
+
}): ScheduledRun {
|
|
286
|
+
return {
|
|
287
|
+
...buildScheduledRun({
|
|
288
|
+
claimedAtMs: args.completedAtMs,
|
|
289
|
+
scheduledForMs: args.scheduledForMs,
|
|
290
|
+
task: args.task,
|
|
291
|
+
}),
|
|
292
|
+
completedAtMs: args.completedAtMs,
|
|
293
|
+
errorMessage: args.errorMessage,
|
|
294
|
+
status: "skipped",
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function isMissedRunTooOld(args: {
|
|
299
|
+
nowMs: number;
|
|
300
|
+
scheduledForMs: number;
|
|
301
|
+
}): boolean {
|
|
302
|
+
return args.scheduledForMs + MISSED_RUN_MAX_AGE_MS < args.nowMs;
|
|
303
|
+
}
|
|
304
|
+
|
|
305
|
+
function normalizedText(value: string | undefined): string {
|
|
306
|
+
return value?.trim().replace(/\s+/g, " ").toLowerCase() ?? "";
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
function taskDedupeFingerprint(task: ScheduledTask): string {
|
|
310
|
+
return JSON.stringify({
|
|
311
|
+
destination: task.destination,
|
|
312
|
+
schedule: {
|
|
313
|
+
kind: task.schedule.kind,
|
|
314
|
+
oneOffAtMs: task.schedule.kind === "one_off" ? task.nextRunAtMs : null,
|
|
315
|
+
recurrence: task.schedule.recurrence
|
|
316
|
+
? {
|
|
317
|
+
dayOfMonth: task.schedule.recurrence.dayOfMonth ?? null,
|
|
318
|
+
frequency: task.schedule.recurrence.frequency,
|
|
319
|
+
interval: task.schedule.recurrence.interval,
|
|
320
|
+
month: task.schedule.recurrence.month ?? null,
|
|
321
|
+
startDate: task.schedule.recurrence.startDate,
|
|
322
|
+
time: task.schedule.recurrence.time,
|
|
323
|
+
weekdays: [...(task.schedule.recurrence.weekdays ?? [])].sort(),
|
|
324
|
+
}
|
|
325
|
+
: null,
|
|
326
|
+
timezone: task.schedule.timezone,
|
|
327
|
+
},
|
|
328
|
+
task: normalizedText(task.task.text),
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
function isEarlierTask(left: ScheduledTask, right: ScheduledTask): boolean {
|
|
333
|
+
return (
|
|
334
|
+
left.createdAtMs < right.createdAtMs ||
|
|
335
|
+
(left.createdAtMs === right.createdAtMs && left.id < right.id)
|
|
336
|
+
);
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
function canFinishRun(
|
|
340
|
+
run: ScheduledRun,
|
|
341
|
+
startedAtMs: number | undefined,
|
|
342
|
+
): boolean {
|
|
343
|
+
if (run.status === "pending") {
|
|
344
|
+
return startedAtMs === undefined;
|
|
345
|
+
}
|
|
346
|
+
return run.status === "running" && run.startedAtMs === startedAtMs;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
class PluginStateSchedulerStore implements SchedulerStore {
|
|
350
|
+
private readonly state: AgentPluginState;
|
|
351
|
+
|
|
352
|
+
constructor(state: AgentPluginState) {
|
|
353
|
+
this.state = state;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
async saveTask(task: ScheduledTask): Promise<void> {
|
|
357
|
+
await withLock(this.state, taskLockKey(task.id), async () => {
|
|
358
|
+
const current =
|
|
359
|
+
(await this.state.get<ScheduledTask>(taskKey(task.id))) ?? undefined;
|
|
360
|
+
await this.saveTaskRecord(task, current);
|
|
361
|
+
});
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
private async saveTaskRecord(
|
|
365
|
+
task: ScheduledTask,
|
|
366
|
+
current: ScheduledTask | undefined,
|
|
367
|
+
): Promise<void> {
|
|
368
|
+
if (
|
|
369
|
+
current?.status === "blocked" &&
|
|
370
|
+
task.status === "active" &&
|
|
371
|
+
typeof task.nextRunAtMs === "number" &&
|
|
372
|
+
Number.isFinite(task.nextRunAtMs)
|
|
373
|
+
) {
|
|
374
|
+
await this.state.delete(claimKey(task.id, task.nextRunAtMs));
|
|
375
|
+
}
|
|
376
|
+
await this.state.set(taskKey(task.id), task, SCHEDULER_RECORD_TTL_MS);
|
|
377
|
+
|
|
378
|
+
if (task.status === "deleted") {
|
|
379
|
+
await removeFromIndex(this.state, globalTaskIndexKey(), task.id);
|
|
380
|
+
await removeFromIndex(
|
|
381
|
+
this.state,
|
|
382
|
+
teamTaskIndexKey(task.destination.teamId),
|
|
383
|
+
task.id,
|
|
384
|
+
);
|
|
385
|
+
if (current && current.destination.teamId !== task.destination.teamId) {
|
|
386
|
+
await removeFromIndex(
|
|
387
|
+
this.state,
|
|
388
|
+
teamTaskIndexKey(current.destination.teamId),
|
|
389
|
+
task.id,
|
|
390
|
+
);
|
|
391
|
+
}
|
|
392
|
+
return;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
await addToIndex(this.state, globalTaskIndexKey(), task.id);
|
|
396
|
+
await addToIndex(
|
|
397
|
+
this.state,
|
|
398
|
+
teamTaskIndexKey(task.destination.teamId),
|
|
399
|
+
task.id,
|
|
400
|
+
);
|
|
401
|
+
if (current && current.destination.teamId !== task.destination.teamId) {
|
|
402
|
+
await removeFromIndex(
|
|
403
|
+
this.state,
|
|
404
|
+
teamTaskIndexKey(current.destination.teamId),
|
|
405
|
+
task.id,
|
|
406
|
+
);
|
|
407
|
+
}
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
async getTask(taskId: string): Promise<ScheduledTask | undefined> {
|
|
411
|
+
return (await this.state.get<ScheduledTask>(taskKey(taskId))) ?? undefined;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
async listTasksForTeam(teamId: string): Promise<ScheduledTask[]> {
|
|
415
|
+
const ids = await getIndex(this.state, teamTaskIndexKey(teamId));
|
|
416
|
+
const tasks = await Promise.all(ids.map((id) => this.getTask(id)));
|
|
417
|
+
return tasks
|
|
418
|
+
.filter((task): task is ScheduledTask => Boolean(task))
|
|
419
|
+
.filter((task) => task.status !== "deleted")
|
|
420
|
+
.sort((a, b) => a.createdAtMs - b.createdAtMs);
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
async claimDueRun(args: {
|
|
424
|
+
nowMs: number;
|
|
425
|
+
}): Promise<ScheduledRun | undefined> {
|
|
426
|
+
const ids = await getIndex(this.state, globalTaskIndexKey());
|
|
427
|
+
|
|
428
|
+
for (const id of ids) {
|
|
429
|
+
const task = await this.getTask(id);
|
|
430
|
+
if (!task || !isDueTask(task, args.nowMs)) {
|
|
431
|
+
continue;
|
|
432
|
+
}
|
|
433
|
+
|
|
434
|
+
const scheduledForMs = getDueRunAtMs(task, args.nowMs);
|
|
435
|
+
if (scheduledForMs === undefined) {
|
|
436
|
+
continue;
|
|
437
|
+
}
|
|
438
|
+
const runId = buildRunId(task.id, scheduledForMs);
|
|
439
|
+
const tryClaimActiveRun = async (): Promise<boolean> =>
|
|
440
|
+
await this.state.setIfNotExists(
|
|
441
|
+
activeRunKey(task.id),
|
|
442
|
+
{ claimedAtMs: args.nowMs, runId, scheduledForMs },
|
|
443
|
+
CLAIM_TTL_MS,
|
|
444
|
+
);
|
|
445
|
+
|
|
446
|
+
let activeClaimed = await tryClaimActiveRun();
|
|
447
|
+
if (!activeClaimed) {
|
|
448
|
+
if (await clearStaleActiveRun(this.state, task.id, args.nowMs)) {
|
|
449
|
+
activeClaimed = await tryClaimActiveRun();
|
|
450
|
+
}
|
|
451
|
+
if (!activeClaimed) {
|
|
452
|
+
continue;
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
if (isMissedRunTooOld({ nowMs: args.nowMs, scheduledForMs })) {
|
|
457
|
+
await this.skipMissedRun({ nowMs: args.nowMs, scheduledForMs, task });
|
|
458
|
+
await clearActiveRun(this.state, task.id, runId);
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
const tryClaimScheduledSlot = async (): Promise<boolean> =>
|
|
463
|
+
await this.state.setIfNotExists(
|
|
464
|
+
claimKey(task.id, scheduledForMs),
|
|
465
|
+
{ claimedAtMs: args.nowMs },
|
|
466
|
+
CLAIM_TTL_MS,
|
|
467
|
+
);
|
|
468
|
+
|
|
469
|
+
let claimed = await tryClaimScheduledSlot();
|
|
470
|
+
if (!claimed) {
|
|
471
|
+
const existingRun = await this.getRun(runId);
|
|
472
|
+
if (isStalePendingRun(existingRun, args.nowMs)) {
|
|
473
|
+
await clearActiveRun(this.state, task.id, runId);
|
|
474
|
+
await this.state.delete(claimKey(task.id, scheduledForMs));
|
|
475
|
+
activeClaimed = await tryClaimActiveRun();
|
|
476
|
+
claimed = activeClaimed ? await tryClaimScheduledSlot() : false;
|
|
477
|
+
}
|
|
478
|
+
if (!claimed) {
|
|
479
|
+
await clearActiveRun(this.state, task.id, runId);
|
|
480
|
+
continue;
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
const run = buildScheduledRun({
|
|
485
|
+
claimedAtMs: args.nowMs,
|
|
486
|
+
scheduledForMs,
|
|
487
|
+
task,
|
|
488
|
+
});
|
|
489
|
+
await this.state.set(runKey(run.id), run, SCHEDULED_RUN_TTL_MS);
|
|
490
|
+
return run;
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
return undefined;
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
private async skipMissedRun(args: {
|
|
497
|
+
nowMs: number;
|
|
498
|
+
scheduledForMs: number;
|
|
499
|
+
task: ScheduledTask;
|
|
500
|
+
}): Promise<void> {
|
|
501
|
+
await withLock(this.state, taskLockKey(args.task.id), async () => {
|
|
502
|
+
const current =
|
|
503
|
+
(await this.state.get<ScheduledTask>(taskKey(args.task.id))) ??
|
|
504
|
+
undefined;
|
|
505
|
+
if (
|
|
506
|
+
!current ||
|
|
507
|
+
current.status !== "active" ||
|
|
508
|
+
getDueRunAtMs(current, args.nowMs) !== args.scheduledForMs
|
|
509
|
+
) {
|
|
510
|
+
return;
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const duplicateOf = await this.findStaleRecoveryCanonicalTask(current);
|
|
514
|
+
const errorMessage = duplicateOf
|
|
515
|
+
? `Duplicate stale scheduled task was skipped without dispatch. Canonical task: ${duplicateOf.id}.`
|
|
516
|
+
: "Scheduled occurrence was more than 24 hours late and was skipped without dispatch.";
|
|
517
|
+
await this.state.set(
|
|
518
|
+
runKey(buildRunId(current.id, args.scheduledForMs)),
|
|
519
|
+
buildSkippedScheduledRun({
|
|
520
|
+
completedAtMs: args.nowMs,
|
|
521
|
+
errorMessage,
|
|
522
|
+
scheduledForMs: args.scheduledForMs,
|
|
523
|
+
task: current,
|
|
524
|
+
}),
|
|
525
|
+
SCHEDULED_RUN_TTL_MS,
|
|
526
|
+
);
|
|
527
|
+
|
|
528
|
+
const isRunNow = current.runNowAtMs === args.scheduledForMs;
|
|
529
|
+
let nextRunAtMs: number | undefined;
|
|
530
|
+
if (!duplicateOf) {
|
|
531
|
+
nextRunAtMs =
|
|
532
|
+
isRunNow && current.nextRunAtMs !== args.scheduledForMs
|
|
533
|
+
? current.nextRunAtMs
|
|
534
|
+
: current.schedule.kind === "recurring"
|
|
535
|
+
? getNextRunAtMs(current, args.scheduledForMs, args.nowMs)
|
|
536
|
+
: undefined;
|
|
537
|
+
}
|
|
538
|
+
const nextStatus = nextRunAtMs ? "active" : "paused";
|
|
539
|
+
|
|
540
|
+
await this.saveTaskRecord(
|
|
541
|
+
{
|
|
542
|
+
...current,
|
|
543
|
+
nextRunAtMs,
|
|
544
|
+
runNowAtMs: isRunNow ? undefined : current.runNowAtMs,
|
|
545
|
+
status: nextStatus,
|
|
546
|
+
statusReason: nextStatus === "paused" ? errorMessage : undefined,
|
|
547
|
+
updatedAtMs: args.nowMs,
|
|
548
|
+
version: current.version + 1,
|
|
549
|
+
},
|
|
550
|
+
current,
|
|
551
|
+
);
|
|
552
|
+
});
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
private async findStaleRecoveryCanonicalTask(
|
|
556
|
+
task: ScheduledTask,
|
|
557
|
+
): Promise<ScheduledTask | undefined> {
|
|
558
|
+
const fingerprint = taskDedupeFingerprint(task);
|
|
559
|
+
const ids = await getIndex(
|
|
560
|
+
this.state,
|
|
561
|
+
teamTaskIndexKey(task.destination.teamId),
|
|
562
|
+
);
|
|
563
|
+
const tasks = await Promise.all(
|
|
564
|
+
ids.filter((id) => id !== task.id).map((id) => this.getTask(id)),
|
|
565
|
+
);
|
|
566
|
+
return tasks
|
|
567
|
+
.filter((candidate): candidate is ScheduledTask => Boolean(candidate))
|
|
568
|
+
.filter(
|
|
569
|
+
(candidate) =>
|
|
570
|
+
candidate.status === "active" &&
|
|
571
|
+
isEarlierTask(candidate, task) &&
|
|
572
|
+
taskDedupeFingerprint(candidate) === fingerprint,
|
|
573
|
+
)
|
|
574
|
+
.sort((a, b) => a.createdAtMs - b.createdAtMs || a.id.localeCompare(b.id))
|
|
575
|
+
.at(0);
|
|
576
|
+
}
|
|
577
|
+
|
|
578
|
+
async getRun(runId: string): Promise<ScheduledRun | undefined> {
|
|
579
|
+
return (await this.state.get<ScheduledRun>(runKey(runId))) ?? undefined;
|
|
580
|
+
}
|
|
581
|
+
|
|
582
|
+
async listIncompleteRuns(): Promise<ScheduledRun[]> {
|
|
583
|
+
const ids = await getIndex(this.state, globalTaskIndexKey());
|
|
584
|
+
const runs: ScheduledRun[] = [];
|
|
585
|
+
for (const taskId of ids) {
|
|
586
|
+
const active = await this.state.get<{ runId?: unknown }>(
|
|
587
|
+
activeRunKey(taskId),
|
|
588
|
+
);
|
|
589
|
+
if (typeof active?.runId !== "string") {
|
|
590
|
+
continue;
|
|
591
|
+
}
|
|
592
|
+
const run = await this.getRun(active.runId);
|
|
593
|
+
if (run && !isFinishedRun(run)) {
|
|
594
|
+
runs.push(run);
|
|
595
|
+
}
|
|
596
|
+
}
|
|
597
|
+
return runs;
|
|
598
|
+
}
|
|
599
|
+
|
|
600
|
+
async markRunDispatched(args: {
|
|
601
|
+
claimedAtMs: number;
|
|
602
|
+
dispatchId: string;
|
|
603
|
+
nowMs: number;
|
|
604
|
+
runId: string;
|
|
605
|
+
}): Promise<ScheduledRun | undefined> {
|
|
606
|
+
return await this.updateRun(args.runId, (run) =>
|
|
607
|
+
run.status === "pending" && run.claimedAtMs === args.claimedAtMs
|
|
608
|
+
? {
|
|
609
|
+
...run,
|
|
610
|
+
dispatchId: args.dispatchId,
|
|
611
|
+
startedAtMs: args.nowMs,
|
|
612
|
+
status: "running",
|
|
613
|
+
}
|
|
614
|
+
: undefined,
|
|
615
|
+
);
|
|
616
|
+
}
|
|
617
|
+
|
|
618
|
+
async markRunCompleted(args: {
|
|
619
|
+
completedAtMs: number;
|
|
620
|
+
resultMessageTs?: string;
|
|
621
|
+
runId: string;
|
|
622
|
+
startedAtMs: number;
|
|
623
|
+
}): Promise<ScheduledRun | undefined> {
|
|
624
|
+
const next = await this.updateRun(args.runId, (run) =>
|
|
625
|
+
canFinishRun(run, args.startedAtMs)
|
|
626
|
+
? {
|
|
627
|
+
...run,
|
|
628
|
+
completedAtMs: args.completedAtMs,
|
|
629
|
+
resultMessageTs: args.resultMessageTs,
|
|
630
|
+
status: "completed",
|
|
631
|
+
}
|
|
632
|
+
: undefined,
|
|
633
|
+
);
|
|
634
|
+
if (next) {
|
|
635
|
+
await clearActiveRun(this.state, next.taskId, next.id);
|
|
636
|
+
}
|
|
637
|
+
return next;
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
async markRunFailed(args: {
|
|
641
|
+
completedAtMs: number;
|
|
642
|
+
errorMessage: string;
|
|
643
|
+
startedAtMs?: number;
|
|
644
|
+
runId: string;
|
|
645
|
+
}): Promise<ScheduledRun | undefined> {
|
|
646
|
+
const next = await this.updateRun(args.runId, (run) =>
|
|
647
|
+
canFinishRun(run, args.startedAtMs)
|
|
648
|
+
? {
|
|
649
|
+
...run,
|
|
650
|
+
completedAtMs: args.completedAtMs,
|
|
651
|
+
errorMessage: args.errorMessage,
|
|
652
|
+
status: "failed",
|
|
653
|
+
}
|
|
654
|
+
: undefined,
|
|
655
|
+
);
|
|
656
|
+
if (next) {
|
|
657
|
+
await clearActiveRun(this.state, next.taskId, next.id);
|
|
658
|
+
}
|
|
659
|
+
return next;
|
|
660
|
+
}
|
|
661
|
+
|
|
662
|
+
async markRunSkipped(args: {
|
|
663
|
+
completedAtMs: number;
|
|
664
|
+
errorMessage: string;
|
|
665
|
+
runId: string;
|
|
666
|
+
}): Promise<ScheduledRun | undefined> {
|
|
667
|
+
const next = await this.updateRun(args.runId, (run) =>
|
|
668
|
+
run.status === "pending"
|
|
669
|
+
? {
|
|
670
|
+
...run,
|
|
671
|
+
completedAtMs: args.completedAtMs,
|
|
672
|
+
errorMessage: args.errorMessage,
|
|
673
|
+
status: "skipped",
|
|
674
|
+
}
|
|
675
|
+
: undefined,
|
|
676
|
+
);
|
|
677
|
+
if (next) {
|
|
678
|
+
await clearActiveRun(this.state, next.taskId, next.id);
|
|
679
|
+
}
|
|
680
|
+
return next;
|
|
681
|
+
}
|
|
682
|
+
|
|
683
|
+
async markRunBlocked(args: {
|
|
684
|
+
completedAtMs: number;
|
|
685
|
+
errorMessage: string;
|
|
686
|
+
runId: string;
|
|
687
|
+
startedAtMs?: number;
|
|
688
|
+
}): Promise<ScheduledRun | undefined> {
|
|
689
|
+
const next = await this.updateRun(args.runId, (run) =>
|
|
690
|
+
canFinishRun(run, args.startedAtMs)
|
|
691
|
+
? {
|
|
692
|
+
...run,
|
|
693
|
+
completedAtMs: args.completedAtMs,
|
|
694
|
+
errorMessage: args.errorMessage,
|
|
695
|
+
status: "blocked",
|
|
696
|
+
}
|
|
697
|
+
: undefined,
|
|
698
|
+
);
|
|
699
|
+
if (next) {
|
|
700
|
+
await clearActiveRun(this.state, next.taskId, next.id);
|
|
701
|
+
}
|
|
702
|
+
return next;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
async updateTaskAfterRun(args: {
|
|
706
|
+
errorMessage?: string;
|
|
707
|
+
nowMs: number;
|
|
708
|
+
run: ScheduledRun;
|
|
709
|
+
status: "blocked" | "completed" | "failed";
|
|
710
|
+
}): Promise<void> {
|
|
711
|
+
await withLock(this.state, taskLockKey(args.run.taskId), async () => {
|
|
712
|
+
const current =
|
|
713
|
+
(await this.state.get<ScheduledTask>(taskKey(args.run.taskId))) ??
|
|
714
|
+
undefined;
|
|
715
|
+
if (!current || current.status === "deleted") {
|
|
716
|
+
return;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
const isRunNow = current.runNowAtMs === args.run.scheduledForMs;
|
|
720
|
+
if (isRunNow) {
|
|
721
|
+
let nextRunAtMs = current.nextRunAtMs;
|
|
722
|
+
if (
|
|
723
|
+
args.status !== "blocked" &&
|
|
724
|
+
typeof current.nextRunAtMs === "number" &&
|
|
725
|
+
current.nextRunAtMs <= args.run.scheduledForMs
|
|
726
|
+
) {
|
|
727
|
+
nextRunAtMs = getNextRunAtMs(
|
|
728
|
+
current,
|
|
729
|
+
current.nextRunAtMs,
|
|
730
|
+
args.nowMs,
|
|
731
|
+
);
|
|
732
|
+
}
|
|
733
|
+
await this.saveTaskRecord(
|
|
734
|
+
{
|
|
735
|
+
...current,
|
|
736
|
+
lastRunAtMs: args.run.scheduledForMs,
|
|
737
|
+
nextRunAtMs,
|
|
738
|
+
runNowAtMs: undefined,
|
|
739
|
+
status:
|
|
740
|
+
args.status === "blocked"
|
|
741
|
+
? "blocked"
|
|
742
|
+
: nextRunAtMs
|
|
743
|
+
? current.status
|
|
744
|
+
: "paused",
|
|
745
|
+
statusReason:
|
|
746
|
+
args.status === "blocked" ? args.errorMessage : undefined,
|
|
747
|
+
updatedAtMs: args.nowMs,
|
|
748
|
+
version: current.version + 1,
|
|
749
|
+
},
|
|
750
|
+
current,
|
|
751
|
+
);
|
|
752
|
+
return;
|
|
753
|
+
}
|
|
754
|
+
|
|
755
|
+
if (
|
|
756
|
+
current.status !== "active" ||
|
|
757
|
+
current.nextRunAtMs !== args.run.scheduledForMs
|
|
758
|
+
) {
|
|
759
|
+
await this.saveTaskRecord(
|
|
760
|
+
{
|
|
761
|
+
...current,
|
|
762
|
+
lastRunAtMs: args.run.scheduledForMs,
|
|
763
|
+
updatedAtMs: args.nowMs,
|
|
764
|
+
version: current.version + 1,
|
|
765
|
+
},
|
|
766
|
+
current,
|
|
767
|
+
);
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
const nextRunAtMs =
|
|
772
|
+
args.status === "blocked"
|
|
773
|
+
? undefined
|
|
774
|
+
: getNextRunAtMs(current, args.run.scheduledForMs, args.nowMs);
|
|
775
|
+
|
|
776
|
+
await this.saveTaskRecord(
|
|
777
|
+
{
|
|
778
|
+
...current,
|
|
779
|
+
lastRunAtMs: args.run.scheduledForMs,
|
|
780
|
+
nextRunAtMs,
|
|
781
|
+
status:
|
|
782
|
+
args.status === "blocked"
|
|
783
|
+
? "blocked"
|
|
784
|
+
: nextRunAtMs
|
|
785
|
+
? "active"
|
|
786
|
+
: "paused",
|
|
787
|
+
statusReason:
|
|
788
|
+
args.status === "blocked" ? args.errorMessage : undefined,
|
|
789
|
+
updatedAtMs: args.nowMs,
|
|
790
|
+
version: current.version + 1,
|
|
791
|
+
},
|
|
792
|
+
current,
|
|
793
|
+
);
|
|
794
|
+
});
|
|
795
|
+
}
|
|
796
|
+
|
|
797
|
+
private async updateRun(
|
|
798
|
+
runId: string,
|
|
799
|
+
update: (run: ScheduledRun) => ScheduledRun | undefined,
|
|
800
|
+
): Promise<ScheduledRun | undefined> {
|
|
801
|
+
return await withLock(this.state, indexLockKey(runKey(runId)), async () => {
|
|
802
|
+
const current = await this.getRun(runId);
|
|
803
|
+
if (!current) {
|
|
804
|
+
return undefined;
|
|
805
|
+
}
|
|
806
|
+
const next = update(current);
|
|
807
|
+
if (!next) {
|
|
808
|
+
return undefined;
|
|
809
|
+
}
|
|
810
|
+
await this.state.set(runKey(runId), next, SCHEDULED_RUN_TTL_MS);
|
|
811
|
+
return next;
|
|
812
|
+
});
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
/** Create a scheduler store backed by this plugin's durable state namespace. */
|
|
817
|
+
export function createSchedulerStore(state: AgentPluginState): SchedulerStore {
|
|
818
|
+
return new PluginStateSchedulerStore(state);
|
|
819
|
+
}
|