@goodandready/dsh-cron 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +144 -6
- package/docs/README.ru.md +147 -5
- package/docs/README.zh.md +144 -6
- package/lib/api.js +533 -0
- package/lib/channels.js +6 -2
- package/lib/client.js +185 -12
- package/lib/config-jobs.js +206 -0
- package/lib/external-api.js +74 -0
- package/lib/failure-inspector.js +107 -0
- package/lib/http-utils.js +24 -0
- package/lib/index.js +176 -388
- package/lib/llm-ask.js +141 -0
- package/lib/metrics.js +97 -0
- package/lib/recipes.js +238 -0
- package/lib/runner.js +210 -171
- package/lib/scheduler.js +373 -192
- package/lib/silent-rule.js +100 -0
- package/lib/store.js +14 -3
- package/lib/task-patch.js +98 -0
- package/lib/task-transfer.js +4 -0
- package/lib/templates.js +4 -0
- package/package.json +3 -2
- package/docs/design/DESIGN.md +0 -81
- package/docs/plans/0.2.5-ui-block.md +0 -50
package/lib/scheduler.js
CHANGED
|
@@ -1,4 +1,18 @@
|
|
|
1
1
|
import { Cron } from 'croner';
|
|
2
|
+
import { applySilentRule } from './silent-rule.js';
|
|
3
|
+
import { inspectFailure } from './failure-inspector.js';
|
|
4
|
+
|
|
5
|
+
/** Task types that talk to a model and can therefore use a fallback model. */
|
|
6
|
+
export const AGENT_TASK_TYPES = ['llm', 'skill', 'workflow'];
|
|
7
|
+
|
|
8
|
+
/** Sum two usage records, so a fallback attempt is not lost in accounting. */
|
|
9
|
+
export function addUsage(a, b) {
|
|
10
|
+
return {
|
|
11
|
+
inputTokens: (a?.inputTokens || 0) + (b?.inputTokens || 0),
|
|
12
|
+
outputTokens: (a?.outputTokens || 0) + (b?.outputTokens || 0),
|
|
13
|
+
cacheReadTokens: (a?.cacheReadTokens || 0) + (b?.cacheReadTokens || 0),
|
|
14
|
+
};
|
|
15
|
+
}
|
|
2
16
|
|
|
3
17
|
function pad(v) {
|
|
4
18
|
return String(v).padStart(2, '0');
|
|
@@ -22,82 +36,78 @@ export function describeCron(cronPattern) {
|
|
|
22
36
|
return cronPattern;
|
|
23
37
|
}
|
|
24
38
|
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
if (!str) throw new Error('Schedule expression must not be empty');
|
|
28
|
-
|
|
29
|
-
// 1. One-shot ISO 8601 timestamp or explicit "at: <ISO/datetime>"
|
|
39
|
+
/** One-shot from an ISO timestamp or an explicit "at:" prefix, else null. */
|
|
40
|
+
function parseAtExpression(str) {
|
|
30
41
|
const atPrefixMatch = str.match(/^(?:at:\s*|at\s+)(.+)$/i);
|
|
31
42
|
const candidateIso = atPrefixMatch ? atPrefixMatch[1].trim() : str;
|
|
32
43
|
const parsedDate = new Date(candidateIso);
|
|
33
|
-
if (
|
|
34
|
-
|
|
35
|
-
return {
|
|
36
|
-
cronPattern: null,
|
|
37
|
-
isOneShot: true,
|
|
38
|
-
targetTimestamp: targetMs,
|
|
39
|
-
humanText: `One-shot at ${parsedDate.toISOString()}`,
|
|
40
|
-
nextRun: targetMs
|
|
41
|
-
};
|
|
44
|
+
if (isNaN(parsedDate.getTime()) || !(candidateIso.includes('-') || candidateIso.includes('T') || atPrefixMatch)) {
|
|
45
|
+
return null;
|
|
42
46
|
}
|
|
47
|
+
const targetMs = parsedDate.getTime();
|
|
48
|
+
return {
|
|
49
|
+
cronPattern: null,
|
|
50
|
+
isOneShot: true,
|
|
51
|
+
targetTimestamp: targetMs,
|
|
52
|
+
humanText: `One-shot at ${parsedDate.toISOString()}`,
|
|
53
|
+
nextRun: targetMs
|
|
54
|
+
};
|
|
55
|
+
}
|
|
43
56
|
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
cronPattern: null,
|
|
65
|
-
isOneShot: true,
|
|
66
|
-
targetTimestamp: targetMs,
|
|
67
|
-
humanText: `One-shot in ${unitText} (at ${new Date(targetMs).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })})`,
|
|
68
|
-
nextRun: targetMs
|
|
69
|
-
};
|
|
57
|
+
/**
|
|
58
|
+
* Relative one-shot: "in 20m", "in 2h", "in 30s", "in 1d" and the accepted
|
|
59
|
+
* Russian input aliases ("через 15 минут").
|
|
60
|
+
*/
|
|
61
|
+
function parseRelativeOneShot(str) {
|
|
62
|
+
const match = str.match(/^(?:in\s+|через\s+)(\d+)\s*(s|sec|seconds?|m|min|minutes?|h|hr|hours?|d|days?|мин|минут|часа?|часов)?$/i);
|
|
63
|
+
if (!match) return null;
|
|
64
|
+
const num = parseInt(match[1], 10);
|
|
65
|
+
const unit = (match[2] || 'm').toLowerCase();
|
|
66
|
+
let delayMs = num * 60 * 1000;
|
|
67
|
+
let unitText = `${num} min`;
|
|
68
|
+
if (unit.startsWith('s')) {
|
|
69
|
+
delayMs = num * 1000;
|
|
70
|
+
unitText = `${num} sec`;
|
|
71
|
+
} else if (unit.startsWith('h') || unit.startsWith('час')) {
|
|
72
|
+
delayMs = num * 3600 * 1000;
|
|
73
|
+
unitText = `${num} h`;
|
|
74
|
+
} else if (unit.startsWith('d')) {
|
|
75
|
+
delayMs = num * 86400 * 1000;
|
|
76
|
+
unitText = `${num} d`;
|
|
70
77
|
}
|
|
78
|
+
const targetMs = Date.now() + delayMs;
|
|
79
|
+
return {
|
|
80
|
+
cronPattern: null,
|
|
81
|
+
isOneShot: true,
|
|
82
|
+
targetTimestamp: targetMs,
|
|
83
|
+
humanText: `One-shot in ${unitText} (at ${new Date(targetMs).toLocaleTimeString([], { hour: '2-digit', minute: '2-digit' })})`,
|
|
84
|
+
nextRun: targetMs
|
|
85
|
+
};
|
|
86
|
+
}
|
|
71
87
|
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
cronPattern: `0 */${num} * * *`,
|
|
87
|
-
humanText: `Every ${num} hours`,
|
|
88
|
-
isInterval: true,
|
|
89
|
-
};
|
|
90
|
-
}
|
|
91
|
-
if (unit.startsWith('d')) {
|
|
92
|
-
return {
|
|
93
|
-
cronPattern: `0 0 */${num} * *`,
|
|
94
|
-
humanText: `Every ${num} days`,
|
|
95
|
-
isInterval: true,
|
|
96
|
-
};
|
|
97
|
-
}
|
|
88
|
+
/** Repeating interval: "every 5m", "every 2h", "every 1d". */
|
|
89
|
+
function parseIntervalExpression(str) {
|
|
90
|
+
const match = str.match(/^every\s+(\d+)\s*(m|min|minute|minutes|h|hr|hour|hours|d|day|days)?$/i);
|
|
91
|
+
if (!match) return null;
|
|
92
|
+
const num = parseInt(match[1], 10);
|
|
93
|
+
const unit = (match[2] || 'm').toLowerCase();
|
|
94
|
+
if (unit.startsWith('m')) {
|
|
95
|
+
return { cronPattern: `*/${num} * * * *`, humanText: `Every ${num} minutes`, isInterval: true };
|
|
96
|
+
}
|
|
97
|
+
if (unit.startsWith('h')) {
|
|
98
|
+
return { cronPattern: `0 */${num} * * *`, humanText: `Every ${num} hours`, isInterval: true };
|
|
99
|
+
}
|
|
100
|
+
if (unit.startsWith('d')) {
|
|
101
|
+
return { cronPattern: `0 0 */${num} * *`, humanText: `Every ${num} days`, isInterval: true };
|
|
98
102
|
}
|
|
103
|
+
return null;
|
|
104
|
+
}
|
|
99
105
|
|
|
100
|
-
|
|
106
|
+
/**
|
|
107
|
+
* @-shorthands (#15) and the friendly aliases, including the Russian input
|
|
108
|
+
* aliases. Russian keywords are accepted input, not display strings.
|
|
109
|
+
*/
|
|
110
|
+
function parseAliasExpression(str) {
|
|
101
111
|
const lower = str.toLowerCase();
|
|
102
112
|
if (lower === '@hourly') return { cronPattern: '0 * * * *', humanText: 'Every hour' };
|
|
103
113
|
if (lower === '@daily' || lower === '@midnight') return { cronPattern: '0 0 * * *', humanText: 'Every day at 00:00' };
|
|
@@ -106,10 +116,9 @@ export function parseScheduleExpression(input) {
|
|
|
106
116
|
if (lower === '@yearly' || lower === '@annually') return { cronPattern: '0 0 1 1 *', humanText: 'Every year on Jan 1 at 00:00' };
|
|
107
117
|
const everyAlias = str.match(/^@every\s+(\d+)\s*(sec|seconds?|m|min|minutes?|h|hr|hours?|d|days?)$/i);
|
|
108
118
|
if (everyAlias) {
|
|
109
|
-
|
|
119
|
+
// "@every N unit" is another spelling of the interval branch.
|
|
120
|
+
return parseIntervalExpression(`every ${everyAlias[1]} ${everyAlias[2]}`);
|
|
110
121
|
}
|
|
111
|
-
|
|
112
|
-
// 5. Friendly recurring aliases (English plus accepted Russian input aliases)
|
|
113
122
|
if (lower === 'daily' || lower === 'каждый день') {
|
|
114
123
|
return { cronPattern: '0 9 * * *', humanText: 'Every day at 09:00' };
|
|
115
124
|
}
|
|
@@ -119,8 +128,11 @@ export function parseScheduleExpression(input) {
|
|
|
119
128
|
if (lower === 'hourly' || lower === 'каждый час') {
|
|
120
129
|
return { cronPattern: '0 * * * *', humanText: 'Every hour' };
|
|
121
130
|
}
|
|
131
|
+
return null;
|
|
132
|
+
}
|
|
122
133
|
|
|
123
|
-
|
|
134
|
+
/** Validate a 5-field cron expression; the only branch that throws. */
|
|
135
|
+
function parseCronExpression(str) {
|
|
124
136
|
try {
|
|
125
137
|
const testJob = new Cron(str);
|
|
126
138
|
const next = testJob.nextRun();
|
|
@@ -134,6 +146,22 @@ export function parseScheduleExpression(input) {
|
|
|
134
146
|
}
|
|
135
147
|
}
|
|
136
148
|
|
|
149
|
+
/**
|
|
150
|
+
* Turn a typed schedule into a cron pattern or a one-shot target. The branch
|
|
151
|
+
* order is the contract: a timestamp, a relative one-shot, an interval and an
|
|
152
|
+
* alias are all more specific than a raw cron expression.
|
|
153
|
+
*/
|
|
154
|
+
export function parseScheduleExpression(input) {
|
|
155
|
+
const str = String(input || '').trim();
|
|
156
|
+
if (!str) throw new Error('Schedule expression must not be empty');
|
|
157
|
+
|
|
158
|
+
return parseAtExpression(str)
|
|
159
|
+
|| parseRelativeOneShot(str)
|
|
160
|
+
|| parseIntervalExpression(str)
|
|
161
|
+
|| parseAliasExpression(str)
|
|
162
|
+
|| parseCronExpression(str);
|
|
163
|
+
}
|
|
164
|
+
|
|
137
165
|
export class TaskScheduler {
|
|
138
166
|
constructor(store, executeFn, options = {}) {
|
|
139
167
|
this.store = store;
|
|
@@ -146,6 +174,7 @@ export class TaskScheduler {
|
|
|
146
174
|
this.timers = new Map(); // taskId -> setTimeout handle (for one-shot tasks)
|
|
147
175
|
this.retryTimers = new Map(); // taskId -> setTimeout handle (for #18 retries)
|
|
148
176
|
this.running = new Map(); // taskId -> { controller, startedAt, queueCount }
|
|
177
|
+
this.runCounters = {}; // status -> finished runs since start (#53, /dsh-cron/metrics)
|
|
149
178
|
}
|
|
150
179
|
|
|
151
180
|
isRunning(taskId) {
|
|
@@ -180,6 +209,7 @@ export class TaskScheduler {
|
|
|
180
209
|
const missedAt = task.nextRunAt;
|
|
181
210
|
const policy = task.misfirePolicy || 'skip'; // #12: skip | runOnce | catchUpAll
|
|
182
211
|
console.warn(`[dsh-cron] Task "${task.title}" (${task.id}) missed scheduled run at ${new Date(missedAt).toISOString()} (misfire: ${policy})`);
|
|
212
|
+
this.countRun('missed');
|
|
183
213
|
this.store.recordRun(task.id, {
|
|
184
214
|
at: missedAt,
|
|
185
215
|
status: 'missed',
|
|
@@ -239,102 +269,92 @@ export class TaskScheduler {
|
|
|
239
269
|
this.running.clear();
|
|
240
270
|
}
|
|
241
271
|
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
if (this.jobs.has(
|
|
245
|
-
this.jobs.get(
|
|
246
|
-
this.jobs.delete(
|
|
272
|
+
/** Drop an armed job or one-shot timer before re-scheduling a task. */
|
|
273
|
+
clearScheduled(taskId) {
|
|
274
|
+
if (this.jobs.has(taskId)) {
|
|
275
|
+
this.jobs.get(taskId).stop();
|
|
276
|
+
this.jobs.delete(taskId);
|
|
277
|
+
}
|
|
278
|
+
if (this.timers.has(taskId)) {
|
|
279
|
+
clearTimeout(this.timers.get(taskId));
|
|
280
|
+
this.timers.delete(taskId);
|
|
247
281
|
}
|
|
248
|
-
|
|
249
|
-
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
/** Arm a one-shot task; a target already in the past fires immediately. */
|
|
285
|
+
scheduleOneShot(task, parsed) {
|
|
286
|
+
task.oneShot = true;
|
|
287
|
+
task.nextRunAt = parsed.targetTimestamp;
|
|
288
|
+
this.store.set(task);
|
|
289
|
+
|
|
290
|
+
const delay = Math.max(0, parsed.targetTimestamp - Date.now());
|
|
291
|
+
const timer = setTimeout(() => {
|
|
250
292
|
this.timers.delete(task.id);
|
|
293
|
+
this.runTask(task.id).catch((err) => {
|
|
294
|
+
console.error(`[dsh-cron] one-shot run of task ${task.id} failed:`, (err && err.message) || err);
|
|
295
|
+
});
|
|
296
|
+
}, delay);
|
|
297
|
+
this.timers.set(task.id, timer);
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
/** Arm a recurring cron job and persist the next run timestamp. */
|
|
301
|
+
scheduleCron(task, parsed) {
|
|
302
|
+
const timezone = task.timezone || this.defaultTimezone || undefined;
|
|
303
|
+
const job = new Cron(parsed.cronPattern, {
|
|
304
|
+
protect: true,
|
|
305
|
+
catch: (err) => console.error(`[dsh-cron] scheduled run of "${task.title}" (${task.id}) failed:`, (err && err.message) || err),
|
|
306
|
+
...(timezone ? { timezone } : {}),
|
|
307
|
+
}, async () => {
|
|
308
|
+
await this.runTask(task.id);
|
|
309
|
+
});
|
|
310
|
+
|
|
311
|
+
this.jobs.set(task.id, job);
|
|
312
|
+
const next = job.nextRun();
|
|
313
|
+
if (next) {
|
|
314
|
+
task.nextRunAt = next.getTime();
|
|
315
|
+
this.store.set(task);
|
|
251
316
|
}
|
|
317
|
+
}
|
|
252
318
|
|
|
319
|
+
scheduleTask(task) {
|
|
320
|
+
this.clearScheduled(task.id);
|
|
253
321
|
if (task.status !== 'active') return;
|
|
254
322
|
this.clearRetryTimer(task.id);
|
|
255
323
|
|
|
256
324
|
try {
|
|
257
325
|
const parsed = parseScheduleExpression(task.schedule);
|
|
258
|
-
const timezone = task.timezone || this.defaultTimezone || undefined;
|
|
259
|
-
|
|
260
|
-
// Handle One-shot task
|
|
261
326
|
if (parsed.isOneShot) {
|
|
262
|
-
task
|
|
263
|
-
const now = Date.now();
|
|
264
|
-
const delay = Math.max(0, parsed.targetTimestamp - now);
|
|
265
|
-
task.nextRunAt = parsed.targetTimestamp;
|
|
266
|
-
this.store.set(task);
|
|
267
|
-
|
|
268
|
-
// If target is in the past, or when delay triggers:
|
|
269
|
-
const timer = setTimeout(() => {
|
|
270
|
-
this.timers.delete(task.id);
|
|
271
|
-
this.runTask(task.id).catch((err) => {
|
|
272
|
-
console.error(`[dsh-cron] one-shot run of task ${task.id} failed:`, (err && err.message) || err);
|
|
273
|
-
});
|
|
274
|
-
}, delay);
|
|
275
|
-
|
|
276
|
-
this.timers.set(task.id, timer);
|
|
327
|
+
this.scheduleOneShot(task, parsed);
|
|
277
328
|
return;
|
|
278
329
|
}
|
|
279
|
-
|
|
280
|
-
// Handle Recurring Cron job
|
|
281
|
-
const job = new Cron(parsed.cronPattern, {
|
|
282
|
-
protect: true,
|
|
283
|
-
catch: (err) => console.error(`[dsh-cron] scheduled run of "${task.title}" (${task.id}) failed:`, (err && err.message) || err),
|
|
284
|
-
...(timezone ? { timezone } : {}),
|
|
285
|
-
}, async () => {
|
|
286
|
-
await this.runTask(task.id);
|
|
287
|
-
});
|
|
288
|
-
|
|
289
|
-
this.jobs.set(task.id, job);
|
|
290
|
-
const next = job.nextRun();
|
|
291
|
-
if (next) {
|
|
292
|
-
task.nextRunAt = next.getTime();
|
|
293
|
-
this.store.set(task);
|
|
294
|
-
}
|
|
330
|
+
this.scheduleCron(task, parsed);
|
|
295
331
|
} catch (err) {
|
|
296
332
|
console.error(`[dsh-cron] failed to schedule task ${task.id}:`, err.message);
|
|
297
333
|
}
|
|
298
334
|
}
|
|
299
335
|
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
336
|
+
/**
|
|
337
|
+
* Pre-flight guards for a run: the global concurrency throttle (#52) and the
|
|
338
|
+
* per-task overlap policy. Returns the run token to continue with, or null
|
|
339
|
+
* when this tick must not execute (the reason is already recorded).
|
|
340
|
+
*/
|
|
341
|
+
beginRun(task, taskId) {
|
|
304
342
|
// Global concurrency throttle (#52): a run that would exceed the limit is
|
|
305
343
|
// skipped and recorded; already-running tasks are never throttled against
|
|
306
344
|
// themselves.
|
|
307
345
|
if (!this.running.has(taskId) && this.maxConcurrent > 0 && this.running.size >= this.maxConcurrent) {
|
|
308
346
|
console.log(`[dsh-cron] Task "${task.title}" (${taskId}) skipped: concurrency limit (${this.maxConcurrent})`);
|
|
309
|
-
this.
|
|
310
|
-
|
|
311
|
-
status: 'skipped',
|
|
312
|
-
durationMs: 0,
|
|
313
|
-
output: `Skipped: concurrency limit reached (${this.maxConcurrent} parallel runs)`,
|
|
314
|
-
error: null,
|
|
315
|
-
usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0 },
|
|
316
|
-
costUsd: 0,
|
|
317
|
-
});
|
|
318
|
-
return;
|
|
347
|
+
this.recordSkipped(taskId, `Skipped: concurrency limit reached (${this.maxConcurrent} parallel runs)`);
|
|
348
|
+
return null;
|
|
319
349
|
}
|
|
320
350
|
|
|
321
351
|
const overlapPolicy = task.overlapPolicy || 'skip';
|
|
322
|
-
|
|
323
|
-
// Check overlap with currently running task
|
|
324
352
|
const active = this.running.get(taskId);
|
|
325
353
|
if (active) {
|
|
326
354
|
if (overlapPolicy === 'skip') {
|
|
327
355
|
console.log(`[dsh-cron] Task "${task.title}" (${taskId}) is already running, skipping run (overlapPolicy: skip)`);
|
|
328
|
-
this.
|
|
329
|
-
|
|
330
|
-
status: 'skipped',
|
|
331
|
-
durationMs: 0,
|
|
332
|
-
output: 'Skipped: the previous run is still in progress (overlapPolicy: skip)',
|
|
333
|
-
error: null,
|
|
334
|
-
usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0 },
|
|
335
|
-
costUsd: 0,
|
|
336
|
-
});
|
|
337
|
-
return;
|
|
356
|
+
this.recordSkipped(taskId, 'Skipped: the previous run is still in progress (overlapPolicy: skip)');
|
|
357
|
+
return null;
|
|
338
358
|
}
|
|
339
359
|
if (overlapPolicy === 'replace') {
|
|
340
360
|
console.log(`[dsh-cron] Task "${task.title}" (${taskId}) is already running, canceling current run (overlapPolicy: replace)`);
|
|
@@ -343,73 +363,233 @@ export class TaskScheduler {
|
|
|
343
363
|
if (overlapPolicy === 'queue') {
|
|
344
364
|
console.log(`[dsh-cron] Task "${task.title}" (${taskId}) is already running, queueing next run (overlapPolicy: queue)`);
|
|
345
365
|
active.queueCount = (active.queueCount || 0) + 1;
|
|
346
|
-
return;
|
|
366
|
+
return null;
|
|
347
367
|
}
|
|
348
368
|
}
|
|
349
369
|
|
|
350
370
|
const controller = new AbortController();
|
|
351
|
-
const currentRun = {
|
|
352
|
-
controller,
|
|
353
|
-
startedAt: Date.now(),
|
|
354
|
-
queueCount: 0,
|
|
355
|
-
};
|
|
371
|
+
const currentRun = { controller, startedAt: Date.now(), queueCount: 0 };
|
|
356
372
|
this.running.set(taskId, currentRun);
|
|
373
|
+
return currentRun;
|
|
374
|
+
}
|
|
357
375
|
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
let costUsd = 0;
|
|
364
|
-
let sessionId = null;
|
|
376
|
+
/** Count a finished run for the metrics endpoint (#53). */
|
|
377
|
+
countRun(status) {
|
|
378
|
+
const key = status || 'unknown';
|
|
379
|
+
this.runCounters[key] = (this.runCounters[key] || 0) + 1;
|
|
380
|
+
}
|
|
365
381
|
|
|
382
|
+
/** Snapshot of the run counters; the caller must not mutate it. */
|
|
383
|
+
getRunCounters() {
|
|
384
|
+
return { ...this.runCounters };
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/** Record a run that never started, with the reason in the history entry. */
|
|
388
|
+
recordSkipped(taskId, reason) {
|
|
389
|
+
this.countRun('skipped');
|
|
390
|
+
this.store.recordRun(taskId, {
|
|
391
|
+
at: Date.now(),
|
|
392
|
+
status: 'skipped',
|
|
393
|
+
durationMs: 0,
|
|
394
|
+
output: reason,
|
|
395
|
+
error: null,
|
|
396
|
+
usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0 },
|
|
397
|
+
costUsd: 0,
|
|
398
|
+
});
|
|
399
|
+
}
|
|
400
|
+
|
|
401
|
+
/** Run the task body once and normalise whatever the executor returned. */
|
|
402
|
+
async executeOnce(task, signal) {
|
|
403
|
+
const result = {
|
|
404
|
+
status: 'success',
|
|
405
|
+
output: '',
|
|
406
|
+
error: null,
|
|
407
|
+
usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0 },
|
|
408
|
+
costUsd: 0,
|
|
409
|
+
sessionId: null,
|
|
410
|
+
model: task.model || '',
|
|
411
|
+
fallback: false,
|
|
412
|
+
};
|
|
413
|
+
if (typeof this.executeFn !== 'function') return result;
|
|
366
414
|
try {
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
} else {
|
|
375
|
-
output = String(res || '');
|
|
376
|
-
}
|
|
377
|
-
}
|
|
378
|
-
} catch (err) {
|
|
379
|
-
error = err.message || String(err);
|
|
380
|
-
if (error.toLowerCase().includes('timed out') || error.toLowerCase().includes('timeout')) {
|
|
381
|
-
status = 'timeout';
|
|
415
|
+
const res = await this.executeFn(task, { signal });
|
|
416
|
+
if (typeof res === 'object' && res !== null) {
|
|
417
|
+
result.output = res.output || '';
|
|
418
|
+
result.usage = res.usage || result.usage;
|
|
419
|
+
result.costUsd = res.costUsd || 0;
|
|
420
|
+
result.sessionId = res.sessionId || null;
|
|
421
|
+
if (res.model) result.model = res.model;
|
|
382
422
|
} else {
|
|
383
|
-
|
|
384
|
-
}
|
|
385
|
-
} finally {
|
|
386
|
-
const durationMs = Date.now() - start;
|
|
387
|
-
const runInfo = {
|
|
388
|
-
at: start,
|
|
389
|
-
status,
|
|
390
|
-
durationMs,
|
|
391
|
-
output,
|
|
392
|
-
error,
|
|
393
|
-
usage,
|
|
394
|
-
costUsd,
|
|
395
|
-
sessionId,
|
|
396
|
-
};
|
|
397
|
-
try {
|
|
398
|
-
this.store.recordRun(taskId, runInfo);
|
|
399
|
-
} catch (recordErr) {
|
|
400
|
-
console.error(`[dsh-cron] failed to record the run of task ${taskId}:`, recordErr.message);
|
|
423
|
+
result.output = String(res || '');
|
|
401
424
|
}
|
|
425
|
+
} catch (err) {
|
|
426
|
+
result.error = err.message || String(err);
|
|
427
|
+
const timedOut = result.error.toLowerCase().includes('timed out') || result.error.toLowerCase().includes('timeout');
|
|
428
|
+
result.status = timedOut ? 'timeout' : 'error';
|
|
429
|
+
}
|
|
430
|
+
return result;
|
|
431
|
+
}
|
|
402
432
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
433
|
+
/**
|
|
434
|
+
* Run the task, retrying once on the configured fallback model when the
|
|
435
|
+
* primary attempt failed (#45).
|
|
436
|
+
*
|
|
437
|
+
* The point is cost: a task can default to the cheap model and still finish
|
|
438
|
+
* on the strong one when the cheap model cannot do the job. Exactly one
|
|
439
|
+
* fallback attempt is made — the ordinary retry backoff still applies
|
|
440
|
+
* afterwards — and usage and cost of BOTH attempts are summed, because both
|
|
441
|
+
* were really spent.
|
|
442
|
+
*/
|
|
443
|
+
async executeWithFallback(task, signal) {
|
|
444
|
+
const primary = await this.executeOnce(task, signal);
|
|
445
|
+
if (!this.shouldUseFallback(task, primary)) return primary;
|
|
446
|
+
|
|
447
|
+
const fallbackTask = {
|
|
448
|
+
...task,
|
|
449
|
+
provider: task.fallbackProvider || task.provider,
|
|
450
|
+
model: task.fallbackModel,
|
|
451
|
+
};
|
|
452
|
+
console.log(`[dsh-cron] task "${task.title}" failed on ${task.model || 'the default model'}, retrying once on ${task.fallbackModel}`);
|
|
453
|
+
const secondary = await this.executeOnce(fallbackTask, signal);
|
|
454
|
+
return {
|
|
455
|
+
...secondary,
|
|
456
|
+
fallback: true,
|
|
457
|
+
usage: addUsage(primary.usage, secondary.usage),
|
|
458
|
+
costUsd: Number((primary.costUsd + secondary.costUsd).toFixed(6)),
|
|
459
|
+
primaryModel: task.model || '',
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
|
|
463
|
+
/** Only agent-mediated types use a model, and only a failed run falls back. */
|
|
464
|
+
shouldUseFallback(task, outcome) {
|
|
465
|
+
if (!task || !task.fallbackModel) return false;
|
|
466
|
+
if (!AGENT_TASK_TYPES.includes(task.type || 'llm')) return false;
|
|
467
|
+
return outcome.status === 'error' || outcome.status === 'timeout';
|
|
468
|
+
}
|
|
408
469
|
|
|
409
|
-
const queuedCount = finishedRun?.queueCount || 0;
|
|
410
470
|
|
|
411
|
-
|
|
412
|
-
|
|
471
|
+
async runTask(taskId) {
|
|
472
|
+
const task = this.store.get(taskId);
|
|
473
|
+
if (!task) return;
|
|
474
|
+
|
|
475
|
+
const currentRun = this.beginRun(task, taskId);
|
|
476
|
+
if (currentRun === null) return;
|
|
477
|
+
|
|
478
|
+
const start = Date.now();
|
|
479
|
+
const outcome = await this.executeWithFallback(task, currentRun.controller.signal);
|
|
480
|
+
await this.completeRun(task, taskId, currentRun, outcome, start);
|
|
481
|
+
}
|
|
482
|
+
|
|
483
|
+
/**
|
|
484
|
+
* Record the run, release the run slot and hand the result to delivery and
|
|
485
|
+
* bookkeeping. Kept separate from runTask so the run flow reads in steps.
|
|
486
|
+
*/
|
|
487
|
+
async completeRun(task, taskId, currentRun, outcome, start) {
|
|
488
|
+
const runInfo = {
|
|
489
|
+
at: start,
|
|
490
|
+
status: outcome.status,
|
|
491
|
+
durationMs: Date.now() - start,
|
|
492
|
+
output: outcome.output,
|
|
493
|
+
error: outcome.error,
|
|
494
|
+
usage: outcome.usage,
|
|
495
|
+
costUsd: outcome.costUsd,
|
|
496
|
+
sessionId: outcome.sessionId,
|
|
497
|
+
// #45: which model actually produced this result, and whether the run
|
|
498
|
+
// only finished thanks to the configured fallback.
|
|
499
|
+
model: outcome.model || task.model || '',
|
|
500
|
+
fallback: Boolean(outcome.fallback),
|
|
501
|
+
};
|
|
502
|
+
|
|
503
|
+
// #43/#44 decide before the run is recorded, so the single history entry
|
|
504
|
+
// already carries the diagnosis and the silence verdict.
|
|
505
|
+
const silent = await this.applyModelAssists(task, runInfo);
|
|
506
|
+
|
|
507
|
+
try {
|
|
508
|
+
this.store.recordRun(taskId, runInfo);
|
|
509
|
+
} catch (recordErr) {
|
|
510
|
+
console.error(`[dsh-cron] failed to record the run of task ${taskId}:`, recordErr.message);
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
// Release the slot before delivery so a long notification never blocks the
|
|
514
|
+
// next tick of this task.
|
|
515
|
+
const finishedRun = this.running.get(taskId);
|
|
516
|
+
if (finishedRun === currentRun) this.running.delete(taskId);
|
|
517
|
+
const queuedCount = finishedRun?.queueCount || 0;
|
|
518
|
+
|
|
519
|
+
if (!silent.skipped) await this.deliverNotifications(task, runInfo);
|
|
520
|
+
this.finishRun(task, taskId, outcome.status, queuedCount);
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
/**
|
|
524
|
+
* Run the model-assisted steps for a finished run: diagnose a failure (#43)
|
|
525
|
+
* and decide whether a successful run should stay silent (#44). Both are
|
|
526
|
+
* fail-open and write their outcome into `runInfo`.
|
|
527
|
+
*/
|
|
528
|
+
async applyModelAssists(task, runInfo) {
|
|
529
|
+
const inspection = await this.inspectFailureRun(task, runInfo);
|
|
530
|
+
if (inspection.inspected) {
|
|
531
|
+
runInfo.diagnosis = inspection.diagnosis;
|
|
532
|
+
runInfo.suggestion = inspection.suggestion;
|
|
533
|
+
runInfo.confidence = inspection.confidence;
|
|
534
|
+
console.log(`[dsh-cron] task "${task.title}" failure diagnosed (${inspection.confidence}): ${inspection.diagnosis}`);
|
|
535
|
+
} else if (inspection.error && task.inspectOnFailure) {
|
|
536
|
+
console.warn(`[dsh-cron] failure inspection for "${task.title}" not applied: ${inspection.error}`);
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
const silent = await this.applySilentRuleToRun(task, runInfo);
|
|
540
|
+
if (silent.skipped) {
|
|
541
|
+
runInfo.silentSkip = true;
|
|
542
|
+
runInfo.silentReason = silent.reason;
|
|
543
|
+
console.log(`[dsh-cron] task "${task.title}" stayed silent by rule: ${silent.reason || 'no reason given'}`);
|
|
544
|
+
} else if (silent.verdictError) {
|
|
545
|
+
console.warn(`[dsh-cron] silent rule for "${task.title}" not applied: ${silent.verdictError}`);
|
|
546
|
+
}
|
|
547
|
+
return silent;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
/** Ask the model to diagnose a failed run (#43). Never throws. */
|
|
551
|
+
async inspectFailureRun(task, runInfo) {
|
|
552
|
+
if (typeof this.askModel !== 'function') return { inspected: false, error: 'no model is available' };
|
|
553
|
+
const settings = this.readSettings();
|
|
554
|
+
try {
|
|
555
|
+
return await inspectFailure({
|
|
556
|
+
task,
|
|
557
|
+
runInfo,
|
|
558
|
+
ask: this.askModel,
|
|
559
|
+
preferredModel: String(settings.inspectorModel || ''),
|
|
560
|
+
timeoutMs: this.inspectTimeoutMs || 25000,
|
|
561
|
+
});
|
|
562
|
+
} catch (err) {
|
|
563
|
+
return { inspected: false, error: err.message };
|
|
564
|
+
}
|
|
565
|
+
}
|
|
566
|
+
|
|
567
|
+
/** Settings snapshot, or an empty object when the store has none. */
|
|
568
|
+
readSettings() {
|
|
569
|
+
try {
|
|
570
|
+
return typeof this.store.getSettings === 'function' ? this.store.getSettings() : {};
|
|
571
|
+
} catch {
|
|
572
|
+
return {};
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
|
|
576
|
+
/**
|
|
577
|
+
* Ask the silent rule whether this run should be delivered (#44).
|
|
578
|
+
* Never throws: any problem means "deliver".
|
|
579
|
+
*/
|
|
580
|
+
async applySilentRuleToRun(task, runInfo) {
|
|
581
|
+
if (typeof this.askModel !== 'function') return { skipped: false, reason: '' };
|
|
582
|
+
const preferredModel = String(this.readSettings().silentRuleModel || '');
|
|
583
|
+
try {
|
|
584
|
+
return await applySilentRule({
|
|
585
|
+
task,
|
|
586
|
+
runInfo,
|
|
587
|
+
ask: this.askModel,
|
|
588
|
+
preferredModel,
|
|
589
|
+
timeoutMs: this.silentRuleTimeoutMs || 20000,
|
|
590
|
+
});
|
|
591
|
+
} catch (err) {
|
|
592
|
+
return { skipped: false, reason: '', verdictError: err.message };
|
|
413
593
|
}
|
|
414
594
|
}
|
|
415
595
|
|
|
@@ -449,6 +629,7 @@ export class TaskScheduler {
|
|
|
449
629
|
* cannot reject the cron callback.
|
|
450
630
|
*/
|
|
451
631
|
finishRun(task, taskId, status, queuedCount) {
|
|
632
|
+
this.countRun(status);
|
|
452
633
|
try {
|
|
453
634
|
if (task.oneShot) {
|
|
454
635
|
task.status = 'completed';
|