@goodandready/dsh-cron 0.2.50 → 0.2.52
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api.js +66 -0
- package/lib/approval.js +143 -0
- package/lib/channels.js +4 -2
- package/lib/client.js +141 -90
- package/lib/cron-tool.js +24 -0
- package/lib/global-budgets.js +137 -0
- package/lib/index.js +4 -1
- package/lib/recipes-catalog.js +107 -0
- package/lib/routes.js +3 -1
- package/lib/scheduler-execution.js +132 -6
- package/lib/scheduler.js +12 -0
- package/lib/settings.js +6 -1
- package/lib/store.js +10 -1
- package/lib/task-create.js +3 -0
- package/lib/task-patch.js +7 -0
- package/lib/task-transfer.js +2 -0
- package/lib/telegram.js +1 -1
- package/lib/validators.js +192 -0
- package/package.json +1 -1
package/lib/cron-tool.js
CHANGED
|
@@ -152,6 +152,30 @@ export const cronToolParameters = {
|
|
|
152
152
|
enum: ['default', 'read-only', 'workspace-write', 'full'],
|
|
153
153
|
description: 'Permission preset applied to the task session (only used for agent session tasks: llm, skill, workflow)',
|
|
154
154
|
},
|
|
155
|
+
requireApproval: {
|
|
156
|
+
type: 'boolean',
|
|
157
|
+
description: 'Require explicit human approval before execution (#318)',
|
|
158
|
+
},
|
|
159
|
+
validators: {
|
|
160
|
+
type: 'array',
|
|
161
|
+
description: 'Declarative post-execution validation checks (json, regex, text, exitCode, file)',
|
|
162
|
+
items: {
|
|
163
|
+
type: 'object',
|
|
164
|
+
properties: {
|
|
165
|
+
type: { type: 'string', enum: ['json', 'regex', 'text', 'exitCode', 'file'] },
|
|
166
|
+
pattern: { type: 'string' },
|
|
167
|
+
text: { type: 'string' },
|
|
168
|
+
negate: { type: 'boolean' },
|
|
169
|
+
statusKey: { type: 'string' },
|
|
170
|
+
expected: { type: 'string' },
|
|
171
|
+
path: { type: 'string' },
|
|
172
|
+
mustExist: { type: 'boolean' },
|
|
173
|
+
minBytes: { type: 'number' },
|
|
174
|
+
target: { type: 'string' },
|
|
175
|
+
},
|
|
176
|
+
additionalProperties: true,
|
|
177
|
+
},
|
|
178
|
+
},
|
|
155
179
|
kanbanMode: {
|
|
156
180
|
type: 'string',
|
|
157
181
|
enum: ['none', 'on_failure', 'always'],
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
import { logger } from './logger.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Global & Provider/Model Budgets and Rate Limits (#320).
|
|
5
|
+
* Manages shared expenditure windows and concurrent reservations
|
|
6
|
+
* across all scheduled tasks.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
export class GlobalBudgetManager {
|
|
10
|
+
constructor() {
|
|
11
|
+
this.reservations = new Map(); // reservationId -> { provider, model, costUsd, at }
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Calculate total expenditure from store history across all tasks
|
|
16
|
+
* filtered by provider/model and window.
|
|
17
|
+
*/
|
|
18
|
+
calculateUsage(store, { provider = null, model = null, windowMs = 86400000, now = Date.now() } = {}) {
|
|
19
|
+
const cutoff = now - windowMs;
|
|
20
|
+
let totalCostUsd = 0;
|
|
21
|
+
let totalTokens = 0;
|
|
22
|
+
let runCount = 0;
|
|
23
|
+
|
|
24
|
+
const allTasks = store.list ? store.list() : [];
|
|
25
|
+
for (const task of allTasks) {
|
|
26
|
+
if (provider && task.provider && task.provider !== provider) continue;
|
|
27
|
+
if (model && task.model && task.model !== model) continue;
|
|
28
|
+
|
|
29
|
+
const runs = (store.history && store.history.get(task.id)) || [];
|
|
30
|
+
for (const r of runs) {
|
|
31
|
+
if (r && typeof r.at === 'number' && r.at >= cutoff) {
|
|
32
|
+
totalCostUsd += Number(r.costUsd) || 0;
|
|
33
|
+
if (typeof r.tokens === 'number') {
|
|
34
|
+
totalTokens += r.tokens;
|
|
35
|
+
} else if (r.usage) {
|
|
36
|
+
totalTokens += (r.usage.inputTokens || 0) + (r.usage.outputTokens || 0) + (r.usage.cacheReadTokens || 0);
|
|
37
|
+
}
|
|
38
|
+
runCount++;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
return {
|
|
44
|
+
totalCostUsd: Number(totalCostUsd.toFixed(6)),
|
|
45
|
+
totalTokens,
|
|
46
|
+
runCount,
|
|
47
|
+
};
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/**
|
|
51
|
+
* Check if global or provider budget limits are exceeded.
|
|
52
|
+
*/
|
|
53
|
+
checkBudget(store, task, settings = {}, now = Date.now()) {
|
|
54
|
+
// 1. Global Monthly / Daily Cost Limit
|
|
55
|
+
const globalDailyCostLimit = Number(settings.globalDailyCostLimitUsd) || 0;
|
|
56
|
+
if (globalDailyCostLimit > 0) {
|
|
57
|
+
const usage = this.calculateUsage(store, { windowMs: 86400000, now });
|
|
58
|
+
const pendingReservationsCost = this.getActiveReservationsCost();
|
|
59
|
+
if ((usage.totalCostUsd + pendingReservationsCost) >= globalDailyCostLimit) {
|
|
60
|
+
return {
|
|
61
|
+
exceeded: true,
|
|
62
|
+
type: 'globalDailyCostLimitUsd',
|
|
63
|
+
reason: `Global daily budget exceeded: $${(usage.totalCostUsd + pendingReservationsCost).toFixed(4)} >= $${globalDailyCostLimit.toFixed(4)}`,
|
|
64
|
+
current: usage.totalCostUsd,
|
|
65
|
+
limit: globalDailyCostLimit,
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
// 2. Provider Rate Limit (max runs per minute / hour)
|
|
71
|
+
const provider = task.provider || 'default';
|
|
72
|
+
const providerHourlyLimit = (settings.providerHourlyRunLimits && Number(settings.providerHourlyRunLimits[provider])) || 0;
|
|
73
|
+
if (providerHourlyLimit > 0) {
|
|
74
|
+
const pUsage = this.calculateUsage(store, { provider, windowMs: 3600000, now });
|
|
75
|
+
if (pUsage.runCount >= providerHourlyLimit) {
|
|
76
|
+
return {
|
|
77
|
+
exceeded: true,
|
|
78
|
+
type: 'providerHourlyLimit',
|
|
79
|
+
reason: `Provider "${provider}" hourly run limit reached: ${pUsage.runCount} >= ${providerHourlyLimit}`,
|
|
80
|
+
current: pUsage.runCount,
|
|
81
|
+
limit: providerHourlyLimit,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// 3. Model Rate Limit
|
|
87
|
+
const model = task.model || 'default';
|
|
88
|
+
const modelHourlyLimit = (settings.modelHourlyRunLimits && Number(settings.modelHourlyRunLimits[model])) || 0;
|
|
89
|
+
if (modelHourlyLimit > 0) {
|
|
90
|
+
const mUsage = this.calculateUsage(store, { model, windowMs: 3600000, now });
|
|
91
|
+
if (mUsage.runCount >= modelHourlyLimit) {
|
|
92
|
+
return {
|
|
93
|
+
exceeded: true,
|
|
94
|
+
type: 'modelHourlyLimit',
|
|
95
|
+
reason: `Model "${model}" hourly run limit reached: ${mUsage.runCount} >= ${modelHourlyLimit}`,
|
|
96
|
+
current: mUsage.runCount,
|
|
97
|
+
limit: modelHourlyLimit,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
return { exceeded: false };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
reserve(task, estimatedCostUsd = 0.05) {
|
|
106
|
+
const id = 'res_' + Math.random().toString(36).slice(2, 9);
|
|
107
|
+
this.reservations.set(id, {
|
|
108
|
+
taskId: task.id,
|
|
109
|
+
provider: task.provider || 'default',
|
|
110
|
+
model: task.model || 'default',
|
|
111
|
+
costUsd: Number(estimatedCostUsd) || 0,
|
|
112
|
+
at: Date.now(),
|
|
113
|
+
});
|
|
114
|
+
return id;
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
settle(reservationId) {
|
|
118
|
+
if (reservationId) {
|
|
119
|
+
this.reservations.delete(reservationId);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
getActiveReservationsCost() {
|
|
124
|
+
let sum = 0;
|
|
125
|
+
const cutoff = Date.now() - 300000; // 5 min TTL on stale reservations
|
|
126
|
+
for (const [id, r] of this.reservations.entries()) {
|
|
127
|
+
if (r.at < cutoff) {
|
|
128
|
+
this.reservations.delete(id);
|
|
129
|
+
} else {
|
|
130
|
+
sum += r.costUsd;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return sum;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
export const globalBudgetManager = new GlobalBudgetManager();
|
package/lib/index.js
CHANGED
|
@@ -165,7 +165,10 @@ export function apply(ctx, rawConfig) {
|
|
|
165
165
|
deliver: (task, runInfo, settings) => deliverRun({
|
|
166
166
|
task,
|
|
167
167
|
runInfo,
|
|
168
|
-
settings
|
|
168
|
+
settings: {
|
|
169
|
+
...settings,
|
|
170
|
+
_webServerPort: ctx.webServer?.port || 3080,
|
|
171
|
+
},
|
|
169
172
|
secrets: {
|
|
170
173
|
botToken: settings.botToken || '',
|
|
171
174
|
resolveSecret: (ref) => resolveCredentialValue(ctx, ref),
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
import { RECIPES as BASE_RECIPES } from './recipes.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Enhanced Verified Recipes Catalog with guided parameters, dependencies, and test runs (#322).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
export const EXTENDED_RECIPES = [
|
|
8
|
+
...BASE_RECIPES,
|
|
9
|
+
{
|
|
10
|
+
id: 'recipe_gitea_issues_digest',
|
|
11
|
+
category: 'maintenance',
|
|
12
|
+
title: 'Gitea Issues Weekly Digest',
|
|
13
|
+
description: 'Summarises open bugs and feature issues from local Gitea and reports triage status.',
|
|
14
|
+
type: 'llm',
|
|
15
|
+
schedule: '0 9 * * 1',
|
|
16
|
+
scheduleText: 'Mondays at 09:00',
|
|
17
|
+
prompt: 'Query open Gitea issues. Group by priority and scope, highlight stale issues, and produce an actionable weekly triage digest.',
|
|
18
|
+
channels: ['telegram'],
|
|
19
|
+
silentRule: 'Stay silent when there are no new or updated issues in the past 7 days.',
|
|
20
|
+
dependencies: {
|
|
21
|
+
integrations: ['gitea'],
|
|
22
|
+
credentials: ['giteaTokenRef'],
|
|
23
|
+
},
|
|
24
|
+
parameters: [
|
|
25
|
+
{ name: 'repo', label: 'Repository (owner/name)', type: 'string', default: 'goodandready/dsh-cron' },
|
|
26
|
+
{ name: 'daysBack', label: 'Days back', type: 'number', default: 7 },
|
|
27
|
+
],
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
id: 'recipe_security_dependency_audit',
|
|
31
|
+
category: 'security',
|
|
32
|
+
title: 'NPM Dependency Security Audit',
|
|
33
|
+
description: 'Runs npm audit in repository directory and alerts on critical or high vulnerabilities.',
|
|
34
|
+
type: 'script',
|
|
35
|
+
schedule: '0 6 * * 1',
|
|
36
|
+
scheduleText: 'Mondays at 06:00',
|
|
37
|
+
prompt: 'npm audit --audit-level=high 2>&1 || { echo "CRITICAL / HIGH vulnerabilities found!"; exit 1; }',
|
|
38
|
+
channels: ['telegram'],
|
|
39
|
+
silentRule: 'Stay silent when npm audit finds 0 high/critical vulnerabilities.',
|
|
40
|
+
dependencies: {
|
|
41
|
+
integrations: ['shell'],
|
|
42
|
+
},
|
|
43
|
+
parameters: [
|
|
44
|
+
{ name: 'cwd', label: 'Working Directory', type: 'string', default: '.' },
|
|
45
|
+
],
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
id: 'recipe_model_cost_summary',
|
|
49
|
+
category: 'system',
|
|
50
|
+
title: 'Daily Model Spend & Token Telemetry',
|
|
51
|
+
description: 'Computes rolling 24h spend across all models and warns if daily cost exceeds threshold.',
|
|
52
|
+
type: 'llm',
|
|
53
|
+
schedule: '0 23 * * *',
|
|
54
|
+
scheduleText: 'Every day at 23:00',
|
|
55
|
+
prompt: 'Analyze token usage ledger across all active cron jobs. Provide cost breakdown per model and identify anomalies.',
|
|
56
|
+
channels: ['telegram'],
|
|
57
|
+
silentRule: 'Stay silent when total daily expenditure is under $0.10.',
|
|
58
|
+
dependencies: {
|
|
59
|
+
integrations: ['cron-store'],
|
|
60
|
+
},
|
|
61
|
+
parameters: [
|
|
62
|
+
{ name: 'thresholdUsd', label: 'Alert Threshold (USD)', type: 'number', default: 1.0 },
|
|
63
|
+
],
|
|
64
|
+
},
|
|
65
|
+
];
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Validate recipe dependencies against host settings.
|
|
69
|
+
* Returns { ok: boolean, missing: string[] }
|
|
70
|
+
*/
|
|
71
|
+
export function validateRecipeDependencies(recipe, settings = {}) {
|
|
72
|
+
if (!recipe || !recipe.dependencies) return { ok: true, missing: [] };
|
|
73
|
+
const missing = [];
|
|
74
|
+
const deps = recipe.dependencies;
|
|
75
|
+
|
|
76
|
+
if (Array.isArray(deps.credentials)) {
|
|
77
|
+
for (const cred of deps.credentials) {
|
|
78
|
+
if (!settings[cred] && !settings[cred.replace('Ref', '')]) {
|
|
79
|
+
missing.push(`Credential: ${cred}`);
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
if (Array.isArray(deps.integrations)) {
|
|
85
|
+
for (const integ of deps.integrations) {
|
|
86
|
+
if (integ === 'gitea' && !settings.giteaBaseUrl) {
|
|
87
|
+
missing.push('Integration: Gitea (giteaBaseUrl not configured)');
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return {
|
|
93
|
+
ok: missing.length === 0,
|
|
94
|
+
missing,
|
|
95
|
+
};
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function listVerifiedRecipes(settings = {}) {
|
|
99
|
+
return EXTENDED_RECIPES.map((r) => {
|
|
100
|
+
const depCheck = validateRecipeDependencies(r, settings);
|
|
101
|
+
return {
|
|
102
|
+
...r,
|
|
103
|
+
dependenciesValid: depCheck.ok,
|
|
104
|
+
missingDependencies: depCheck.missing,
|
|
105
|
+
};
|
|
106
|
+
});
|
|
107
|
+
}
|
package/lib/routes.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { listVerifiedRecipes } from './recipes-catalog.js';
|
|
1
2
|
import { randomUUID } from 'node:crypto';
|
|
2
3
|
import { createCronApiHandler } from './api.js';
|
|
3
4
|
import { createMetricsHandler } from './metrics.js';
|
|
@@ -93,8 +94,9 @@ export function registerRoutes(ctx, { store, scheduler, recommendations }) {
|
|
|
93
94
|
sendJson(res, 405, { ok: false, error: 'Method not allowed' });
|
|
94
95
|
return;
|
|
95
96
|
}
|
|
97
|
+
const settings = store.getSettings ? store.getSettings() : {};
|
|
96
98
|
const categories = recipesByCategory();
|
|
97
|
-
const recipes =
|
|
99
|
+
const recipes = listVerifiedRecipes(settings);
|
|
98
100
|
sendJson(res, 200, { ok: true, categories, recipes });
|
|
99
101
|
} catch (err) {
|
|
100
102
|
sendJson(res, err.statusCode || 500, { ok: false, error: err.message });
|
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { approvalManager } from './approval.js';
|
|
2
|
+
import { globalBudgetManager } from './global-budgets.js';
|
|
3
|
+
import { validateRunResult } from './validators.js';
|
|
1
4
|
import { logger } from './logger.js';
|
|
2
5
|
import { checkAndApplyBurnGuard, checkTaskBudgetLimits } from './burn-guard.js';
|
|
3
6
|
import { resolveTaskForExecution } from './prompt-interpolation.js';
|
|
@@ -308,6 +311,30 @@ export async function executeTask(scheduler, taskId, options = {}) {
|
|
|
308
311
|
};
|
|
309
312
|
}
|
|
310
313
|
|
|
314
|
+
// 1. Approval Gate (#318): Human-in-the-loop check before executing actions requiring approval
|
|
315
|
+
if (task.requireApproval && !options.approvedBy) {
|
|
316
|
+
let pending = approvalManager.getPendingProposalForTask(taskId);
|
|
317
|
+
if (!pending) {
|
|
318
|
+
pending = approvalManager.createProposal(taskId, null, {
|
|
319
|
+
actionType: 'run',
|
|
320
|
+
summary: `Execution of task "${task.title}" requires human approval`,
|
|
321
|
+
details: { prompt: task.prompt, type: task.type, model: task.model }
|
|
322
|
+
});
|
|
323
|
+
}
|
|
324
|
+
logger.info(`[dsh-cron] Task "${task.title}" (${taskId}) paused awaiting human approval (${pending.id})`);
|
|
325
|
+
scheduler.recordSkipped(taskId, `Awaiting human approval: proposal ${pending.id}`);
|
|
326
|
+
return { ok: false, status: 'waiting_approval', proposalId: pending.id, stoppedReason: 'Awaiting human approval' };
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
// 2. Global & Provider/Model Budgets (#320)
|
|
330
|
+
const settings = typeof scheduler.store.getSettings === 'function' ? scheduler.store.getSettings() : {};
|
|
331
|
+
const globalBudget = globalBudgetManager.checkBudget(scheduler.store, task, settings);
|
|
332
|
+
if (globalBudget.exceeded) {
|
|
333
|
+
logger.warn(`[dsh-cron] Global budget limit blocked task "${task.title}" (${taskId}): ${globalBudget.reason}`);
|
|
334
|
+
scheduler.recordSkipped(taskId, `Budget limit reached: ${globalBudget.reason}`);
|
|
335
|
+
return { ok: false, status: 'error', error: globalBudget.reason, budgetBlocked: true };
|
|
336
|
+
}
|
|
337
|
+
|
|
311
338
|
// Pre-flight Burn Guard check (#181): prevent executing task if budget already breached
|
|
312
339
|
const runs = (scheduler.store.history && scheduler.store.history.get(taskId)) || [];
|
|
313
340
|
const guard = checkTaskBudgetLimits(task, runs);
|
|
@@ -327,7 +354,9 @@ export async function executeTask(scheduler, taskId, options = {}) {
|
|
|
327
354
|
const currentRun = scheduler.beginRun(task, taskId, options);
|
|
328
355
|
if (currentRun === null) return null;
|
|
329
356
|
|
|
357
|
+
const preflightStart = Date.now();
|
|
330
358
|
const preflight = await executePreflight(task);
|
|
359
|
+
const preflightDurationMs = Date.now() - preflightStart;
|
|
331
360
|
if (!preflight.ok) {
|
|
332
361
|
logger.info(`[dsh-cron] task "${task.title}" (${taskId}) skipped: ${preflight.reason}`);
|
|
333
362
|
scheduler.running.delete(taskId);
|
|
@@ -336,8 +365,17 @@ export async function executeTask(scheduler, taskId, options = {}) {
|
|
|
336
365
|
}
|
|
337
366
|
|
|
338
367
|
const start = Date.now();
|
|
339
|
-
const
|
|
340
|
-
|
|
368
|
+
const reservationId = globalBudgetManager.reserve(task);
|
|
369
|
+
let outcome;
|
|
370
|
+
try {
|
|
371
|
+
outcome = await scheduler.executeWithFallback(task, currentRun.controller.signal, options);
|
|
372
|
+
} finally {
|
|
373
|
+
globalBudgetManager.settle(reservationId);
|
|
374
|
+
}
|
|
375
|
+
await scheduler.completeRun(task, taskId, currentRun, outcome, start, {
|
|
376
|
+
...options,
|
|
377
|
+
preflightDurationMs,
|
|
378
|
+
});
|
|
341
379
|
return outcome;
|
|
342
380
|
}
|
|
343
381
|
|
|
@@ -429,7 +467,7 @@ export async function deliverNotifications(task, runInfo, { store, resolveSecret
|
|
|
429
467
|
logger.warn('[dsh-cron] credential resolution failed:', secErr.message);
|
|
430
468
|
}
|
|
431
469
|
}
|
|
432
|
-
if (typeof deliver !== 'function') return;
|
|
470
|
+
if (typeof deliver !== 'function') return { status: 'skipped', delivered: [], failures: [] };
|
|
433
471
|
const result = await deliver(task, runInfo, settings);
|
|
434
472
|
if (result && Array.isArray(result.delivered) && result.delivered.length) {
|
|
435
473
|
logger.info(`[dsh-cron] delivered "${task.title}" to: ${result.delivered.map((d) => d.channel).join(', ')}`);
|
|
@@ -437,8 +475,16 @@ export async function deliverNotifications(task, runInfo, { store, resolveSecret
|
|
|
437
475
|
if (result && Array.isArray(result.failures) && result.failures.length) {
|
|
438
476
|
logger.warn(`[dsh-cron] delivery failures for "${task.title}": ${result.failures.map((f) => `${f.channel}: ${f.error}`).join('; ')}`);
|
|
439
477
|
}
|
|
478
|
+
const delivered = Array.isArray(result?.delivered) ? result.delivered : [];
|
|
479
|
+
const failures = Array.isArray(result?.failures) ? result.failures : [];
|
|
480
|
+
let status = 'skipped';
|
|
481
|
+
if (delivered.length > 0 && failures.length === 0) status = 'delivered';
|
|
482
|
+
else if (delivered.length > 0 && failures.length > 0) status = 'partial';
|
|
483
|
+
else if (delivered.length === 0 && failures.length > 0) status = 'failed';
|
|
484
|
+
return { status, delivered, failures };
|
|
440
485
|
} catch (notifyErr) {
|
|
441
486
|
logger.error('[dsh-cron] notification error:', notifyErr.message);
|
|
487
|
+
return { status: 'failed', delivered: [], failures: [{ channel: 'all', error: notifyErr.message }] };
|
|
442
488
|
}
|
|
443
489
|
}
|
|
444
490
|
|
|
@@ -522,13 +568,41 @@ export function finishRun(scheduler, taskSnapshot, taskId, status, queuedCount,
|
|
|
522
568
|
/**
|
|
523
569
|
* Handle complete run lifecycle: metrics, self-healing, diagnosis, silent rule, storage, delivery, chaining.
|
|
524
570
|
*/
|
|
571
|
+
function computeStoppedReason(status, valRes, error) {
|
|
572
|
+
if (status === 'validation_failed') return `Validation failed: ${valRes?.error || 'Check failed'}`;
|
|
573
|
+
if (status === 'validator_error') return `Validator configuration error: ${valRes?.error || 'Check failed'}`;
|
|
574
|
+
if (status === 'timeout') return 'Execution timed out';
|
|
575
|
+
if (status === 'error') return error ? `Execution error: ${error}` : 'Execution error';
|
|
576
|
+
if (status === 'success') return 'Completed successfully';
|
|
577
|
+
if (status === 'skipped') return error || 'Run skipped';
|
|
578
|
+
return null;
|
|
579
|
+
}
|
|
580
|
+
|
|
525
581
|
export async function handleCompleteRun(scheduler, task, taskId, currentRun, outcome, start, options = {}) {
|
|
582
|
+
// Post-execution declarative validation (#316)
|
|
583
|
+
let valRes = { ok: true, skipped: true };
|
|
584
|
+
let valDurationMs = 0;
|
|
585
|
+
if (outcome.status === 'success' && Array.isArray(task.validators) && task.validators.length > 0) {
|
|
586
|
+
const vStart = Date.now();
|
|
587
|
+
valRes = validateRunResult(task, outcome, { cwd: task.cwd });
|
|
588
|
+
valDurationMs = Date.now() - vStart;
|
|
589
|
+
if (!valRes.ok) {
|
|
590
|
+
outcome.status = valRes.status === 'error' ? 'validator_error' : 'validation_failed';
|
|
591
|
+
outcome.error = valRes.error || 'Validation failed';
|
|
592
|
+
}
|
|
593
|
+
}
|
|
594
|
+
|
|
526
595
|
const isSuccess = outcome.status === 'success';
|
|
527
|
-
const isFailure = outcome.status === 'error' || outcome.status === 'timeout';
|
|
596
|
+
const isFailure = outcome.status === 'error' || outcome.status === 'timeout' || outcome.status === 'validation_failed' || outcome.status === 'validator_error';
|
|
597
|
+
const executionMs = Date.now() - start;
|
|
598
|
+
const queueWaitMs = (currentRun && currentRun.startedAt && currentRun.queuedAt) ? (currentRun.startedAt - currentRun.queuedAt) : 0;
|
|
599
|
+
const preflightMs = options.preflightDurationMs || 0;
|
|
600
|
+
|
|
528
601
|
const runInfo = {
|
|
602
|
+
runId: currentRun?.id || ('run_' + Math.random().toString(36).slice(2, 9) + '_' + Date.now()),
|
|
529
603
|
at: start,
|
|
530
604
|
status: outcome.status,
|
|
531
|
-
durationMs:
|
|
605
|
+
durationMs: executionMs,
|
|
532
606
|
output: outcome.output,
|
|
533
607
|
error: outcome.error,
|
|
534
608
|
usage: outcome.usage,
|
|
@@ -536,6 +610,7 @@ export async function handleCompleteRun(scheduler, task, taskId, currentRun, out
|
|
|
536
610
|
sessionId: outcome.sessionId,
|
|
537
611
|
model: outcome.model || task.model || '',
|
|
538
612
|
fallback: Boolean(outcome.fallback),
|
|
613
|
+
validation: valRes.skipped ? null : valRes,
|
|
539
614
|
};
|
|
540
615
|
|
|
541
616
|
if (isFailure && task.selfHealingCommand) {
|
|
@@ -573,6 +648,58 @@ export async function handleCompleteRun(scheduler, task, taskId, currentRun, out
|
|
|
573
648
|
|
|
574
649
|
const silent = await scheduler.applyModelAssists(task, runInfo);
|
|
575
650
|
|
|
651
|
+
let deliveryResult = { status: 'skipped', delivered: [], failures: [] };
|
|
652
|
+
let delivStart = Date.now();
|
|
653
|
+
if (!silent.skipped) {
|
|
654
|
+
deliveryResult = (await scheduler.deliverNotifications(task, runInfo)) || deliveryResult;
|
|
655
|
+
}
|
|
656
|
+
const delivDurationMs = Date.now() - delivStart;
|
|
657
|
+
|
|
658
|
+
const stages = [
|
|
659
|
+
{
|
|
660
|
+
name: 'queue',
|
|
661
|
+
status: 'success',
|
|
662
|
+
durationMs: queueWaitMs,
|
|
663
|
+
},
|
|
664
|
+
{
|
|
665
|
+
name: 'preflight',
|
|
666
|
+
status: 'success',
|
|
667
|
+
durationMs: preflightMs,
|
|
668
|
+
error: null,
|
|
669
|
+
},
|
|
670
|
+
{
|
|
671
|
+
name: 'execution',
|
|
672
|
+
status: (outcome.status === 'error' || outcome.status === 'timeout') ? 'failed' : 'success',
|
|
673
|
+
durationMs: executionMs,
|
|
674
|
+
error: (outcome.status === 'error' || outcome.status === 'timeout') ? (outcome.error || null) : null,
|
|
675
|
+
},
|
|
676
|
+
{
|
|
677
|
+
name: 'validation',
|
|
678
|
+
status: valRes.skipped ? 'skipped' : (valRes.ok ? 'success' : 'failed'),
|
|
679
|
+
durationMs: valDurationMs,
|
|
680
|
+
error: valRes.ok ? null : (valRes.error || null),
|
|
681
|
+
},
|
|
682
|
+
{
|
|
683
|
+
name: 'delivery',
|
|
684
|
+
status: deliveryResult?.status || 'skipped',
|
|
685
|
+
durationMs: delivDurationMs,
|
|
686
|
+
error: (deliveryResult?.failures && deliveryResult.failures.length > 0)
|
|
687
|
+
? deliveryResult.failures.map(f => `${f.channel}: ${f.error}`).join('; ')
|
|
688
|
+
: null,
|
|
689
|
+
delivered: deliveryResult?.delivered || [],
|
|
690
|
+
failures: deliveryResult?.failures || [],
|
|
691
|
+
},
|
|
692
|
+
];
|
|
693
|
+
|
|
694
|
+
runInfo.stages = stages;
|
|
695
|
+
runInfo.delivery = deliveryResult;
|
|
696
|
+
runInfo.cost = {
|
|
697
|
+
amount: Number(outcome.costUsd) || 0,
|
|
698
|
+
currency: 'USD',
|
|
699
|
+
kind: (Number(outcome.costUsd) > 0) ? 'exact' : ((task.type === 'llm' || task.type === 'skill' || task.type === 'workflow') ? 'unknown' : 'exact'),
|
|
700
|
+
};
|
|
701
|
+
runInfo.stoppedReason = computeStoppedReason(outcome.status, valRes, outcome.error);
|
|
702
|
+
|
|
576
703
|
try {
|
|
577
704
|
scheduler.store.recordRun(taskId, runInfo);
|
|
578
705
|
} catch (recordErr) {
|
|
@@ -585,7 +712,6 @@ export async function handleCompleteRun(scheduler, task, taskId, currentRun, out
|
|
|
585
712
|
const queuedRuns = isCurrentRun ? (finishedRun?.queuedRuns || []) : [];
|
|
586
713
|
const queuedCount = isCurrentRun ? (finishedRun?.queueCount || 0) : 0;
|
|
587
714
|
|
|
588
|
-
if (!silent.skipped) await scheduler.deliverNotifications(task, runInfo);
|
|
589
715
|
scheduler.finishRun(task, taskId, outcome.status, queuedCount, {
|
|
590
716
|
isCurrentRun,
|
|
591
717
|
runId: currentRun?.id,
|
package/lib/scheduler.js
CHANGED
|
@@ -548,7 +548,9 @@ export class TaskScheduler {
|
|
|
548
548
|
|
|
549
549
|
recordSkipped(taskId, reason) {
|
|
550
550
|
this.countRun('skipped');
|
|
551
|
+
const runId = 'run_' + Math.random().toString(36).slice(2, 9) + '_' + Date.now();
|
|
551
552
|
this.store.recordRun(taskId, {
|
|
553
|
+
runId,
|
|
552
554
|
at: Date.now(),
|
|
553
555
|
status: 'skipped',
|
|
554
556
|
durationMs: 0,
|
|
@@ -556,6 +558,16 @@ export class TaskScheduler {
|
|
|
556
558
|
error: null,
|
|
557
559
|
usage: { inputTokens: 0, outputTokens: 0, cacheReadTokens: 0 },
|
|
558
560
|
costUsd: 0,
|
|
561
|
+
stoppedReason: reason,
|
|
562
|
+
stages: [
|
|
563
|
+
{ name: 'queue', status: 'skipped', durationMs: 0 },
|
|
564
|
+
{ name: 'preflight', status: 'skipped', durationMs: 0, error: reason },
|
|
565
|
+
{ name: 'execution', status: 'skipped', durationMs: 0 },
|
|
566
|
+
{ name: 'validation', status: 'skipped', durationMs: 0 },
|
|
567
|
+
{ name: 'delivery', status: 'skipped', durationMs: 0 },
|
|
568
|
+
],
|
|
569
|
+
cost: { amount: 0, currency: 'USD', kind: 'exact' },
|
|
570
|
+
delivery: { status: 'skipped', delivered: [], failures: [] },
|
|
559
571
|
});
|
|
560
572
|
}
|
|
561
573
|
|
package/lib/settings.js
CHANGED
|
@@ -40,13 +40,17 @@ export const Config = z.object({
|
|
|
40
40
|
barkKey: z.string().default('').description('Bark device key').volatile(),
|
|
41
41
|
pushplusTokenRef: z.string().default('').description('Credential name for the PushPlus token').volatile(),
|
|
42
42
|
pushplusUrl: z.string().default('https://www.pushplus.plus/send').description('PushPlus send endpoint (override for a self-hosted proxy)').volatile(),
|
|
43
|
-
ttsBaseUrl: z.string().default('
|
|
43
|
+
ttsBaseUrl: z.string().default('').description('Base URL of the harness used to reach the dsh-tts speak route (empty = auto-resolved from webServer port or http://127.0.0.1:3080)').volatile(),
|
|
44
44
|
giteaBaseUrl: z.string().default('').description('Gitea base URL for alert issues').volatile(),
|
|
45
45
|
giteaRepo: z.string().default('').description('Gitea repository (owner/name) for alert issues').volatile(),
|
|
46
46
|
giteaTokenRef: z.string().default('').description('Credential name for the Gitea API token').volatile(),
|
|
47
47
|
giteaLabels: z.any().default([]).description('Gitea label names or IDs for alert issues').volatile(),
|
|
48
48
|
giteaLabelIds: z.any().default({}).description('Mapping of label name to integer ID for Gitea issues').volatile(),
|
|
49
49
|
prReviewerEnabled: z.boolean().default(false).description('Enable automated PR Reviewer scheduled agent (#33)').volatile(),
|
|
50
|
+
// --- Global Budgets (#320) ---
|
|
51
|
+
globalDailyCostLimitUsd: z.number().default(0).description('Global daily expenditure limit in USD across all tasks (0 = unlimited)').volatile(),
|
|
52
|
+
providerHourlyRunLimits: z.object({}).default({}).description('Max runs per hour keyed by provider').volatile(),
|
|
53
|
+
modelHourlyRunLimits: z.object({}).default({}).description('Max runs per hour keyed by model').volatile(),
|
|
50
54
|
// --- Declarative jobs (#50, ADR-0001) ---
|
|
51
55
|
jobs: z.array(z.any()).default([]).description('Static jobs owned by the config; each entry needs id, title, schedule and (for agent types) prompt'),
|
|
52
56
|
// --- External REST API (#54, ADR-0001) ---
|
|
@@ -83,6 +87,7 @@ export const SETTINGS_SYNC_KEYS = [
|
|
|
83
87
|
'ttsBaseUrl',
|
|
84
88
|
'giteaBaseUrl', 'giteaRepo', 'giteaTokenRef', 'giteaLabels', 'giteaLabelIds', 'prReviewerEnabled',
|
|
85
89
|
'apiToken',
|
|
90
|
+
'globalDailyCostLimitUsd', 'providerHourlyRunLimits', 'modelHourlyRunLimits',
|
|
86
91
|
];
|
|
87
92
|
|
|
88
93
|
/**
|
package/lib/store.js
CHANGED
|
@@ -689,7 +689,7 @@ export class TaskStore {
|
|
|
689
689
|
const prevRuns = this.history.get(id) || [];
|
|
690
690
|
const runs = [
|
|
691
691
|
{
|
|
692
|
-
id: 'run_' + Math.random().toString(36).slice(2, 9),
|
|
692
|
+
id: runInfo.runId || runInfo.id || ('run_' + Math.random().toString(36).slice(2, 9)),
|
|
693
693
|
at: task.lastRunAt,
|
|
694
694
|
status: task.lastStatus,
|
|
695
695
|
durationMs: task.lastDurationMs,
|
|
@@ -697,6 +697,15 @@ export class TaskStore {
|
|
|
697
697
|
error: runInfo.error ? String(runInfo.error).slice(0, 1000) : null,
|
|
698
698
|
usage,
|
|
699
699
|
costUsd,
|
|
700
|
+
stages: Array.isArray(runInfo.stages) ? runInfo.stages : null,
|
|
701
|
+
stoppedReason: runInfo.stoppedReason || null,
|
|
702
|
+
cost: runInfo.cost || {
|
|
703
|
+
amount: costUsd,
|
|
704
|
+
currency: 'USD',
|
|
705
|
+
kind: costUsd > 0 ? 'exact' : ((task.type === 'llm' || task.type === 'skill' || task.type === 'workflow') ? 'unknown' : 'exact')
|
|
706
|
+
},
|
|
707
|
+
delivery: runInfo.delivery || null,
|
|
708
|
+
validation: runInfo.validation || null,
|
|
700
709
|
sessionId: runInfo.sessionId || null,
|
|
701
710
|
// #45: which model produced the run, and whether it only finished thanks
|
|
702
711
|
// to the configured fallback.
|
package/lib/task-create.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { sanitizeValidators } from './validators.js';
|
|
1
2
|
import { normalizeTaskType, EXTERNAL_RUNTIMES } from './runtimes.js';
|
|
2
3
|
import { validateTaskType } from './task-transfer.js';
|
|
3
4
|
import { parseScheduleExpression } from './scheduler.js';
|
|
@@ -72,6 +73,8 @@ export function executeCreateTask(store, scheduler, args) {
|
|
|
72
73
|
costLimitUsd: Number(args.costLimitUsd) > 0 ? Number(args.costLimitUsd) : undefined,
|
|
73
74
|
dailyCostLimitUsd: Number(args.dailyCostLimitUsd) > 0 ? Number(args.dailyCostLimitUsd) : undefined,
|
|
74
75
|
tokenLimit: Number(args.tokenLimit) > 0 ? Number(args.tokenLimit) : undefined,
|
|
76
|
+
validators: Array.isArray(args.validators) ? sanitizeValidators(args.validators) : undefined,
|
|
77
|
+
requireApproval: Boolean(args.requireApproval),
|
|
75
78
|
};
|
|
76
79
|
// Issue #310 / GH #5: strip any undefined properties before store.set()
|
|
77
80
|
for (const k of Object.keys(taskData)) {
|
package/lib/task-patch.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { sanitizeValidators } from './validators.js';
|
|
1
2
|
/**
|
|
2
3
|
* Partial task updates, shared by the HTTP PATCH route and the cron tool (action: 'update')
|
|
3
4
|
* tool (#49).
|
|
@@ -86,6 +87,12 @@ export function buildTaskPatch(current, body) {
|
|
|
86
87
|
return { ok: false, error: err.message };
|
|
87
88
|
}
|
|
88
89
|
}
|
|
90
|
+
if (patch.requireApproval !== undefined) {
|
|
91
|
+
patch.requireApproval = Boolean(patch.requireApproval);
|
|
92
|
+
}
|
|
93
|
+
if (patch.validators !== undefined) {
|
|
94
|
+
patch.validators = Array.isArray(patch.validators) ? sanitizeValidators(patch.validators) : [];
|
|
95
|
+
}
|
|
89
96
|
if (patch.costLimitUsd !== undefined) {
|
|
90
97
|
patch.costLimitUsd = patch.costLimitUsd === null ? null : (Number(patch.costLimitUsd) > 0 ? Number(patch.costLimitUsd) : null);
|
|
91
98
|
}
|
package/lib/task-transfer.js
CHANGED
package/lib/telegram.js
CHANGED
|
@@ -210,7 +210,7 @@ export function shouldNotifyTask(task, runInfo, globalSettings = {}) {
|
|
|
210
210
|
const isEnabled = task.notifyTelegram ?? globalSettings.notifyTelegram ?? false;
|
|
211
211
|
if (!isEnabled) return false;
|
|
212
212
|
|
|
213
|
-
const failed = runInfo.status === 'error' || runInfo.status === 'timeout' || runInfo.status === 'missed';
|
|
213
|
+
const failed = runInfo.status === 'error' || runInfo.status === 'timeout' || runInfo.status === 'missed' || runInfo.status === 'validation_failed' || runInfo.status === 'validator_error';
|
|
214
214
|
const onlyOnFail = task.onlyOnFailure ?? globalSettings.onlyOnFailure ?? false;
|
|
215
215
|
if (onlyOnFail && !failed) {
|
|
216
216
|
return false;
|