@goodandready/dsh-cron 0.3.24 → 0.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api-actions.js +1 -317
- package/lib/api-collection.js +1 -250
- package/lib/api-helpers.js +1 -100
- package/lib/api-import-export.js +1 -155
- package/lib/api-item.js +1 -190
- package/lib/api-task-webhook.js +1 -247
- package/lib/api-webhook.js +1 -183
- package/lib/api.js +1 -178
- package/lib/artifacts.js +1 -200
- package/lib/chain-checkpoints.js +1 -223
- package/lib/client-src/00-open.js +62 -0
- package/lib/client-src/10-locales.js +484 -0
- package/lib/client-src/20-styles.js +64 -0
- package/lib/client-src/30-utils.js +248 -0
- package/lib/client-src/40-delivery-settings.js +96 -0
- package/lib/client-src/45-dag-graph-view.js +287 -0
- package/lib/client-src/46-live-logs-modal.js +108 -0
- package/lib/client-src/47-analytics-digest-modal.js +166 -0
- package/lib/client-src/48-timeline-heatmap-view.js +176 -0
- package/lib/client-src/50-cron-screen-state.js +447 -0
- package/lib/client-src/52-cron-screen-actions.js +330 -0
- package/lib/client-src/53-cron-screen-modal-actions.js +640 -0
- package/lib/client-src/54-cron-screen-analytics-actions.js +68 -0
- package/lib/client-src/55-cron-screen-view.js +378 -0
- package/lib/client-src/60-modal-task-form-head.js +580 -0
- package/lib/client-src/62-modal-task-form-tail.js +466 -0
- package/lib/client-src/65-modal-dialogs.js +350 -0
- package/lib/client-src/70-settings-card.js +420 -0
- package/lib/client-src/80-sidebar-jobs.js +200 -0
- package/lib/client-src/90-close.js +200 -0
- package/lib/client.js +48 -3
- package/lib/commands.js +200 -0
- package/lib/durable-journal.js +1 -188
- package/lib/external-api.js +1 -74
- package/lib/index.js +17 -0
- package/lib/log-streamer.js +10 -2
- package/lib/metrics.js +1 -125
- package/lib/routes/api-actions.js +317 -0
- package/lib/routes/api-collection.js +250 -0
- package/lib/routes/api-helpers.js +100 -0
- package/lib/routes/api-import-export.js +155 -0
- package/lib/routes/api-item.js +191 -0
- package/lib/routes/api-task-webhook.js +247 -0
- package/lib/routes/api-webhook.js +183 -0
- package/lib/routes/api.js +178 -0
- package/lib/routes/external-api.js +74 -0
- package/lib/routes/index.js +11 -0
- package/lib/routes/metrics.js +125 -0
- package/lib/routes/routes.js +549 -0
- package/lib/routes.js +1 -549
- package/lib/runner-external.js +1 -352
- package/lib/runner-worktree.js +1 -90
- package/lib/runner.js +1 -881
- package/lib/runtimes/index.js +4 -0
- package/lib/runtimes/runner-external.js +352 -0
- package/lib/runtimes/runner-worktree.js +90 -0
- package/lib/runtimes/runner.js +881 -0
- package/lib/runtimes/runtimes.js +198 -0
- package/lib/runtimes.js +1 -198
- package/lib/scheduler/index.js +6 -0
- package/lib/scheduler/scheduler-assists.js +142 -0
- package/lib/scheduler/scheduler-completion.js +417 -0
- package/lib/scheduler/scheduler-execution.js +360 -0
- package/lib/scheduler/scheduler-lifecycle.js +157 -0
- package/lib/scheduler/scheduler-preflight.js +195 -0
- package/lib/scheduler/scheduler.js +751 -0
- package/lib/scheduler-assists.js +1 -142
- package/lib/scheduler-completion.js +1 -417
- package/lib/scheduler-execution.js +1 -360
- package/lib/scheduler-lifecycle.js +1 -157
- package/lib/scheduler-preflight.js +1 -195
- package/lib/scheduler.js +1 -751
- package/lib/store/artifacts.js +200 -0
- package/lib/store/chain-checkpoints.js +223 -0
- package/lib/store/durable-journal.js +188 -0
- package/lib/store/index.js +6 -0
- package/lib/store/store.js +1179 -0
- package/lib/store/task-schema.js +180 -0
- package/lib/store/task-transfer.js +310 -0
- package/lib/store.js +2 -1179
- package/lib/task-schema.js +1 -180
- package/lib/task-transfer.js +1 -310
- package/lib/tui-settings.js +157 -0
- package/package.json +6 -3
package/lib/scheduler-assists.js
CHANGED
|
@@ -1,142 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { applySilentRule } from './silent-rule.js';
|
|
3
|
-
import { logger } from './logger.js';
|
|
4
|
-
import { shouldAutoHeal, applyDoctorFix } from './doctor.js';
|
|
5
|
-
|
|
6
|
-
/**
|
|
7
|
-
* Ask the model to diagnose a failed run (#43). Never throws.
|
|
8
|
-
*/
|
|
9
|
-
export async function inspectFailureRun(task, runInfo, { askModel, readSettingsFn, inspectTimeoutMs }) {
|
|
10
|
-
if (typeof askModel !== 'function') return { inspected: false, error: 'no model is available' };
|
|
11
|
-
const settings = readSettingsFn ? readSettingsFn() : {};
|
|
12
|
-
try {
|
|
13
|
-
return await inspectFailure({
|
|
14
|
-
task,
|
|
15
|
-
runInfo,
|
|
16
|
-
ask: askModel,
|
|
17
|
-
preferredModel: String(settings.inspectorModel || ''),
|
|
18
|
-
timeoutMs: inspectTimeoutMs || 25000,
|
|
19
|
-
});
|
|
20
|
-
} catch (err) {
|
|
21
|
-
return { inspected: false, error: err.message };
|
|
22
|
-
}
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* Ask the silent rule whether this run should be delivered (#44).
|
|
27
|
-
* Never throws: any problem means "deliver".
|
|
28
|
-
*/
|
|
29
|
-
export async function applySilentRuleToRun(task, runInfo, { askModel, readSettingsFn, silentRuleTimeoutMs }) {
|
|
30
|
-
if (typeof askModel !== 'function') return { skipped: false, reason: '' };
|
|
31
|
-
const preferredModel = String((readSettingsFn ? readSettingsFn() : {}).silentRuleModel || '');
|
|
32
|
-
try {
|
|
33
|
-
return await applySilentRule({
|
|
34
|
-
task,
|
|
35
|
-
runInfo,
|
|
36
|
-
ask: askModel,
|
|
37
|
-
preferredModel,
|
|
38
|
-
timeoutMs: silentRuleTimeoutMs || 20000,
|
|
39
|
-
});
|
|
40
|
-
} catch (err) {
|
|
41
|
-
return { skipped: false, reason: '', verdictError: err.message };
|
|
42
|
-
}
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
/**
|
|
46
|
-
* Run the model-assisted steps for a finished run.
|
|
47
|
-
*/
|
|
48
|
-
export async function applyModelAssists(task, runInfo, options = {}) {
|
|
49
|
-
const inspection = await inspectFailureRun(task, runInfo, options);
|
|
50
|
-
if (inspection.inspected) {
|
|
51
|
-
runInfo.diagnosis = inspection.diagnosis;
|
|
52
|
-
runInfo.suggestion = inspection.suggestion;
|
|
53
|
-
runInfo.confidence = inspection.confidence;
|
|
54
|
-
runInfo.doctorReport = inspection.doctorReport || {
|
|
55
|
-
category: inspection.category || 'runtime_error',
|
|
56
|
-
cause: inspection.diagnosis,
|
|
57
|
-
suggestedFix: inspection.suggestion,
|
|
58
|
-
confidence: inspection.confidence,
|
|
59
|
-
diff: inspection.diff || '',
|
|
60
|
-
timestamp: Date.now(),
|
|
61
|
-
applied: false,
|
|
62
|
-
autoHealed: false,
|
|
63
|
-
};
|
|
64
|
-
logger.info(`[dsh-cron] task "${task.title}" failure diagnosed (${inspection.confidence}): ${inspection.diagnosis}`);
|
|
65
|
-
|
|
66
|
-
// Safe Auto-Heal Check (#353)
|
|
67
|
-
if (shouldAutoHeal(task, runInfo.doctorReport)) {
|
|
68
|
-
try {
|
|
69
|
-
const healResult = await applyDoctorFix(task, runInfo.doctorReport, {
|
|
70
|
-
store: options.store,
|
|
71
|
-
user: 'scheduler:auto-heal',
|
|
72
|
-
isAutoHeal: true,
|
|
73
|
-
});
|
|
74
|
-
if (healResult.ok) {
|
|
75
|
-
logger.info(`[dsh-cron] task "${task.title}" auto-healed successfully with doctor fix!`);
|
|
76
|
-
runInfo.autoHealed = true;
|
|
77
|
-
if (runInfo.doctorReport) runInfo.doctorReport.autoHealed = true;
|
|
78
|
-
}
|
|
79
|
-
} catch (autoHealErr) {
|
|
80
|
-
logger.warn(`[dsh-cron] failed to auto-heal task "${task.title}":`, autoHealErr.message);
|
|
81
|
-
}
|
|
82
|
-
}
|
|
83
|
-
} else if (inspection.error && (task.inspectOnFailure || task.autoDiagnose || task.autoHeal)) {
|
|
84
|
-
logger.warn(`[dsh-cron] failure inspection for "${task.title}" not applied: ${inspection.error}`);
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
const silent = await applySilentRuleToRun(task, runInfo, options);
|
|
88
|
-
if (silent.skipped) {
|
|
89
|
-
runInfo.silentSkip = true;
|
|
90
|
-
runInfo.silentReason = silent.reason;
|
|
91
|
-
logger.info(`[dsh-cron] task "${task.title}" stayed silent by rule: ${silent.reason || 'no reason given'}`);
|
|
92
|
-
} else if (silent.verdictError) {
|
|
93
|
-
logger.warn(`[dsh-cron] silent rule for "${task.title}" not applied: ${silent.verdictError}`);
|
|
94
|
-
}
|
|
95
|
-
return silent;
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
/**
|
|
99
|
-
* Deliver a finished run to every configured channel (#26).
|
|
100
|
-
*/
|
|
101
|
-
export async function deliverNotifications(task, runInfo, { store, resolveSecrets, deliver, secretTimeoutMs = 5000 }) {
|
|
102
|
-
try {
|
|
103
|
-
let settings = typeof store?.getSettings === 'function' ? store.getSettings() : {};
|
|
104
|
-
if (resolveSecrets) {
|
|
105
|
-
try {
|
|
106
|
-
let timer;
|
|
107
|
-
const deadline = new Promise((_, reject) => {
|
|
108
|
-
timer = setTimeout(() => reject(new Error(`credential resolution timed out after ${secretTimeoutMs}ms`)), secretTimeoutMs);
|
|
109
|
-
if (typeof timer.unref === 'function') timer.unref();
|
|
110
|
-
});
|
|
111
|
-
const resolved = await Promise.race([
|
|
112
|
-
Promise.resolve(resolveSecrets(settings)).finally(() => {
|
|
113
|
-
if (timer) clearTimeout(timer);
|
|
114
|
-
}),
|
|
115
|
-
deadline,
|
|
116
|
-
]);
|
|
117
|
-
settings = { ...settings, ...(resolved || {}) };
|
|
118
|
-
} catch (secErr) {
|
|
119
|
-
logger.warn('[dsh-cron] credential resolution failed:', secErr.message);
|
|
120
|
-
}
|
|
121
|
-
}
|
|
122
|
-
if (typeof deliver !== 'function') return { status: 'skipped', delivered: [], failures: [] };
|
|
123
|
-
const result = await deliver(task, runInfo, settings);
|
|
124
|
-
if (result && Array.isArray(result.delivered) && result.delivered.length) {
|
|
125
|
-
logger.info(`[dsh-cron] delivered "${task.title}" to: ${result.delivered.map((d) => d.channel).join(', ')}`);
|
|
126
|
-
}
|
|
127
|
-
if (result && Array.isArray(result.failures) && result.failures.length) {
|
|
128
|
-
logger.warn(`[dsh-cron] delivery failures for "${task.title}": ${result.failures.map((f) => `${f.channel}: ${f.error}`).join('; ')}`);
|
|
129
|
-
}
|
|
130
|
-
const delivered = Array.isArray(result?.delivered) ? result.delivered : [];
|
|
131
|
-
const failures = Array.isArray(result?.failures) ? result.failures : [];
|
|
132
|
-
let status = 'skipped';
|
|
133
|
-
if (delivered.length > 0 && failures.length === 0) status = 'delivered';
|
|
134
|
-
else if (delivered.length > 0 && failures.length > 0) status = 'partial';
|
|
135
|
-
else if (delivered.length === 0 && failures.length > 0) status = 'failed';
|
|
136
|
-
return { status, delivered, failures };
|
|
137
|
-
} catch (notifyErr) {
|
|
138
|
-
logger.error('[dsh-cron] notification error:', notifyErr.message);
|
|
139
|
-
return { status: 'failed', delivered: [], failures: [{ channel: 'all', error: notifyErr.message }] };
|
|
140
|
-
}
|
|
141
|
-
}
|
|
142
|
-
|
|
1
|
+
export * from './scheduler/scheduler-assists.js';
|
|
@@ -1,417 +1 @@
|
|
|
1
|
-
|
|
2
|
-
import { captureGitDiff } from './git-diff.js';
|
|
3
|
-
import { resolveBranchTargets } from './branching.js';
|
|
4
|
-
import { shouldAutoHeal, applyDoctorFix } from './doctor.js';
|
|
5
|
-
import { approvalManager } from './approval.js';
|
|
6
|
-
import { validateRunResult } from './validators.js';
|
|
7
|
-
import { logger } from './logger.js';
|
|
8
|
-
import { checkAndApplyBurnGuard } from './burn-guard.js';
|
|
9
|
-
import { bestEffort } from './best-effort.js';
|
|
10
|
-
import { parseLlmActionDirectives, executeLlmActionDirectives } from './llm-actions.js';
|
|
11
|
-
import { parseChainTargets, sanitizeDependsOn } from './dag.js';
|
|
12
|
-
import { applyModelAssists, deliverNotifications } from './scheduler-assists.js';
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Handle complete run lifecycle: metrics, self-healing, diagnosis, silent rule, storage, delivery, chaining.
|
|
16
|
-
*/
|
|
17
|
-
function computeStoppedReason(status, valRes, error) {
|
|
18
|
-
if (status === 'validation_failed') return `Validation failed: ${valRes?.error || 'Check failed'}`;
|
|
19
|
-
if (status === 'validator_error') return `Validator configuration error: ${valRes?.error || 'Check failed'}`;
|
|
20
|
-
if (status === 'timeout') return 'Execution timed out';
|
|
21
|
-
if (status === 'error') return error ? `Execution error: ${error}` : 'Execution error';
|
|
22
|
-
if (status === 'success') return 'Completed successfully';
|
|
23
|
-
if (status === 'skipped') return error || 'Run skipped';
|
|
24
|
-
return null;
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
export async function handleCompleteRun(scheduler, task, taskId, currentRun, outcome, start, options = {}) {
|
|
28
|
-
// Post-execution declarative validation (#316)
|
|
29
|
-
let valRes = { ok: true, skipped: true };
|
|
30
|
-
let valDurationMs = 0;
|
|
31
|
-
if (outcome.status === 'success' && Array.isArray(task.validators) && task.validators.length > 0) {
|
|
32
|
-
const vStart = Date.now();
|
|
33
|
-
valRes = validateRunResult(task, outcome, { cwd: task.cwd });
|
|
34
|
-
valDurationMs = Date.now() - vStart;
|
|
35
|
-
if (!valRes.ok) {
|
|
36
|
-
outcome.status = valRes.status === 'error' ? 'validator_error' : 'validation_failed';
|
|
37
|
-
outcome.error = valRes.error || 'Validation failed';
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
const isSuccess = outcome.status === 'success';
|
|
42
|
-
const isFailure = outcome.status === 'error' || outcome.status === 'timeout' || outcome.status === 'validation_failed' || outcome.status === 'validator_error';
|
|
43
|
-
const executionMs = Date.now() - start;
|
|
44
|
-
const queueWaitMs = (currentRun && currentRun.startedAt && currentRun.queuedAt) ? (currentRun.startedAt - currentRun.queuedAt) : 0;
|
|
45
|
-
const preflightMs = options.preflightDurationMs || 0;
|
|
46
|
-
|
|
47
|
-
const runInfo = {
|
|
48
|
-
runId: currentRun?.id || ('run_' + Math.random().toString(36).slice(2, 9) + '_' + Date.now()),
|
|
49
|
-
at: start,
|
|
50
|
-
status: outcome.status,
|
|
51
|
-
durationMs: executionMs,
|
|
52
|
-
output: outcome.output,
|
|
53
|
-
error: outcome.error,
|
|
54
|
-
usage: outcome.usage,
|
|
55
|
-
costUsd: outcome.costUsd,
|
|
56
|
-
sessionId: outcome.sessionId,
|
|
57
|
-
model: outcome.model || task.model || '',
|
|
58
|
-
fallback: Boolean(outcome.fallback),
|
|
59
|
-
validation: valRes.skipped ? null : valRes,
|
|
60
|
-
};
|
|
61
|
-
|
|
62
|
-
if (isFailure && task.selfHealingCommand) {
|
|
63
|
-
logger.info(`[dsh-cron] running self-healing command for task "${task.title}": ${task.selfHealingCommand}`);
|
|
64
|
-
try {
|
|
65
|
-
await new Promise((resolve) => {
|
|
66
|
-
exec(task.selfHealingCommand, { timeout: 30000 }, (shErr, stdout, stderr) => {
|
|
67
|
-
runInfo.selfHealing = {
|
|
68
|
-
command: task.selfHealingCommand,
|
|
69
|
-
success: !shErr,
|
|
70
|
-
output: (stdout + (stderr ? '\n' + stderr : '')).trim().slice(0, 500),
|
|
71
|
-
};
|
|
72
|
-
resolve();
|
|
73
|
-
});
|
|
74
|
-
});
|
|
75
|
-
} catch (shErr) {
|
|
76
|
-
runInfo.selfHealing = { command: task.selfHealingCommand, success: false, error: shErr.message };
|
|
77
|
-
}
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
if (isFailure && task.autoDiagnose && typeof scheduler.executeFn === 'function') {
|
|
81
|
-
try {
|
|
82
|
-
const diagRes = await scheduler.executeFn({
|
|
83
|
-
type: 'llm',
|
|
84
|
-
prompt: `Diagnose this task failure in 1-2 concise sentences. Error: ${outcome.error || 'unknown'}. Recent output: ${(outcome.output || '').slice(-300)}`,
|
|
85
|
-
model: task.fallbackModel || task.model,
|
|
86
|
-
}, { signal: AbortSignal.timeout(10000) });
|
|
87
|
-
if (diagRes && diagRes.output) {
|
|
88
|
-
runInfo.autoDiagnosis = diagRes.output.trim();
|
|
89
|
-
}
|
|
90
|
-
} catch (diagErr) {
|
|
91
|
-
bestEffort('auto-diagnose', () => {}, scheduler.logger);
|
|
92
|
-
}
|
|
93
|
-
}
|
|
94
|
-
|
|
95
|
-
const silent = await scheduler.applyModelAssists(task, runInfo);
|
|
96
|
-
|
|
97
|
-
let deliveryResult = { status: 'skipped', delivered: [], failures: [] };
|
|
98
|
-
let delivStart = Date.now();
|
|
99
|
-
if (!silent.skipped) {
|
|
100
|
-
deliveryResult = (await scheduler.deliverNotifications(task, runInfo)) || deliveryResult;
|
|
101
|
-
}
|
|
102
|
-
const delivDurationMs = Date.now() - delivStart;
|
|
103
|
-
|
|
104
|
-
const stages = [
|
|
105
|
-
{
|
|
106
|
-
name: 'queue',
|
|
107
|
-
status: 'success',
|
|
108
|
-
durationMs: queueWaitMs,
|
|
109
|
-
},
|
|
110
|
-
{
|
|
111
|
-
name: 'preflight',
|
|
112
|
-
status: 'success',
|
|
113
|
-
durationMs: preflightMs,
|
|
114
|
-
error: null,
|
|
115
|
-
},
|
|
116
|
-
{
|
|
117
|
-
name: 'execution',
|
|
118
|
-
status: (outcome.status === 'error' || outcome.status === 'timeout') ? 'failed' : 'success',
|
|
119
|
-
durationMs: executionMs,
|
|
120
|
-
error: (outcome.status === 'error' || outcome.status === 'timeout') ? (outcome.error || null) : null,
|
|
121
|
-
},
|
|
122
|
-
{
|
|
123
|
-
name: 'validation',
|
|
124
|
-
status: valRes.skipped ? 'skipped' : (valRes.ok ? 'success' : 'failed'),
|
|
125
|
-
durationMs: valDurationMs,
|
|
126
|
-
error: valRes.ok ? null : (valRes.error || null),
|
|
127
|
-
},
|
|
128
|
-
{
|
|
129
|
-
name: 'delivery',
|
|
130
|
-
status: deliveryResult?.status || 'skipped',
|
|
131
|
-
durationMs: delivDurationMs,
|
|
132
|
-
error: (deliveryResult?.failures && deliveryResult.failures.length > 0)
|
|
133
|
-
? deliveryResult.failures.map(f => `${f.channel}: ${f.error}`).join('; ')
|
|
134
|
-
: null,
|
|
135
|
-
delivered: deliveryResult?.delivered || [],
|
|
136
|
-
failures: deliveryResult?.failures || [],
|
|
137
|
-
},
|
|
138
|
-
];
|
|
139
|
-
|
|
140
|
-
if (outcome.subtaskRuns) {
|
|
141
|
-
runInfo.subtaskRuns = outcome.subtaskRuns;
|
|
142
|
-
runInfo.swarmStats = outcome.swarmStats;
|
|
143
|
-
}
|
|
144
|
-
|
|
145
|
-
// Capture git diff if task runs in a worktree or explicitly opted in (#351, #379, GH #17)
|
|
146
|
-
const explicitCapture = (typeof task.captureGitDiff === 'boolean')
|
|
147
|
-
? task.captureGitDiff
|
|
148
|
-
: ((typeof task.captureDiff === 'boolean') ? task.captureDiff : null);
|
|
149
|
-
const shouldCaptureDiff = explicitCapture !== null ? explicitCapture : Boolean(task.worktree);
|
|
150
|
-
const diffDir = shouldCaptureDiff ? (task.worktree ? (runInfo.worktreePath || task.cwd) : task.cwd) : null;
|
|
151
|
-
if (diffDir) {
|
|
152
|
-
try {
|
|
153
|
-
const diffResult = await captureGitDiff(diffDir);
|
|
154
|
-
if (diffResult && diffResult.hasChanges) {
|
|
155
|
-
runInfo.gitDiff = diffResult;
|
|
156
|
-
}
|
|
157
|
-
} catch (diffErr) {
|
|
158
|
-
bestEffort('log-diff-error', () => scheduler?.logger?.debug?.(`[dsh-cron] git diff capture failed: ${diffErr?.message}`));
|
|
159
|
-
}
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
runInfo.stages = stages;
|
|
163
|
-
runInfo.delivery = deliveryResult;
|
|
164
|
-
runInfo.cost = {
|
|
165
|
-
amount: Number(outcome.costUsd) || 0,
|
|
166
|
-
currency: 'USD',
|
|
167
|
-
kind: (Number(outcome.costUsd) > 0) ? 'exact' : ((task.type === 'llm' || task.type === 'skill' || task.type === 'workflow') ? 'unknown' : 'exact'),
|
|
168
|
-
};
|
|
169
|
-
runInfo.stoppedReason = computeStoppedReason(outcome.status, valRes, outcome.error);
|
|
170
|
-
// Durable journal & Idempotency (#317)
|
|
171
|
-
runInfo.idempotencyKey = options.idempotencyKey || currentRun?.idempotencyKey || null;
|
|
172
|
-
|
|
173
|
-
// Artifacts capture (#321)
|
|
174
|
-
if (Array.isArray(outcome.artifacts) && scheduler.store.artifactManager) {
|
|
175
|
-
for (const art of outcome.artifacts) {
|
|
176
|
-
if (art && art.name && (art.content || art.data)) {
|
|
177
|
-
try {
|
|
178
|
-
scheduler.store.artifactManager.saveArtifact(runInfo.runId, taskId, art.name, art.content || art.data);
|
|
179
|
-
} catch (artErr) {
|
|
180
|
-
logger.warn(`[dsh-cron] failed to save artifact ${art.name}:`, artErr.message);
|
|
181
|
-
}
|
|
182
|
-
}
|
|
183
|
-
}
|
|
184
|
-
}
|
|
185
|
-
if (scheduler.store.getArtifactsForRun) {
|
|
186
|
-
runInfo.artifacts = scheduler.store.getArtifactsForRun(runInfo.runId);
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
// Chain Checkpoints & Decision Nodes (#319, #328, #352)
|
|
190
|
-
const branchResult = resolveBranchTargets(task, outcome, runInfo);
|
|
191
|
-
if (branchResult && branchResult.decision) {
|
|
192
|
-
runInfo.branchDecision = branchResult.decision;
|
|
193
|
-
}
|
|
194
|
-
const chainedTargets = branchResult
|
|
195
|
-
? branchResult.targets
|
|
196
|
-
: parseChainTargets(outcome.status === 'success' ? task.onSuccess : task.onFailure);
|
|
197
|
-
let hasDependents = false;
|
|
198
|
-
if (scheduler.store?.tasks) {
|
|
199
|
-
for (const t of Object.values(scheduler.store.tasks)) {
|
|
200
|
-
if (t && sanitizeDependsOn(t.dependsOn).includes(taskId)) {
|
|
201
|
-
hasDependents = true;
|
|
202
|
-
break;
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
const chainRunId = options.chainRunId || ((chainedTargets.length > 0 || hasDependents) ? ('chain_' + Math.random().toString(36).slice(2, 10)) : null);
|
|
207
|
-
if (chainRunId) {
|
|
208
|
-
runInfo.chainRunId = chainRunId;
|
|
209
|
-
if (scheduler.store.chainCheckpointManager) {
|
|
210
|
-
scheduler.store.chainCheckpointManager.recordStepSnapshot(chainRunId, taskId, {
|
|
211
|
-
status: outcome.status,
|
|
212
|
-
input: { prompt: task.prompt, prevOutput: options.prevOutput || options.chainPrevOutput || '', prevOutputs: options.prevOutputs || {} },
|
|
213
|
-
output: outcome.output || outcome.error || '',
|
|
214
|
-
durationMs: executionMs,
|
|
215
|
-
});
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
|
|
219
|
-
// Update Durable Journal (#317)
|
|
220
|
-
if (scheduler.store.durableJournal) {
|
|
221
|
-
const journalState = outcome.status === 'success' ? 'completed' : (outcome.status === 'uncertain' ? 'uncertain' : 'failed');
|
|
222
|
-
scheduler.store.durableJournal.transition(taskId, runInfo.runId, journalState, runInfo);
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
try {
|
|
227
|
-
scheduler.store.recordRun(taskId, runInfo);
|
|
228
|
-
} catch (recordErr) {
|
|
229
|
-
logger.error(`[dsh-cron] failed to record the run of task ${taskId}:`, recordErr.message);
|
|
230
|
-
}
|
|
231
|
-
|
|
232
|
-
const finishedRun = scheduler.running.get(taskId);
|
|
233
|
-
const isCurrentRun = (finishedRun === currentRun);
|
|
234
|
-
if (isCurrentRun) scheduler.running.delete(taskId);
|
|
235
|
-
const queuedRuns = isCurrentRun ? (finishedRun?.queuedRuns || []) : [];
|
|
236
|
-
const queuedCount = isCurrentRun ? (finishedRun?.queueCount || 0) : 0;
|
|
237
|
-
|
|
238
|
-
scheduler.finishRun(task, taskId, outcome.status, queuedCount, {
|
|
239
|
-
isCurrentRun,
|
|
240
|
-
runId: currentRun?.id,
|
|
241
|
-
queuedRuns,
|
|
242
|
-
outcome,
|
|
243
|
-
runInfo,
|
|
244
|
-
silent,
|
|
245
|
-
});
|
|
246
|
-
|
|
247
|
-
if (scheduler.isStopped) return;
|
|
248
|
-
|
|
249
|
-
const guard = await checkAndApplyBurnGuard(scheduler, taskId, outcome);
|
|
250
|
-
if (guard && guard.exceeded) return;
|
|
251
|
-
|
|
252
|
-
if (!scheduler.isStopped && scheduler.queue.length > 0 && (scheduler.maxConcurrent === 0 || scheduler.running.size < scheduler.maxConcurrent)) {
|
|
253
|
-
let admittedIndex = -1;
|
|
254
|
-
for (let i = 0; i < scheduler.queue.length; i++) {
|
|
255
|
-
const candidate = scheduler.queue[i];
|
|
256
|
-
const queuedTask = scheduler.store.get(candidate.taskId);
|
|
257
|
-
if (!queuedTask || queuedTask.status !== 'active') {
|
|
258
|
-
scheduler.queue.splice(i, 1);
|
|
259
|
-
i--;
|
|
260
|
-
continue;
|
|
261
|
-
}
|
|
262
|
-
const qGroup = queuedTask.concurrencyGroup && queuedTask.concurrencyGroup !== 'default'
|
|
263
|
-
? String(queuedTask.concurrencyGroup).trim()
|
|
264
|
-
: null;
|
|
265
|
-
const qLimit = scheduler.getGroupLimit ? scheduler.getGroupLimit(qGroup) : (qGroup ? 1 : 0);
|
|
266
|
-
if (qGroup && qLimit > 0) {
|
|
267
|
-
const inGroup = scheduler.getRunningGroupCount ? scheduler.getRunningGroupCount(qGroup) : 0;
|
|
268
|
-
if (inGroup >= qLimit) {
|
|
269
|
-
continue;
|
|
270
|
-
}
|
|
271
|
-
}
|
|
272
|
-
admittedIndex = i;
|
|
273
|
-
break;
|
|
274
|
-
}
|
|
275
|
-
|
|
276
|
-
if (admittedIndex !== -1) {
|
|
277
|
-
const nextItem = scheduler.queue.splice(admittedIndex, 1)[0];
|
|
278
|
-
setImmediate(() => {
|
|
279
|
-
if (!scheduler.isStopped) {
|
|
280
|
-
scheduler.runTask(nextItem.taskId, nextItem.options || {}).catch((runErr) => {
|
|
281
|
-
bestEffort('drain-queued-task', () => {}, scheduler.logger);
|
|
282
|
-
});
|
|
283
|
-
}
|
|
284
|
-
});
|
|
285
|
-
}
|
|
286
|
-
}
|
|
287
|
-
|
|
288
|
-
// 1. Structured LLM Actions (#137)
|
|
289
|
-
if (task.type === 'llm' && outcome.status === 'success' && outcome.output) {
|
|
290
|
-
try {
|
|
291
|
-
const settings = typeof scheduler.store.getSettings === 'function' ? scheduler.store.getSettings() : {};
|
|
292
|
-
if (settings.llmActionsEnabled) {
|
|
293
|
-
const directives = parseLlmActionDirectives(outcome.output);
|
|
294
|
-
if (directives.length > 0) {
|
|
295
|
-
const actionResults = await executeLlmActionDirectives({
|
|
296
|
-
directives,
|
|
297
|
-
task,
|
|
298
|
-
runInfo,
|
|
299
|
-
scheduler,
|
|
300
|
-
store: scheduler.store,
|
|
301
|
-
settings,
|
|
302
|
-
options,
|
|
303
|
-
});
|
|
304
|
-
if (actionResults && actionResults.length > 0) {
|
|
305
|
-
runInfo.actionResults = actionResults;
|
|
306
|
-
}
|
|
307
|
-
}
|
|
308
|
-
}
|
|
309
|
-
} catch (actErr) {
|
|
310
|
-
logger.warn(`[dsh-cron] structured action error for ${taskId}:`, actErr.message);
|
|
311
|
-
}
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
// 2. Task Chaining & Decision Nodes (#137, #328, #352 Fan-Out / Fan-In)
|
|
315
|
-
const targets = branchResult
|
|
316
|
-
? branchResult.targets
|
|
317
|
-
: parseChainTargets(isSuccess ? task.onSuccess : (isFailure ? task.onFailure : null));
|
|
318
|
-
const currentDepth = options.chainDepth || 0;
|
|
319
|
-
|
|
320
|
-
if (branchResult && branchResult.decision) {
|
|
321
|
-
logger.info(`[dsh-cron] decision node in "${task.title}": routed to ${targets.join(', ') || 'none'} (${branchResult.decision.isElse ? 'fallback else' : (branchResult.decision.matched ? `rule #${branchResult.decision.matchedIndex + 1}` : 'unmatched')})`);
|
|
322
|
-
if (chainRunId && scheduler.store.chainCheckpointManager && Array.isArray(task.branchRules)) {
|
|
323
|
-
for (const rule of task.branchRules) {
|
|
324
|
-
if (rule && rule.targetTask && !targets.includes(rule.targetTask)) {
|
|
325
|
-
scheduler.store.chainCheckpointManager.recordStepSnapshot(chainRunId, rule.targetTask, {
|
|
326
|
-
status: 'skipped',
|
|
327
|
-
skippedReason: 'condition not met',
|
|
328
|
-
durationMs: 0,
|
|
329
|
-
});
|
|
330
|
-
}
|
|
331
|
-
}
|
|
332
|
-
}
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
if (currentDepth < 4) {
|
|
336
|
-
// 2.1 Trigger explicit targets (Fan-Out)
|
|
337
|
-
for (const targetId of targets) {
|
|
338
|
-
const nextTask = scheduler.store.get(targetId);
|
|
339
|
-
if (!nextTask) {
|
|
340
|
-
logger.warn(`[dsh-cron] chained task target not found: ${targetId}`);
|
|
341
|
-
continue;
|
|
342
|
-
}
|
|
343
|
-
const deps = sanitizeDependsOn(nextTask.dependsOn);
|
|
344
|
-
if (deps.length > 0 && scheduler.store.chainCheckpointManager) {
|
|
345
|
-
const barrier = scheduler.store.chainCheckpointManager.checkJoinBarrier(chainRunId, nextTask);
|
|
346
|
-
if (barrier.ready && scheduler.store.chainCheckpointManager.canTriggerBarrier(chainRunId, nextTask.id)) {
|
|
347
|
-
scheduler.store.chainCheckpointManager.markBarrierTriggered(chainRunId, nextTask.id);
|
|
348
|
-
logger.info(`[dsh-cron] triggering joined task "${nextTask.title}" (${targetId}) (barrier satisfied)`);
|
|
349
|
-
scheduler.runNow(targetId, {
|
|
350
|
-
chainRunId,
|
|
351
|
-
chainDepth: currentDepth + 1,
|
|
352
|
-
prevOutputs: barrier.outputs,
|
|
353
|
-
prevOutput: barrier.latestOutput,
|
|
354
|
-
prevTaskId: task.id,
|
|
355
|
-
prevStatus: outcome.status,
|
|
356
|
-
prevDurationMs: Date.now() - start,
|
|
357
|
-
prevCostUsd: outcome.costUsd || 0,
|
|
358
|
-
}).catch((err) => {
|
|
359
|
-
logger.warn(`[dsh-cron] chained joined task ${targetId} failed to trigger:`, err.message);
|
|
360
|
-
});
|
|
361
|
-
} else if (barrier.pending) {
|
|
362
|
-
logger.info(`[dsh-cron] task "${nextTask.title}" (${targetId}) waiting for dependencies: ${barrier.waitingFor?.join(', ')}`);
|
|
363
|
-
} else if (barrier.failed) {
|
|
364
|
-
logger.warn(`[dsh-cron] task "${nextTask.title}" (${targetId}) barrier failed due to dependency: ${barrier.failedDep}`);
|
|
365
|
-
}
|
|
366
|
-
} else {
|
|
367
|
-
logger.info(`[dsh-cron] triggering chained task "${nextTask.title}" (${targetId}) from "${task.title}" (depth ${currentDepth + 1})`);
|
|
368
|
-
scheduler.runNow(targetId, {
|
|
369
|
-
chainRunId,
|
|
370
|
-
chainDepth: currentDepth + 1,
|
|
371
|
-
prevOutput: outcome.output || outcome.error || '',
|
|
372
|
-
prevTaskId: task.id,
|
|
373
|
-
prevStatus: outcome.status,
|
|
374
|
-
prevDurationMs: Date.now() - start,
|
|
375
|
-
prevCostUsd: outcome.costUsd || 0,
|
|
376
|
-
}).catch((err) => {
|
|
377
|
-
logger.warn(`[dsh-cron] chained task ${targetId} failed to trigger:`, err.message);
|
|
378
|
-
});
|
|
379
|
-
}
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
// 2.2 Trigger any other tasks in store that depend on this task via dependsOn (Fan-In join step)
|
|
383
|
-
if (chainRunId && scheduler.store?.tasks && scheduler.store.chainCheckpointManager) {
|
|
384
|
-
for (const candidate of Object.values(scheduler.store.tasks)) {
|
|
385
|
-
if (!candidate || candidate.id === taskId || targets.includes(candidate.id)) continue;
|
|
386
|
-
const deps = sanitizeDependsOn(candidate.dependsOn);
|
|
387
|
-
if (deps.includes(taskId)) {
|
|
388
|
-
const barrier = scheduler.store.chainCheckpointManager.checkJoinBarrier(chainRunId, candidate);
|
|
389
|
-
if (barrier.ready && scheduler.store.chainCheckpointManager.canTriggerBarrier(chainRunId, candidate.id)) {
|
|
390
|
-
scheduler.store.chainCheckpointManager.markBarrierTriggered(chainRunId, candidate.id);
|
|
391
|
-
logger.info(`[dsh-cron] triggering dependent joined task "${candidate.title}" (${candidate.id}) (barrier satisfied)`);
|
|
392
|
-
scheduler.runNow(candidate.id, {
|
|
393
|
-
chainRunId,
|
|
394
|
-
chainDepth: currentDepth + 1,
|
|
395
|
-
prevOutputs: barrier.outputs,
|
|
396
|
-
prevOutput: barrier.latestOutput,
|
|
397
|
-
prevTaskId: task.id,
|
|
398
|
-
prevStatus: outcome.status,
|
|
399
|
-
prevDurationMs: Date.now() - start,
|
|
400
|
-
prevCostUsd: outcome.costUsd || 0,
|
|
401
|
-
}).catch((err) => {
|
|
402
|
-
logger.warn(`[dsh-cron] dependent joined task ${candidate.id} failed to trigger:`, err.message);
|
|
403
|
-
});
|
|
404
|
-
} else if (barrier.pending) {
|
|
405
|
-
logger.info(`[dsh-cron] task "${candidate.title}" (${candidate.id}) waiting for dependencies: ${barrier.waitingFor?.join(', ')}`);
|
|
406
|
-
} else if (barrier.failed) {
|
|
407
|
-
logger.warn(`[dsh-cron] task "${candidate.title}" (${candidate.id}) barrier failed due to dependency: ${barrier.failedDep}`);
|
|
408
|
-
}
|
|
409
|
-
}
|
|
410
|
-
}
|
|
411
|
-
}
|
|
412
|
-
} else {
|
|
413
|
-
logger.warn(`[dsh-cron] chained task depth limit (5) reached for task ${task.id}. Loop prevented.`);
|
|
414
|
-
}
|
|
415
|
-
}
|
|
416
|
-
|
|
417
|
-
|
|
1
|
+
export * from './scheduler/scheduler-completion.js';
|