@goodandready/dsh-cron 0.2.52 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api.js +24 -0
- package/lib/artifacts.js +200 -0
- package/lib/chain-checkpoints.js +125 -0
- package/lib/client.js +89 -91
- package/lib/durable-journal.js +188 -0
- package/lib/prompt-interpolation.js +19 -0
- package/lib/routes.js +84 -0
- package/lib/runner.js +99 -1
- package/lib/scheduler-execution.js +58 -0
- package/lib/scheduler-lifecycle.js +6 -0
- package/lib/scheduler.js +35 -0
- package/lib/secrets.js +114 -0
- package/lib/settings.js +3 -1
- package/lib/store.js +50 -1
- package/lib/task-create.js +17 -0
- package/lib/task-patch.js +4 -0
- package/lib/task-transfer.js +1 -0
- package/lib/watcher.js +249 -0
- package/package.json +1 -1
package/lib/routes.js
CHANGED
|
@@ -44,6 +44,21 @@ export function registerRoutes(ctx, { store, scheduler, recommendations }) {
|
|
|
44
44
|
handler: createCronApiHandler(store, scheduler, null)
|
|
45
45
|
}), 'dsh-cron: /schedule[...]');
|
|
46
46
|
|
|
47
|
+
|
|
48
|
+
// Chain retry from failed step (#319)
|
|
49
|
+
ctx.effect(() => ctx.webServer.register({
|
|
50
|
+
kind: 'prefix',
|
|
51
|
+
path: '/dsh-cron/chains',
|
|
52
|
+
handler: (req, res) => handleChainRoutes(store, scheduler, req, res)
|
|
53
|
+
}), 'dsh-cron: /chains[...]');
|
|
54
|
+
|
|
55
|
+
// Run artifacts listing, download and safe preview (#321)
|
|
56
|
+
ctx.effect(() => ctx.webServer.register({
|
|
57
|
+
kind: 'prefix',
|
|
58
|
+
path: '/dsh-cron/runs',
|
|
59
|
+
handler: (req, res) => handleRunArtifactRoutes(store, req, res)
|
|
60
|
+
}), 'dsh-cron: /runs[...]');
|
|
61
|
+
|
|
47
62
|
// GET /dsh-cron/metrics (#53) — Prometheus text exposition
|
|
48
63
|
ctx.effect(() => ctx.webServer.register({
|
|
49
64
|
kind: 'exact',
|
|
@@ -198,3 +213,72 @@ export function registerRoutes(ctx, { store, scheduler, recommendations }) {
|
|
|
198
213
|
}), 'dsh-cron: /kanban/test');
|
|
199
214
|
}
|
|
200
215
|
|
|
216
|
+
|
|
217
|
+
/**
|
|
218
|
+
* Handler for chain step retries (#319)
|
|
219
|
+
*/
|
|
220
|
+
export async function handleChainRoutes(store, scheduler, req, res) {
|
|
221
|
+
if (req.method === 'POST' && rejectCrossOrigin(req, res)) return;
|
|
222
|
+
const url = new URL(req.url, 'http://localhost');
|
|
223
|
+
const m = url.pathname.match(/^\/dsh-cron\/chains\/([^\/]+)\/retry-from\/([^\/]+)$/);
|
|
224
|
+
if (!m) {
|
|
225
|
+
sendJson(res, 404, { ok: false, error: 'Route not found' });
|
|
226
|
+
return;
|
|
227
|
+
}
|
|
228
|
+
if (req.method !== 'POST') {
|
|
229
|
+
sendJson(res, 405, { ok: false, error: 'Method not allowed' });
|
|
230
|
+
return;
|
|
231
|
+
}
|
|
232
|
+
const [, chainRunId, stepTaskId] = m;
|
|
233
|
+
const result = await store.chainCheckpointManager.retryChainFromStep(scheduler, chainRunId, stepTaskId);
|
|
234
|
+
if (!result.ok) {
|
|
235
|
+
sendJson(res, 400, result);
|
|
236
|
+
return;
|
|
237
|
+
}
|
|
238
|
+
sendJson(res, 200, result);
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Handler for run artifacts (#321)
|
|
243
|
+
*/
|
|
244
|
+
export async function handleRunArtifactRoutes(store, req, res) {
|
|
245
|
+
const url = new URL(req.url, 'http://localhost');
|
|
246
|
+
const m = url.pathname.match(/^\/dsh-cron\/runs\/([^\/]+)\/artifacts(?:\/([^\/]+)(?:\/(preview))?)?$/);
|
|
247
|
+
if (!m) {
|
|
248
|
+
sendJson(res, 404, { ok: false, error: 'Route not found' });
|
|
249
|
+
return;
|
|
250
|
+
}
|
|
251
|
+
if (req.method !== 'GET') {
|
|
252
|
+
sendJson(res, 405, { ok: false, error: 'Method not allowed' });
|
|
253
|
+
return;
|
|
254
|
+
}
|
|
255
|
+
const [, runId, artifactId, action] = m;
|
|
256
|
+
if (!artifactId) {
|
|
257
|
+
const artifacts = store.getArtifactsForRun(runId);
|
|
258
|
+
sendJson(res, 200, { ok: true, runId, artifacts });
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
const artifacts = store.getArtifactsForRun(runId);
|
|
262
|
+
const record = artifacts.find((a) => a.id === artifactId);
|
|
263
|
+
if (!record) {
|
|
264
|
+
sendJson(res, 404, { ok: false, error: 'Artifact record not found' });
|
|
265
|
+
return;
|
|
266
|
+
}
|
|
267
|
+
if (action === 'preview') {
|
|
268
|
+
const preview = store.artifactManager.generatePreview(artifactId, record.name);
|
|
269
|
+
sendJson(res, preview.ok ? 200 : 404, preview);
|
|
270
|
+
return;
|
|
271
|
+
}
|
|
272
|
+
const buf = store.artifactManager.readArtifact(artifactId, record.name);
|
|
273
|
+
if (!buf) {
|
|
274
|
+
sendJson(res, 404, { ok: false, error: 'Artifact file missing from disk (expired or removed)' });
|
|
275
|
+
return;
|
|
276
|
+
}
|
|
277
|
+
res.writeHead(200, {
|
|
278
|
+
'Content-Type': record.mimeType || 'application/octet-stream',
|
|
279
|
+
'Content-Disposition': `attachment; filename="${encodeURIComponent(record.name)}"`,
|
|
280
|
+
'Content-Length': buf.length,
|
|
281
|
+
'X-Content-Type-Options': 'nosniff',
|
|
282
|
+
});
|
|
283
|
+
res.end(buf);
|
|
284
|
+
}
|
package/lib/runner.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { logger } from './logger.js';
|
|
2
2
|
import { bestEffort } from './best-effort.js';
|
|
3
3
|
import { estimateTokenCost } from './integrations.js';
|
|
4
|
+
import { resolveDynamicCredentialsInText, resolveDynamicCredentialsInObject } from './secrets.js';
|
|
4
5
|
import {
|
|
5
6
|
EXTERNAL_RUNTIMES,
|
|
6
7
|
normalizeTaskType,
|
|
@@ -144,7 +145,104 @@ export class SessionRunner {
|
|
|
144
145
|
async _run(task, signal, options = {}) {
|
|
145
146
|
logger.info(`[dsh-cron] executing task "${task.title}" (${task.id})`);
|
|
146
147
|
|
|
147
|
-
//
|
|
148
|
+
// Dynamic credentials resolution in-memory (#335)
|
|
149
|
+
let effectiveTask = task;
|
|
150
|
+
const resolvedSecrets = [];
|
|
151
|
+
const settings = (this.store && typeof this.store.getSettings === 'function')
|
|
152
|
+
? this.store.getSettings()
|
|
153
|
+
: {};
|
|
154
|
+
|
|
155
|
+
if (settings.enableDynamicCredentials) {
|
|
156
|
+
effectiveTask = { ...task };
|
|
157
|
+
if (typeof effectiveTask.prompt === 'string') {
|
|
158
|
+
const res = await resolveDynamicCredentialsInText(effectiveTask.prompt, this.ctx);
|
|
159
|
+
effectiveTask.prompt = res.text;
|
|
160
|
+
resolvedSecrets.push(...res.resolvedSecrets);
|
|
161
|
+
}
|
|
162
|
+
if (typeof effectiveTask.script === 'string') {
|
|
163
|
+
const res = await resolveDynamicCredentialsInText(effectiveTask.script, this.ctx);
|
|
164
|
+
effectiveTask.script = res.text;
|
|
165
|
+
resolvedSecrets.push(...res.resolvedSecrets);
|
|
166
|
+
}
|
|
167
|
+
if (typeof effectiveTask.command === 'string') {
|
|
168
|
+
const res = await resolveDynamicCredentialsInText(effectiveTask.command, this.ctx);
|
|
169
|
+
effectiveTask.command = res.text;
|
|
170
|
+
resolvedSecrets.push(...res.resolvedSecrets);
|
|
171
|
+
}
|
|
172
|
+
if (typeof effectiveTask.httpUrl === 'string') {
|
|
173
|
+
const res = await resolveDynamicCredentialsInText(effectiveTask.httpUrl, this.ctx);
|
|
174
|
+
effectiveTask.httpUrl = res.text;
|
|
175
|
+
resolvedSecrets.push(...res.resolvedSecrets);
|
|
176
|
+
}
|
|
177
|
+
if (typeof effectiveTask.httpBody === 'string') {
|
|
178
|
+
const res = await resolveDynamicCredentialsInText(effectiveTask.httpBody, this.ctx);
|
|
179
|
+
effectiveTask.httpBody = res.text;
|
|
180
|
+
resolvedSecrets.push(...res.resolvedSecrets);
|
|
181
|
+
}
|
|
182
|
+
if (effectiveTask.httpHeaders) {
|
|
183
|
+
const res = await resolveDynamicCredentialsInObject(effectiveTask.httpHeaders, this.ctx);
|
|
184
|
+
effectiveTask.httpHeaders = res.data;
|
|
185
|
+
resolvedSecrets.push(...res.resolvedSecrets);
|
|
186
|
+
}
|
|
187
|
+
if (effectiveTask.env) {
|
|
188
|
+
const res = await resolveDynamicCredentialsInObject(effectiveTask.env, this.ctx);
|
|
189
|
+
effectiveTask.env = res.data;
|
|
190
|
+
resolvedSecrets.push(...res.resolvedSecrets);
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const maskSecrets = (result) => {
|
|
195
|
+
if (!result || typeof result !== 'object') return result;
|
|
196
|
+
if (resolvedSecrets.length > 0 && typeof result.output === 'string') {
|
|
197
|
+
let out = result.output;
|
|
198
|
+
for (const secret of resolvedSecrets) {
|
|
199
|
+
if (secret && String(secret).length >= 4 && out.includes(secret)) {
|
|
200
|
+
out = out.replaceAll(secret, '[REDACTED_CREDENTIAL]');
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
result.output = out;
|
|
204
|
+
}
|
|
205
|
+
return result;
|
|
206
|
+
};
|
|
207
|
+
|
|
208
|
+
const runInternal = async () => {
|
|
209
|
+
// 1. Non-LLM runtimes: shell, Node.js, Python, HTTP, SSH, Docker (#4 #5 #7 #8 #9)
|
|
210
|
+
const runtime = normalizeTaskType(effectiveTask.type);
|
|
211
|
+
if (EXTERNAL_RUNTIMES.includes(runtime) || effectiveTask.mode === 'no-llm') {
|
|
212
|
+
return this._runExternal(effectiveTask, runtime === 'llm' ? 'script' : runtime, signal, options);
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
// 2. Agent session, when the harness exposes one
|
|
216
|
+
if (this.ctx && this.ctx.agents && typeof this.ctx.agents.create === 'function') {
|
|
217
|
+
return this._runAgentSession(effectiveTask, runtime, signal, options);
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// 3. Legacy session service
|
|
221
|
+
if (this.ctx && this.ctx.session && typeof this.ctx.session.create === 'function') {
|
|
222
|
+
return this._runLegacySession(effectiveTask, options);
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
// 4. Nothing to run through: report the prompt as triggered
|
|
226
|
+
return this._runBare(effectiveTask, options);
|
|
227
|
+
};
|
|
228
|
+
|
|
229
|
+
try {
|
|
230
|
+
const outcome = await runInternal();
|
|
231
|
+
return maskSecrets(outcome);
|
|
232
|
+
} catch (err) {
|
|
233
|
+
if (resolvedSecrets.length > 0 && err && err.message) {
|
|
234
|
+
for (const secret of resolvedSecrets) {
|
|
235
|
+
if (secret && String(secret).length >= 4 && err.message.includes(secret)) {
|
|
236
|
+
err.message = err.message.replaceAll(secret, '[REDACTED_CREDENTIAL]');
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
throw err;
|
|
241
|
+
}
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Legacy dispatch replaced above; original runner helpers remain unchanged below:
|
|
245
|
+
async _runOriginalDispatch(task, signal, options = {}) {
|
|
148
246
|
const runtime = normalizeTaskType(task.type);
|
|
149
247
|
if (EXTERNAL_RUNTIMES.includes(runtime) || task.mode === 'no-llm') {
|
|
150
248
|
return this._runExternal(task, runtime === 'llm' ? 'script' : runtime, signal, options);
|
|
@@ -310,6 +310,23 @@ export async function executeTask(scheduler, taskId, options = {}) {
|
|
|
310
310
|
model: outcome.model || task.model || '',
|
|
311
311
|
};
|
|
312
312
|
}
|
|
313
|
+
// 0. Idempotency Check (#317): deduplicate runs with an already completed idempotency key
|
|
314
|
+
if (options.idempotencyKey && scheduler.store.durableJournal) {
|
|
315
|
+
const existing = scheduler.store.durableJournal.isIdempotencyKeyExecuted(taskId, options.idempotencyKey);
|
|
316
|
+
if (existing) {
|
|
317
|
+
logger.info(`[dsh-cron] Idempotency key "${options.idempotencyKey}" already executed for task "${taskId}", returning cached run`);
|
|
318
|
+
return {
|
|
319
|
+
ok: existing.status === 'completed' || existing.status === 'success',
|
|
320
|
+
idempotentReplay: true,
|
|
321
|
+
runId: existing.id,
|
|
322
|
+
status: existing.status,
|
|
323
|
+
output: existing.output,
|
|
324
|
+
costUsd: existing.costUsd,
|
|
325
|
+
durationMs: existing.durationMs,
|
|
326
|
+
};
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
|
|
313
330
|
|
|
314
331
|
// 1. Approval Gate (#318): Human-in-the-loop check before executing actions requiring approval
|
|
315
332
|
if (task.requireApproval && !options.approvedBy) {
|
|
@@ -699,6 +716,46 @@ export async function handleCompleteRun(scheduler, task, taskId, currentRun, out
|
|
|
699
716
|
kind: (Number(outcome.costUsd) > 0) ? 'exact' : ((task.type === 'llm' || task.type === 'skill' || task.type === 'workflow') ? 'unknown' : 'exact'),
|
|
700
717
|
};
|
|
701
718
|
runInfo.stoppedReason = computeStoppedReason(outcome.status, valRes, outcome.error);
|
|
719
|
+
// Durable journal & Idempotency (#317)
|
|
720
|
+
runInfo.idempotencyKey = options.idempotencyKey || currentRun?.idempotencyKey || null;
|
|
721
|
+
|
|
722
|
+
// Artifacts capture (#321)
|
|
723
|
+
if (Array.isArray(outcome.artifacts) && scheduler.store.artifactManager) {
|
|
724
|
+
for (const art of outcome.artifacts) {
|
|
725
|
+
if (art && art.name && (art.content || art.data)) {
|
|
726
|
+
try {
|
|
727
|
+
scheduler.store.artifactManager.saveArtifact(runInfo.runId, taskId, art.name, art.content || art.data);
|
|
728
|
+
} catch (artErr) {
|
|
729
|
+
logger.warn(`[dsh-cron] failed to save artifact ${art.name}:`, artErr.message);
|
|
730
|
+
}
|
|
731
|
+
}
|
|
732
|
+
}
|
|
733
|
+
}
|
|
734
|
+
if (scheduler.store.getArtifactsForRun) {
|
|
735
|
+
runInfo.artifacts = scheduler.store.getArtifactsForRun(runInfo.runId);
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
// Chain Checkpoints (#319)
|
|
739
|
+
const chainedTarget = outcome.status === 'success' ? task.onSuccess : task.onFailure;
|
|
740
|
+
const chainRunId = options.chainRunId || (chainedTarget ? ('chain_' + Math.random().toString(36).slice(2, 10)) : null);
|
|
741
|
+
if (chainRunId) {
|
|
742
|
+
runInfo.chainRunId = chainRunId;
|
|
743
|
+
if (scheduler.store.chainCheckpointManager) {
|
|
744
|
+
scheduler.store.chainCheckpointManager.recordStepSnapshot(chainRunId, taskId, {
|
|
745
|
+
status: outcome.status,
|
|
746
|
+
input: { prompt: task.prompt, prevOutput: options.prevOutput || options.chainPrevOutput || '' },
|
|
747
|
+
output: outcome.output || outcome.error || '',
|
|
748
|
+
durationMs: executionMs,
|
|
749
|
+
});
|
|
750
|
+
}
|
|
751
|
+
}
|
|
752
|
+
|
|
753
|
+
// Update Durable Journal (#317)
|
|
754
|
+
if (scheduler.store.durableJournal) {
|
|
755
|
+
const journalState = outcome.status === 'success' ? 'completed' : (outcome.status === 'uncertain' ? 'uncertain' : 'failed');
|
|
756
|
+
scheduler.store.durableJournal.transition(taskId, runInfo.runId, journalState, runInfo);
|
|
757
|
+
}
|
|
758
|
+
|
|
702
759
|
|
|
703
760
|
try {
|
|
704
761
|
scheduler.store.recordRun(taskId, runInfo);
|
|
@@ -795,6 +852,7 @@ export async function handleCompleteRun(scheduler, task, taskId, currentRun, out
|
|
|
795
852
|
if (nextTask) {
|
|
796
853
|
logger.info(`[dsh-cron] triggering chained task "${nextTask.title}" (${targetId}) from "${task.title}" (depth ${currentDepth + 1})`);
|
|
797
854
|
scheduler.runNow(targetId, {
|
|
855
|
+
chainRunId,
|
|
798
856
|
chainDepth: currentDepth + 1,
|
|
799
857
|
prevOutput: outcome.output || outcome.error || '',
|
|
800
858
|
prevTaskId: task.id,
|
|
@@ -92,6 +92,9 @@ export function pauseTask(scheduler, taskId, reason = null) {
|
|
|
92
92
|
clearTimeout(scheduler.timers.get(taskId));
|
|
93
93
|
scheduler.timers.delete(taskId);
|
|
94
94
|
}
|
|
95
|
+
if (scheduler.watcherManager) {
|
|
96
|
+
scheduler.watcherManager.unwatchTask(taskId);
|
|
97
|
+
}
|
|
95
98
|
const running = scheduler.running.get(taskId);
|
|
96
99
|
if (running) {
|
|
97
100
|
bestEffort('abort-paused-task', () => running.controller.abort(new Error('Task paused')), scheduler.logger);
|
|
@@ -115,6 +118,9 @@ export function removeTask(scheduler, taskId) {
|
|
|
115
118
|
clearTimeout(scheduler.timers.get(taskId));
|
|
116
119
|
scheduler.timers.delete(taskId);
|
|
117
120
|
}
|
|
121
|
+
if (scheduler.watcherManager) {
|
|
122
|
+
scheduler.watcherManager.unwatchTask(taskId);
|
|
123
|
+
}
|
|
118
124
|
const running = scheduler.running.get(taskId);
|
|
119
125
|
if (running) {
|
|
120
126
|
bestEffort('abort-deleted-task', () => running.controller.abort(new Error('Task deleted')), scheduler.logger);
|
package/lib/scheduler.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { logger } from './logger.js';
|
|
2
2
|
import { bestEffort } from './best-effort.js';
|
|
3
3
|
import { Cron } from 'croner';
|
|
4
|
+
import { FileSystemWatcherManager } from './watcher.js';
|
|
4
5
|
|
|
5
6
|
const MAX_TIMEOUT_MS = 2147483647; // 2^31 - 1 ms (~24.85 days)
|
|
6
7
|
import {
|
|
@@ -51,6 +52,7 @@ export {
|
|
|
51
52
|
removeTask,
|
|
52
53
|
resumeTask,
|
|
53
54
|
toggleTask,
|
|
55
|
+
FileSystemWatcherManager,
|
|
54
56
|
};
|
|
55
57
|
|
|
56
58
|
function pad(v) {
|
|
@@ -320,6 +322,7 @@ export class TaskScheduler {
|
|
|
320
322
|
this.queue = [];
|
|
321
323
|
this.groupLimits = opts.groupLimits instanceof Map ? opts.groupLimits : new Map(Object.entries(opts.groupLimits || {}));
|
|
322
324
|
this.heartbeatTimer = null;
|
|
325
|
+
this.watcherManager = new FileSystemWatcherManager(this);
|
|
323
326
|
this.logger = opts.logger || null;
|
|
324
327
|
this.isStopped = false;
|
|
325
328
|
this.generation = 0;
|
|
@@ -361,6 +364,26 @@ export class TaskScheduler {
|
|
|
361
364
|
start() {
|
|
362
365
|
this.isStopped = false;
|
|
363
366
|
this.generation = (this.generation || 0) + 1;
|
|
367
|
+
|
|
368
|
+
// Recovery & cleanup on restart (#317, #321)
|
|
369
|
+
if (this.store && this.store.durableJournal) {
|
|
370
|
+
try {
|
|
371
|
+
const recovered = this.store.durableJournal.recoverPendingRuns(this);
|
|
372
|
+
if (recovered.uncertain.length > 0) {
|
|
373
|
+
logger.warn(`[dsh-cron] Recovered ${recovered.uncertain.length} in-flight run(s) interrupted by restart -> status marked uncertain`);
|
|
374
|
+
}
|
|
375
|
+
} catch (recErr) {
|
|
376
|
+
logger.error('[dsh-cron] restart recovery failed:', recErr.message);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
if (this.store && this.store.artifactManager) {
|
|
380
|
+
try {
|
|
381
|
+
this.store.artifactManager.cleanupArtifacts(this.store);
|
|
382
|
+
} catch (cleanErr) {
|
|
383
|
+
logger.warn('[dsh-cron] artifact cleanup failed:', cleanErr.message);
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
|
|
364
387
|
const tasks = this.store.list();
|
|
365
388
|
const now = Date.now();
|
|
366
389
|
for (const task of tasks) {
|
|
@@ -396,12 +419,18 @@ export class TaskScheduler {
|
|
|
396
419
|
this.scheduleTask(task);
|
|
397
420
|
}
|
|
398
421
|
this.startHeartbeatWatcher();
|
|
422
|
+
if (this.watcherManager) {
|
|
423
|
+
this.watcherManager.syncAll(tasks);
|
|
424
|
+
}
|
|
399
425
|
}
|
|
400
426
|
|
|
401
427
|
stopAll() {
|
|
402
428
|
this.isStopped = true;
|
|
403
429
|
this.generation = (this.generation || 0) + 1;
|
|
404
430
|
this.stopHeartbeatWatcher();
|
|
431
|
+
if (this.watcherManager) {
|
|
432
|
+
this.watcherManager.stopAll();
|
|
433
|
+
}
|
|
405
434
|
|
|
406
435
|
// Clear concurrency queue (#217)
|
|
407
436
|
this.queue = [];
|
|
@@ -432,6 +461,9 @@ export class TaskScheduler {
|
|
|
432
461
|
}
|
|
433
462
|
|
|
434
463
|
clearScheduled(taskId) {
|
|
464
|
+
if (this.watcherManager) {
|
|
465
|
+
this.watcherManager.unwatchTask(taskId);
|
|
466
|
+
}
|
|
435
467
|
if (this.jobs.has(taskId)) {
|
|
436
468
|
this.jobs.get(taskId).stop();
|
|
437
469
|
this.jobs.delete(taskId);
|
|
@@ -496,6 +528,9 @@ export class TaskScheduler {
|
|
|
496
528
|
this.clearScheduled(task.id);
|
|
497
529
|
if (task.status !== 'active') return;
|
|
498
530
|
this.clearRetryTimer(task.id);
|
|
531
|
+
if (this.watcherManager) {
|
|
532
|
+
this.watcherManager.watchTask(task);
|
|
533
|
+
}
|
|
499
534
|
|
|
500
535
|
try {
|
|
501
536
|
const parsed = parseScheduleExpression(task.schedule);
|
package/lib/secrets.js
CHANGED
|
@@ -120,3 +120,117 @@ export async function resolveTelegramSecrets(ctx, settings = {}) {
|
|
|
120
120
|
out.chatId = String(settings.chatId || '').trim();
|
|
121
121
|
return out;
|
|
122
122
|
}
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
// --- Dynamic credential resolution (#335) ---
|
|
126
|
+
export const CREDENTIAL_URI_REGEX = /^credential:\/\/(?:([a-zA-Z0-9_-]+)\/)?([a-zA-Z0-9_.-]+)$/;
|
|
127
|
+
export const CREDENTIAL_URI_GLOBAL_REGEX = /credential:\/\/(?:[a-zA-Z0-9_-]+\/)?[a-zA-Z0-9_.-]+/g;
|
|
128
|
+
|
|
129
|
+
/** Check if a string is a credential:// reference URI. */
|
|
130
|
+
export function isCredentialUri(value) {
|
|
131
|
+
return typeof value === 'string' && CREDENTIAL_URI_REGEX.test(value.trim());
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** Parse a credential://[domain/]<key> URI into domain and key components. */
|
|
135
|
+
export function parseCredentialUri(value) {
|
|
136
|
+
const match = String(value || '').trim().match(CREDENTIAL_URI_REGEX);
|
|
137
|
+
if (!match) return null;
|
|
138
|
+
return {
|
|
139
|
+
domain: match[1] || null,
|
|
140
|
+
key: match[2],
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* Resolve a single credential:// URI using harness credentials service or environment.
|
|
146
|
+
* If not resolvable, returns the original URI.
|
|
147
|
+
*/
|
|
148
|
+
export async function resolveCredentialReference(uri, ctx) {
|
|
149
|
+
if (typeof uri !== 'string') return uri;
|
|
150
|
+
const parsed = parseCredentialUri(uri);
|
|
151
|
+
if (!parsed) return uri;
|
|
152
|
+
|
|
153
|
+
try {
|
|
154
|
+
const creds = (ctx && typeof ctx.get === 'function' ? ctx.get('credentials') : null)
|
|
155
|
+
|| (ctx && ctx.credentials);
|
|
156
|
+
if (creds && typeof creds.resolve === 'function') {
|
|
157
|
+
const hit = await creds.resolve(parsed.domain ? { domain: parsed.domain, name: parsed.key } : parsed.key);
|
|
158
|
+
if (hit && hit.value) return String(hit.value);
|
|
159
|
+
}
|
|
160
|
+
} catch {
|
|
161
|
+
// credentials service unavailable
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// Fallback: resolve by key name
|
|
165
|
+
const fallback = await resolveCredentialValue(ctx, parsed.key);
|
|
166
|
+
if (fallback) return fallback;
|
|
167
|
+
|
|
168
|
+
// Fallback: check environment variable with DOMAIN_KEY
|
|
169
|
+
if (parsed.domain) {
|
|
170
|
+
const prefixedEnv = process.env[`${parsed.domain.toUpperCase()}_${parsed.key.toUpperCase()}`];
|
|
171
|
+
if (prefixedEnv) return String(prefixedEnv);
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
return uri;
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
/**
|
|
178
|
+
* Replace all credential://... references in a string with resolved secret values.
|
|
179
|
+
* Returns { text: string, resolvedSecrets: string[] }.
|
|
180
|
+
*/
|
|
181
|
+
export async function resolveDynamicCredentialsInText(text, ctx) {
|
|
182
|
+
if (typeof text !== 'string' || !text.includes('credential://')) {
|
|
183
|
+
return { text, resolvedSecrets: [] };
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
const matches = text.match(CREDENTIAL_URI_GLOBAL_REGEX);
|
|
187
|
+
if (!matches || !matches.length) {
|
|
188
|
+
return { text, resolvedSecrets: [] };
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
let result = text;
|
|
192
|
+
const resolvedSecrets = [];
|
|
193
|
+
|
|
194
|
+
for (const match of matches) {
|
|
195
|
+
const resolved = await resolveCredentialReference(match, ctx);
|
|
196
|
+
if (resolved && resolved !== match) {
|
|
197
|
+
result = result.replaceAll(match, resolved);
|
|
198
|
+
resolvedSecrets.push(resolved);
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
return { text: result, resolvedSecrets };
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* Recursively resolve credential://... references in strings, arrays, or objects.
|
|
207
|
+
* Returns { data: any, resolvedSecrets: string[] }.
|
|
208
|
+
*/
|
|
209
|
+
export async function resolveDynamicCredentialsInObject(obj, ctx) {
|
|
210
|
+
if (obj === null || obj === undefined) return { data: obj, resolvedSecrets: [] };
|
|
211
|
+
if (typeof obj === 'string') {
|
|
212
|
+
const textRes = await resolveDynamicCredentialsInText(obj, ctx);
|
|
213
|
+
return { data: textRes.text, resolvedSecrets: textRes.resolvedSecrets };
|
|
214
|
+
}
|
|
215
|
+
if (Array.isArray(obj)) {
|
|
216
|
+
const outArr = [];
|
|
217
|
+
const allSecrets = [];
|
|
218
|
+
for (const item of obj) {
|
|
219
|
+
const res = await resolveDynamicCredentialsInObject(item, ctx);
|
|
220
|
+
outArr.push(res.data);
|
|
221
|
+
allSecrets.push(...res.resolvedSecrets);
|
|
222
|
+
}
|
|
223
|
+
return { data: outArr, resolvedSecrets: allSecrets };
|
|
224
|
+
}
|
|
225
|
+
if (typeof obj === 'object') {
|
|
226
|
+
const outObj = {};
|
|
227
|
+
const allSecrets = [];
|
|
228
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
229
|
+
const res = await resolveDynamicCredentialsInObject(v, ctx);
|
|
230
|
+
outObj[k] = res.data;
|
|
231
|
+
allSecrets.push(...res.resolvedSecrets);
|
|
232
|
+
}
|
|
233
|
+
return { data: outObj, resolvedSecrets: allSecrets };
|
|
234
|
+
}
|
|
235
|
+
return { data: obj, resolvedSecrets: [] };
|
|
236
|
+
}
|
package/lib/settings.js
CHANGED
|
@@ -47,6 +47,8 @@ export const Config = z.object({
|
|
|
47
47
|
giteaLabels: z.any().default([]).description('Gitea label names or IDs for alert issues').volatile(),
|
|
48
48
|
giteaLabelIds: z.any().default({}).description('Mapping of label name to integer ID for Gitea issues').volatile(),
|
|
49
49
|
prReviewerEnabled: z.boolean().default(false).description('Enable automated PR Reviewer scheduled agent (#33)').volatile(),
|
|
50
|
+
// --- Dynamic credentials resolution (#335) ---
|
|
51
|
+
enableDynamicCredentials: z.boolean().default(false).description('Enable dynamic credential://... reference resolution at runtime (#335)').volatile(),
|
|
50
52
|
// --- Global Budgets (#320) ---
|
|
51
53
|
globalDailyCostLimitUsd: z.number().default(0).description('Global daily expenditure limit in USD across all tasks (0 = unlimited)').volatile(),
|
|
52
54
|
providerHourlyRunLimits: z.object({}).default({}).description('Max runs per hour keyed by provider').volatile(),
|
|
@@ -85,7 +87,7 @@ export const SETTINGS_SYNC_KEYS = [
|
|
|
85
87
|
'pushplusTokenRef',
|
|
86
88
|
'pushplusUrl',
|
|
87
89
|
'ttsBaseUrl',
|
|
88
|
-
'giteaBaseUrl', 'giteaRepo', 'giteaTokenRef', 'giteaLabels', 'giteaLabelIds', 'prReviewerEnabled',
|
|
90
|
+
'giteaBaseUrl', 'giteaRepo', 'giteaTokenRef', 'giteaLabels', 'giteaLabelIds', 'prReviewerEnabled', 'enableDynamicCredentials',
|
|
89
91
|
'apiToken',
|
|
90
92
|
'globalDailyCostLimitUsd', 'providerHourlyRunLimits', 'modelHourlyRunLimits',
|
|
91
93
|
];
|
package/lib/store.js
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { DurableJournal } from './durable-journal.js';
|
|
2
|
+
import { ChainCheckpointManager } from './chain-checkpoints.js';
|
|
3
|
+
import { ArtifactManager } from './artifacts.js';
|
|
1
4
|
import { logger } from './logger.js';
|
|
2
5
|
import { bestEffort } from './best-effort.js';
|
|
3
6
|
import fs from 'node:fs';
|
|
@@ -36,6 +39,12 @@ export class TaskStore {
|
|
|
36
39
|
this.tasks = new Map();
|
|
37
40
|
this.history = new Map();
|
|
38
41
|
this.settings = {};
|
|
42
|
+
this.chainCheckpoints = {};
|
|
43
|
+
this.artifacts = {};
|
|
44
|
+
const storeDir = path.dirname(this.filePath);
|
|
45
|
+
this.durableJournal = new DurableJournal(this);
|
|
46
|
+
this.chainCheckpointManager = new ChainCheckpointManager(this);
|
|
47
|
+
this.artifactManager = new ArtifactManager(path.join(storeDir, "artifacts"));
|
|
39
48
|
this._lastBackupTime = 0;
|
|
40
49
|
this._saveTimer = null;
|
|
41
50
|
this._savePromise = Promise.resolve();
|
|
@@ -85,6 +94,12 @@ export class TaskStore {
|
|
|
85
94
|
if (data.settings && typeof data.settings === 'object') {
|
|
86
95
|
this.settings = data.settings;
|
|
87
96
|
}
|
|
97
|
+
if (data.chainCheckpoints && typeof data.chainCheckpoints === 'object') {
|
|
98
|
+
this.chainCheckpoints = data.chainCheckpoints;
|
|
99
|
+
}
|
|
100
|
+
if (data.artifacts && typeof data.artifacts === 'object') {
|
|
101
|
+
this.artifacts = data.artifacts;
|
|
102
|
+
}
|
|
88
103
|
};
|
|
89
104
|
|
|
90
105
|
if (fs.existsSync(this.filePath)) {
|
|
@@ -189,6 +204,8 @@ export class TaskStore {
|
|
|
189
204
|
tasks: tasksSnapshot,
|
|
190
205
|
history: historySnapshot,
|
|
191
206
|
settings: this.settings ? { ...this.settings } : {},
|
|
207
|
+
chainCheckpoints: this.chainCheckpoints ? { ...this.chainCheckpoints } : {},
|
|
208
|
+
artifacts: this.artifacts ? { ...this.artifacts } : {},
|
|
192
209
|
};
|
|
193
210
|
}
|
|
194
211
|
|
|
@@ -347,7 +364,10 @@ export class TaskStore {
|
|
|
347
364
|
yield ']';
|
|
348
365
|
await new Promise((r) => setImmediate(r));
|
|
349
366
|
}
|
|
350
|
-
yield '}
|
|
367
|
+
yield '}';
|
|
368
|
+
yield ',"chainCheckpoints":' + JSON.stringify(data.chainCheckpoints || {});
|
|
369
|
+
yield ',"artifacts":' + JSON.stringify(data.artifacts || {});
|
|
370
|
+
yield '}';
|
|
351
371
|
}
|
|
352
372
|
|
|
353
373
|
try {
|
|
@@ -407,6 +427,32 @@ export class TaskStore {
|
|
|
407
427
|
}
|
|
408
428
|
}
|
|
409
429
|
|
|
430
|
+
getRun(runId) {
|
|
431
|
+
if (!runId) return null;
|
|
432
|
+
for (const [taskId, runs] of this.history.entries()) {
|
|
433
|
+
if (Array.isArray(runs)) {
|
|
434
|
+
const found = runs.find((r) => r.id === runId);
|
|
435
|
+
if (found) return { ...found, taskId };
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
return null;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
recordArtifact(runIdOrRecord, maybeRecord) {
|
|
442
|
+
const runId = maybeRecord ? runIdOrRecord : (runIdOrRecord && runIdOrRecord.runId);
|
|
443
|
+
const artifactRecord = maybeRecord ? maybeRecord : runIdOrRecord;
|
|
444
|
+
if (!runId || !artifactRecord) return;
|
|
445
|
+
this.artifacts = this.artifacts || {};
|
|
446
|
+
this.artifacts[runId] = this.artifacts[runId] || [];
|
|
447
|
+
this.artifacts[runId].push(artifactRecord);
|
|
448
|
+
this.scheduleSave(50);
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
getArtifactsForRun(runId) {
|
|
452
|
+
if (!runId || !this.artifacts) return [];
|
|
453
|
+
return this.artifacts[runId] || [];
|
|
454
|
+
}
|
|
455
|
+
|
|
410
456
|
getSettings() {
|
|
411
457
|
const defaults = getDshDefaultTelegramCredentials();
|
|
412
458
|
const base = { ...this.settings };
|
|
@@ -706,6 +752,9 @@ export class TaskStore {
|
|
|
706
752
|
},
|
|
707
753
|
delivery: runInfo.delivery || null,
|
|
708
754
|
validation: runInfo.validation || null,
|
|
755
|
+
idempotencyKey: runInfo.idempotencyKey || null,
|
|
756
|
+
chainRunId: runInfo.chainRunId || null,
|
|
757
|
+
artifacts: runInfo.artifacts || null,
|
|
709
758
|
sessionId: runInfo.sessionId || null,
|
|
710
759
|
// #45: which model produced the run, and whether it only finished thanks
|
|
711
760
|
// to the configured fallback.
|
package/lib/task-create.js
CHANGED
|
@@ -5,6 +5,22 @@ import { parseScheduleExpression } from './scheduler.js';
|
|
|
5
5
|
import { CHANNEL_IDS } from './channels.js';
|
|
6
6
|
import { redactTaskSecrets } from './http-utils.js';
|
|
7
7
|
|
|
8
|
+
export function sanitizeFileWatch(fw) {
|
|
9
|
+
if (!fw || typeof fw !== 'object') return undefined;
|
|
10
|
+
const p = String(fw.path || '').trim();
|
|
11
|
+
if (!p) return undefined;
|
|
12
|
+
return {
|
|
13
|
+
enabled: Boolean(fw.enabled !== false),
|
|
14
|
+
path: p,
|
|
15
|
+
pattern: fw.pattern ? String(fw.pattern).trim() : '*',
|
|
16
|
+
events: Array.isArray(fw.events) && fw.events.length > 0
|
|
17
|
+
? fw.events.map(String).filter((e) => ['create', 'modify', 'delete'].includes(e))
|
|
18
|
+
: ['create', 'modify'],
|
|
19
|
+
debounceMs: Math.max(50, Number(fw.debounceMs) || 500),
|
|
20
|
+
recursive: Boolean(fw.recursive),
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
|
|
8
24
|
export function executeCreateTask(store, scheduler, args) {
|
|
9
25
|
const taskType = normalizeTaskType(args.type);
|
|
10
26
|
const typeError = validateTaskType(taskType, args);
|
|
@@ -75,6 +91,7 @@ export function executeCreateTask(store, scheduler, args) {
|
|
|
75
91
|
tokenLimit: Number(args.tokenLimit) > 0 ? Number(args.tokenLimit) : undefined,
|
|
76
92
|
validators: Array.isArray(args.validators) ? sanitizeValidators(args.validators) : undefined,
|
|
77
93
|
requireApproval: Boolean(args.requireApproval),
|
|
94
|
+
fileWatch: args.fileWatch && typeof args.fileWatch === 'object' ? sanitizeFileWatch(args.fileWatch) : undefined,
|
|
78
95
|
};
|
|
79
96
|
// Issue #310 / GH #5: strip any undefined properties before store.set()
|
|
80
97
|
for (const k of Object.keys(taskData)) {
|
package/lib/task-patch.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { sanitizeValidators } from './validators.js';
|
|
2
|
+
import { sanitizeFileWatch } from './task-create.js';
|
|
2
3
|
/**
|
|
3
4
|
* Partial task updates, shared by the HTTP PATCH route and the cron tool (action: 'update')
|
|
4
5
|
* tool (#49).
|
|
@@ -90,6 +91,9 @@ export function buildTaskPatch(current, body) {
|
|
|
90
91
|
if (patch.requireApproval !== undefined) {
|
|
91
92
|
patch.requireApproval = Boolean(patch.requireApproval);
|
|
92
93
|
}
|
|
94
|
+
if (patch.fileWatch !== undefined) {
|
|
95
|
+
patch.fileWatch = patch.fileWatch && typeof patch.fileWatch === 'object' ? sanitizeFileWatch(patch.fileWatch) : null;
|
|
96
|
+
}
|
|
93
97
|
if (patch.validators !== undefined) {
|
|
94
98
|
patch.validators = Array.isArray(patch.validators) ? sanitizeValidators(patch.validators) : [];
|
|
95
99
|
}
|