@goodandready/dsh-cron 0.3.23 → 0.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/api-item.js +2 -1
- package/lib/client.js +48 -3
- package/lib/index.js +2 -1
- package/lib/log-streamer.js +10 -2
- package/lib/routes.js +50 -0
- package/lib/runner-external.js +91 -6
- package/lib/runtimes.js +8 -2
- package/lib/store.js +171 -1
- package/lib/task-schema.js +6 -1
- package/lib/task-transfer.js +5 -0
- package/package.json +5 -3
package/lib/api-item.js
CHANGED
|
@@ -21,7 +21,8 @@ export async function handleTaskItem({ store, scheduler, req, res, url, id, acti
|
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
if (req.method === 'GET') {
|
|
24
|
-
|
|
24
|
+
// Both /dsh-cron/tasks/:id/stream and /dsh-cron/tasks/:id/logs/stream (#392)
|
|
25
|
+
if (action === 'stream' || (action === 'logs' && subAction === 'stream')) {
|
|
25
26
|
logStreamer.handleSseRequest(id, req, res);
|
|
26
27
|
return;
|
|
27
28
|
}
|
package/lib/client.js
CHANGED
|
@@ -2736,14 +2736,46 @@ if (window.__dshLiveLogsEvtSource) {
|
|
|
2736
2736
|
try { window.__dshLiveLogsEvtSource.close(); } catch (e) { void e; }
|
|
2737
2737
|
window.__dshLiveLogsEvtSource = null;
|
|
2738
2738
|
}
|
|
2739
|
+
if (window.__dshLiveLogsPollTimer) {
|
|
2740
|
+
clearInterval(window.__dshLiveLogsPollTimer);
|
|
2741
|
+
window.__dshLiveLogsPollTimer = null;
|
|
2742
|
+
}
|
|
2739
2743
|
setLiveLogsTaskId(task.id);
|
|
2740
2744
|
setLiveLogsTaskTitle(task.title || task.id);
|
|
2741
2745
|
setLiveLogsLines([]);
|
|
2742
2746
|
setLiveLogsStatus(task.running ? 'connecting' : 'idle');
|
|
2743
2747
|
setLiveLogsCompleted(false);
|
|
2744
2748
|
setLiveLogsModalOpen(true);
|
|
2749
|
+
const startPollingFallback = () => {
|
|
2750
|
+
if (window.__dshLiveLogsPollTimer) return;
|
|
2751
|
+
const pollFn = () => {
|
|
2752
|
+
fetchJsonWithTimeout('/dsh-cron/tasks/' + encodeURIComponent(task.id) + '/history?limit=1')
|
|
2753
|
+
.then((data) => {
|
|
2754
|
+
if (data && data.history && data.history.length > 0) {
|
|
2755
|
+
const lastRun = data.history[0];
|
|
2756
|
+
if (lastRun.output || lastRun.error) {
|
|
2757
|
+
const text = lastRun.output || lastRun.error;
|
|
2758
|
+
setLiveLogsLines([{ chunk: text, stream: lastRun.error ? 'stderr' : 'stdout', ts: lastRun.at }]);
|
|
2759
|
+
}
|
|
2760
|
+
if (lastRun.status) {
|
|
2761
|
+
setLiveLogsStatus(lastRun.status);
|
|
2762
|
+
setLiveLogsCompleted(true);
|
|
2763
|
+
if (window.__dshLiveLogsPollTimer) {
|
|
2764
|
+
clearInterval(window.__dshLiveLogsPollTimer);
|
|
2765
|
+
window.__dshLiveLogsPollTimer = null;
|
|
2766
|
+
}
|
|
2767
|
+
}
|
|
2768
|
+
}
|
|
2769
|
+
})
|
|
2770
|
+
.catch(() => {});
|
|
2771
|
+
};
|
|
2772
|
+
pollFn();
|
|
2773
|
+
window.__dshLiveLogsPollTimer = setInterval(pollFn, 2000);
|
|
2774
|
+
};
|
|
2775
|
+
if (typeof EventSource !== 'undefined') {
|
|
2745
2776
|
try {
|
|
2746
|
-
const
|
|
2777
|
+
const streamUrl = '/dsh-cron/tasks/' + encodeURIComponent(task.id) + '/logs/stream';
|
|
2778
|
+
const es = new EventSource(streamUrl);
|
|
2747
2779
|
window.__dshLiveLogsEvtSource = es;
|
|
2748
2780
|
es.addEventListener('init', (e) => {
|
|
2749
2781
|
try {
|
|
@@ -2767,12 +2799,21 @@ try { es.close(); } catch (e) { void e; }
|
|
|
2767
2799
|
window.__dshLiveLogsEvtSource = null;
|
|
2768
2800
|
});
|
|
2769
2801
|
es.onerror = () => {
|
|
2770
|
-
setLiveLogsStatus((curr) => curr === 'running' ? 'completed' : 'closed');
|
|
2771
2802
|
try { es.close(); } catch (e) { void e; }
|
|
2772
2803
|
window.__dshLiveLogsEvtSource = null;
|
|
2804
|
+
setLiveLogsStatus((curr) => {
|
|
2805
|
+
if (curr === 'running' || curr === 'connecting') {
|
|
2806
|
+
startPollingFallback();
|
|
2807
|
+
return curr;
|
|
2808
|
+
}
|
|
2809
|
+
return 'completed';
|
|
2810
|
+
});
|
|
2773
2811
|
};
|
|
2774
2812
|
} catch (err) {
|
|
2775
|
-
|
|
2813
|
+
startPollingFallback();
|
|
2814
|
+
}
|
|
2815
|
+
} else {
|
|
2816
|
+
startPollingFallback();
|
|
2776
2817
|
}
|
|
2777
2818
|
}
|
|
2778
2819
|
function handleCloseLiveLogs() {
|
|
@@ -2780,6 +2821,10 @@ if (window.__dshLiveLogsEvtSource) {
|
|
|
2780
2821
|
try { window.__dshLiveLogsEvtSource.close(); } catch (e) { void e; }
|
|
2781
2822
|
window.__dshLiveLogsEvtSource = null;
|
|
2782
2823
|
}
|
|
2824
|
+
if (window.__dshLiveLogsPollTimer) {
|
|
2825
|
+
clearInterval(window.__dshLiveLogsPollTimer);
|
|
2826
|
+
window.__dshLiveLogsPollTimer = null;
|
|
2827
|
+
}
|
|
2783
2828
|
setLiveLogsModalOpen(false);
|
|
2784
2829
|
}
|
|
2785
2830
|
const openAnalyticsDigest = async (days = 7) => {
|
package/lib/index.js
CHANGED
|
@@ -17,7 +17,8 @@ import { bestEffort } from './best-effort.js';
|
|
|
17
17
|
import { setLogger, logger } from './logger.js';
|
|
18
18
|
import { registerPluginUpdater } from './updater.js';
|
|
19
19
|
import { credentialRefStatus } from './secrets.js';
|
|
20
|
-
import { TaskStore } from './store.js';
|
|
20
|
+
import { TaskStore, CURRENT_SCHEMA_VERSION, runStoreMigrations } from './store.js';
|
|
21
|
+
export { TaskStore, CURRENT_SCHEMA_VERSION, runStoreMigrations };
|
|
21
22
|
import { TaskScheduler, parseScheduleExpression } from './scheduler.js';
|
|
22
23
|
import { SessionRunner } from './runner.js';
|
|
23
24
|
import { getModelsHandler } from './models-handler.js';
|
package/lib/log-streamer.js
CHANGED
|
@@ -50,12 +50,15 @@ export class LogStreamer extends EventEmitter {
|
|
|
50
50
|
ts: Date.now(),
|
|
51
51
|
};
|
|
52
52
|
|
|
53
|
+
entry.eventSeq = (entry.eventSeq || 0) + 1;
|
|
54
|
+
item.id = entry.eventSeq;
|
|
55
|
+
|
|
53
56
|
entry.buffer.push(item);
|
|
54
57
|
if (entry.buffer.length > MAX_BUFFER_LINES) {
|
|
55
58
|
entry.buffer.shift();
|
|
56
59
|
}
|
|
57
60
|
|
|
58
|
-
const ssePayload = `
|
|
61
|
+
const ssePayload = `id: ${item.id}\nevent: log\ndata: ${JSON.stringify(item)}\n\n`;
|
|
59
62
|
for (const res of entry.subscribers) {
|
|
60
63
|
try {
|
|
61
64
|
res.write(ssePayload);
|
|
@@ -111,12 +114,17 @@ export class LogStreamer extends EventEmitter {
|
|
|
111
114
|
|
|
112
115
|
const entry = this.streams.get(taskId);
|
|
113
116
|
|
|
117
|
+
// Support W3C Last-Event-ID for reconnection (#392)
|
|
118
|
+
const lastEventIdStr = req?.headers ? req.headers['last-event-id'] : undefined;
|
|
119
|
+
const lastEventId = lastEventIdStr ? parseInt(lastEventIdStr, 10) : 0;
|
|
120
|
+
const effectiveLogs = (entry?.buffer || []).filter((item) => !lastEventId || (item.id && item.id > lastEventId));
|
|
121
|
+
|
|
114
122
|
// Initial event with current state and buffered logs
|
|
115
123
|
const initData = {
|
|
116
124
|
taskId,
|
|
117
125
|
runId: entry?.runId || null,
|
|
118
126
|
status: entry ? (entry.completed ? 'completed' : 'running') : 'idle',
|
|
119
|
-
bufferedLogs:
|
|
127
|
+
bufferedLogs: effectiveLogs,
|
|
120
128
|
};
|
|
121
129
|
res.write(`event: init\ndata: ${JSON.stringify(initData)}\n\n`);
|
|
122
130
|
|
package/lib/routes.js
CHANGED
|
@@ -22,6 +22,56 @@ const LOCALES_JSON = readFileSync(new URL('./locales.json', import.meta.url), 'u
|
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
export function registerRoutes(ctx, { store, scheduler, recommendations }) {
|
|
25
|
+
// Monthly history archives collection route (#391)
|
|
26
|
+
ctx.effect(() => ctx.webServer.register({
|
|
27
|
+
kind: 'exact',
|
|
28
|
+
path: '/dsh-cron/history/archives',
|
|
29
|
+
handler: (req, res) => {
|
|
30
|
+
if (req.method !== 'GET') {
|
|
31
|
+
res.writeHead(405, { 'content-type': 'text/plain; charset=utf-8' });
|
|
32
|
+
res.end('GET only');
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
res.writeHead(200, {
|
|
36
|
+
'content-type': 'application/json; charset=utf-8',
|
|
37
|
+
'cache-control': 'no-store',
|
|
38
|
+
});
|
|
39
|
+
const archives = typeof store?.listMonthlyArchives === 'function' ? store.listMonthlyArchives() : [];
|
|
40
|
+
res.end(JSON.stringify({ ok: true, archives }));
|
|
41
|
+
}
|
|
42
|
+
}), 'dsh-cron: /history/archives');
|
|
43
|
+
|
|
44
|
+
// Specific monthly history archive route (#391)
|
|
45
|
+
ctx.effect(() => ctx.webServer.register({
|
|
46
|
+
kind: 'prefix',
|
|
47
|
+
path: '/dsh-cron/history/archive/',
|
|
48
|
+
handler: (req, res) => {
|
|
49
|
+
if (req.method !== 'GET') {
|
|
50
|
+
res.writeHead(405, { 'content-type': 'text/plain; charset=utf-8' });
|
|
51
|
+
res.end('GET only');
|
|
52
|
+
return;
|
|
53
|
+
}
|
|
54
|
+
const url = new URL(req.url, 'http://127.0.0.1');
|
|
55
|
+
const month = url.pathname.replace(/^\/dsh-cron\/history\/archive\//, '').replace(/\/$/, '');
|
|
56
|
+
if (!/^\d{4}-\d{2}$/.test(month)) {
|
|
57
|
+
res.writeHead(400, { 'content-type': 'application/json; charset=utf-8' });
|
|
58
|
+
res.end(JSON.stringify({ ok: false, error: 'Invalid month format, expected YYYY-MM' }));
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
const archive = typeof store?.getMonthlyArchive === 'function' ? store.getMonthlyArchive(month) : null;
|
|
62
|
+
if (!archive) {
|
|
63
|
+
res.writeHead(404, { 'content-type': 'application/json; charset=utf-8' });
|
|
64
|
+
res.end(JSON.stringify({ ok: false, error: `Archive for month "${month}" not found` }));
|
|
65
|
+
return;
|
|
66
|
+
}
|
|
67
|
+
res.writeHead(200, {
|
|
68
|
+
'content-type': 'application/json; charset=utf-8',
|
|
69
|
+
'cache-control': 'public, max-age=3600',
|
|
70
|
+
});
|
|
71
|
+
res.end(JSON.stringify({ ok: true, month, archive }));
|
|
72
|
+
}
|
|
73
|
+
}), 'dsh-cron: /history/archive/:month');
|
|
74
|
+
|
|
25
75
|
// Health probe route (#384)
|
|
26
76
|
ctx.effect(() => ctx.webServer.register({
|
|
27
77
|
kind: 'exact',
|
package/lib/runner-external.js
CHANGED
|
@@ -176,11 +176,22 @@ export async function runExternal(task, runtime, signal, execOptions = {}, { ctx
|
|
|
176
176
|
return new Promise((resolve, reject) => {
|
|
177
177
|
let child = null;
|
|
178
178
|
let settled = false;
|
|
179
|
+
let timedOut = false;
|
|
179
180
|
let stdout = '';
|
|
180
181
|
let stderr = '';
|
|
181
|
-
|
|
182
|
+
let truncated = false;
|
|
183
|
+
// Issue #393: maxLogBytesPerRun cap (default 5MB)
|
|
184
|
+
const maxLogBytes = Number(task.maxLogBytesPerRun) > 0 ? Number(task.maxLogBytesPerRun) : 5 * 1024 * 1024;
|
|
182
185
|
const isPosix = process.platform !== 'win32';
|
|
183
186
|
|
|
187
|
+
// Issue #393: Execution timeout guardrail (default 15 minutes = 900,000ms, or task.executionTimeoutMs / timeoutSeconds)
|
|
188
|
+
const effectiveTimeoutMs = Number(task.executionTimeoutMs) > 0
|
|
189
|
+
? Number(task.executionTimeoutMs)
|
|
190
|
+
: (Number(task.timeoutSeconds) > 0 ? Number(task.timeoutSeconds) * 1000 : 15 * 60 * 1000);
|
|
191
|
+
|
|
192
|
+
let timeoutTimer = null;
|
|
193
|
+
let forceKillTimer = null;
|
|
194
|
+
|
|
184
195
|
const killProcessTree = (sig = 'SIGTERM') => {
|
|
185
196
|
terminateProcessTree(child, sig, {
|
|
186
197
|
isPosix,
|
|
@@ -190,13 +201,37 @@ export async function runExternal(task, runtime, signal, execOptions = {}, { ctx
|
|
|
190
201
|
});
|
|
191
202
|
};
|
|
192
203
|
|
|
204
|
+
const cleanupTimers = () => {
|
|
205
|
+
if (timeoutTimer) { clearTimeout(timeoutTimer); timeoutTimer = null; }
|
|
206
|
+
if (forceKillTimer) { clearTimeout(forceKillTimer); forceKillTimer = null; }
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
if (effectiveTimeoutMs > 0) {
|
|
210
|
+
timeoutTimer = setTimeout(() => {
|
|
211
|
+
if (settled) return;
|
|
212
|
+
timedOut = true;
|
|
213
|
+
logger.warn(`[dsh-cron] task "${task.title || task.id}" timed out after ${effectiveTimeoutMs}ms, sending SIGTERM`);
|
|
214
|
+
killProcessTree('SIGTERM');
|
|
215
|
+
forceKillTimer = setTimeout(() => {
|
|
216
|
+
if (!settled) {
|
|
217
|
+
logger.warn(`[dsh-cron] task "${task.title || task.id}" force killing with SIGKILL`);
|
|
218
|
+
killProcessTree('SIGKILL');
|
|
219
|
+
}
|
|
220
|
+
}, 3000);
|
|
221
|
+
if (typeof forceKillTimer.unref === 'function') forceKillTimer.unref();
|
|
222
|
+
}, effectiveTimeoutMs);
|
|
223
|
+
if (typeof timeoutTimer.unref === 'function') timeoutTimer.unref();
|
|
224
|
+
}
|
|
225
|
+
|
|
193
226
|
const abortHandler = () => {
|
|
227
|
+
cleanupTimers();
|
|
194
228
|
killProcessTree('SIGTERM');
|
|
195
229
|
setTimeout(() => killProcessTree('SIGKILL'), 300);
|
|
196
230
|
};
|
|
197
231
|
|
|
198
232
|
if (signal) {
|
|
199
233
|
if (signal.aborted) {
|
|
234
|
+
cleanupTimers();
|
|
200
235
|
return reject(signal.reason || new Error('Command aborted'));
|
|
201
236
|
}
|
|
202
237
|
signal.addEventListener('abort', abortHandler, { once: true });
|
|
@@ -218,23 +253,67 @@ export async function runExternal(task, runtime, signal, execOptions = {}, { ctx
|
|
|
218
253
|
if (invocation.file) {
|
|
219
254
|
child = spawn(invocation.file, invocation.args || [], options);
|
|
220
255
|
} else {
|
|
256
|
+
let command = invocation.shellCommand;
|
|
257
|
+
if (isPosix && Number(task.maxMemoryMb) > 0) {
|
|
258
|
+
command = `ulimit -v ${Number(task.maxMemoryMb) * 1024} 2>/dev/null || true; ${command}`;
|
|
259
|
+
}
|
|
221
260
|
const shellPath = isPosix ? (process.env.SHELL || '/bin/bash') : (process.env.ComSpec || 'cmd.exe');
|
|
222
|
-
const shellArgs = isPosix ? ['-c',
|
|
261
|
+
const shellArgs = isPosix ? ['-c', command] : ['/d', '/s', '/c', command];
|
|
223
262
|
child = spawn(shellPath, shellArgs, options);
|
|
224
263
|
}
|
|
225
264
|
|
|
226
265
|
child.stdout.on('data', (chunk) => {
|
|
227
|
-
|
|
228
|
-
|
|
266
|
+
const remaining = maxLogBytes - (stdout.length + stderr.length);
|
|
267
|
+
const text = chunk.toString('utf-8');
|
|
268
|
+
if (remaining > 0) {
|
|
269
|
+
if (text.length <= remaining) {
|
|
270
|
+
stdout += text;
|
|
271
|
+
if (!truncated) logStreamer.appendLog(task.id, text, 'stdout');
|
|
272
|
+
} else {
|
|
273
|
+
stdout += text.slice(0, remaining);
|
|
274
|
+
if (!truncated) {
|
|
275
|
+
truncated = true;
|
|
276
|
+
const note = `\n[TRUNCATED: log output exceeded maxLogBytesPerRun (${Math.round(maxLogBytes / 1024 / 1024)}MB)]`;
|
|
277
|
+
stdout += note;
|
|
278
|
+
logStreamer.appendLog(task.id, text.slice(0, remaining) + note, 'stdout');
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
} else if (!truncated) {
|
|
282
|
+
truncated = true;
|
|
283
|
+
const note = `\n[TRUNCATED: log output exceeded maxLogBytesPerRun (${Math.round(maxLogBytes / 1024 / 1024)}MB)]`;
|
|
284
|
+
stdout += note;
|
|
285
|
+
logStreamer.appendLog(task.id, note, 'stdout');
|
|
286
|
+
}
|
|
229
287
|
});
|
|
288
|
+
|
|
230
289
|
child.stderr.on('data', (chunk) => {
|
|
231
|
-
|
|
232
|
-
|
|
290
|
+
const remaining = maxLogBytes - (stdout.length + stderr.length);
|
|
291
|
+
const text = chunk.toString('utf-8');
|
|
292
|
+
if (remaining > 0) {
|
|
293
|
+
if (text.length <= remaining) {
|
|
294
|
+
stderr += text;
|
|
295
|
+
if (!truncated) logStreamer.appendLog(task.id, text, 'stderr');
|
|
296
|
+
} else {
|
|
297
|
+
stderr += text.slice(0, remaining);
|
|
298
|
+
if (!truncated) {
|
|
299
|
+
truncated = true;
|
|
300
|
+
const note = `\n[TRUNCATED: log output exceeded maxLogBytesPerRun (${Math.round(maxLogBytes / 1024 / 1024)}MB)]`;
|
|
301
|
+
stderr += note;
|
|
302
|
+
logStreamer.appendLog(task.id, text.slice(0, remaining) + note, 'stderr');
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
} else if (!truncated) {
|
|
306
|
+
truncated = true;
|
|
307
|
+
const note = `\n[TRUNCATED: log output exceeded maxLogBytesPerRun (${Math.round(maxLogBytes / 1024 / 1024)}MB)]`;
|
|
308
|
+
stderr += note;
|
|
309
|
+
logStreamer.appendLog(task.id, note, 'stderr');
|
|
310
|
+
}
|
|
233
311
|
});
|
|
234
312
|
|
|
235
313
|
child.on('error', (err) => {
|
|
236
314
|
if (settled) return;
|
|
237
315
|
settled = true;
|
|
316
|
+
cleanupTimers();
|
|
238
317
|
if (signal) signal.removeEventListener('abort', abortHandler);
|
|
239
318
|
logStreamer.completeStream(task.id, { status: 'error', error: err.message, exitCode: 1 });
|
|
240
319
|
reject(err);
|
|
@@ -243,8 +322,14 @@ export async function runExternal(task, runtime, signal, execOptions = {}, { ctx
|
|
|
243
322
|
child.on('close', (code, sig) => {
|
|
244
323
|
if (settled) return;
|
|
245
324
|
settled = true;
|
|
325
|
+
cleanupTimers();
|
|
246
326
|
if (signal) signal.removeEventListener('abort', abortHandler);
|
|
247
327
|
|
|
328
|
+
if (timedOut) {
|
|
329
|
+
logStreamer.completeStream(task.id, { status: 'timeout', error: `Execution timed out after ${effectiveTimeoutMs}ms`, exitCode: 124 });
|
|
330
|
+
return reject(new Error(`Execution timed out after ${effectiveTimeoutMs}ms (SIGTERM/SIGKILL sent)`));
|
|
331
|
+
}
|
|
332
|
+
|
|
248
333
|
if (signal && signal.aborted) {
|
|
249
334
|
return reject(signal.reason || new Error('Command aborted'));
|
|
250
335
|
}
|
package/lib/runtimes.js
CHANGED
|
@@ -48,12 +48,16 @@ function looksLikeScriptFile(token, extensions) {
|
|
|
48
48
|
export function buildNodeInvocation(task, cwd) {
|
|
49
49
|
const nodeExecutable = task.nodePath || process.execPath;
|
|
50
50
|
const { path: filePath, args } = splitCommandLine(task.prompt);
|
|
51
|
+
const flags = [];
|
|
52
|
+
if (Number(task.maxMemoryMb) > 0) {
|
|
53
|
+
flags.push(`--max-old-space-size=${Number(task.maxMemoryMb)}`);
|
|
54
|
+
}
|
|
51
55
|
if (looksLikeScriptFile(filePath, ['.js', '.mjs', '.cjs'])) {
|
|
52
|
-
return { file: nodeExecutable, args: [filePath, ...args] };
|
|
56
|
+
return { file: nodeExecutable, args: [...flags, filePath, ...args] };
|
|
53
57
|
}
|
|
54
58
|
const code = String(task.prompt || '');
|
|
55
59
|
const isEsm = /\bimport\s|\bexport\s/.test(code);
|
|
56
|
-
const nodeArgs = isEsm ? ['--input-type=module', '-e', code] : ['-e', code];
|
|
60
|
+
const nodeArgs = isEsm ? [...flags, '--input-type=module', '-e', code] : [...flags, '-e', code];
|
|
57
61
|
return { file: nodeExecutable, args: nodeArgs };
|
|
58
62
|
}
|
|
59
63
|
|
|
@@ -147,6 +151,8 @@ export function buildDockerInvocation(task, explicitEnv) {
|
|
|
147
151
|
if (value === null || value === undefined) continue;
|
|
148
152
|
args.push('-e', `${key}=${value}`);
|
|
149
153
|
}
|
|
154
|
+
if (Number(task.maxMemoryMb) > 0) args.push(`--memory=${Number(task.maxMemoryMb)}m`);
|
|
155
|
+
if (Number(task.cpus) > 0) args.push(`--cpus=${Number(task.cpus)}`);
|
|
150
156
|
if (task.cwd) args.push('-v', `${task.cwd}:/workspace`, '-w', '/workspace');
|
|
151
157
|
args.push(image, 'sh', '-lc', String(task.prompt || 'true'));
|
|
152
158
|
return { file: 'docker', args, env: envMap };
|
package/lib/store.js
CHANGED
|
@@ -4,6 +4,7 @@ import { ArtifactManager } from './artifacts.js';
|
|
|
4
4
|
import { logger } from './logger.js';
|
|
5
5
|
import { bestEffort } from './best-effort.js';
|
|
6
6
|
import fs from 'node:fs';
|
|
7
|
+
import { gzipSync, gunzipSync } from 'node:zlib';
|
|
7
8
|
import { pipeline } from 'node:stream/promises';
|
|
8
9
|
import path from 'node:path';
|
|
9
10
|
import { getDshDefaultTelegramCredentials } from './telegram.js';
|
|
@@ -17,6 +18,36 @@ import { getDshDefaultTelegramCredentials } from './telegram.js';
|
|
|
17
18
|
* home's data directory, which is how an isolated test cycle once leaked a
|
|
18
19
|
* task into the wrong profile.
|
|
19
20
|
*/
|
|
21
|
+
|
|
22
|
+
export const CURRENT_SCHEMA_VERSION = 2;
|
|
23
|
+
|
|
24
|
+
export function runStoreMigrations(data, fromVersion = 1, toVersion = CURRENT_SCHEMA_VERSION) {
|
|
25
|
+
if (!data || typeof data !== 'object') return data;
|
|
26
|
+
let current = { ...data };
|
|
27
|
+
|
|
28
|
+
// Migration from version 1 to 2 (Issue #390 & #393)
|
|
29
|
+
if (fromVersion < 2 && toVersion >= 2) {
|
|
30
|
+
if (Array.isArray(current.tasks)) {
|
|
31
|
+
current.tasks = current.tasks.map((task) => {
|
|
32
|
+
if (!task || typeof task !== 'object') return task;
|
|
33
|
+
return {
|
|
34
|
+
...task,
|
|
35
|
+
maxMemoryMb: task.maxMemoryMb !== undefined ? Number(task.maxMemoryMb) : 0,
|
|
36
|
+
executionTimeoutMs: task.executionTimeoutMs !== undefined
|
|
37
|
+
? Number(task.executionTimeoutMs)
|
|
38
|
+
: (Number(task.timeoutSeconds) > 0 ? Number(task.timeoutSeconds) * 1000 : 900000),
|
|
39
|
+
maxLogBytesPerRun: task.maxLogBytesPerRun !== undefined ? Number(task.maxLogBytesPerRun) : (5 * 1024 * 1024),
|
|
40
|
+
cpus: task.cpus !== undefined ? Number(task.cpus) : 0,
|
|
41
|
+
maxArchiveMonths: task.maxArchiveMonths !== undefined ? Number(task.maxArchiveMonths) : 6,
|
|
42
|
+
};
|
|
43
|
+
});
|
|
44
|
+
}
|
|
45
|
+
current.schemaVersion = 2;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
return current;
|
|
49
|
+
}
|
|
50
|
+
|
|
20
51
|
export function getDefaultStorePath() {
|
|
21
52
|
const base = process.env.DSH_DATA_DIR
|
|
22
53
|
|| (process.env.DSH_HOME
|
|
@@ -64,7 +95,21 @@ export class TaskStore {
|
|
|
64
95
|
const bakPath = `${this.filePath}.bak`;
|
|
65
96
|
|
|
66
97
|
const loadData = (raw) => {
|
|
67
|
-
|
|
98
|
+
let data = JSON.parse(raw);
|
|
99
|
+
const storedVersion = Number(data.schemaVersion) || 1;
|
|
100
|
+
if (storedVersion < CURRENT_SCHEMA_VERSION) {
|
|
101
|
+
// Pre-migration backup (#390)
|
|
102
|
+
try {
|
|
103
|
+
const preBakPath = `${this.filePath}.bak-v${storedVersion}-${Date.now()}`;
|
|
104
|
+
fs.writeFileSync(preBakPath, raw, { encoding: 'utf-8', mode: 0o600 });
|
|
105
|
+
bestEffort('chmod-pre-bak', () => fs.chmodSync(preBakPath, 0o600));
|
|
106
|
+
logger.info(`[dsh-cron] pre-migration backup saved to ${preBakPath}`);
|
|
107
|
+
} catch (bakErr) {
|
|
108
|
+
logger.warn('[dsh-cron] failed to write pre-migration backup:', bakErr.message);
|
|
109
|
+
}
|
|
110
|
+
data = runStoreMigrations(data, storedVersion, CURRENT_SCHEMA_VERSION);
|
|
111
|
+
this._needsMigrationFlush = true;
|
|
112
|
+
}
|
|
68
113
|
this.tasks.clear();
|
|
69
114
|
this.history.clear();
|
|
70
115
|
if (Array.isArray(data.tasks)) {
|
|
@@ -151,6 +196,11 @@ export class TaskStore {
|
|
|
151
196
|
logger.error('[dsh-cron] failed to restore tasks store from .bak:', bakErr.message);
|
|
152
197
|
}
|
|
153
198
|
}
|
|
199
|
+
if (this._needsMigrationFlush) {
|
|
200
|
+
this._needsMigrationFlush = false;
|
|
201
|
+
this.scheduleSave(100);
|
|
202
|
+
}
|
|
203
|
+
bestEffort('prune-old-archives', () => this._pruneOldMonthlyArchives(dir, 6));
|
|
154
204
|
}
|
|
155
205
|
|
|
156
206
|
/**
|
|
@@ -201,6 +251,7 @@ export class TaskStore {
|
|
|
201
251
|
}
|
|
202
252
|
return {
|
|
203
253
|
version: 1,
|
|
254
|
+
schemaVersion: CURRENT_SCHEMA_VERSION,
|
|
204
255
|
updatedAt: new Date().toISOString(),
|
|
205
256
|
tasks: tasksSnapshot,
|
|
206
257
|
history: historySnapshot,
|
|
@@ -358,6 +409,7 @@ export class TaskStore {
|
|
|
358
409
|
}
|
|
359
410
|
|
|
360
411
|
yield '{"version":' + JSON.stringify(data.version || 1);
|
|
412
|
+
yield ',"schemaVersion":' + JSON.stringify(data.schemaVersion || CURRENT_SCHEMA_VERSION);
|
|
361
413
|
yield ',"updatedAt":' + JSON.stringify(data.updatedAt || new Date().toISOString());
|
|
362
414
|
yield ',"settings":' + JSON.stringify(data.settings || {});
|
|
363
415
|
|
|
@@ -651,6 +703,11 @@ export class TaskStore {
|
|
|
651
703
|
? (task.onlyOnFailure === null || task.onlyOnFailure === 'inherit' ? undefined : Boolean(task.onlyOnFailure))
|
|
652
704
|
: prev.onlyOnFailure,
|
|
653
705
|
timeoutSeconds: task.timeoutSeconds !== undefined ? Number(task.timeoutSeconds) : (prev.timeoutSeconds || 1800),
|
|
706
|
+
maxMemoryMb: task.maxMemoryMb !== undefined ? Number(task.maxMemoryMb) : (prev.maxMemoryMb || 0),
|
|
707
|
+
executionTimeoutMs: task.executionTimeoutMs !== undefined ? Number(task.executionTimeoutMs) : (prev.executionTimeoutMs || 0),
|
|
708
|
+
maxLogBytesPerRun: task.maxLogBytesPerRun !== undefined ? Number(task.maxLogBytesPerRun) : (prev.maxLogBytesPerRun || (5 * 1024 * 1024)),
|
|
709
|
+
cpus: task.cpus !== undefined ? Number(task.cpus) : (prev.cpus || 0),
|
|
710
|
+
maxArchiveMonths: task.maxArchiveMonths !== undefined ? Number(task.maxArchiveMonths) : (prev.maxArchiveMonths || 6),
|
|
654
711
|
overlapPolicy: task.overlapPolicy !== undefined ? String(task.overlapPolicy) : (prev.overlapPolicy || 'skip'),
|
|
655
712
|
kanbanMode: task.kanbanMode !== undefined ? String(task.kanbanMode) : (prev.kanbanMode || 'none'), // 'none' | 'on_failure' | 'always'
|
|
656
713
|
onSuccess: task.onSuccess !== undefined ? String(task.onSuccess).trim() : (prev.onSuccess || ''),
|
|
@@ -922,11 +979,124 @@ export class TaskStore {
|
|
|
922
979
|
bestEffort('chmod-tmp', () => fs.chmodSync(tmp, 0o600));
|
|
923
980
|
fs.renameSync(tmp, archivePath);
|
|
924
981
|
bestEffort('chmod-archive', () => fs.chmodSync(archivePath, 0o600));
|
|
982
|
+
|
|
983
|
+
// Partition runs into monthly buckets for compressed archives (#391)
|
|
984
|
+
const monthlyBuckets = new Map();
|
|
985
|
+
for (const run of sanitizedOverflow) {
|
|
986
|
+
const at = run.at || run.timestamp || run.startedAt || Date.now();
|
|
987
|
+
const date = new Date(at);
|
|
988
|
+
const monthKey = !isNaN(date.getTime())
|
|
989
|
+
? `${date.getUTCFullYear()}-${String(date.getUTCMonth() + 1).padStart(2, '0')}`
|
|
990
|
+
: new Date().toISOString().slice(0, 7);
|
|
991
|
+
if (!monthlyBuckets.has(monthKey)) {
|
|
992
|
+
monthlyBuckets.set(monthKey, []);
|
|
993
|
+
}
|
|
994
|
+
monthlyBuckets.get(monthKey).push(run);
|
|
995
|
+
}
|
|
996
|
+
|
|
997
|
+
for (const [monthKey, runsForMonth] of monthlyBuckets.entries()) {
|
|
998
|
+
const gzPath = path.join(dir, `tasks-history-${monthKey}.json.gz`);
|
|
999
|
+
let monthArchive = {};
|
|
1000
|
+
if (fs.existsSync(gzPath)) {
|
|
1001
|
+
try {
|
|
1002
|
+
const rawGz = fs.readFileSync(gzPath);
|
|
1003
|
+
const decompressed = gunzipSync(rawGz).toString('utf-8');
|
|
1004
|
+
monthArchive = JSON.parse(decompressed) || {};
|
|
1005
|
+
} catch (gzReadErr) {
|
|
1006
|
+
logger.warn(`[dsh-cron] failed to read existing gzip archive for ${monthKey}:`, gzReadErr.message);
|
|
1007
|
+
monthArchive = {};
|
|
1008
|
+
}
|
|
1009
|
+
}
|
|
1010
|
+
if (!Array.isArray(monthArchive[id])) {
|
|
1011
|
+
monthArchive[id] = [];
|
|
1012
|
+
}
|
|
1013
|
+
monthArchive[id].push(...runsForMonth);
|
|
1014
|
+
const compressed = gzipSync(Buffer.from(JSON.stringify(monthArchive), 'utf-8'), { level: 9 });
|
|
1015
|
+
const randGz = Math.random().toString(36).slice(2, 8);
|
|
1016
|
+
const tmpGz = `${gzPath}.tmp.${process.pid}_${Date.now()}_${randGz}`;
|
|
1017
|
+
fs.writeFileSync(tmpGz, compressed, { mode: 0o600 });
|
|
1018
|
+
bestEffort('chmod-tmp-gz', () => fs.chmodSync(tmpGz, 0o600));
|
|
1019
|
+
fs.renameSync(tmpGz, gzPath);
|
|
1020
|
+
bestEffort('chmod-archive-gz', () => fs.chmodSync(gzPath, 0o600));
|
|
1021
|
+
}
|
|
1022
|
+
|
|
1023
|
+
this._pruneOldMonthlyArchives(dir, 6);
|
|
925
1024
|
} catch (archiveErr) {
|
|
926
1025
|
logger.error('[dsh-cron] history archive error:', archiveErr.message);
|
|
927
1026
|
}
|
|
928
1027
|
}
|
|
929
1028
|
|
|
1029
|
+
listMonthlyArchives() {
|
|
1030
|
+
try {
|
|
1031
|
+
const dir = path.dirname(this.filePath);
|
|
1032
|
+
if (!fs.existsSync(dir)) return [];
|
|
1033
|
+
const files = fs.readdirSync(dir);
|
|
1034
|
+
const archives = [];
|
|
1035
|
+
for (const file of files) {
|
|
1036
|
+
const match = file.match(/^tasks-history-(\d{4}-\d{2})\.json\.gz$/);
|
|
1037
|
+
if (match) {
|
|
1038
|
+
const month = match[1];
|
|
1039
|
+
const fullPath = path.join(dir, file);
|
|
1040
|
+
try {
|
|
1041
|
+
const stat = fs.statSync(fullPath);
|
|
1042
|
+
archives.push({
|
|
1043
|
+
month,
|
|
1044
|
+
filename: file,
|
|
1045
|
+
sizeBytes: stat.size,
|
|
1046
|
+
mtime: stat.mtime.toISOString(),
|
|
1047
|
+
path: fullPath,
|
|
1048
|
+
});
|
|
1049
|
+
} catch (_err) {
|
|
1050
|
+
// ignore unreadable stat
|
|
1051
|
+
}
|
|
1052
|
+
}
|
|
1053
|
+
}
|
|
1054
|
+
return archives.sort((a, b) => b.month.localeCompare(a.month));
|
|
1055
|
+
} catch (err) {
|
|
1056
|
+
logger.error('[dsh-cron] listMonthlyArchives error:', err.message);
|
|
1057
|
+
return [];
|
|
1058
|
+
}
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
getMonthlyArchive(month) {
|
|
1062
|
+
if (!month || !/^\d{4}-\d{2}$/.test(month)) return null;
|
|
1063
|
+
try {
|
|
1064
|
+
const dir = path.dirname(this.filePath);
|
|
1065
|
+
const gzPath = path.join(dir, `tasks-history-${month}.json.gz`);
|
|
1066
|
+
if (!fs.existsSync(gzPath)) return null;
|
|
1067
|
+
const rawGz = fs.readFileSync(gzPath);
|
|
1068
|
+
const decompressed = gunzipSync(rawGz).toString('utf-8');
|
|
1069
|
+
return JSON.parse(decompressed);
|
|
1070
|
+
} catch (err) {
|
|
1071
|
+
logger.error(`[dsh-cron] getMonthlyArchive error for ${month}:`, err.message);
|
|
1072
|
+
return null;
|
|
1073
|
+
}
|
|
1074
|
+
}
|
|
1075
|
+
|
|
1076
|
+
_pruneOldMonthlyArchives(dir, maxMonths = 6) {
|
|
1077
|
+
try {
|
|
1078
|
+
if (!fs.existsSync(dir) || maxMonths < 0) return;
|
|
1079
|
+
const files = fs.readdirSync(dir);
|
|
1080
|
+
const monthlyFiles = [];
|
|
1081
|
+
for (const file of files) {
|
|
1082
|
+
const match = file.match(/^tasks-history-(\d{4}-\d{2})\.json\.gz$/);
|
|
1083
|
+
if (match) {
|
|
1084
|
+
monthlyFiles.push({ month: match[1], file, fullPath: path.join(dir, file) });
|
|
1085
|
+
}
|
|
1086
|
+
}
|
|
1087
|
+
monthlyFiles.sort((a, b) => b.month.localeCompare(a.month));
|
|
1088
|
+
if (monthlyFiles.length > maxMonths) {
|
|
1089
|
+
const toDelete = monthlyFiles.slice(maxMonths);
|
|
1090
|
+
for (const item of toDelete) {
|
|
1091
|
+
bestEffort('delete-old-archive', () => fs.unlinkSync(item.fullPath));
|
|
1092
|
+
logger.info(`[dsh-cron] pruned old monthly history archive: ${item.file}`);
|
|
1093
|
+
}
|
|
1094
|
+
}
|
|
1095
|
+
} catch (err) {
|
|
1096
|
+
bestEffort('log-prune-err', () => logger.debug?.(`[dsh-cron] prune old monthly archives error: ${err.message}`));
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
|
|
930
1100
|
getHistory(id, limit = 20) {
|
|
931
1101
|
const runs = this.history.get(id) || [];
|
|
932
1102
|
return runs.slice(0, limit);
|
package/lib/task-schema.js
CHANGED
|
@@ -95,7 +95,12 @@ export const TaskSchema = z.object({
|
|
|
95
95
|
previousScript: z.string(),
|
|
96
96
|
swarm: z.any(),
|
|
97
97
|
gitDiff: z.any(),
|
|
98
|
-
|
|
98
|
+
maxMemoryMb: z.number().min(0).default(0),
|
|
99
|
+
executionTimeoutMs: z.number().min(0).default(900000),
|
|
100
|
+
maxLogBytesPerRun: z.number().min(0).default(5 * 1024 * 1024),
|
|
101
|
+
cpus: z.number().min(0).default(0),
|
|
102
|
+
maxArchiveMonths: z.number().min(1).default(6),
|
|
103
|
+
schemaVersion: z.number().default(2),
|
|
99
104
|
createdAt: z.number(),
|
|
100
105
|
updatedAt: z.number(),
|
|
101
106
|
});
|
package/lib/task-transfer.js
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@goodandready/dsh-cron",
|
|
3
|
-
"version": "0.3.
|
|
3
|
+
"version": "0.3.25",
|
|
4
4
|
"description": "Background automation runner for DSH: isolated agent runs, script/HTTP/SSH/Docker runtimes, cost guard, notifications, heartbeats.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "lib/index.js",
|
|
@@ -23,7 +23,8 @@
|
|
|
23
23
|
"build:client": "node scripts/build-client.mjs",
|
|
24
24
|
"pretest": "node scripts/build-client.mjs",
|
|
25
25
|
"preflight": "node scripts/ci-preflight.mjs",
|
|
26
|
-
"test:engine": "node scripts/test-engines.mjs"
|
|
26
|
+
"test:engine": "node scripts/test-engines.mjs",
|
|
27
|
+
"build:client:check": "node scripts/build-client.mjs --check"
|
|
27
28
|
},
|
|
28
29
|
"keywords": [
|
|
29
30
|
"dsh",
|
|
@@ -108,6 +109,7 @@
|
|
|
108
109
|
"@deepseek-ai/dsh-session": ">=0.1.0-rc.6 <0.1.1 || >=0.1.1-rc.1 <0.1.2 || >=0.1.2-alpha.1 <1.0.0",
|
|
109
110
|
"@deepseek-ai/dsh-settings": ">=0.1.0-rc.6 <0.1.1 || >=0.1.1-rc.1 <0.1.2 || >=0.1.2-alpha.1 <1.0.0",
|
|
110
111
|
"@deepseek-ai/dsh-tools": ">=0.1.0-rc.6 <0.1.1 || >=0.1.1-rc.1 <0.1.2 || >=0.1.2-alpha.1 <1.0.0",
|
|
111
|
-
"@deepseek-ai/schemastery": "^3.18.1"
|
|
112
|
+
"@deepseek-ai/schemastery": "^3.18.1",
|
|
113
|
+
"esbuild": "^0.28.2"
|
|
112
114
|
}
|
|
113
115
|
}
|