@karmaniverous/jeeves-runner 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +28 -0
- package/README.md +401 -0
- package/dist/cli/jeeves-runner/index.js +880 -0
- package/dist/db/migrations/001-initial.sql +61 -0
- package/dist/index.d.ts +270 -0
- package/dist/mjs/index.js +917 -0
- package/package.json +141 -0
|
@@ -0,0 +1,880 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { mkdirSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { dirname, resolve } from 'node:path';
|
|
4
|
+
import { Command } from 'commander';
|
|
5
|
+
import { DatabaseSync } from 'node:sqlite';
|
|
6
|
+
import { pino } from 'pino';
|
|
7
|
+
import Fastify from 'fastify';
|
|
8
|
+
import { request } from 'node:https';
|
|
9
|
+
import { spawn } from 'node:child_process';
|
|
10
|
+
import { Cron } from 'croner';
|
|
11
|
+
import { z } from 'zod';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* SQLite connection manager. Creates DB file with parent directories, enables WAL mode for concurrency.
|
|
15
|
+
*/
|
|
16
|
+
/**
|
|
17
|
+
* Create and configure a SQLite database connection.
|
|
18
|
+
* Ensures parent directories exist and enables WAL mode for better concurrency.
|
|
19
|
+
*/
|
|
20
|
+
function createConnection(dbPath) {
|
|
21
|
+
// Ensure parent directory exists
|
|
22
|
+
const dir = dirname(dbPath);
|
|
23
|
+
mkdirSync(dir, { recursive: true });
|
|
24
|
+
// Open database
|
|
25
|
+
const db = new DatabaseSync(dbPath);
|
|
26
|
+
// Enable WAL mode for better concurrency
|
|
27
|
+
db.exec('PRAGMA journal_mode = WAL;');
|
|
28
|
+
db.exec('PRAGMA foreign_keys = ON;');
|
|
29
|
+
return db;
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* Close a database connection cleanly.
|
|
33
|
+
*/
|
|
34
|
+
function closeConnection(db) {
|
|
35
|
+
db.close();
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Schema migration runner. Tracks applied migrations via schema_version table, applies pending migrations idempotently.
|
|
40
|
+
*/
|
|
41
|
+
/** Initial schema migration SQL (embedded to avoid runtime file resolution issues). */
|
|
42
|
+
const MIGRATION_001 = `
|
|
43
|
+
CREATE TABLE IF NOT EXISTS jobs (
|
|
44
|
+
id TEXT PRIMARY KEY,
|
|
45
|
+
name TEXT NOT NULL,
|
|
46
|
+
schedule TEXT NOT NULL,
|
|
47
|
+
script TEXT NOT NULL,
|
|
48
|
+
type TEXT DEFAULT 'script',
|
|
49
|
+
description TEXT,
|
|
50
|
+
enabled INTEGER DEFAULT 1,
|
|
51
|
+
timeout_ms INTEGER,
|
|
52
|
+
overlap_policy TEXT DEFAULT 'skip',
|
|
53
|
+
on_failure TEXT,
|
|
54
|
+
on_success TEXT,
|
|
55
|
+
created_at TEXT DEFAULT (datetime('now')),
|
|
56
|
+
updated_at TEXT DEFAULT (datetime('now'))
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
CREATE TABLE IF NOT EXISTS runs (
|
|
60
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
61
|
+
job_id TEXT NOT NULL REFERENCES jobs(id),
|
|
62
|
+
status TEXT NOT NULL,
|
|
63
|
+
started_at TEXT,
|
|
64
|
+
finished_at TEXT,
|
|
65
|
+
duration_ms INTEGER,
|
|
66
|
+
exit_code INTEGER,
|
|
67
|
+
tokens INTEGER,
|
|
68
|
+
result_meta TEXT,
|
|
69
|
+
error TEXT,
|
|
70
|
+
stdout_tail TEXT,
|
|
71
|
+
stderr_tail TEXT,
|
|
72
|
+
trigger TEXT DEFAULT 'schedule'
|
|
73
|
+
);
|
|
74
|
+
|
|
75
|
+
CREATE INDEX IF NOT EXISTS idx_runs_job_started ON runs(job_id, started_at DESC);
|
|
76
|
+
CREATE INDEX IF NOT EXISTS idx_runs_status ON runs(status);
|
|
77
|
+
|
|
78
|
+
CREATE TABLE IF NOT EXISTS cursors (
|
|
79
|
+
namespace TEXT NOT NULL,
|
|
80
|
+
key TEXT NOT NULL,
|
|
81
|
+
value TEXT,
|
|
82
|
+
expires_at TEXT,
|
|
83
|
+
updated_at TEXT DEFAULT (datetime('now')),
|
|
84
|
+
PRIMARY KEY (namespace, key)
|
|
85
|
+
);
|
|
86
|
+
|
|
87
|
+
CREATE INDEX IF NOT EXISTS idx_cursors_expires ON cursors(expires_at) WHERE expires_at IS NOT NULL;
|
|
88
|
+
|
|
89
|
+
CREATE TABLE IF NOT EXISTS queues (
|
|
90
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
91
|
+
queue TEXT NOT NULL,
|
|
92
|
+
payload TEXT NOT NULL,
|
|
93
|
+
status TEXT DEFAULT 'pending',
|
|
94
|
+
priority INTEGER DEFAULT 0,
|
|
95
|
+
attempts INTEGER DEFAULT 0,
|
|
96
|
+
max_attempts INTEGER DEFAULT 1,
|
|
97
|
+
error TEXT,
|
|
98
|
+
created_at TEXT DEFAULT (datetime('now')),
|
|
99
|
+
claimed_at TEXT,
|
|
100
|
+
finished_at TEXT
|
|
101
|
+
);
|
|
102
|
+
|
|
103
|
+
CREATE INDEX IF NOT EXISTS idx_queues_poll ON queues(queue, status, priority DESC, created_at);
|
|
104
|
+
`;
|
|
105
|
+
/** Registry of all migrations keyed by version number. */
|
|
106
|
+
const MIGRATIONS = {
|
|
107
|
+
1: MIGRATION_001,
|
|
108
|
+
};
|
|
109
|
+
/**
|
|
110
|
+
* Run all pending migrations. Creates schema_version table if needed, applies migrations in order.
|
|
111
|
+
*/
|
|
112
|
+
function runMigrations(db) {
|
|
113
|
+
// Create schema_version table if it doesn't exist
|
|
114
|
+
db.exec(`
|
|
115
|
+
CREATE TABLE IF NOT EXISTS schema_version (
|
|
116
|
+
version INTEGER PRIMARY KEY,
|
|
117
|
+
applied_at TEXT DEFAULT (datetime('now'))
|
|
118
|
+
);
|
|
119
|
+
`);
|
|
120
|
+
// Get current version
|
|
121
|
+
const currentVersionRow = db
|
|
122
|
+
.prepare('SELECT MAX(version) as version FROM schema_version')
|
|
123
|
+
.get();
|
|
124
|
+
const currentVersion = currentVersionRow?.version ?? 0;
|
|
125
|
+
// Apply pending migrations
|
|
126
|
+
const pendingVersions = Object.keys(MIGRATIONS)
|
|
127
|
+
.map(Number)
|
|
128
|
+
.filter((v) => v > currentVersion)
|
|
129
|
+
.sort((a, b) => a - b);
|
|
130
|
+
for (const version of pendingVersions) {
|
|
131
|
+
db.exec(MIGRATIONS[version]);
|
|
132
|
+
db.prepare('INSERT INTO schema_version (version) VALUES (?)').run(version);
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* Fastify API routes for job management and monitoring. Provides endpoints for job CRUD, run history, manual triggers, and system stats.
|
|
138
|
+
*/
|
|
139
|
+
/**
|
|
140
|
+
* Register all API routes on the Fastify instance.
|
|
141
|
+
*/
|
|
142
|
+
function registerRoutes(app, deps) {
|
|
143
|
+
const { db, scheduler } = deps;
|
|
144
|
+
/** GET /health — Health check. */
|
|
145
|
+
app.get('/health', () => {
|
|
146
|
+
return { ok: true, uptime: process.uptime() };
|
|
147
|
+
});
|
|
148
|
+
/** GET /jobs — List all jobs with last run status. */
|
|
149
|
+
app.get('/jobs', () => {
|
|
150
|
+
const rows = db
|
|
151
|
+
.prepare(`SELECT j.*,
|
|
152
|
+
(SELECT status FROM runs WHERE job_id = j.id ORDER BY started_at DESC LIMIT 1) as last_status,
|
|
153
|
+
(SELECT started_at FROM runs WHERE job_id = j.id ORDER BY started_at DESC LIMIT 1) as last_run
|
|
154
|
+
FROM jobs j`)
|
|
155
|
+
.all();
|
|
156
|
+
return { jobs: rows };
|
|
157
|
+
});
|
|
158
|
+
/** GET /jobs/:id — Single job detail. */
|
|
159
|
+
app.get('/jobs/:id', async (request, reply) => {
|
|
160
|
+
const job = db
|
|
161
|
+
.prepare('SELECT * FROM jobs WHERE id = ?')
|
|
162
|
+
.get(request.params.id);
|
|
163
|
+
if (!job) {
|
|
164
|
+
reply.code(404);
|
|
165
|
+
return { error: 'Job not found' };
|
|
166
|
+
}
|
|
167
|
+
return { job };
|
|
168
|
+
});
|
|
169
|
+
/** GET /jobs/:id/runs — Run history for a job. */
|
|
170
|
+
app.get('/jobs/:id/runs', (request) => {
|
|
171
|
+
const limit = parseInt(request.query.limit ?? '50', 10);
|
|
172
|
+
const runs = db
|
|
173
|
+
.prepare('SELECT * FROM runs WHERE job_id = ? ORDER BY started_at DESC LIMIT ?')
|
|
174
|
+
.all(request.params.id, limit);
|
|
175
|
+
return { runs };
|
|
176
|
+
});
|
|
177
|
+
/** POST /jobs/:id/run — Trigger manual job run. */
|
|
178
|
+
app.post('/jobs/:id/run', async (request, reply) => {
|
|
179
|
+
try {
|
|
180
|
+
const result = await scheduler.triggerJob(request.params.id);
|
|
181
|
+
return { result };
|
|
182
|
+
}
|
|
183
|
+
catch (err) {
|
|
184
|
+
reply.code(404);
|
|
185
|
+
return { error: err instanceof Error ? err.message : 'Unknown error' };
|
|
186
|
+
}
|
|
187
|
+
});
|
|
188
|
+
/** POST /jobs/:id/enable — Enable a job. */
|
|
189
|
+
app.post('/jobs/:id/enable', (request, reply) => {
|
|
190
|
+
const result = db
|
|
191
|
+
.prepare('UPDATE jobs SET enabled = 1 WHERE id = ?')
|
|
192
|
+
.run(request.params.id);
|
|
193
|
+
if (result.changes === 0) {
|
|
194
|
+
reply.code(404);
|
|
195
|
+
return { error: 'Job not found' };
|
|
196
|
+
}
|
|
197
|
+
return { ok: true };
|
|
198
|
+
});
|
|
199
|
+
/** POST /jobs/:id/disable — Disable a job. */
|
|
200
|
+
app.post('/jobs/:id/disable', (request, reply) => {
|
|
201
|
+
const result = db
|
|
202
|
+
.prepare('UPDATE jobs SET enabled = 0 WHERE id = ?')
|
|
203
|
+
.run(request.params.id);
|
|
204
|
+
if (result.changes === 0) {
|
|
205
|
+
reply.code(404);
|
|
206
|
+
return { error: 'Job not found' };
|
|
207
|
+
}
|
|
208
|
+
return { ok: true };
|
|
209
|
+
});
|
|
210
|
+
/** GET /stats — Aggregate job statistics. */
|
|
211
|
+
app.get('/stats', () => {
|
|
212
|
+
const totalJobs = db
|
|
213
|
+
.prepare('SELECT COUNT(*) as count FROM jobs')
|
|
214
|
+
.get();
|
|
215
|
+
const runningCount = scheduler.getRunningJobs().length;
|
|
216
|
+
const okLastHour = db
|
|
217
|
+
.prepare(`SELECT COUNT(*) as count FROM runs
|
|
218
|
+
WHERE status = 'ok' AND started_at > datetime('now', '-1 hour')`)
|
|
219
|
+
.get();
|
|
220
|
+
const errorsLastHour = db
|
|
221
|
+
.prepare(`SELECT COUNT(*) as count FROM runs
|
|
222
|
+
WHERE status IN ('error', 'timeout') AND started_at > datetime('now', '-1 hour')`)
|
|
223
|
+
.get();
|
|
224
|
+
return {
|
|
225
|
+
totalJobs: totalJobs.count,
|
|
226
|
+
running: runningCount,
|
|
227
|
+
okLastHour: okLastHour.count,
|
|
228
|
+
errorsLastHour: errorsLastHour.count,
|
|
229
|
+
};
|
|
230
|
+
});
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* Fastify HTTP server for runner API. Creates server instance with logging, registers routes, listens on configured port (localhost only).
|
|
235
|
+
*/
|
|
236
|
+
/**
|
|
237
|
+
* Create and configure the Fastify server. Routes are registered but server is not started.
|
|
238
|
+
*/
|
|
239
|
+
function createServer(config, deps) {
|
|
240
|
+
const app = Fastify({
|
|
241
|
+
logger: {
|
|
242
|
+
level: config.log.level,
|
|
243
|
+
...(config.log.file
|
|
244
|
+
? {
|
|
245
|
+
transport: {
|
|
246
|
+
target: 'pino/file',
|
|
247
|
+
options: { destination: config.log.file },
|
|
248
|
+
},
|
|
249
|
+
}
|
|
250
|
+
: {}),
|
|
251
|
+
},
|
|
252
|
+
});
|
|
253
|
+
registerRoutes(app, deps);
|
|
254
|
+
return app;
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Database maintenance tasks: run retention pruning and expired cursor cleanup.
|
|
259
|
+
*/
|
|
260
|
+
/** Delete runs older than the configured retention period. */
|
|
261
|
+
function pruneOldRuns(db, days, logger) {
|
|
262
|
+
const result = db
|
|
263
|
+
.prepare(`DELETE FROM runs WHERE started_at < datetime('now', '-${String(days)} days')`)
|
|
264
|
+
.run();
|
|
265
|
+
if (result.changes > 0) {
|
|
266
|
+
logger.info({ deleted: result.changes }, 'Pruned old runs');
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
/** Delete expired cursor entries. */
|
|
270
|
+
function cleanExpiredCursors(db, logger) {
|
|
271
|
+
const result = db
|
|
272
|
+
.prepare(`DELETE FROM cursors WHERE expires_at IS NOT NULL AND expires_at < datetime('now')`)
|
|
273
|
+
.run();
|
|
274
|
+
if (result.changes > 0) {
|
|
275
|
+
logger.info({ deleted: result.changes }, 'Cleaned expired cursors');
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
/**
|
|
279
|
+
* Create the maintenance controller. Runs cleanup tasks on startup and at configured intervals.
|
|
280
|
+
*/
|
|
281
|
+
function createMaintenance(db, config, logger) {
|
|
282
|
+
let interval = null;
|
|
283
|
+
function runAll() {
|
|
284
|
+
pruneOldRuns(db, config.runRetentionDays, logger);
|
|
285
|
+
cleanExpiredCursors(db, logger);
|
|
286
|
+
}
|
|
287
|
+
return {
|
|
288
|
+
start() {
|
|
289
|
+
// Run immediately on startup
|
|
290
|
+
runAll();
|
|
291
|
+
// Then run periodically
|
|
292
|
+
interval = setInterval(runAll, config.cursorCleanupIntervalMs);
|
|
293
|
+
},
|
|
294
|
+
stop() {
|
|
295
|
+
if (interval) {
|
|
296
|
+
clearInterval(interval);
|
|
297
|
+
interval = null;
|
|
298
|
+
}
|
|
299
|
+
},
|
|
300
|
+
runNow() {
|
|
301
|
+
runAll();
|
|
302
|
+
},
|
|
303
|
+
};
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
/**
|
|
307
|
+
* Slack notification module. Sends job completion/failure messages via Slack Web API (chat.postMessage). Falls back gracefully if no token.
|
|
308
|
+
*/
|
|
309
|
+
/** Post a message to Slack via chat.postMessage API. */
|
|
310
|
+
function postToSlack(token, channel, text) {
|
|
311
|
+
return new Promise((resolve, reject) => {
|
|
312
|
+
const payload = JSON.stringify({ channel, text });
|
|
313
|
+
const req = request('https://slack.com/api/chat.postMessage', {
|
|
314
|
+
method: 'POST',
|
|
315
|
+
headers: {
|
|
316
|
+
'Content-Type': 'application/json',
|
|
317
|
+
Authorization: `Bearer ${token}`,
|
|
318
|
+
'Content-Length': Buffer.byteLength(payload),
|
|
319
|
+
},
|
|
320
|
+
}, (res) => {
|
|
321
|
+
let body = '';
|
|
322
|
+
res.on('data', (chunk) => {
|
|
323
|
+
body += chunk.toString();
|
|
324
|
+
});
|
|
325
|
+
res.on('end', () => {
|
|
326
|
+
if (res.statusCode === 200) {
|
|
327
|
+
resolve();
|
|
328
|
+
}
|
|
329
|
+
else {
|
|
330
|
+
reject(new Error(`Slack API returned ${String(res.statusCode)}: ${body}`));
|
|
331
|
+
}
|
|
332
|
+
});
|
|
333
|
+
});
|
|
334
|
+
req.on('error', reject);
|
|
335
|
+
req.write(payload);
|
|
336
|
+
req.end();
|
|
337
|
+
});
|
|
338
|
+
}
|
|
339
|
+
/**
|
|
340
|
+
* Create a notifier that sends Slack messages for job events. If no token, logs warning and returns silently.
|
|
341
|
+
*/
|
|
342
|
+
function createNotifier(config) {
|
|
343
|
+
const { slackToken } = config;
|
|
344
|
+
return {
|
|
345
|
+
async notifySuccess(jobName, durationMs, channel) {
|
|
346
|
+
if (!slackToken) {
|
|
347
|
+
console.warn(`No Slack token configured — skipping success notification for ${jobName}`);
|
|
348
|
+
return;
|
|
349
|
+
}
|
|
350
|
+
const durationSec = (durationMs / 1000).toFixed(1);
|
|
351
|
+
const text = `✅ *${jobName}* completed (${durationSec}s)`;
|
|
352
|
+
await postToSlack(slackToken, channel, text);
|
|
353
|
+
},
|
|
354
|
+
async notifyFailure(jobName, durationMs, error, channel) {
|
|
355
|
+
if (!slackToken) {
|
|
356
|
+
console.warn(`No Slack token configured — skipping failure notification for ${jobName}`);
|
|
357
|
+
return;
|
|
358
|
+
}
|
|
359
|
+
const durationSec = (durationMs / 1000).toFixed(1);
|
|
360
|
+
const errorMsg = error ? `: ${error}` : '';
|
|
361
|
+
const text = `⚠️ *${jobName}* failed (${durationSec}s)${errorMsg}`;
|
|
362
|
+
await postToSlack(slackToken, channel, text);
|
|
363
|
+
},
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* Job executor. Spawns job scripts as child processes, captures output, parses result metadata, enforces timeouts.
|
|
369
|
+
*/
|
|
370
|
+
/** Ring buffer for capturing last N lines of output. */
|
|
371
|
+
class RingBuffer {
|
|
372
|
+
maxLines;
|
|
373
|
+
lines = [];
|
|
374
|
+
constructor(maxLines) {
|
|
375
|
+
this.maxLines = maxLines;
|
|
376
|
+
}
|
|
377
|
+
append(line) {
|
|
378
|
+
this.lines.push(line);
|
|
379
|
+
if (this.lines.length > this.maxLines) {
|
|
380
|
+
this.lines.shift();
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
getAll() {
|
|
384
|
+
return this.lines.join('\n');
|
|
385
|
+
}
|
|
386
|
+
}
|
|
387
|
+
/** Parse JR_RESULT:\{json\} lines from stdout to extract tokens and resultMeta. */
|
|
388
|
+
function parseResultLines(stdout) {
|
|
389
|
+
const lines = stdout.split('\n');
|
|
390
|
+
let tokens = null;
|
|
391
|
+
let resultMeta = null;
|
|
392
|
+
for (const line of lines) {
|
|
393
|
+
const match = /^JR_RESULT:(.+)$/.exec(line.trim());
|
|
394
|
+
if (match) {
|
|
395
|
+
try {
|
|
396
|
+
const data = JSON.parse(match[1]);
|
|
397
|
+
if (data.tokens !== undefined)
|
|
398
|
+
tokens = data.tokens;
|
|
399
|
+
if (data.meta !== undefined)
|
|
400
|
+
resultMeta = data.meta;
|
|
401
|
+
}
|
|
402
|
+
catch {
|
|
403
|
+
// Ignore parse errors
|
|
404
|
+
}
|
|
405
|
+
}
|
|
406
|
+
}
|
|
407
|
+
return { tokens, resultMeta };
|
|
408
|
+
}
|
|
409
|
+
/**
|
|
410
|
+
* Execute a job script as a child process. Captures output, parses metadata, enforces timeout.
|
|
411
|
+
*/
|
|
412
|
+
function executeJob(options) {
|
|
413
|
+
const { script, dbPath, jobId, runId, timeoutMs } = options;
|
|
414
|
+
const startTime = Date.now();
|
|
415
|
+
return new Promise((resolve) => {
|
|
416
|
+
const stdoutBuffer = new RingBuffer(100);
|
|
417
|
+
const stderrBuffer = new RingBuffer(100);
|
|
418
|
+
const child = spawn('node', [script], {
|
|
419
|
+
env: {
|
|
420
|
+
...process.env,
|
|
421
|
+
JR_DB_PATH: dbPath,
|
|
422
|
+
JR_JOB_ID: jobId,
|
|
423
|
+
JR_RUN_ID: String(runId),
|
|
424
|
+
},
|
|
425
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
426
|
+
});
|
|
427
|
+
let timedOut = false;
|
|
428
|
+
let timeoutHandle = null;
|
|
429
|
+
if (timeoutMs) {
|
|
430
|
+
timeoutHandle = setTimeout(() => {
|
|
431
|
+
timedOut = true;
|
|
432
|
+
child.kill('SIGTERM');
|
|
433
|
+
setTimeout(() => child.kill('SIGKILL'), 5000); // Force kill after 5s
|
|
434
|
+
}, timeoutMs);
|
|
435
|
+
}
|
|
436
|
+
child.stdout.on('data', (chunk) => {
|
|
437
|
+
const lines = chunk.toString().split('\n');
|
|
438
|
+
for (const line of lines) {
|
|
439
|
+
if (line.trim())
|
|
440
|
+
stdoutBuffer.append(line);
|
|
441
|
+
}
|
|
442
|
+
});
|
|
443
|
+
child.stderr.on('data', (chunk) => {
|
|
444
|
+
const lines = chunk.toString().split('\n');
|
|
445
|
+
for (const line of lines) {
|
|
446
|
+
if (line.trim())
|
|
447
|
+
stderrBuffer.append(line);
|
|
448
|
+
}
|
|
449
|
+
});
|
|
450
|
+
child.on('close', (exitCode) => {
|
|
451
|
+
if (timeoutHandle)
|
|
452
|
+
clearTimeout(timeoutHandle);
|
|
453
|
+
const durationMs = Date.now() - startTime;
|
|
454
|
+
const stdoutTail = stdoutBuffer.getAll();
|
|
455
|
+
const stderrTail = stderrBuffer.getAll();
|
|
456
|
+
const { tokens, resultMeta } = parseResultLines(stdoutTail);
|
|
457
|
+
if (timedOut) {
|
|
458
|
+
resolve({
|
|
459
|
+
status: 'timeout',
|
|
460
|
+
exitCode: null,
|
|
461
|
+
durationMs,
|
|
462
|
+
tokens: null,
|
|
463
|
+
resultMeta: null,
|
|
464
|
+
stdoutTail,
|
|
465
|
+
stderrTail,
|
|
466
|
+
error: `Job timed out after ${String(timeoutMs)}ms`,
|
|
467
|
+
});
|
|
468
|
+
}
|
|
469
|
+
else if (exitCode === 0) {
|
|
470
|
+
resolve({
|
|
471
|
+
status: 'ok',
|
|
472
|
+
exitCode,
|
|
473
|
+
durationMs,
|
|
474
|
+
tokens,
|
|
475
|
+
resultMeta,
|
|
476
|
+
stdoutTail,
|
|
477
|
+
stderrTail,
|
|
478
|
+
error: null,
|
|
479
|
+
});
|
|
480
|
+
}
|
|
481
|
+
else {
|
|
482
|
+
resolve({
|
|
483
|
+
status: 'error',
|
|
484
|
+
exitCode,
|
|
485
|
+
durationMs,
|
|
486
|
+
tokens,
|
|
487
|
+
resultMeta,
|
|
488
|
+
stdoutTail,
|
|
489
|
+
stderrTail,
|
|
490
|
+
error: stderrTail || `Exit code ${String(exitCode)}`,
|
|
491
|
+
});
|
|
492
|
+
}
|
|
493
|
+
});
|
|
494
|
+
child.on('error', (err) => {
|
|
495
|
+
if (timeoutHandle)
|
|
496
|
+
clearTimeout(timeoutHandle);
|
|
497
|
+
const durationMs = Date.now() - startTime;
|
|
498
|
+
resolve({
|
|
499
|
+
status: 'error',
|
|
500
|
+
exitCode: null,
|
|
501
|
+
durationMs,
|
|
502
|
+
tokens: null,
|
|
503
|
+
resultMeta: null,
|
|
504
|
+
stdoutTail: stdoutBuffer.getAll(),
|
|
505
|
+
stderrTail: stderrBuffer.getAll(),
|
|
506
|
+
error: err.message,
|
|
507
|
+
});
|
|
508
|
+
});
|
|
509
|
+
});
|
|
510
|
+
}
|
|
511
|
+
|
|
512
|
+
/**
|
|
513
|
+
* Croner-based job scheduler. Loads enabled jobs, creates cron instances, manages execution, respects overlap policies and concurrency limits.
|
|
514
|
+
*/
|
|
515
|
+
/**
|
|
516
|
+
* Create the job scheduler. Manages cron schedules, job execution, overlap policies, and notifications.
|
|
517
|
+
*/
|
|
518
|
+
function createScheduler(deps) {
|
|
519
|
+
const { db, executor, notifier, config, logger } = deps;
|
|
520
|
+
const crons = new Map();
|
|
521
|
+
const runningJobs = new Set();
|
|
522
|
+
/** Insert a run record and return its ID. */
|
|
523
|
+
function createRun(jobId, trigger) {
|
|
524
|
+
const result = db
|
|
525
|
+
.prepare(`INSERT INTO runs (job_id, status, started_at, trigger)
|
|
526
|
+
VALUES (?, 'running', datetime('now'), ?)`)
|
|
527
|
+
.run(jobId, trigger);
|
|
528
|
+
return result.lastInsertRowid;
|
|
529
|
+
}
|
|
530
|
+
/** Update run record with completion data. */
|
|
531
|
+
function finishRun(runId, execResult) {
|
|
532
|
+
db.prepare(`UPDATE runs SET status = ?, finished_at = datetime('now'), duration_ms = ?,
|
|
533
|
+
exit_code = ?, tokens = ?, result_meta = ?, error = ?, stdout_tail = ?, stderr_tail = ?
|
|
534
|
+
WHERE id = ?`).run(execResult.status, execResult.durationMs, execResult.exitCode, execResult.tokens, execResult.resultMeta, execResult.error, execResult.stdoutTail, execResult.stderrTail, runId);
|
|
535
|
+
}
|
|
536
|
+
/** Execute a job: create run record, run script, update record, send notifications. */
|
|
537
|
+
async function runJob(job, trigger) {
|
|
538
|
+
const { id, name, script, timeout_ms, on_success, on_failure } = job;
|
|
539
|
+
// Check concurrency limit
|
|
540
|
+
if (runningJobs.size >= config.maxConcurrency) {
|
|
541
|
+
logger.warn({ jobId: id }, 'Max concurrency reached, skipping job');
|
|
542
|
+
throw new Error('Max concurrency reached');
|
|
543
|
+
}
|
|
544
|
+
runningJobs.add(id);
|
|
545
|
+
const runId = createRun(id, trigger);
|
|
546
|
+
logger.info({ jobId: id, runId, trigger }, 'Starting job');
|
|
547
|
+
try {
|
|
548
|
+
const result = await executor({
|
|
549
|
+
script,
|
|
550
|
+
dbPath: config.dbPath,
|
|
551
|
+
jobId: id,
|
|
552
|
+
runId,
|
|
553
|
+
timeoutMs: timeout_ms ?? undefined,
|
|
554
|
+
});
|
|
555
|
+
finishRun(runId, result);
|
|
556
|
+
logger.info({ jobId: id, runId, status: result.status }, 'Job finished');
|
|
557
|
+
// Send notifications
|
|
558
|
+
if (result.status === 'ok' && on_success) {
|
|
559
|
+
await notifier
|
|
560
|
+
.notifySuccess(name, result.durationMs, on_success)
|
|
561
|
+
.catch((err) => {
|
|
562
|
+
logger.error({ jobId: id, err }, 'Notification failed');
|
|
563
|
+
});
|
|
564
|
+
}
|
|
565
|
+
else if (result.status !== 'ok' && on_failure) {
|
|
566
|
+
await notifier
|
|
567
|
+
.notifyFailure(name, result.durationMs, result.error, on_failure)
|
|
568
|
+
.catch((err) => {
|
|
569
|
+
logger.error({ jobId: id, err }, 'Notification failed');
|
|
570
|
+
});
|
|
571
|
+
}
|
|
572
|
+
return result;
|
|
573
|
+
}
|
|
574
|
+
finally {
|
|
575
|
+
runningJobs.delete(id);
|
|
576
|
+
}
|
|
577
|
+
}
|
|
578
|
+
/** Handle scheduled job fire. */
|
|
579
|
+
async function onScheduledRun(job) {
|
|
580
|
+
const { id, overlap_policy } = job;
|
|
581
|
+
// Check overlap policy
|
|
582
|
+
if (runningJobs.has(id)) {
|
|
583
|
+
if (overlap_policy === 'skip') {
|
|
584
|
+
logger.info({ jobId: id }, 'Job already running, skipping (overlap_policy=skip)');
|
|
585
|
+
return;
|
|
586
|
+
}
|
|
587
|
+
else if (overlap_policy === 'queue') {
|
|
588
|
+
logger.info({ jobId: id }, 'Job already running, queueing (overlap_policy=queue)');
|
|
589
|
+
// In a real implementation, we'd queue this. For now, just skip.
|
|
590
|
+
return;
|
|
591
|
+
}
|
|
592
|
+
// 'allow' policy: proceed
|
|
593
|
+
}
|
|
594
|
+
await runJob(job, 'schedule').catch((err) => {
|
|
595
|
+
logger.error({ jobId: id, err }, 'Job execution failed');
|
|
596
|
+
});
|
|
597
|
+
}
|
|
598
|
+
return {
|
|
599
|
+
start() {
|
|
600
|
+
// Load all enabled jobs
|
|
601
|
+
const jobs = db
|
|
602
|
+
.prepare('SELECT * FROM jobs WHERE enabled = 1')
|
|
603
|
+
.all();
|
|
604
|
+
logger.info({ count: jobs.length }, 'Loading jobs');
|
|
605
|
+
for (const job of jobs) {
|
|
606
|
+
try {
|
|
607
|
+
const cron = new Cron(job.schedule, () => {
|
|
608
|
+
void onScheduledRun(job);
|
|
609
|
+
});
|
|
610
|
+
crons.set(job.id, cron);
|
|
611
|
+
logger.info({ jobId: job.id, schedule: job.schedule }, 'Scheduled job');
|
|
612
|
+
}
|
|
613
|
+
catch (err) {
|
|
614
|
+
logger.error({ jobId: job.id, err }, 'Failed to schedule job');
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
},
|
|
618
|
+
stop() {
|
|
619
|
+
logger.info('Stopping scheduler');
|
|
620
|
+
// Stop all crons
|
|
621
|
+
for (const cron of crons.values()) {
|
|
622
|
+
cron.stop();
|
|
623
|
+
}
|
|
624
|
+
crons.clear();
|
|
625
|
+
// Wait for running jobs (simple poll with timeout)
|
|
626
|
+
const deadline = Date.now() + config.shutdownGraceMs;
|
|
627
|
+
const checkInterval = setInterval(() => {
|
|
628
|
+
if (runningJobs.size === 0 || Date.now() > deadline) {
|
|
629
|
+
clearInterval(checkInterval);
|
|
630
|
+
if (runningJobs.size > 0) {
|
|
631
|
+
logger.warn({ count: runningJobs.size }, 'Forced shutdown with running jobs');
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
}, 100);
|
|
635
|
+
},
|
|
636
|
+
async triggerJob(jobId) {
|
|
637
|
+
const job = db.prepare('SELECT * FROM jobs WHERE id = ?').get(jobId);
|
|
638
|
+
if (!job)
|
|
639
|
+
throw new Error(`Job not found: ${jobId}`);
|
|
640
|
+
return runJob(job, 'manual');
|
|
641
|
+
},
|
|
642
|
+
getRunningJobs() {
|
|
643
|
+
return Array.from(runningJobs);
|
|
644
|
+
},
|
|
645
|
+
};
|
|
646
|
+
}
|
|
647
|
+
|
|
648
|
+
/**
|
|
649
|
+
* Main runner orchestrator. Wires up database, scheduler, API server, and handles graceful shutdown on SIGTERM/SIGINT.
|
|
650
|
+
*/
|
|
651
|
+
/**
|
|
652
|
+
* Create the runner. Initializes database, scheduler, API server, and sets up graceful shutdown.
|
|
653
|
+
*/
|
|
654
|
+
function createRunner(config) {
|
|
655
|
+
let db = null;
|
|
656
|
+
let scheduler = null;
|
|
657
|
+
let server = null;
|
|
658
|
+
let maintenance = null;
|
|
659
|
+
const logger = pino({
|
|
660
|
+
level: config.log.level,
|
|
661
|
+
...(config.log.file
|
|
662
|
+
? {
|
|
663
|
+
transport: {
|
|
664
|
+
target: 'pino/file',
|
|
665
|
+
options: { destination: config.log.file },
|
|
666
|
+
},
|
|
667
|
+
}
|
|
668
|
+
: {}),
|
|
669
|
+
});
|
|
670
|
+
return {
|
|
671
|
+
async start() {
|
|
672
|
+
logger.info('Starting runner');
|
|
673
|
+
// Database
|
|
674
|
+
db = createConnection(config.dbPath);
|
|
675
|
+
runMigrations(db);
|
|
676
|
+
logger.info({ dbPath: config.dbPath }, 'Database ready');
|
|
677
|
+
// Notifier
|
|
678
|
+
const slackToken = config.notifications.slackTokenPath
|
|
679
|
+
? readFileSync(config.notifications.slackTokenPath, 'utf-8').trim()
|
|
680
|
+
: null;
|
|
681
|
+
const notifier = createNotifier({ slackToken });
|
|
682
|
+
// Maintenance (run retention pruning + cursor cleanup)
|
|
683
|
+
maintenance = createMaintenance(db, {
|
|
684
|
+
runRetentionDays: config.runRetentionDays,
|
|
685
|
+
cursorCleanupIntervalMs: config.cursorCleanupIntervalMs,
|
|
686
|
+
}, logger);
|
|
687
|
+
maintenance.start();
|
|
688
|
+
logger.info('Maintenance tasks started');
|
|
689
|
+
// Scheduler
|
|
690
|
+
scheduler = createScheduler({
|
|
691
|
+
db,
|
|
692
|
+
executor: executeJob,
|
|
693
|
+
notifier,
|
|
694
|
+
config,
|
|
695
|
+
logger,
|
|
696
|
+
});
|
|
697
|
+
scheduler.start();
|
|
698
|
+
logger.info('Scheduler started');
|
|
699
|
+
// API server
|
|
700
|
+
server = createServer(config, { db, scheduler });
|
|
701
|
+
await server.listen({ port: config.port, host: '127.0.0.1' });
|
|
702
|
+
logger.info({ port: config.port }, 'API server listening');
|
|
703
|
+
// Graceful shutdown
|
|
704
|
+
const shutdown = async (signal) => {
|
|
705
|
+
logger.info({ signal }, 'Received shutdown signal');
|
|
706
|
+
await this.stop();
|
|
707
|
+
process.exit(0);
|
|
708
|
+
};
|
|
709
|
+
process.on('SIGTERM', () => {
|
|
710
|
+
void shutdown('SIGTERM');
|
|
711
|
+
});
|
|
712
|
+
process.on('SIGINT', () => {
|
|
713
|
+
void shutdown('SIGINT');
|
|
714
|
+
});
|
|
715
|
+
},
|
|
716
|
+
async stop() {
|
|
717
|
+
logger.info('Stopping runner');
|
|
718
|
+
if (maintenance) {
|
|
719
|
+
maintenance.stop();
|
|
720
|
+
logger.info('Maintenance stopped');
|
|
721
|
+
}
|
|
722
|
+
if (scheduler) {
|
|
723
|
+
scheduler.stop();
|
|
724
|
+
logger.info('Scheduler stopped');
|
|
725
|
+
}
|
|
726
|
+
if (server) {
|
|
727
|
+
await server.close();
|
|
728
|
+
logger.info('API server stopped');
|
|
729
|
+
}
|
|
730
|
+
if (db) {
|
|
731
|
+
closeConnection(db);
|
|
732
|
+
logger.info('Database closed');
|
|
733
|
+
}
|
|
734
|
+
},
|
|
735
|
+
};
|
|
736
|
+
}
|
|
737
|
+
|
|
738
|
+
/**
|
|
739
|
+
* Runner configuration schema and types.
|
|
740
|
+
*
|
|
741
|
+
* @module
|
|
742
|
+
*/
|
|
743
|
+
/** Notification configuration sub-schema. */
|
|
744
|
+
const notificationsSchema = z.object({
|
|
745
|
+
slackTokenPath: z.string().optional(),
|
|
746
|
+
defaultOnFailure: z.string().nullable().default(null),
|
|
747
|
+
defaultOnSuccess: z.string().nullable().default(null),
|
|
748
|
+
});
|
|
749
|
+
/** Log configuration sub-schema. */
|
|
750
|
+
const logSchema = z.object({
|
|
751
|
+
level: z
|
|
752
|
+
.enum(['trace', 'debug', 'info', 'warn', 'error', 'fatal'])
|
|
753
|
+
.default('info'),
|
|
754
|
+
file: z.string().optional(),
|
|
755
|
+
});
|
|
756
|
+
/** Full runner configuration schema. Validates and provides defaults. */
|
|
757
|
+
const runnerConfigSchema = z.object({
|
|
758
|
+
port: z.number().default(3100),
|
|
759
|
+
dbPath: z.string().default('./data/runner.sqlite'),
|
|
760
|
+
maxConcurrency: z.number().default(4),
|
|
761
|
+
runRetentionDays: z.number().default(30),
|
|
762
|
+
cursorCleanupIntervalMs: z.number().default(3600000),
|
|
763
|
+
shutdownGraceMs: z.number().default(30000),
|
|
764
|
+
notifications: notificationsSchema.default({
|
|
765
|
+
defaultOnFailure: null,
|
|
766
|
+
defaultOnSuccess: null,
|
|
767
|
+
}),
|
|
768
|
+
log: logSchema.default({ level: 'info' }),
|
|
769
|
+
});
|
|
770
|
+
|
|
771
|
+
/**
|
|
772
|
+
* CLI entry point for jeeves-runner.
|
|
773
|
+
*
|
|
774
|
+
* @module
|
|
775
|
+
*/
|
|
776
|
+
/** Load and validate config from a JSON file path, or return defaults. */
|
|
777
|
+
function loadConfig(configPath) {
|
|
778
|
+
if (configPath) {
|
|
779
|
+
const raw = readFileSync(resolve(configPath), 'utf-8');
|
|
780
|
+
return runnerConfigSchema.parse(JSON.parse(raw));
|
|
781
|
+
}
|
|
782
|
+
return runnerConfigSchema.parse({});
|
|
783
|
+
}
|
|
784
|
+
const program = new Command();
|
|
785
|
+
program
|
|
786
|
+
.name('jeeves-runner')
|
|
787
|
+
.description('Graph-aware job execution engine with SQLite state')
|
|
788
|
+
.version('0.0.0');
|
|
789
|
+
program
|
|
790
|
+
.command('start')
|
|
791
|
+
.description('Start the runner daemon')
|
|
792
|
+
.option('-c, --config <path>', 'Path to config file')
|
|
793
|
+
.action((options) => {
|
|
794
|
+
const config = loadConfig(options.config);
|
|
795
|
+
const runner = createRunner(config);
|
|
796
|
+
void runner.start();
|
|
797
|
+
});
|
|
798
|
+
program
|
|
799
|
+
.command('status')
|
|
800
|
+
.description('Show runner status')
|
|
801
|
+
.option('-c, --config <path>', 'Path to config file')
|
|
802
|
+
.action((options) => {
|
|
803
|
+
const config = loadConfig(options.config);
|
|
804
|
+
void (async () => {
|
|
805
|
+
try {
|
|
806
|
+
const resp = await fetch(`http://127.0.0.1:${String(config.port)}/stats`);
|
|
807
|
+
const stats = (await resp.json());
|
|
808
|
+
console.log(JSON.stringify(stats, null, 2));
|
|
809
|
+
}
|
|
810
|
+
catch {
|
|
811
|
+
console.error(`Runner not reachable on port ${String(config.port)}. Is it running?`);
|
|
812
|
+
process.exit(1);
|
|
813
|
+
}
|
|
814
|
+
})();
|
|
815
|
+
});
|
|
816
|
+
program
|
|
817
|
+
.command('add-job')
|
|
818
|
+
.description('Add a new job')
|
|
819
|
+
.requiredOption('-i, --id <id>', 'Job ID')
|
|
820
|
+
.requiredOption('-n, --name <name>', 'Job name')
|
|
821
|
+
.requiredOption('-s, --schedule <schedule>', 'Cron schedule')
|
|
822
|
+
.requiredOption('--script <script>', 'Absolute path to script')
|
|
823
|
+
.option('-t, --type <type>', 'Job type (script|session)', 'script')
|
|
824
|
+
.option('-d, --description <desc>', 'Job description')
|
|
825
|
+
.option('--timeout <ms>', 'Timeout in ms')
|
|
826
|
+
.option('--overlap <policy>', 'Overlap policy (skip|queue|allow)', 'skip')
|
|
827
|
+
.option('--on-failure <channel>', 'Slack channel for failure alerts')
|
|
828
|
+
.option('--on-success <channel>', 'Slack channel for success alerts')
|
|
829
|
+
.option('-c, --config <path>', 'Path to config file')
|
|
830
|
+
.action((options) => {
|
|
831
|
+
const config = loadConfig(options.config);
|
|
832
|
+
const db = createConnection(config.dbPath);
|
|
833
|
+
runMigrations(db);
|
|
834
|
+
db.prepare(`INSERT INTO jobs (id, name, schedule, script, type, description, timeout_ms, overlap_policy, on_failure, on_success)
|
|
835
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(options.id, options.name, options.schedule, resolve(options.script), options.type, options.description ?? null, options.timeout ? parseInt(options.timeout, 10) : null, options.overlap, options.onFailure ?? null, options.onSuccess ?? null);
|
|
836
|
+
console.log(`Job '${options.id}' added.`);
|
|
837
|
+
db.close();
|
|
838
|
+
});
|
|
839
|
+
program
|
|
840
|
+
.command('list-jobs')
|
|
841
|
+
.description('List all jobs')
|
|
842
|
+
.option('-c, --config <path>', 'Path to config file')
|
|
843
|
+
.action((options) => {
|
|
844
|
+
const config = loadConfig(options.config);
|
|
845
|
+
const db = createConnection(config.dbPath);
|
|
846
|
+
runMigrations(db);
|
|
847
|
+
const rows = db
|
|
848
|
+
.prepare('SELECT id, name, schedule, enabled FROM jobs')
|
|
849
|
+
.all();
|
|
850
|
+
if (rows.length === 0) {
|
|
851
|
+
console.log('No jobs configured.');
|
|
852
|
+
}
|
|
853
|
+
else {
|
|
854
|
+
for (const row of rows) {
|
|
855
|
+
const status = row.enabled ? '✅' : '⏸️';
|
|
856
|
+
console.log(`${status} ${row.id} ${row.schedule} ${row.name}`);
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
db.close();
|
|
860
|
+
});
|
|
861
|
+
program
|
|
862
|
+
.command('trigger')
|
|
863
|
+
.description('Manually trigger a job')
|
|
864
|
+
.requiredOption('-i, --id <id>', 'Job ID to trigger')
|
|
865
|
+
.option('-c, --config <path>', 'Path to config file')
|
|
866
|
+
.action((options) => {
|
|
867
|
+
const config = loadConfig(options.config);
|
|
868
|
+
void (async () => {
|
|
869
|
+
try {
|
|
870
|
+
const resp = await fetch(`http://127.0.0.1:${String(config.port)}/jobs/${options.id}/run`, { method: 'POST' });
|
|
871
|
+
const result = (await resp.json());
|
|
872
|
+
console.log(JSON.stringify(result, null, 2));
|
|
873
|
+
}
|
|
874
|
+
catch {
|
|
875
|
+
console.error(`Runner not reachable on port ${String(config.port)}. Is it running?`);
|
|
876
|
+
process.exit(1);
|
|
877
|
+
}
|
|
878
|
+
})();
|
|
879
|
+
});
|
|
880
|
+
program.parse();
|