@donkeylabs/server 0.5.0 → 0.6.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/external-jobs.md +131 -11
- package/docs/router.md +93 -0
- package/examples/external-jobs/python/donkeylabs_job.py +366 -0
- package/examples/external-jobs/shell/donkeylabs-job.sh +264 -0
- package/examples/external-jobs/shell/example-job.sh +47 -0
- package/package.json +3 -2
- package/src/client/base.ts +6 -4
- package/src/core/external-job-socket.ts +142 -21
- package/src/core/index.ts +29 -0
- package/src/core/job-adapter-sqlite.ts +287 -0
- package/src/core/jobs.ts +36 -3
- package/src/core/process-adapter-sqlite.ts +282 -0
- package/src/core/process-socket.ts +521 -0
- package/src/core/processes.ts +758 -0
- package/src/core.ts +75 -4
- package/src/harness.ts +3 -0
- package/src/index.ts +12 -0
- package/src/server.ts +32 -3
|
@@ -0,0 +1,287 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in SQLite Job Adapter
|
|
3
|
+
*
|
|
4
|
+
* Provides automatic persistence for jobs, enabling server restart resilience
|
|
5
|
+
* for external jobs without requiring user configuration.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { Database } from "bun:sqlite";
|
|
9
|
+
import { mkdir } from "node:fs/promises";
|
|
10
|
+
import { dirname } from "node:path";
|
|
11
|
+
import type { Job, JobAdapter, JobStatus } from "./jobs";
|
|
12
|
+
import type { ExternalJobProcessState } from "./external-jobs";
|
|
13
|
+
|
|
14
|
+
export interface SqliteJobAdapterConfig {
|
|
15
|
+
/** Path to SQLite database file (default: .donkeylabs/jobs.db) */
|
|
16
|
+
path?: string;
|
|
17
|
+
/** Auto-cleanup completed jobs older than N days (default: 7, 0 to disable) */
|
|
18
|
+
cleanupDays?: number;
|
|
19
|
+
/** Cleanup interval in ms (default: 3600000 = 1 hour) */
|
|
20
|
+
cleanupInterval?: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export class SqliteJobAdapter implements JobAdapter {
|
|
24
|
+
private db: Database;
|
|
25
|
+
private initialized = false;
|
|
26
|
+
private cleanupTimer?: ReturnType<typeof setInterval>;
|
|
27
|
+
private cleanupDays: number;
|
|
28
|
+
|
|
29
|
+
constructor(config: SqliteJobAdapterConfig = {}) {
|
|
30
|
+
const dbPath = config.path ?? ".donkeylabs/jobs.db";
|
|
31
|
+
this.cleanupDays = config.cleanupDays ?? 7;
|
|
32
|
+
|
|
33
|
+
// Ensure directory exists
|
|
34
|
+
this.ensureDir(dbPath);
|
|
35
|
+
|
|
36
|
+
this.db = new Database(dbPath);
|
|
37
|
+
this.init();
|
|
38
|
+
|
|
39
|
+
// Start cleanup timer
|
|
40
|
+
if (this.cleanupDays > 0) {
|
|
41
|
+
const interval = config.cleanupInterval ?? 3600000; // 1 hour
|
|
42
|
+
this.cleanupTimer = setInterval(() => this.cleanup(), interval);
|
|
43
|
+
// Run cleanup on startup
|
|
44
|
+
this.cleanup();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
private ensureDir(dbPath: string): void {
|
|
49
|
+
const dir = dirname(dbPath);
|
|
50
|
+
if (dir && dir !== ".") {
|
|
51
|
+
// Sync mkdir for constructor
|
|
52
|
+
try {
|
|
53
|
+
Bun.spawnSync(["mkdir", "-p", dir]);
|
|
54
|
+
} catch {
|
|
55
|
+
// Directory may already exist
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
private init(): void {
|
|
61
|
+
if (this.initialized) return;
|
|
62
|
+
|
|
63
|
+
this.db.run(`
|
|
64
|
+
CREATE TABLE IF NOT EXISTS jobs (
|
|
65
|
+
id TEXT PRIMARY KEY,
|
|
66
|
+
name TEXT NOT NULL,
|
|
67
|
+
data TEXT NOT NULL,
|
|
68
|
+
status TEXT NOT NULL DEFAULT 'pending',
|
|
69
|
+
created_at TEXT NOT NULL,
|
|
70
|
+
run_at TEXT,
|
|
71
|
+
started_at TEXT,
|
|
72
|
+
completed_at TEXT,
|
|
73
|
+
result TEXT,
|
|
74
|
+
error TEXT,
|
|
75
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
76
|
+
max_attempts INTEGER NOT NULL DEFAULT 3,
|
|
77
|
+
-- External job fields
|
|
78
|
+
external INTEGER DEFAULT 0,
|
|
79
|
+
pid INTEGER,
|
|
80
|
+
socket_path TEXT,
|
|
81
|
+
tcp_port INTEGER,
|
|
82
|
+
last_heartbeat TEXT,
|
|
83
|
+
process_state TEXT
|
|
84
|
+
)
|
|
85
|
+
`);
|
|
86
|
+
|
|
87
|
+
// Indexes for efficient queries
|
|
88
|
+
this.db.run(`CREATE INDEX IF NOT EXISTS idx_jobs_status ON jobs(status)`);
|
|
89
|
+
this.db.run(`CREATE INDEX IF NOT EXISTS idx_jobs_name ON jobs(name)`);
|
|
90
|
+
this.db.run(`CREATE INDEX IF NOT EXISTS idx_jobs_external ON jobs(external, status)`);
|
|
91
|
+
this.db.run(`CREATE INDEX IF NOT EXISTS idx_jobs_scheduled ON jobs(status, run_at)`);
|
|
92
|
+
|
|
93
|
+
this.initialized = true;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
async create(job: Omit<Job, "id">): Promise<Job> {
|
|
97
|
+
const id = `job_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
|
|
98
|
+
|
|
99
|
+
this.db.run(
|
|
100
|
+
`INSERT INTO jobs (
|
|
101
|
+
id, name, data, status, created_at, run_at, attempts, max_attempts,
|
|
102
|
+
external, process_state
|
|
103
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
104
|
+
[
|
|
105
|
+
id,
|
|
106
|
+
job.name,
|
|
107
|
+
JSON.stringify(job.data),
|
|
108
|
+
job.status,
|
|
109
|
+
job.createdAt.toISOString(),
|
|
110
|
+
job.runAt?.toISOString() ?? null,
|
|
111
|
+
job.attempts,
|
|
112
|
+
job.maxAttempts,
|
|
113
|
+
job.external ? 1 : 0,
|
|
114
|
+
job.processState ?? null,
|
|
115
|
+
]
|
|
116
|
+
);
|
|
117
|
+
|
|
118
|
+
return { ...job, id };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
async get(jobId: string): Promise<Job | null> {
|
|
122
|
+
const row = this.db.query(`SELECT * FROM jobs WHERE id = ?`).get(jobId) as any;
|
|
123
|
+
if (!row) return null;
|
|
124
|
+
return this.rowToJob(row);
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async update(jobId: string, updates: Partial<Job>): Promise<void> {
|
|
128
|
+
const sets: string[] = [];
|
|
129
|
+
const values: any[] = [];
|
|
130
|
+
|
|
131
|
+
if (updates.status !== undefined) {
|
|
132
|
+
sets.push("status = ?");
|
|
133
|
+
values.push(updates.status);
|
|
134
|
+
}
|
|
135
|
+
if (updates.startedAt !== undefined) {
|
|
136
|
+
sets.push("started_at = ?");
|
|
137
|
+
values.push(updates.startedAt?.toISOString() ?? null);
|
|
138
|
+
}
|
|
139
|
+
if (updates.completedAt !== undefined) {
|
|
140
|
+
sets.push("completed_at = ?");
|
|
141
|
+
values.push(updates.completedAt?.toISOString() ?? null);
|
|
142
|
+
}
|
|
143
|
+
if (updates.result !== undefined) {
|
|
144
|
+
sets.push("result = ?");
|
|
145
|
+
values.push(JSON.stringify(updates.result));
|
|
146
|
+
}
|
|
147
|
+
if (updates.error !== undefined) {
|
|
148
|
+
sets.push("error = ?");
|
|
149
|
+
values.push(updates.error);
|
|
150
|
+
}
|
|
151
|
+
if (updates.attempts !== undefined) {
|
|
152
|
+
sets.push("attempts = ?");
|
|
153
|
+
values.push(updates.attempts);
|
|
154
|
+
}
|
|
155
|
+
// External job fields
|
|
156
|
+
if (updates.pid !== undefined) {
|
|
157
|
+
sets.push("pid = ?");
|
|
158
|
+
values.push(updates.pid);
|
|
159
|
+
}
|
|
160
|
+
if (updates.socketPath !== undefined) {
|
|
161
|
+
sets.push("socket_path = ?");
|
|
162
|
+
values.push(updates.socketPath);
|
|
163
|
+
}
|
|
164
|
+
if (updates.tcpPort !== undefined) {
|
|
165
|
+
sets.push("tcp_port = ?");
|
|
166
|
+
values.push(updates.tcpPort);
|
|
167
|
+
}
|
|
168
|
+
if (updates.lastHeartbeat !== undefined) {
|
|
169
|
+
sets.push("last_heartbeat = ?");
|
|
170
|
+
values.push(updates.lastHeartbeat?.toISOString() ?? null);
|
|
171
|
+
}
|
|
172
|
+
if (updates.processState !== undefined) {
|
|
173
|
+
sets.push("process_state = ?");
|
|
174
|
+
values.push(updates.processState);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
if (sets.length === 0) return;
|
|
178
|
+
|
|
179
|
+
values.push(jobId);
|
|
180
|
+
this.db.run(`UPDATE jobs SET ${sets.join(", ")} WHERE id = ?`, values);
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
async delete(jobId: string): Promise<boolean> {
|
|
184
|
+
const result = this.db.run(`DELETE FROM jobs WHERE id = ?`, [jobId]);
|
|
185
|
+
return result.changes > 0;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
async getPending(limit: number = 100): Promise<Job[]> {
|
|
189
|
+
const rows = this.db
|
|
190
|
+
.query(`SELECT * FROM jobs WHERE status = 'pending' ORDER BY created_at LIMIT ?`)
|
|
191
|
+
.all(limit) as any[];
|
|
192
|
+
return rows.map((r) => this.rowToJob(r));
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
async getScheduledReady(now: Date): Promise<Job[]> {
|
|
196
|
+
const rows = this.db
|
|
197
|
+
.query(`SELECT * FROM jobs WHERE status = 'scheduled' AND run_at <= ? ORDER BY run_at`)
|
|
198
|
+
.all(now.toISOString()) as any[];
|
|
199
|
+
return rows.map((r) => this.rowToJob(r));
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async getByName(name: string, status?: JobStatus): Promise<Job[]> {
|
|
203
|
+
let query = `SELECT * FROM jobs WHERE name = ?`;
|
|
204
|
+
const params: any[] = [name];
|
|
205
|
+
|
|
206
|
+
if (status) {
|
|
207
|
+
query += ` AND status = ?`;
|
|
208
|
+
params.push(status);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
query += ` ORDER BY created_at DESC`;
|
|
212
|
+
|
|
213
|
+
const rows = this.db.query(query).all(...params) as any[];
|
|
214
|
+
return rows.map((r) => this.rowToJob(r));
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
async getRunningExternal(): Promise<Job[]> {
|
|
218
|
+
const rows = this.db
|
|
219
|
+
.query(`SELECT * FROM jobs WHERE external = 1 AND status = 'running'`)
|
|
220
|
+
.all() as any[];
|
|
221
|
+
return rows.map((r) => this.rowToJob(r));
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async getOrphanedExternal(): Promise<Job[]> {
|
|
225
|
+
// Get external jobs that were running when server died
|
|
226
|
+
const rows = this.db
|
|
227
|
+
.query(
|
|
228
|
+
`SELECT * FROM jobs WHERE external = 1 AND status = 'running'
|
|
229
|
+
AND (process_state = 'running' OR process_state = 'orphaned' OR process_state = 'spawning')`
|
|
230
|
+
)
|
|
231
|
+
.all() as any[];
|
|
232
|
+
return rows.map((r) => this.rowToJob(r));
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
private rowToJob(row: any): Job {
|
|
236
|
+
return {
|
|
237
|
+
id: row.id,
|
|
238
|
+
name: row.name,
|
|
239
|
+
data: JSON.parse(row.data),
|
|
240
|
+
status: row.status as JobStatus,
|
|
241
|
+
createdAt: new Date(row.created_at),
|
|
242
|
+
runAt: row.run_at ? new Date(row.run_at) : undefined,
|
|
243
|
+
startedAt: row.started_at ? new Date(row.started_at) : undefined,
|
|
244
|
+
completedAt: row.completed_at ? new Date(row.completed_at) : undefined,
|
|
245
|
+
result: row.result ? JSON.parse(row.result) : undefined,
|
|
246
|
+
error: row.error ?? undefined,
|
|
247
|
+
attempts: row.attempts,
|
|
248
|
+
maxAttempts: row.max_attempts,
|
|
249
|
+
// External job fields
|
|
250
|
+
external: row.external === 1 ? true : undefined,
|
|
251
|
+
pid: row.pid ?? undefined,
|
|
252
|
+
socketPath: row.socket_path ?? undefined,
|
|
253
|
+
tcpPort: row.tcp_port ?? undefined,
|
|
254
|
+
lastHeartbeat: row.last_heartbeat ? new Date(row.last_heartbeat) : undefined,
|
|
255
|
+
processState: row.process_state as ExternalJobProcessState | undefined,
|
|
256
|
+
};
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
/** Clean up old completed/failed jobs */
|
|
260
|
+
private cleanup(): void {
|
|
261
|
+
if (this.cleanupDays <= 0) return;
|
|
262
|
+
|
|
263
|
+
try {
|
|
264
|
+
const cutoff = new Date();
|
|
265
|
+
cutoff.setDate(cutoff.getDate() - this.cleanupDays);
|
|
266
|
+
|
|
267
|
+
const result = this.db.run(
|
|
268
|
+
`DELETE FROM jobs WHERE (status = 'completed' OR status = 'failed') AND completed_at < ?`,
|
|
269
|
+
[cutoff.toISOString()]
|
|
270
|
+
);
|
|
271
|
+
|
|
272
|
+
if (result.changes > 0) {
|
|
273
|
+
console.log(`[Jobs] Cleaned up ${result.changes} old jobs`);
|
|
274
|
+
}
|
|
275
|
+
} catch (err) {
|
|
276
|
+
console.error("[Jobs] Cleanup error:", err);
|
|
277
|
+
}
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
/** Stop the adapter and cleanup timer */
|
|
281
|
+
stop(): void {
|
|
282
|
+
if (this.cleanupTimer) {
|
|
283
|
+
clearInterval(this.cleanupTimer);
|
|
284
|
+
this.cleanupTimer = undefined;
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
}
|
package/src/core/jobs.ts
CHANGED
|
@@ -25,6 +25,7 @@ import {
|
|
|
25
25
|
createExternalJobSocketServer,
|
|
26
26
|
type ExternalJobSocketServer,
|
|
27
27
|
} from "./external-job-socket";
|
|
28
|
+
import { SqliteJobAdapter } from "./job-adapter-sqlite";
|
|
28
29
|
|
|
29
30
|
export type JobStatus = "pending" | "running" | "completed" | "failed" | "scheduled";
|
|
30
31
|
|
|
@@ -82,6 +83,13 @@ export interface JobsConfig {
|
|
|
82
83
|
maxAttempts?: number; // Default retry attempts, default 3
|
|
83
84
|
/** External jobs configuration */
|
|
84
85
|
external?: ExternalJobsConfig;
|
|
86
|
+
/**
|
|
87
|
+
* Use SQLite for persistence (default: true when external jobs are used)
|
|
88
|
+
* Set to false to use MemoryJobAdapter (not recommended for production)
|
|
89
|
+
*/
|
|
90
|
+
persist?: boolean;
|
|
91
|
+
/** SQLite database path (default: .donkeylabs/jobs.db) */
|
|
92
|
+
dbPath?: string;
|
|
85
93
|
}
|
|
86
94
|
|
|
87
95
|
export interface Jobs {
|
|
@@ -188,6 +196,7 @@ export class MemoryJobAdapter implements JobAdapter {
|
|
|
188
196
|
|
|
189
197
|
class JobsImpl implements Jobs {
|
|
190
198
|
private adapter: JobAdapter;
|
|
199
|
+
private sqliteAdapter?: SqliteJobAdapter;
|
|
191
200
|
private events?: Events;
|
|
192
201
|
private handlers = new Map<string, JobHandler>();
|
|
193
202
|
private running = false;
|
|
@@ -197,6 +206,8 @@ class JobsImpl implements Jobs {
|
|
|
197
206
|
private concurrency: number;
|
|
198
207
|
private pollInterval: number;
|
|
199
208
|
private defaultMaxAttempts: number;
|
|
209
|
+
private usePersistence: boolean;
|
|
210
|
+
private dbPath?: string;
|
|
200
211
|
|
|
201
212
|
// External jobs support
|
|
202
213
|
private externalConfigs = new Map<string, ExternalJobConfig>();
|
|
@@ -205,12 +216,23 @@ class JobsImpl implements Jobs {
|
|
|
205
216
|
private externalProcesses = new Map<string, { pid: number; timeout?: ReturnType<typeof setTimeout> }>();
|
|
206
217
|
|
|
207
218
|
constructor(config: JobsConfig = {}) {
|
|
208
|
-
this.adapter = config.adapter ?? new MemoryJobAdapter();
|
|
209
219
|
this.events = config.events;
|
|
210
220
|
this.concurrency = config.concurrency ?? 5;
|
|
211
221
|
this.pollInterval = config.pollInterval ?? 1000;
|
|
212
222
|
this.defaultMaxAttempts = config.maxAttempts ?? 3;
|
|
213
223
|
this.externalConfig = config.external ?? {};
|
|
224
|
+
this.usePersistence = config.persist ?? true; // Default to SQLite persistence
|
|
225
|
+
this.dbPath = config.dbPath;
|
|
226
|
+
|
|
227
|
+
// Use provided adapter, or create SQLite adapter if persistence enabled
|
|
228
|
+
if (config.adapter) {
|
|
229
|
+
this.adapter = config.adapter;
|
|
230
|
+
} else if (this.usePersistence) {
|
|
231
|
+
this.sqliteAdapter = new SqliteJobAdapter({ path: this.dbPath });
|
|
232
|
+
this.adapter = this.sqliteAdapter;
|
|
233
|
+
} else {
|
|
234
|
+
this.adapter = new MemoryJobAdapter();
|
|
235
|
+
}
|
|
214
236
|
}
|
|
215
237
|
|
|
216
238
|
register<T = any, R = any>(name: string, handler: JobHandler<T, R>): void {
|
|
@@ -362,6 +384,11 @@ class JobsImpl implements Jobs {
|
|
|
362
384
|
this.socketServer = null;
|
|
363
385
|
}
|
|
364
386
|
|
|
387
|
+
// Stop SQLite adapter cleanup timer
|
|
388
|
+
if (this.sqliteAdapter) {
|
|
389
|
+
this.sqliteAdapter.stop();
|
|
390
|
+
}
|
|
391
|
+
|
|
365
392
|
// Wait for active in-process jobs to complete (with timeout)
|
|
366
393
|
const maxWait = 30000; // 30 seconds
|
|
367
394
|
const startTime = Date.now();
|
|
@@ -476,6 +503,9 @@ class JobsImpl implements Jobs {
|
|
|
476
503
|
console.log(`[Jobs] Found orphaned job ${job.id} with PID ${job.pid}, attempting reconnect`);
|
|
477
504
|
activeJobIds.add(job.id);
|
|
478
505
|
|
|
506
|
+
// Reserve the socket path/port to prevent new jobs from using it
|
|
507
|
+
this.socketServer?.reserve(job.id, job.socketPath, job.tcpPort);
|
|
508
|
+
|
|
479
509
|
// Try to reconnect to the socket
|
|
480
510
|
const reconnected = await this.socketServer?.reconnect(
|
|
481
511
|
job.id,
|
|
@@ -496,7 +526,7 @@ class JobsImpl implements Jobs {
|
|
|
496
526
|
});
|
|
497
527
|
}
|
|
498
528
|
} else {
|
|
499
|
-
// Mark as orphaned, but keep tracking
|
|
529
|
+
// Mark as orphaned, but keep tracking (reservation remains)
|
|
500
530
|
await this.adapter.update(job.id, { processState: "orphaned" });
|
|
501
531
|
|
|
502
532
|
if (this.events) {
|
|
@@ -507,7 +537,7 @@ class JobsImpl implements Jobs {
|
|
|
507
537
|
}
|
|
508
538
|
}
|
|
509
539
|
} else {
|
|
510
|
-
// Process is dead, mark job as failed
|
|
540
|
+
// Process is dead, mark job as failed and release any reservations
|
|
511
541
|
console.log(`[Jobs] Orphaned job ${job.id} process (PID ${job.pid}) is dead`);
|
|
512
542
|
await this.adapter.update(job.id, {
|
|
513
543
|
status: "failed",
|
|
@@ -515,6 +545,9 @@ class JobsImpl implements Jobs {
|
|
|
515
545
|
completedAt: new Date(),
|
|
516
546
|
});
|
|
517
547
|
|
|
548
|
+
// Release reservation since the job is done
|
|
549
|
+
this.socketServer?.release(job.id);
|
|
550
|
+
|
|
518
551
|
if (this.events) {
|
|
519
552
|
await this.events.emit("job.failed", {
|
|
520
553
|
jobId: job.id,
|
|
@@ -0,0 +1,282 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in SQLite Process Adapter
|
|
3
|
+
*
|
|
4
|
+
* Provides automatic persistence for managed processes, enabling server restart resilience
|
|
5
|
+
* and orphan recovery without requiring user configuration.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { Database } from "bun:sqlite";
|
|
9
|
+
import { mkdir } from "node:fs/promises";
|
|
10
|
+
import { dirname } from "node:path";
|
|
11
|
+
import type { ProcessStatus, ManagedProcess, ProcessConfig } from "./processes";
|
|
12
|
+
|
|
13
|
+
export interface SqliteProcessAdapterConfig {
|
|
14
|
+
/** Path to SQLite database file (default: .donkeylabs/processes.db) */
|
|
15
|
+
path?: string;
|
|
16
|
+
/** Auto-cleanup stopped processes older than N days (default: 7, 0 to disable) */
|
|
17
|
+
cleanupDays?: number;
|
|
18
|
+
/** Cleanup interval in ms (default: 3600000 = 1 hour) */
|
|
19
|
+
cleanupInterval?: number;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface ProcessAdapter {
|
|
23
|
+
/** Create a new process record */
|
|
24
|
+
create(process: Omit<ManagedProcess, "id">): Promise<ManagedProcess>;
|
|
25
|
+
/** Get a process by ID */
|
|
26
|
+
get(processId: string): Promise<ManagedProcess | null>;
|
|
27
|
+
/** Update a process record */
|
|
28
|
+
update(processId: string, updates: Partial<ManagedProcess>): Promise<void>;
|
|
29
|
+
/** Delete a process record */
|
|
30
|
+
delete(processId: string): Promise<boolean>;
|
|
31
|
+
/** Get all processes by name */
|
|
32
|
+
getByName(name: string): Promise<ManagedProcess[]>;
|
|
33
|
+
/** Get all running processes */
|
|
34
|
+
getRunning(): Promise<ManagedProcess[]>;
|
|
35
|
+
/** Get orphaned processes (status="running" from before crash) */
|
|
36
|
+
getOrphaned(): Promise<ManagedProcess[]>;
|
|
37
|
+
/** Stop the adapter and cleanup timer */
|
|
38
|
+
stop(): void;
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export class SqliteProcessAdapter implements ProcessAdapter {
|
|
42
|
+
private db: Database;
|
|
43
|
+
private initialized = false;
|
|
44
|
+
private cleanupTimer?: ReturnType<typeof setInterval>;
|
|
45
|
+
private cleanupDays: number;
|
|
46
|
+
|
|
47
|
+
constructor(config: SqliteProcessAdapterConfig = {}) {
|
|
48
|
+
const dbPath = config.path ?? ".donkeylabs/processes.db";
|
|
49
|
+
this.cleanupDays = config.cleanupDays ?? 7;
|
|
50
|
+
|
|
51
|
+
// Ensure directory exists
|
|
52
|
+
this.ensureDir(dbPath);
|
|
53
|
+
|
|
54
|
+
this.db = new Database(dbPath);
|
|
55
|
+
this.init();
|
|
56
|
+
|
|
57
|
+
// Start cleanup timer
|
|
58
|
+
if (this.cleanupDays > 0) {
|
|
59
|
+
const interval = config.cleanupInterval ?? 3600000; // 1 hour
|
|
60
|
+
this.cleanupTimer = setInterval(() => this.cleanup(), interval);
|
|
61
|
+
// Run cleanup on startup
|
|
62
|
+
this.cleanup();
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
private ensureDir(dbPath: string): void {
|
|
67
|
+
const dir = dirname(dbPath);
|
|
68
|
+
if (dir && dir !== ".") {
|
|
69
|
+
// Sync mkdir for constructor
|
|
70
|
+
try {
|
|
71
|
+
Bun.spawnSync(["mkdir", "-p", dir]);
|
|
72
|
+
} catch {
|
|
73
|
+
// Directory may already exist
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
private init(): void {
|
|
79
|
+
if (this.initialized) return;
|
|
80
|
+
|
|
81
|
+
this.db.run(`
|
|
82
|
+
CREATE TABLE IF NOT EXISTS processes (
|
|
83
|
+
id TEXT PRIMARY KEY,
|
|
84
|
+
name TEXT NOT NULL,
|
|
85
|
+
pid INTEGER,
|
|
86
|
+
socket_path TEXT,
|
|
87
|
+
tcp_port INTEGER,
|
|
88
|
+
status TEXT NOT NULL DEFAULT 'stopped',
|
|
89
|
+
config TEXT NOT NULL,
|
|
90
|
+
metadata TEXT,
|
|
91
|
+
created_at TEXT NOT NULL,
|
|
92
|
+
started_at TEXT,
|
|
93
|
+
stopped_at TEXT,
|
|
94
|
+
last_heartbeat TEXT,
|
|
95
|
+
restart_count INTEGER DEFAULT 0,
|
|
96
|
+
consecutive_failures INTEGER DEFAULT 0,
|
|
97
|
+
error TEXT
|
|
98
|
+
)
|
|
99
|
+
`);
|
|
100
|
+
|
|
101
|
+
// Indexes for efficient queries
|
|
102
|
+
this.db.run(`CREATE INDEX IF NOT EXISTS idx_processes_status ON processes(status)`);
|
|
103
|
+
this.db.run(`CREATE INDEX IF NOT EXISTS idx_processes_name ON processes(name)`);
|
|
104
|
+
this.db.run(`CREATE INDEX IF NOT EXISTS idx_processes_name_status ON processes(name, status)`);
|
|
105
|
+
|
|
106
|
+
this.initialized = true;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
async create(process: Omit<ManagedProcess, "id">): Promise<ManagedProcess> {
|
|
110
|
+
const id = `proc_${Date.now()}_${Math.random().toString(36).slice(2, 9)}`;
|
|
111
|
+
|
|
112
|
+
this.db.run(
|
|
113
|
+
`INSERT INTO processes (
|
|
114
|
+
id, name, pid, socket_path, tcp_port, status, config, metadata,
|
|
115
|
+
created_at, started_at, stopped_at, last_heartbeat,
|
|
116
|
+
restart_count, consecutive_failures, error
|
|
117
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
118
|
+
[
|
|
119
|
+
id,
|
|
120
|
+
process.name,
|
|
121
|
+
process.pid ?? null,
|
|
122
|
+
process.socketPath ?? null,
|
|
123
|
+
process.tcpPort ?? null,
|
|
124
|
+
process.status,
|
|
125
|
+
JSON.stringify(process.config),
|
|
126
|
+
process.metadata ? JSON.stringify(process.metadata) : null,
|
|
127
|
+
process.createdAt.toISOString(),
|
|
128
|
+
process.startedAt?.toISOString() ?? null,
|
|
129
|
+
process.stoppedAt?.toISOString() ?? null,
|
|
130
|
+
process.lastHeartbeat?.toISOString() ?? null,
|
|
131
|
+
process.restartCount ?? 0,
|
|
132
|
+
process.consecutiveFailures ?? 0,
|
|
133
|
+
process.error ?? null,
|
|
134
|
+
]
|
|
135
|
+
);
|
|
136
|
+
|
|
137
|
+
return { ...process, id };
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
async get(processId: string): Promise<ManagedProcess | null> {
|
|
141
|
+
const row = this.db.query(`SELECT * FROM processes WHERE id = ?`).get(processId) as any;
|
|
142
|
+
if (!row) return null;
|
|
143
|
+
return this.rowToProcess(row);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
async update(processId: string, updates: Partial<ManagedProcess>): Promise<void> {
|
|
147
|
+
const sets: string[] = [];
|
|
148
|
+
const values: any[] = [];
|
|
149
|
+
|
|
150
|
+
if (updates.pid !== undefined) {
|
|
151
|
+
sets.push("pid = ?");
|
|
152
|
+
values.push(updates.pid);
|
|
153
|
+
}
|
|
154
|
+
if (updates.socketPath !== undefined) {
|
|
155
|
+
sets.push("socket_path = ?");
|
|
156
|
+
values.push(updates.socketPath);
|
|
157
|
+
}
|
|
158
|
+
if (updates.tcpPort !== undefined) {
|
|
159
|
+
sets.push("tcp_port = ?");
|
|
160
|
+
values.push(updates.tcpPort);
|
|
161
|
+
}
|
|
162
|
+
if (updates.status !== undefined) {
|
|
163
|
+
sets.push("status = ?");
|
|
164
|
+
values.push(updates.status);
|
|
165
|
+
}
|
|
166
|
+
if (updates.config !== undefined) {
|
|
167
|
+
sets.push("config = ?");
|
|
168
|
+
values.push(JSON.stringify(updates.config));
|
|
169
|
+
}
|
|
170
|
+
if (updates.metadata !== undefined) {
|
|
171
|
+
sets.push("metadata = ?");
|
|
172
|
+
values.push(updates.metadata ? JSON.stringify(updates.metadata) : null);
|
|
173
|
+
}
|
|
174
|
+
if (updates.startedAt !== undefined) {
|
|
175
|
+
sets.push("started_at = ?");
|
|
176
|
+
values.push(updates.startedAt?.toISOString() ?? null);
|
|
177
|
+
}
|
|
178
|
+
if (updates.stoppedAt !== undefined) {
|
|
179
|
+
sets.push("stopped_at = ?");
|
|
180
|
+
values.push(updates.stoppedAt?.toISOString() ?? null);
|
|
181
|
+
}
|
|
182
|
+
if (updates.lastHeartbeat !== undefined) {
|
|
183
|
+
sets.push("last_heartbeat = ?");
|
|
184
|
+
values.push(updates.lastHeartbeat?.toISOString() ?? null);
|
|
185
|
+
}
|
|
186
|
+
if (updates.restartCount !== undefined) {
|
|
187
|
+
sets.push("restart_count = ?");
|
|
188
|
+
values.push(updates.restartCount);
|
|
189
|
+
}
|
|
190
|
+
if (updates.consecutiveFailures !== undefined) {
|
|
191
|
+
sets.push("consecutive_failures = ?");
|
|
192
|
+
values.push(updates.consecutiveFailures);
|
|
193
|
+
}
|
|
194
|
+
if (updates.error !== undefined) {
|
|
195
|
+
sets.push("error = ?");
|
|
196
|
+
values.push(updates.error);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
if (sets.length === 0) return;
|
|
200
|
+
|
|
201
|
+
values.push(processId);
|
|
202
|
+
this.db.run(`UPDATE processes SET ${sets.join(", ")} WHERE id = ?`, values);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
async delete(processId: string): Promise<boolean> {
|
|
206
|
+
const result = this.db.run(`DELETE FROM processes WHERE id = ?`, [processId]);
|
|
207
|
+
return result.changes > 0;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
async getByName(name: string): Promise<ManagedProcess[]> {
|
|
211
|
+
const rows = this.db
|
|
212
|
+
.query(`SELECT * FROM processes WHERE name = ? ORDER BY created_at DESC`)
|
|
213
|
+
.all(name) as any[];
|
|
214
|
+
return rows.map((r) => this.rowToProcess(r));
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
async getRunning(): Promise<ManagedProcess[]> {
|
|
218
|
+
const rows = this.db
|
|
219
|
+
.query(`SELECT * FROM processes WHERE status = 'running' OR status = 'spawning'`)
|
|
220
|
+
.all() as any[];
|
|
221
|
+
return rows.map((r) => this.rowToProcess(r));
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
async getOrphaned(): Promise<ManagedProcess[]> {
|
|
225
|
+
// Get processes that were running or spawning when server died
|
|
226
|
+
const rows = this.db
|
|
227
|
+
.query(
|
|
228
|
+
`SELECT * FROM processes WHERE status IN ('running', 'spawning', 'orphaned')`
|
|
229
|
+
)
|
|
230
|
+
.all() as any[];
|
|
231
|
+
return rows.map((r) => this.rowToProcess(r));
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
private rowToProcess(row: any): ManagedProcess {
|
|
235
|
+
return {
|
|
236
|
+
id: row.id,
|
|
237
|
+
name: row.name,
|
|
238
|
+
pid: row.pid ?? undefined,
|
|
239
|
+
socketPath: row.socket_path ?? undefined,
|
|
240
|
+
tcpPort: row.tcp_port ?? undefined,
|
|
241
|
+
status: row.status as ProcessStatus,
|
|
242
|
+
config: JSON.parse(row.config),
|
|
243
|
+
metadata: row.metadata ? JSON.parse(row.metadata) : undefined,
|
|
244
|
+
createdAt: new Date(row.created_at),
|
|
245
|
+
startedAt: row.started_at ? new Date(row.started_at) : undefined,
|
|
246
|
+
stoppedAt: row.stopped_at ? new Date(row.stopped_at) : undefined,
|
|
247
|
+
lastHeartbeat: row.last_heartbeat ? new Date(row.last_heartbeat) : undefined,
|
|
248
|
+
restartCount: row.restart_count ?? 0,
|
|
249
|
+
consecutiveFailures: row.consecutive_failures ?? 0,
|
|
250
|
+
error: row.error ?? undefined,
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
/** Clean up old stopped/crashed processes */
|
|
255
|
+
private cleanup(): void {
|
|
256
|
+
if (this.cleanupDays <= 0) return;
|
|
257
|
+
|
|
258
|
+
try {
|
|
259
|
+
const cutoff = new Date();
|
|
260
|
+
cutoff.setDate(cutoff.getDate() - this.cleanupDays);
|
|
261
|
+
|
|
262
|
+
const result = this.db.run(
|
|
263
|
+
`DELETE FROM processes WHERE status IN ('stopped', 'crashed', 'dead') AND stopped_at < ?`,
|
|
264
|
+
[cutoff.toISOString()]
|
|
265
|
+
);
|
|
266
|
+
|
|
267
|
+
if (result.changes > 0) {
|
|
268
|
+
console.log(`[Processes] Cleaned up ${result.changes} old process records`);
|
|
269
|
+
}
|
|
270
|
+
} catch (err) {
|
|
271
|
+
console.error("[Processes] Cleanup error:", err);
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/** Stop the adapter and cleanup timer */
|
|
276
|
+
stop(): void {
|
|
277
|
+
if (this.cleanupTimer) {
|
|
278
|
+
clearInterval(this.cleanupTimer);
|
|
279
|
+
this.cleanupTimer = undefined;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
}
|