@donkeylabs/server 2.0.35 → 2.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/docs/code-organization.md +424 -0
- package/docs/project-structure.md +37 -26
- package/docs/swift-adapter.md +293 -0
- package/docs/versioning.md +351 -0
- package/package.json +1 -1
- package/src/client/base.ts +18 -5
- package/src/core/events.ts +54 -7
- package/src/core/health.ts +165 -0
- package/src/core/index.ts +12 -0
- package/src/core/jobs.ts +11 -4
- package/src/core/logs-adapter-kysely.ts +7 -0
- package/src/core/logs.ts +3 -0
- package/src/core/migrations/workflows/002_add_metadata_column.ts +41 -7
- package/src/core/processes.ts +7 -3
- package/src/core/subprocess-bootstrap.ts +3 -0
- package/src/core/watchdog-executor.ts +58 -20
- package/src/core/workflow-executor.ts +1 -0
- package/src/core/workflow-state-machine.ts +6 -5
- package/src/core/workflows.ts +3 -2
- package/src/core.ts +9 -1
- package/src/generator/index.ts +4 -0
- package/src/harness.ts +17 -5
- package/src/index.ts +21 -0
- package/src/router.ts +22 -2
- package/src/server.ts +469 -118
- package/src/versioning.ts +154 -0
package/src/core/events.ts
CHANGED
|
@@ -11,20 +11,33 @@ export interface Subscription {
|
|
|
11
11
|
unsubscribe(): void;
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
export interface EventMetadata {
|
|
15
|
+
traceId?: string;
|
|
16
|
+
source?: string;
|
|
17
|
+
[key: string]: any;
|
|
18
|
+
}
|
|
19
|
+
|
|
14
20
|
export interface EventRecord {
|
|
15
21
|
event: string;
|
|
16
22
|
data: any;
|
|
17
23
|
timestamp: Date;
|
|
24
|
+
metadata?: EventMetadata;
|
|
18
25
|
}
|
|
19
26
|
|
|
20
27
|
export interface EventAdapter {
|
|
21
|
-
publish(event: string, data: any): Promise<void>;
|
|
28
|
+
publish(event: string, data: any, metadata?: EventMetadata): Promise<void>;
|
|
22
29
|
getHistory(event: string, limit?: number): Promise<EventRecord[]>;
|
|
30
|
+
/** Subscribe to events from other instances (for distributed adapters) */
|
|
31
|
+
subscribe?(callback: (event: string, data: any, metadata?: EventMetadata) => void): Promise<void>;
|
|
32
|
+
/** Stop the adapter and clean up resources */
|
|
33
|
+
stop?(): Promise<void>;
|
|
23
34
|
}
|
|
24
35
|
|
|
25
36
|
export interface EventsConfig {
|
|
26
37
|
adapter?: EventAdapter;
|
|
27
38
|
maxHistorySize?: number;
|
|
39
|
+
/** SSE service for auto-propagating distributed events to SSE clients */
|
|
40
|
+
sse?: import("./sse").SSE;
|
|
28
41
|
}
|
|
29
42
|
|
|
30
43
|
/**
|
|
@@ -41,11 +54,11 @@ export interface Events {
|
|
|
41
54
|
/**
|
|
42
55
|
* Emit a typed event (when EventRegistry is augmented)
|
|
43
56
|
*/
|
|
44
|
-
emit<K extends keyof EventRegistry>(event: K, data: EventRegistry[K]): Promise<void>;
|
|
57
|
+
emit<K extends keyof EventRegistry>(event: K, data: EventRegistry[K], metadata?: EventMetadata): Promise<void>;
|
|
45
58
|
/**
|
|
46
59
|
* Emit an untyped event (fallback for dynamic event names)
|
|
47
60
|
*/
|
|
48
|
-
emit<T = any>(event: string, data: T): Promise<void>;
|
|
61
|
+
emit<T = any>(event: string, data: T, metadata?: EventMetadata): Promise<void>;
|
|
49
62
|
|
|
50
63
|
/**
|
|
51
64
|
* Subscribe to a typed event (when EventRegistry is augmented)
|
|
@@ -67,6 +80,8 @@ export interface Events {
|
|
|
67
80
|
|
|
68
81
|
off(event: string, handler?: EventHandler): void;
|
|
69
82
|
getHistory(event: string, limit?: number): Promise<EventRecord[]>;
|
|
83
|
+
/** Stop the event bus and clean up resources */
|
|
84
|
+
stop(): Promise<void>;
|
|
70
85
|
}
|
|
71
86
|
|
|
72
87
|
// In-memory event adapter with history
|
|
@@ -78,11 +93,12 @@ export class MemoryEventAdapter implements EventAdapter {
|
|
|
78
93
|
this.maxHistorySize = maxHistorySize;
|
|
79
94
|
}
|
|
80
95
|
|
|
81
|
-
async publish(event: string, data: any): Promise<void> {
|
|
96
|
+
async publish(event: string, data: any, metadata?: EventMetadata): Promise<void> {
|
|
82
97
|
const record: EventRecord = {
|
|
83
98
|
event,
|
|
84
99
|
data,
|
|
85
100
|
timestamp: new Date(),
|
|
101
|
+
metadata,
|
|
86
102
|
};
|
|
87
103
|
|
|
88
104
|
this.history.push(record);
|
|
@@ -103,16 +119,41 @@ export class MemoryEventAdapter implements EventAdapter {
|
|
|
103
119
|
class EventsImpl implements Events {
|
|
104
120
|
private handlers = new Map<string, Set<EventHandler>>();
|
|
105
121
|
private adapter: EventAdapter;
|
|
122
|
+
private sse?: import("./sse").SSE;
|
|
123
|
+
private stopped = false;
|
|
106
124
|
|
|
107
125
|
constructor(config: EventsConfig = {}) {
|
|
108
126
|
this.adapter = config.adapter ?? new MemoryEventAdapter(config.maxHistorySize);
|
|
127
|
+
this.sse = config.sse;
|
|
128
|
+
|
|
129
|
+
// If adapter supports distributed subscriptions, set up the callback
|
|
130
|
+
if (this.adapter.subscribe) {
|
|
131
|
+
this.adapter.subscribe((event, data, metadata) => {
|
|
132
|
+
// Dispatch to local handlers without re-publishing to adapter
|
|
133
|
+
this.dispatchToLocalHandlers(event, data, metadata);
|
|
134
|
+
// Propagate to SSE clients so browser subscribers see distributed events
|
|
135
|
+
this.sse?.broadcastAll(event, data);
|
|
136
|
+
});
|
|
137
|
+
}
|
|
109
138
|
}
|
|
110
139
|
|
|
111
|
-
async emit<T = any>(event: string, data: T): Promise<void> {
|
|
140
|
+
async emit<T = any>(event: string, data: T, metadata?: EventMetadata): Promise<void> {
|
|
141
|
+
if (this.stopped) return;
|
|
142
|
+
|
|
112
143
|
// Store in adapter (for history/persistence)
|
|
113
|
-
await this.adapter.publish(event, data);
|
|
144
|
+
await this.adapter.publish(event, data, metadata);
|
|
145
|
+
|
|
146
|
+
// Dispatch to local handlers
|
|
147
|
+
await this.dispatchToLocalHandlers(event, data, metadata);
|
|
148
|
+
}
|
|
114
149
|
|
|
115
|
-
|
|
150
|
+
/**
|
|
151
|
+
* Dispatch an event to locally registered handlers.
|
|
152
|
+
* Separated from emit() so distributed adapters can deliver remote events
|
|
153
|
+
* without re-publishing to the adapter.
|
|
154
|
+
*/
|
|
155
|
+
private async dispatchToLocalHandlers(event: string, data: any, _metadata?: EventMetadata): Promise<void> {
|
|
156
|
+
// Notify exact-match handlers
|
|
116
157
|
const eventHandlers = this.handlers.get(event);
|
|
117
158
|
if (eventHandlers) {
|
|
118
159
|
const promises: Promise<void>[] = [];
|
|
@@ -185,6 +226,12 @@ class EventsImpl implements Events {
|
|
|
185
226
|
return this.adapter.getHistory(event, limit);
|
|
186
227
|
}
|
|
187
228
|
|
|
229
|
+
async stop(): Promise<void> {
|
|
230
|
+
this.stopped = true;
|
|
231
|
+
await this.adapter.stop?.();
|
|
232
|
+
this.handlers.clear();
|
|
233
|
+
}
|
|
234
|
+
|
|
188
235
|
private matchPattern(event: string, pattern: string): boolean {
|
|
189
236
|
// Convert glob pattern to regex (e.g., "user.*" -> /^user\..*$/)
|
|
190
237
|
const regex = new RegExp(
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
// Core Health Check Service
|
|
2
|
+
// Provides liveness and readiness probes for production deployments
|
|
3
|
+
|
|
4
|
+
import type { Kysely } from "kysely";
|
|
5
|
+
|
|
6
|
+
export type HealthStatus = "healthy" | "degraded" | "unhealthy";
|
|
7
|
+
|
|
8
|
+
export interface HealthCheckResult {
|
|
9
|
+
status: HealthStatus;
|
|
10
|
+
message?: string;
|
|
11
|
+
latencyMs?: number;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface HealthCheck {
|
|
15
|
+
name: string;
|
|
16
|
+
check: () => Promise<HealthCheckResult> | HealthCheckResult;
|
|
17
|
+
/** Whether failure of this check marks the service as unhealthy (default: true) */
|
|
18
|
+
critical?: boolean;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface HealthResponse {
|
|
22
|
+
status: HealthStatus;
|
|
23
|
+
uptime: number;
|
|
24
|
+
timestamp: string;
|
|
25
|
+
checks: Record<string, HealthCheckResult>;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
export interface HealthConfig {
|
|
29
|
+
checks?: HealthCheck[];
|
|
30
|
+
/** Path for liveness probe (default: "/_health") */
|
|
31
|
+
livenessPath?: string;
|
|
32
|
+
/** Path for readiness probe (default: "/_ready") */
|
|
33
|
+
readinessPath?: string;
|
|
34
|
+
/** Timeout per check in ms (default: 5000) */
|
|
35
|
+
checkTimeout?: number;
|
|
36
|
+
/** Whether to register a built-in database check (default: true) */
|
|
37
|
+
dbCheck?: boolean;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export interface Health {
|
|
41
|
+
/** Register a health check */
|
|
42
|
+
register(check: HealthCheck): void;
|
|
43
|
+
/** Run all readiness checks */
|
|
44
|
+
check(): Promise<HealthResponse>;
|
|
45
|
+
/** Fast liveness probe (no external checks) */
|
|
46
|
+
liveness(isShuttingDown: boolean): HealthResponse;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
class HealthImpl implements Health {
|
|
50
|
+
private checks: HealthCheck[] = [];
|
|
51
|
+
private startTime = Date.now();
|
|
52
|
+
private checkTimeout: number;
|
|
53
|
+
|
|
54
|
+
constructor(config: HealthConfig = {}) {
|
|
55
|
+
this.checkTimeout = config.checkTimeout ?? 5000;
|
|
56
|
+
|
|
57
|
+
if (config.checks) {
|
|
58
|
+
for (const check of config.checks) {
|
|
59
|
+
this.checks.push(check);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
register(check: HealthCheck): void {
|
|
65
|
+
this.checks.push(check);
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
async check(): Promise<HealthResponse> {
|
|
69
|
+
const results: Record<string, HealthCheckResult> = {};
|
|
70
|
+
let overallStatus: HealthStatus = "healthy";
|
|
71
|
+
|
|
72
|
+
await Promise.all(
|
|
73
|
+
this.checks.map(async (check) => {
|
|
74
|
+
const start = Date.now();
|
|
75
|
+
try {
|
|
76
|
+
const result = await Promise.race([
|
|
77
|
+
Promise.resolve(check.check()),
|
|
78
|
+
new Promise<HealthCheckResult>((_, reject) =>
|
|
79
|
+
setTimeout(() => reject(new Error("Health check timed out")), this.checkTimeout)
|
|
80
|
+
),
|
|
81
|
+
]);
|
|
82
|
+
results[check.name] = {
|
|
83
|
+
...result,
|
|
84
|
+
latencyMs: result.latencyMs ?? Date.now() - start,
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
const isCritical = check.critical !== false;
|
|
88
|
+
if (result.status === "unhealthy" && isCritical) {
|
|
89
|
+
overallStatus = "unhealthy";
|
|
90
|
+
} else if (result.status === "degraded" && overallStatus !== "unhealthy") {
|
|
91
|
+
overallStatus = "degraded";
|
|
92
|
+
} else if (result.status === "unhealthy" && !isCritical && overallStatus !== "unhealthy") {
|
|
93
|
+
overallStatus = "degraded";
|
|
94
|
+
}
|
|
95
|
+
} catch (err) {
|
|
96
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
97
|
+
results[check.name] = {
|
|
98
|
+
status: "unhealthy",
|
|
99
|
+
message,
|
|
100
|
+
latencyMs: Date.now() - start,
|
|
101
|
+
};
|
|
102
|
+
const isCritical = check.critical !== false;
|
|
103
|
+
if (isCritical) {
|
|
104
|
+
overallStatus = "unhealthy";
|
|
105
|
+
} else if (overallStatus !== "unhealthy") {
|
|
106
|
+
overallStatus = "degraded";
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
})
|
|
110
|
+
);
|
|
111
|
+
|
|
112
|
+
return {
|
|
113
|
+
status: overallStatus,
|
|
114
|
+
uptime: Math.floor((Date.now() - this.startTime) / 1000),
|
|
115
|
+
timestamp: new Date().toISOString(),
|
|
116
|
+
checks: results,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
liveness(isShuttingDown: boolean): HealthResponse {
|
|
121
|
+
return {
|
|
122
|
+
status: isShuttingDown ? "unhealthy" : "healthy",
|
|
123
|
+
uptime: Math.floor((Date.now() - this.startTime) / 1000),
|
|
124
|
+
timestamp: new Date().toISOString(),
|
|
125
|
+
checks: {},
|
|
126
|
+
};
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Create a built-in database health check.
|
|
132
|
+
*/
|
|
133
|
+
export function createDbHealthCheck(db: Kysely<any>): HealthCheck {
|
|
134
|
+
return {
|
|
135
|
+
name: "database",
|
|
136
|
+
critical: true,
|
|
137
|
+
check: async () => {
|
|
138
|
+
const start = Date.now();
|
|
139
|
+
try {
|
|
140
|
+
await db.selectFrom(db.dynamic.ref("sqlite_master") as any)
|
|
141
|
+
.select(db.dynamic.ref("1") as any)
|
|
142
|
+
.execute()
|
|
143
|
+
.catch(async () => {
|
|
144
|
+
// Fallback for non-SQLite databases
|
|
145
|
+
const { sql } = await import("kysely");
|
|
146
|
+
await sql`SELECT 1`.execute(db);
|
|
147
|
+
});
|
|
148
|
+
return {
|
|
149
|
+
status: "healthy" as const,
|
|
150
|
+
latencyMs: Date.now() - start,
|
|
151
|
+
};
|
|
152
|
+
} catch (err) {
|
|
153
|
+
return {
|
|
154
|
+
status: "unhealthy" as const,
|
|
155
|
+
message: err instanceof Error ? err.message : String(err),
|
|
156
|
+
latencyMs: Date.now() - start,
|
|
157
|
+
};
|
|
158
|
+
}
|
|
159
|
+
},
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
export function createHealth(config?: HealthConfig): Health {
|
|
164
|
+
return new HealthImpl(config);
|
|
165
|
+
}
|
package/src/core/index.ts
CHANGED
|
@@ -21,6 +21,7 @@ export {
|
|
|
21
21
|
export {
|
|
22
22
|
type Events,
|
|
23
23
|
type EventHandler,
|
|
24
|
+
type EventMetadata,
|
|
24
25
|
type Subscription,
|
|
25
26
|
type EventRecord,
|
|
26
27
|
type EventAdapter,
|
|
@@ -302,3 +303,14 @@ export {
|
|
|
302
303
|
PersistentTransport,
|
|
303
304
|
type PersistentTransportConfig,
|
|
304
305
|
} from "./logs-transport";
|
|
306
|
+
|
|
307
|
+
export {
|
|
308
|
+
type Health,
|
|
309
|
+
type HealthCheck,
|
|
310
|
+
type HealthCheckResult,
|
|
311
|
+
type HealthConfig,
|
|
312
|
+
type HealthResponse,
|
|
313
|
+
type HealthStatus,
|
|
314
|
+
createHealth,
|
|
315
|
+
createDbHealthCheck,
|
|
316
|
+
} from "./health";
|
package/src/core/jobs.ts
CHANGED
|
@@ -56,6 +56,8 @@ export interface Job {
|
|
|
56
56
|
lastHeartbeat?: Date;
|
|
57
57
|
/** Current process state */
|
|
58
58
|
processState?: ExternalJobProcessState;
|
|
59
|
+
/** Trace ID for distributed tracing */
|
|
60
|
+
traceId?: string;
|
|
59
61
|
}
|
|
60
62
|
|
|
61
63
|
export interface JobHandler<T = any, R = any> {
|
|
@@ -66,6 +68,8 @@ export interface JobHandlerContext {
|
|
|
66
68
|
logger?: Logger;
|
|
67
69
|
emit?: (event: string, data?: Record<string, any>) => Promise<void>;
|
|
68
70
|
log?: (level: LogLevel, message: string, data?: Record<string, any>) => void;
|
|
71
|
+
/** Trace ID for distributed tracing */
|
|
72
|
+
traceId?: string;
|
|
69
73
|
}
|
|
70
74
|
|
|
71
75
|
/** Options for listing all jobs */
|
|
@@ -128,9 +132,9 @@ export interface Jobs {
|
|
|
128
132
|
/** Register an external job (Python, Go, Shell, etc.) */
|
|
129
133
|
registerExternal(name: string, config: ExternalJobConfig): void;
|
|
130
134
|
/** Enqueue a job (works for both in-process and external jobs) */
|
|
131
|
-
enqueue<T = any>(name: string, data: T, options?: { maxAttempts?: number }): Promise<string>;
|
|
135
|
+
enqueue<T = any>(name: string, data: T, options?: { maxAttempts?: number; traceId?: string }): Promise<string>;
|
|
132
136
|
/** Schedule a job to run at a specific time */
|
|
133
|
-
schedule<T = any>(name: string, data: T, runAt: Date, options?: { maxAttempts?: number }): Promise<string>;
|
|
137
|
+
schedule<T = any>(name: string, data: T, runAt: Date, options?: { maxAttempts?: number; traceId?: string }): Promise<string>;
|
|
134
138
|
/** Get a job by ID */
|
|
135
139
|
get(jobId: string): Promise<Job | null>;
|
|
136
140
|
/** Cancel a pending job */
|
|
@@ -339,7 +343,7 @@ class JobsImpl implements Jobs {
|
|
|
339
343
|
return this.externalConfigs.has(name);
|
|
340
344
|
}
|
|
341
345
|
|
|
342
|
-
async enqueue<T = any>(name: string, data: T, options: { maxAttempts?: number } = {}): Promise<string> {
|
|
346
|
+
async enqueue<T = any>(name: string, data: T, options: { maxAttempts?: number; traceId?: string } = {}): Promise<string> {
|
|
343
347
|
const isExternal = this.isExternalJob(name);
|
|
344
348
|
|
|
345
349
|
if (!isExternal && !this.handlers.has(name)) {
|
|
@@ -355,6 +359,7 @@ class JobsImpl implements Jobs {
|
|
|
355
359
|
maxAttempts: options.maxAttempts ?? this.defaultMaxAttempts,
|
|
356
360
|
external: isExternal || undefined,
|
|
357
361
|
processState: isExternal ? "spawning" : undefined,
|
|
362
|
+
traceId: options.traceId,
|
|
358
363
|
});
|
|
359
364
|
|
|
360
365
|
return job.id;
|
|
@@ -364,7 +369,7 @@ class JobsImpl implements Jobs {
|
|
|
364
369
|
name: string,
|
|
365
370
|
data: T,
|
|
366
371
|
runAt: Date,
|
|
367
|
-
options: { maxAttempts?: number } = {}
|
|
372
|
+
options: { maxAttempts?: number; traceId?: string } = {}
|
|
368
373
|
): Promise<string> {
|
|
369
374
|
const isExternal = this.isExternalJob(name);
|
|
370
375
|
|
|
@@ -382,6 +387,7 @@ class JobsImpl implements Jobs {
|
|
|
382
387
|
maxAttempts: options.maxAttempts ?? this.defaultMaxAttempts,
|
|
383
388
|
external: isExternal || undefined,
|
|
384
389
|
processState: isExternal ? "spawning" : undefined,
|
|
390
|
+
traceId: options.traceId,
|
|
385
391
|
});
|
|
386
392
|
|
|
387
393
|
return job.id;
|
|
@@ -1147,6 +1153,7 @@ class JobsImpl implements Jobs {
|
|
|
1147
1153
|
logger: scopedLogger,
|
|
1148
1154
|
emit,
|
|
1149
1155
|
log,
|
|
1156
|
+
traceId: job.traceId,
|
|
1150
1157
|
});
|
|
1151
1158
|
|
|
1152
1159
|
await this.adapter.update(job.id, {
|
|
@@ -49,6 +49,8 @@ const LOG_LEVEL_VALUES: Record<LogLevel, number> = {
|
|
|
49
49
|
export interface KyselyLogsAdapterConfig {
|
|
50
50
|
/** Database file path (default: ".donkeylabs/logs.db") */
|
|
51
51
|
dbPath?: string;
|
|
52
|
+
/** Use an existing Kysely instance */
|
|
53
|
+
db?: Kysely<Database>;
|
|
52
54
|
}
|
|
53
55
|
|
|
54
56
|
// ============================================
|
|
@@ -61,6 +63,11 @@ export class KyselyLogsAdapter implements LogsAdapter {
|
|
|
61
63
|
private ensureTablePromise: Promise<void> | null = null;
|
|
62
64
|
|
|
63
65
|
constructor(config: KyselyLogsAdapterConfig = {}) {
|
|
66
|
+
if (config.db) {
|
|
67
|
+
this.db = config.db;
|
|
68
|
+
return;
|
|
69
|
+
}
|
|
70
|
+
|
|
64
71
|
const dbPath = config.dbPath ?? ".donkeylabs/logs.db";
|
|
65
72
|
|
|
66
73
|
// Ensure directory exists
|
package/src/core/logs.ts
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
|
|
9
9
|
import type { Events } from "./events";
|
|
10
10
|
import type { LogLevel } from "./logger";
|
|
11
|
+
import type { Kysely } from "kysely";
|
|
11
12
|
|
|
12
13
|
// ============================================
|
|
13
14
|
// Types
|
|
@@ -72,6 +73,8 @@ export interface LogsConfig {
|
|
|
72
73
|
maxBufferSize?: number;
|
|
73
74
|
/** Database path (default: ".donkeylabs/logs.db") */
|
|
74
75
|
dbPath?: string;
|
|
76
|
+
/** Use an existing Kysely instance for logs storage */
|
|
77
|
+
db?: Kysely<any>;
|
|
75
78
|
}
|
|
76
79
|
|
|
77
80
|
// ============================================
|
|
@@ -7,13 +7,8 @@
|
|
|
7
7
|
import { sql, type Kysely } from "kysely";
|
|
8
8
|
|
|
9
9
|
export async function up(db: Kysely<any>): Promise<void> {
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
const tableInfo = await sql<{ name: string }>`
|
|
13
|
-
PRAGMA table_info(__donkeylabs_workflow_instances__)
|
|
14
|
-
`.execute(db);
|
|
15
|
-
|
|
16
|
-
const hasMetadataColumn = tableInfo.rows.some((row) => row.name === "metadata");
|
|
10
|
+
const dialect = getDialectName(db);
|
|
11
|
+
const hasMetadataColumn = await columnExists(db, dialect);
|
|
17
12
|
|
|
18
13
|
if (!hasMetadataColumn) {
|
|
19
14
|
await sql`
|
|
@@ -26,3 +21,42 @@ export async function down(db: Kysely<any>): Promise<void> {
|
|
|
26
21
|
// SQLite doesn't support DROP COLUMN directly
|
|
27
22
|
// In practice, we don't need to remove it - the column can stay
|
|
28
23
|
}
|
|
24
|
+
|
|
25
|
+
async function columnExists(db: Kysely<any>, dialect: string): Promise<boolean> {
|
|
26
|
+
if (dialect.includes("postgres")) {
|
|
27
|
+
const result = await sql<{ name: string }>`
|
|
28
|
+
SELECT column_name as name
|
|
29
|
+
FROM information_schema.columns
|
|
30
|
+
WHERE table_name = '__donkeylabs_workflow_instances__'
|
|
31
|
+
AND column_name = 'metadata'
|
|
32
|
+
AND table_schema = current_schema()
|
|
33
|
+
`.execute(db);
|
|
34
|
+
return result.rows.length > 0;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
if (dialect.includes("mysql")) {
|
|
38
|
+
const result = await sql<{ name: string }>`
|
|
39
|
+
SELECT column_name as name
|
|
40
|
+
FROM information_schema.columns
|
|
41
|
+
WHERE table_name = '__donkeylabs_workflow_instances__'
|
|
42
|
+
AND column_name = 'metadata'
|
|
43
|
+
AND table_schema = database()
|
|
44
|
+
`.execute(db);
|
|
45
|
+
return result.rows.length > 0;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const tableInfo = await sql<{ name: string }>`
|
|
49
|
+
PRAGMA table_info(__donkeylabs_workflow_instances__)
|
|
50
|
+
`.execute(db);
|
|
51
|
+
|
|
52
|
+
return tableInfo.rows.some((row) => row.name === "metadata");
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function getDialectName(db: Kysely<any>): string {
|
|
56
|
+
try {
|
|
57
|
+
const adapter = (db as any).getExecutor?.().adapter;
|
|
58
|
+
return adapter?.constructor?.name?.toLowerCase() ?? "";
|
|
59
|
+
} catch {
|
|
60
|
+
return "";
|
|
61
|
+
}
|
|
62
|
+
}
|
package/src/core/processes.ts
CHANGED
|
@@ -170,8 +170,8 @@ export interface SpawnOptions {
|
|
|
170
170
|
// ============================================
|
|
171
171
|
|
|
172
172
|
export interface ProcessesConfig {
|
|
173
|
-
/**
|
|
174
|
-
adapter?: SqliteProcessAdapterConfig;
|
|
173
|
+
/** Adapter instance or sqlite adapter configuration */
|
|
174
|
+
adapter?: ProcessAdapter | SqliteProcessAdapterConfig;
|
|
175
175
|
/** Socket server configuration */
|
|
176
176
|
socket?: ProcessSocketConfig;
|
|
177
177
|
/** Events service for emitting process events */
|
|
@@ -278,7 +278,11 @@ export class ProcessesImpl implements Processes {
|
|
|
278
278
|
private isShuttingDown = false;
|
|
279
279
|
|
|
280
280
|
constructor(config: ProcessesConfig = {}) {
|
|
281
|
-
|
|
281
|
+
if (config.adapter && typeof (config.adapter as ProcessAdapter).get === "function") {
|
|
282
|
+
this.adapter = config.adapter as ProcessAdapter;
|
|
283
|
+
} else {
|
|
284
|
+
this.adapter = new SqliteProcessAdapter(config.adapter as SqliteProcessAdapterConfig | undefined);
|
|
285
|
+
}
|
|
282
286
|
this.events = config.events;
|
|
283
287
|
this.heartbeatCheckInterval = config.heartbeatCheckInterval ?? 10000;
|
|
284
288
|
this.autoRecoverOrphans = config.autoRecoverOrphans ?? true;
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
KyselyWorkflowAdapter,
|
|
21
21
|
MemoryAuditAdapter,
|
|
22
22
|
MemoryLogsAdapter,
|
|
23
|
+
createHealth,
|
|
23
24
|
} from "./index";
|
|
24
25
|
import { PluginManager, type CoreServices, type ConfiguredPlugin } from "../core";
|
|
25
26
|
|
|
@@ -96,6 +97,7 @@ export async function bootstrapSubprocess(
|
|
|
96
97
|
const audit = createAudit({ adapter: auditAdapter });
|
|
97
98
|
const websocket = createWebSocket();
|
|
98
99
|
const storage = createStorage();
|
|
100
|
+
const health = createHealth({ dbCheck: false });
|
|
99
101
|
|
|
100
102
|
const core: CoreServices = {
|
|
101
103
|
db,
|
|
@@ -114,6 +116,7 @@ export async function bootstrapSubprocess(
|
|
|
114
116
|
websocket,
|
|
115
117
|
storage,
|
|
116
118
|
logs,
|
|
119
|
+
health,
|
|
117
120
|
};
|
|
118
121
|
|
|
119
122
|
workflows.setCore(core);
|
|
@@ -1,10 +1,11 @@
|
|
|
1
|
-
import { Kysely } from "kysely";
|
|
1
|
+
import { Kysely, PostgresDialect, MysqlDialect } from "kysely";
|
|
2
2
|
import { BunSqliteDialect } from "kysely-bun-sqlite";
|
|
3
3
|
import Database from "bun:sqlite";
|
|
4
4
|
import { ProcessClient } from "./process-client";
|
|
5
5
|
import { KyselyWorkflowAdapter } from "./workflow-adapter-kysely";
|
|
6
6
|
import { KyselyJobAdapter } from "./job-adapter-kysely";
|
|
7
7
|
import { SqliteProcessAdapter } from "./process-adapter-sqlite";
|
|
8
|
+
import { KyselyProcessAdapter } from "./process-adapter-kysely";
|
|
8
9
|
import { WatchdogRunner, type WatchdogRunnerConfig } from "./watchdog-runner";
|
|
9
10
|
|
|
10
11
|
interface WatchdogConfig extends WatchdogRunnerConfig {
|
|
@@ -17,6 +18,10 @@ interface WatchdogConfig extends WatchdogRunnerConfig {
|
|
|
17
18
|
synchronous?: "OFF" | "NORMAL" | "FULL" | "EXTRA";
|
|
18
19
|
journalMode?: "DELETE" | "TRUNCATE" | "PERSIST" | "MEMORY" | "WAL" | "OFF";
|
|
19
20
|
};
|
|
21
|
+
database?: {
|
|
22
|
+
type: "sqlite" | "postgres" | "mysql";
|
|
23
|
+
connectionString: string;
|
|
24
|
+
};
|
|
20
25
|
}
|
|
21
26
|
|
|
22
27
|
const raw = process.env.DONKEYLABS_WATCHDOG_CONFIG;
|
|
@@ -27,19 +32,21 @@ if (!raw) {
|
|
|
27
32
|
const config: WatchdogConfig = JSON.parse(raw);
|
|
28
33
|
const client = await ProcessClient.connect();
|
|
29
34
|
|
|
30
|
-
const workflowAdapter = config.workflows?.dbPath
|
|
31
|
-
? new KyselyWorkflowAdapter(createDb(config.workflows
|
|
35
|
+
const workflowAdapter = config.workflows?.dbPath || config.database
|
|
36
|
+
? new KyselyWorkflowAdapter(await createDb(config.workflows?.dbPath, config), {
|
|
32
37
|
cleanupDays: 0,
|
|
33
38
|
})
|
|
34
39
|
: undefined;
|
|
35
|
-
const jobAdapter = config.jobs?.dbPath
|
|
36
|
-
? new KyselyJobAdapter(createDb(config.jobs
|
|
40
|
+
const jobAdapter = config.jobs?.dbPath || config.database
|
|
41
|
+
? new KyselyJobAdapter(await createDb(config.jobs?.dbPath, config), {
|
|
37
42
|
cleanupDays: 0,
|
|
38
43
|
})
|
|
39
44
|
: undefined;
|
|
40
45
|
const processAdapter = config.processes?.dbPath
|
|
41
46
|
? new SqliteProcessAdapter({ path: config.processes.dbPath, cleanupDays: 0 })
|
|
42
|
-
:
|
|
47
|
+
: config.database
|
|
48
|
+
? new KyselyProcessAdapter(await createDb(undefined, config))
|
|
49
|
+
: undefined;
|
|
43
50
|
|
|
44
51
|
const runner = new WatchdogRunner(config, {
|
|
45
52
|
workflowsAdapter: workflowAdapter,
|
|
@@ -60,21 +67,52 @@ process.on("SIGTERM", async () => {
|
|
|
60
67
|
client.disconnect();
|
|
61
68
|
});
|
|
62
69
|
|
|
63
|
-
function createDb(
|
|
64
|
-
dbPath: string,
|
|
65
|
-
|
|
66
|
-
): Kysely<any
|
|
67
|
-
const
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
70
|
+
async function createDb(
|
|
71
|
+
dbPath: string | undefined,
|
|
72
|
+
config: WatchdogConfig
|
|
73
|
+
): Promise<Kysely<any>> {
|
|
74
|
+
const dbConfig = config.database;
|
|
75
|
+
|
|
76
|
+
if (!dbConfig || dbConfig.type === "sqlite") {
|
|
77
|
+
const sqlitePath = dbConfig?.connectionString ?? dbPath;
|
|
78
|
+
if (!sqlitePath) {
|
|
79
|
+
throw new Error("SQLite dbPath or connectionString is required for watchdog");
|
|
80
|
+
}
|
|
81
|
+
const sqlite = new Database(sqlitePath);
|
|
82
|
+
const pragmas = config.sqlitePragmas;
|
|
83
|
+
const busyTimeout = pragmas?.busyTimeout ?? 5000;
|
|
84
|
+
sqlite.run(`PRAGMA busy_timeout = ${busyTimeout}`);
|
|
85
|
+
if (pragmas?.journalMode) {
|
|
86
|
+
sqlite.run(`PRAGMA journal_mode = ${pragmas.journalMode}`);
|
|
87
|
+
}
|
|
88
|
+
if (pragmas?.synchronous) {
|
|
89
|
+
sqlite.run(`PRAGMA synchronous = ${pragmas.synchronous}`);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
return new Kysely<any>({
|
|
93
|
+
dialect: new BunSqliteDialect({ database: sqlite }),
|
|
94
|
+
});
|
|
72
95
|
}
|
|
73
|
-
|
|
74
|
-
|
|
96
|
+
|
|
97
|
+
if (dbConfig.type === "postgres") {
|
|
98
|
+
// @ts-ignore optional dependency
|
|
99
|
+
const { Pool: PGPool } = await import("pg");
|
|
100
|
+
return new Kysely<any>({
|
|
101
|
+
dialect: new PostgresDialect({
|
|
102
|
+
pool: new PGPool({ connectionString: dbConfig.connectionString }),
|
|
103
|
+
}),
|
|
104
|
+
});
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
if (dbConfig.type === "mysql") {
|
|
108
|
+
// @ts-ignore optional dependency
|
|
109
|
+
const { createPool: createMySQLPool } = await import("mysql2");
|
|
110
|
+
return new Kysely<any>({
|
|
111
|
+
dialect: new MysqlDialect({
|
|
112
|
+
pool: createMySQLPool(dbConfig.connectionString),
|
|
113
|
+
}),
|
|
114
|
+
});
|
|
75
115
|
}
|
|
76
116
|
|
|
77
|
-
|
|
78
|
-
dialect: new BunSqliteDialect({ database: sqlite }),
|
|
79
|
-
});
|
|
117
|
+
throw new Error(`Unsupported database type: ${dbConfig.type}`);
|
|
80
118
|
}
|
|
@@ -751,12 +751,13 @@ export class WorkflowStateMachine {
|
|
|
751
751
|
|
|
752
752
|
this.events.onStepCompleted(instanceId, stepName, output, nextStep);
|
|
753
753
|
|
|
754
|
-
// Calculate progress
|
|
754
|
+
// Calculate progress — re-fetch after persist to get accurate count
|
|
755
|
+
const updated = await this.adapter.getInstance(instanceId);
|
|
755
756
|
const totalSteps = definition.steps.size;
|
|
756
|
-
const completedSteps =
|
|
757
|
-
(r) => r.status === "completed"
|
|
758
|
-
|
|
759
|
-
const progress = Math.round((completedSteps / totalSteps) * 100);
|
|
757
|
+
const completedSteps = updated
|
|
758
|
+
? Object.values(updated.stepResults).filter((r) => r.status === "completed").length
|
|
759
|
+
: 1;
|
|
760
|
+
const progress = Math.min(100, Math.round((completedSteps / totalSteps) * 100));
|
|
760
761
|
|
|
761
762
|
this.events.onProgress(instanceId, progress, stepName, completedSteps, totalSteps);
|
|
762
763
|
}
|