@kb-labs/workflow-daemon 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +23 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +2535 -0
- package/dist/index.js.map +1 -0
- package/package.json +65 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,2535 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { createServiceBootstrap, platform } from '@kb-labs/core-runtime';
|
|
3
|
+
import { WorkflowEngine, WorkflowService } from '@kb-labs/workflow-engine';
|
|
4
|
+
import { createCorrelatedLogger, registerOpenAPI, HttpObservabilityCollector, createServiceReadyResponse, metricLine } from '@kb-labs/shared-http';
|
|
5
|
+
import { logDiagnosticEvent } from '@kb-labs/core-platform';
|
|
6
|
+
import { PluginCronJobSchema, UserCronJobSchema, evaluateExpression, interpolateObject, resolveValue } from '@kb-labs/workflow-contracts';
|
|
7
|
+
import { SandboxRunner } from '@kb-labs/workflow-runtime';
|
|
8
|
+
import * as cron from 'node-cron';
|
|
9
|
+
import { CronExpressionParser } from 'cron-parser';
|
|
10
|
+
import { stat, readdir, readFile } from 'fs/promises';
|
|
11
|
+
import { join, extname, basename } from 'path';
|
|
12
|
+
import YAML from 'yaml';
|
|
13
|
+
import Fastify from 'fastify';
|
|
14
|
+
import cors from '@fastify/cors';
|
|
15
|
+
import { createRegistry } from '@kb-labs/core-registry';
|
|
16
|
+
import { findRepoRoot } from '@kb-labs/core-sys';
|
|
17
|
+
import { randomUUID } from 'crypto';
|
|
18
|
+
|
|
19
|
+
async function createWorkflowWorker(options) {
|
|
20
|
+
const {
|
|
21
|
+
engine,
|
|
22
|
+
cliApi,
|
|
23
|
+
logger,
|
|
24
|
+
platform: platform2,
|
|
25
|
+
workspaceRoot,
|
|
26
|
+
concurrency = 5,
|
|
27
|
+
defaultTimeout = 12e4,
|
|
28
|
+
analytics
|
|
29
|
+
} = options;
|
|
30
|
+
let isRunning = false;
|
|
31
|
+
let stopRequested = false;
|
|
32
|
+
const runningJobs = /* @__PURE__ */ new Map();
|
|
33
|
+
const claimedJobs = /* @__PURE__ */ new Set();
|
|
34
|
+
const executionBackend = platform2.executionBackend;
|
|
35
|
+
const runner = new SandboxRunner({
|
|
36
|
+
backend: executionBackend,
|
|
37
|
+
// ExecutionBackend type from plugin-execution
|
|
38
|
+
cliApi,
|
|
39
|
+
workspaceRoot,
|
|
40
|
+
defaultTimeout
|
|
41
|
+
});
|
|
42
|
+
async function processJob() {
|
|
43
|
+
const entry = await engine.nextJob();
|
|
44
|
+
if (!entry) {
|
|
45
|
+
return false;
|
|
46
|
+
}
|
|
47
|
+
const run = await engine.getRun(entry.runId);
|
|
48
|
+
if (!run) {
|
|
49
|
+
logger.error("Data inconsistency: Run not found for job entry", void 0, {
|
|
50
|
+
runId: entry.runId,
|
|
51
|
+
jobId: entry.jobId
|
|
52
|
+
});
|
|
53
|
+
return true;
|
|
54
|
+
}
|
|
55
|
+
const job = run.jobs.find((j) => j.id === entry.jobId);
|
|
56
|
+
if (!job) {
|
|
57
|
+
logger.error("Data inconsistency: Job not found in run", void 0, {
|
|
58
|
+
runId: run.id,
|
|
59
|
+
jobId: entry.jobId
|
|
60
|
+
});
|
|
61
|
+
return true;
|
|
62
|
+
}
|
|
63
|
+
const jobKey = `${run.id}:${job.id}`;
|
|
64
|
+
if (claimedJobs.has(jobKey)) {
|
|
65
|
+
return true;
|
|
66
|
+
}
|
|
67
|
+
claimedJobs.add(jobKey);
|
|
68
|
+
const jobStartTime = Date.now();
|
|
69
|
+
const jobLogger = createCorrelatedLogger(logger, {
|
|
70
|
+
serviceId: "workflow",
|
|
71
|
+
logsSource: "workflow",
|
|
72
|
+
layer: "workflow",
|
|
73
|
+
service: "worker",
|
|
74
|
+
requestId: run.id,
|
|
75
|
+
traceId: run.id,
|
|
76
|
+
operation: "workflow.job",
|
|
77
|
+
bindings: {
|
|
78
|
+
workflowId: run.id,
|
|
79
|
+
runId: run.id,
|
|
80
|
+
jobId: job.id
|
|
81
|
+
}
|
|
82
|
+
});
|
|
83
|
+
jobLogger.info("Processing job", {
|
|
84
|
+
runId: run.id,
|
|
85
|
+
jobId: job.id,
|
|
86
|
+
jobName: job.jobName
|
|
87
|
+
});
|
|
88
|
+
await engine.markJobStarted(run.id, job.id);
|
|
89
|
+
analytics?.track("workflow.worker.job.started", {
|
|
90
|
+
runId: run.id,
|
|
91
|
+
jobId: job.id,
|
|
92
|
+
jobName: job.jobName,
|
|
93
|
+
stepCount: job.steps.length
|
|
94
|
+
}).catch(() => {
|
|
95
|
+
});
|
|
96
|
+
const runTarget = run.metadata?.target;
|
|
97
|
+
const jobTarget = job.target;
|
|
98
|
+
const target = jobTarget ?? runTarget;
|
|
99
|
+
const wsProvider = platform2.getAdapter("workspace");
|
|
100
|
+
let runWorkspace = workspaceRoot;
|
|
101
|
+
let provisionedWorkspaceId;
|
|
102
|
+
if (wsProvider) {
|
|
103
|
+
const wsId = `wt_${run.id.slice(0, 8)}`;
|
|
104
|
+
try {
|
|
105
|
+
const ws = await wsProvider.materialize({
|
|
106
|
+
workspaceId: wsId,
|
|
107
|
+
sourceRef: "main",
|
|
108
|
+
metadata: { runId: run.id, jobId: job.id },
|
|
109
|
+
onProgress: (event) => {
|
|
110
|
+
jobLogger.info(`[workspace] ${event.stage}: ${event.message}`, {
|
|
111
|
+
stage: event.stage,
|
|
112
|
+
progress: event.progress
|
|
113
|
+
});
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
if (ws.rootPath) {
|
|
117
|
+
runWorkspace = ws.rootPath;
|
|
118
|
+
provisionedWorkspaceId = ws.workspaceId;
|
|
119
|
+
jobLogger.info("Workspace provisioned", {
|
|
120
|
+
workspaceId: ws.workspaceId,
|
|
121
|
+
provider: ws.provider,
|
|
122
|
+
rootPath: ws.rootPath
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
} catch (wsError) {
|
|
126
|
+
const msg = wsError instanceof Error ? wsError.message : String(wsError);
|
|
127
|
+
logDiagnosticEvent(jobLogger, {
|
|
128
|
+
domain: "workflow",
|
|
129
|
+
event: "workflow.workspace.provision",
|
|
130
|
+
level: "error",
|
|
131
|
+
reasonCode: inferWorkspaceProvisionReasonCode(msg),
|
|
132
|
+
message: "Workspace provisioning failed",
|
|
133
|
+
outcome: "failed",
|
|
134
|
+
error: wsError instanceof Error ? wsError : new Error(String(wsError)),
|
|
135
|
+
serviceId: "workflow",
|
|
136
|
+
stage: "materialize",
|
|
137
|
+
evidence: {
|
|
138
|
+
runId: run.id,
|
|
139
|
+
jobId: job.id,
|
|
140
|
+
workspaceId: wsId
|
|
141
|
+
}
|
|
142
|
+
});
|
|
143
|
+
throw new Error(`Workspace provisioning failed: ${msg}`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
const jobPromise = (async () => {
|
|
147
|
+
try {
|
|
148
|
+
for (const step of job.steps) {
|
|
149
|
+
if (step.status === "success") {
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
const freshRun = await engine.getRun(run.id);
|
|
153
|
+
const exprCtx = {
|
|
154
|
+
env: freshRun?.env ?? {},
|
|
155
|
+
trigger: freshRun?.trigger ?? { type: "manual" },
|
|
156
|
+
steps: {}
|
|
157
|
+
};
|
|
158
|
+
if (freshRun) {
|
|
159
|
+
for (const j of freshRun.jobs) {
|
|
160
|
+
for (const s of j.steps) {
|
|
161
|
+
if (s.status === "success" && s.spec.id) {
|
|
162
|
+
exprCtx.steps[s.spec.id] = {
|
|
163
|
+
outputs: s.outputs ?? {}
|
|
164
|
+
};
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
if (step.spec.if) {
|
|
170
|
+
const condition = step.spec.if;
|
|
171
|
+
const rawExpr = condition.trim().replace(/^\$\{\{\s*/, "").replace(/\s*\}\}$/, "");
|
|
172
|
+
const shouldRun = evaluateExpression(rawExpr, exprCtx);
|
|
173
|
+
if (!shouldRun) {
|
|
174
|
+
jobLogger.info("Step skipped (condition false)", {
|
|
175
|
+
runId: run.id,
|
|
176
|
+
jobId: job.id,
|
|
177
|
+
stepId: step.id,
|
|
178
|
+
condition
|
|
179
|
+
});
|
|
180
|
+
await engine.markStepCompleted(run.id, job.id, step.id, { skipped: true });
|
|
181
|
+
continue;
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
const interpolatedWith = step.spec.with ? interpolateObject(step.spec.with, exprCtx) : void 0;
|
|
185
|
+
const stepExecutionId = `wf-${run.id}-${job.id}-${step.id}-${Date.now()}`;
|
|
186
|
+
const stepLogger = jobLogger.child({
|
|
187
|
+
operation: "workflow.step",
|
|
188
|
+
stepId: step.id,
|
|
189
|
+
attempt: 1,
|
|
190
|
+
executionId: stepExecutionId,
|
|
191
|
+
spanId: stepExecutionId,
|
|
192
|
+
invocationId: stepExecutionId
|
|
193
|
+
});
|
|
194
|
+
stepLogger.info("Executing step", {
|
|
195
|
+
runId: run.id,
|
|
196
|
+
jobId: job.id,
|
|
197
|
+
stepId: step.id,
|
|
198
|
+
uses: step.spec.uses
|
|
199
|
+
});
|
|
200
|
+
if (step.spec.uses === "builtin:approval") {
|
|
201
|
+
if (step.status !== "waiting_approval") {
|
|
202
|
+
if (interpolatedWith) {
|
|
203
|
+
const stateStore = engine.getStateStore();
|
|
204
|
+
await stateStore.updateStep(run.id, job.id, step.id, (draft) => {
|
|
205
|
+
draft.spec = { ...draft.spec, with: interpolatedWith };
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
await engine.markStepWaitingApproval(run.id, job.id, step.id);
|
|
209
|
+
}
|
|
210
|
+
stepLogger.info("Waiting for approval", {
|
|
211
|
+
runId: run.id,
|
|
212
|
+
jobId: job.id,
|
|
213
|
+
stepId: step.id,
|
|
214
|
+
context: interpolatedWith
|
|
215
|
+
});
|
|
216
|
+
while (!stopRequested) {
|
|
217
|
+
await sleep(2e3);
|
|
218
|
+
const currentRun = await engine.getRun(run.id);
|
|
219
|
+
const currentJob = currentRun?.jobs.find((j) => j.id === job.id);
|
|
220
|
+
const currentStep = currentJob?.steps.find((s) => s.id === step.id);
|
|
221
|
+
if (!currentStep || currentStep.status === "success") {
|
|
222
|
+
stepLogger.info("Approval granted", { runId: run.id, stepId: step.id });
|
|
223
|
+
break;
|
|
224
|
+
}
|
|
225
|
+
if (currentStep.status === "failed") {
|
|
226
|
+
const rejectMsg = currentStep.error?.message ?? "Approval rejected";
|
|
227
|
+
stepLogger.info("Approval rejected", { runId: run.id, stepId: step.id });
|
|
228
|
+
throw new Error(rejectMsg);
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
if (stopRequested) {
|
|
232
|
+
stepLogger.info("Approval wait interrupted by shutdown", { stepId: step.id });
|
|
233
|
+
return;
|
|
234
|
+
}
|
|
235
|
+
continue;
|
|
236
|
+
}
|
|
237
|
+
if (step.spec.uses === "builtin:gate") {
|
|
238
|
+
const gateInput = interpolatedWith ?? {};
|
|
239
|
+
const decisionPath = gateInput.decision;
|
|
240
|
+
const maxIterations = gateInput.maxIterations ?? 3;
|
|
241
|
+
const decisionValue = resolveValue(decisionPath, exprCtx);
|
|
242
|
+
const decisionKey = String(decisionValue);
|
|
243
|
+
const route = gateInput.routes[decisionKey] ?? gateInput.routes[decisionValue];
|
|
244
|
+
const action = route ?? gateInput.default ?? "fail";
|
|
245
|
+
stepLogger.info("Gate evaluation", {
|
|
246
|
+
runId: run.id,
|
|
247
|
+
stepId: step.id,
|
|
248
|
+
decision: decisionPath,
|
|
249
|
+
decisionValue,
|
|
250
|
+
action: typeof action === "string" ? action : "restart"
|
|
251
|
+
});
|
|
252
|
+
const iterationKey = `gate:${step.spec.id ?? step.id}:iterations`;
|
|
253
|
+
const metadata = freshRun?.metadata ?? {};
|
|
254
|
+
const currentIteration = metadata[iterationKey] ?? 0;
|
|
255
|
+
if (action === "continue") {
|
|
256
|
+
await engine.markStepCompleted(run.id, job.id, step.id, {
|
|
257
|
+
decisionValue,
|
|
258
|
+
action: "continue",
|
|
259
|
+
iteration: currentIteration
|
|
260
|
+
});
|
|
261
|
+
continue;
|
|
262
|
+
}
|
|
263
|
+
if (action === "fail") {
|
|
264
|
+
const error = new Error(`Gate failed: decision=${decisionKey}`);
|
|
265
|
+
await engine.markStepFailed(run.id, job.id, step.id, error, {
|
|
266
|
+
decisionValue,
|
|
267
|
+
action: "fail",
|
|
268
|
+
iteration: currentIteration
|
|
269
|
+
});
|
|
270
|
+
throw error;
|
|
271
|
+
}
|
|
272
|
+
const restartAction = action;
|
|
273
|
+
const nextIteration = currentIteration + 1;
|
|
274
|
+
if (nextIteration >= maxIterations) {
|
|
275
|
+
const error = new Error(
|
|
276
|
+
`Gate max iterations reached (${maxIterations}) for step ${step.spec.id ?? step.id}`
|
|
277
|
+
);
|
|
278
|
+
await engine.markStepFailed(run.id, job.id, step.id, error, {
|
|
279
|
+
decisionValue,
|
|
280
|
+
action: "fail",
|
|
281
|
+
maxIterationsReached: true,
|
|
282
|
+
iteration: currentIteration,
|
|
283
|
+
maxIterations
|
|
284
|
+
});
|
|
285
|
+
throw error;
|
|
286
|
+
}
|
|
287
|
+
stepLogger.info("Gate triggering restart", {
|
|
288
|
+
restartFrom: restartAction.restartFrom,
|
|
289
|
+
iteration: nextIteration,
|
|
290
|
+
maxIterations
|
|
291
|
+
});
|
|
292
|
+
await engine.markStepCompleted(run.id, job.id, step.id, {
|
|
293
|
+
decisionValue,
|
|
294
|
+
action: "restart",
|
|
295
|
+
restartFrom: restartAction.restartFrom,
|
|
296
|
+
iteration: nextIteration
|
|
297
|
+
});
|
|
298
|
+
await engine.updateRun(run.id, (draft) => {
|
|
299
|
+
const md = draft.metadata ?? {};
|
|
300
|
+
md[iterationKey] = nextIteration;
|
|
301
|
+
draft.metadata = md;
|
|
302
|
+
if (restartAction.context) {
|
|
303
|
+
const payload = draft.trigger.payload ?? {};
|
|
304
|
+
Object.assign(payload, restartAction.context);
|
|
305
|
+
draft.trigger.payload = payload;
|
|
306
|
+
}
|
|
307
|
+
return draft;
|
|
308
|
+
});
|
|
309
|
+
const stateStore = engine.getStateStore();
|
|
310
|
+
const scheduler = engine.getScheduler();
|
|
311
|
+
let foundTarget = false;
|
|
312
|
+
for (const s of job.steps) {
|
|
313
|
+
if (s.spec.id === restartAction.restartFrom || s.id === restartAction.restartFrom) {
|
|
314
|
+
foundTarget = true;
|
|
315
|
+
}
|
|
316
|
+
if (foundTarget) {
|
|
317
|
+
await stateStore.updateStep(run.id, job.id, s.id, (draft) => {
|
|
318
|
+
draft.status = "queued";
|
|
319
|
+
draft.startedAt = void 0;
|
|
320
|
+
draft.finishedAt = void 0;
|
|
321
|
+
draft.error = void 0;
|
|
322
|
+
draft.outputs = void 0;
|
|
323
|
+
});
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
await stateStore.updateJob(run.id, job.id, (draft) => {
|
|
327
|
+
draft.status = "queued";
|
|
328
|
+
draft.startedAt = void 0;
|
|
329
|
+
draft.finishedAt = void 0;
|
|
330
|
+
});
|
|
331
|
+
const updatedRun = await engine.getRun(run.id);
|
|
332
|
+
const updatedJob = updatedRun?.jobs.find((j) => j.id === job.id);
|
|
333
|
+
if (updatedJob) {
|
|
334
|
+
await scheduler.enqueueJob(run.id, updatedJob, updatedJob.priority ?? "normal");
|
|
335
|
+
}
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
await engine.markStepStarted(run.id, job.id, step.id);
|
|
339
|
+
const interpolatedSpec = interpolatedWith ? { ...step.spec, with: interpolatedWith } : step.spec;
|
|
340
|
+
const result = await runner.execute({
|
|
341
|
+
spec: interpolatedSpec,
|
|
342
|
+
context: {
|
|
343
|
+
runId: run.id,
|
|
344
|
+
jobId: job.id,
|
|
345
|
+
stepId: step.id,
|
|
346
|
+
attempt: 1,
|
|
347
|
+
env: freshRun?.env || {},
|
|
348
|
+
secrets: {},
|
|
349
|
+
// TODO: map run.secrets array to Record
|
|
350
|
+
logger: {
|
|
351
|
+
debug: (message, meta) => stepLogger.debug(message, meta),
|
|
352
|
+
info: (message, meta) => stepLogger.info(message, meta),
|
|
353
|
+
warn: (message, meta) => stepLogger.warn(message, meta),
|
|
354
|
+
error: (message, meta) => stepLogger.error(message, void 0, meta)
|
|
355
|
+
},
|
|
356
|
+
trace: {
|
|
357
|
+
traceId: run.id,
|
|
358
|
+
spanId: stepExecutionId,
|
|
359
|
+
parentSpanId: job.id
|
|
360
|
+
},
|
|
361
|
+
onLog: (entry2) => {
|
|
362
|
+
void engine.publishLog(run.id, job.id, step.id, entry2);
|
|
363
|
+
}
|
|
364
|
+
},
|
|
365
|
+
workspace: runWorkspace,
|
|
366
|
+
target
|
|
367
|
+
});
|
|
368
|
+
if (result.status === "failed") {
|
|
369
|
+
const error = new Error(result.error?.message ?? "Step execution failed");
|
|
370
|
+
await engine.markStepFailed(run.id, job.id, step.id, error);
|
|
371
|
+
stepLogger.error("Step failed", error, {
|
|
372
|
+
runId: run.id,
|
|
373
|
+
jobId: job.id,
|
|
374
|
+
stepId: step.id
|
|
375
|
+
});
|
|
376
|
+
throw error;
|
|
377
|
+
}
|
|
378
|
+
await engine.markStepCompleted(run.id, job.id, step.id, result.status === "success" ? result.outputs : void 0);
|
|
379
|
+
stepLogger.info("Step completed", {
|
|
380
|
+
runId: run.id,
|
|
381
|
+
jobId: job.id,
|
|
382
|
+
stepId: step.id
|
|
383
|
+
});
|
|
384
|
+
}
|
|
385
|
+
await engine.markJobCompleted(run.id, job.id);
|
|
386
|
+
const jobDuration = Date.now() - jobStartTime;
|
|
387
|
+
jobLogger.info("Job completed successfully", {
|
|
388
|
+
runId: run.id,
|
|
389
|
+
jobId: job.id
|
|
390
|
+
});
|
|
391
|
+
analytics?.track("workflow.worker.job.completed", {
|
|
392
|
+
runId: run.id,
|
|
393
|
+
jobId: job.id,
|
|
394
|
+
jobName: job.jobName,
|
|
395
|
+
durationMs: jobDuration,
|
|
396
|
+
stepCount: job.steps.length
|
|
397
|
+
}).catch(() => {
|
|
398
|
+
});
|
|
399
|
+
if (provisionedWorkspaceId && wsProvider) {
|
|
400
|
+
try {
|
|
401
|
+
await wsProvider.release(provisionedWorkspaceId);
|
|
402
|
+
jobLogger.info("Workspace released", { workspaceId: provisionedWorkspaceId });
|
|
403
|
+
} catch (releaseErr) {
|
|
404
|
+
jobLogger.warn("Workspace release failed", {
|
|
405
|
+
workspaceId: provisionedWorkspaceId,
|
|
406
|
+
error: releaseErr instanceof Error ? releaseErr.message : String(releaseErr)
|
|
407
|
+
});
|
|
408
|
+
}
|
|
409
|
+
}
|
|
410
|
+
} catch (error) {
|
|
411
|
+
const err = error instanceof Error ? error : new Error(String(error));
|
|
412
|
+
const jobDuration = Date.now() - jobStartTime;
|
|
413
|
+
await engine.markJobFailed(run.id, job.id, err);
|
|
414
|
+
analytics?.track("workflow.worker.job.failed", {
|
|
415
|
+
runId: run.id,
|
|
416
|
+
jobId: job.id,
|
|
417
|
+
jobName: job.jobName,
|
|
418
|
+
errorMessage: err.message,
|
|
419
|
+
durationMs: jobDuration
|
|
420
|
+
}).catch(() => {
|
|
421
|
+
});
|
|
422
|
+
if (provisionedWorkspaceId) {
|
|
423
|
+
jobLogger.warn("Workspace kept for debugging", {
|
|
424
|
+
workspaceId: provisionedWorkspaceId,
|
|
425
|
+
path: runWorkspace
|
|
426
|
+
});
|
|
427
|
+
}
|
|
428
|
+
} finally {
|
|
429
|
+
runningJobs.delete(jobKey);
|
|
430
|
+
claimedJobs.delete(jobKey);
|
|
431
|
+
}
|
|
432
|
+
})();
|
|
433
|
+
runningJobs.set(jobKey, jobPromise);
|
|
434
|
+
await jobPromise;
|
|
435
|
+
return true;
|
|
436
|
+
}
|
|
437
|
+
async function workerLoop() {
|
|
438
|
+
while (isRunning && !stopRequested) {
|
|
439
|
+
try {
|
|
440
|
+
const processed = await processJob();
|
|
441
|
+
if (!processed) {
|
|
442
|
+
await sleep(1e3);
|
|
443
|
+
}
|
|
444
|
+
} catch (error) {
|
|
445
|
+
logDiagnosticEvent(logger, {
|
|
446
|
+
domain: "workflow",
|
|
447
|
+
event: "workflow.worker.loop",
|
|
448
|
+
level: "error",
|
|
449
|
+
reasonCode: "worker_loop_error",
|
|
450
|
+
message: "Worker loop error",
|
|
451
|
+
outcome: "failed",
|
|
452
|
+
error: error instanceof Error ? error : new Error(String(error)),
|
|
453
|
+
serviceId: "workflow",
|
|
454
|
+
evidence: {
|
|
455
|
+
errorMessage: error instanceof Error ? error.message : String(error),
|
|
456
|
+
errorStack: error instanceof Error ? error.stack : void 0
|
|
457
|
+
}
|
|
458
|
+
});
|
|
459
|
+
await sleep(5e3);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
462
|
+
logger.info("Worker loop stopped");
|
|
463
|
+
}
|
|
464
|
+
return {
|
|
465
|
+
async start() {
|
|
466
|
+
if (isRunning) {
|
|
467
|
+
logger.warn("Worker already running");
|
|
468
|
+
return;
|
|
469
|
+
}
|
|
470
|
+
logger.info("Starting workflow worker", { concurrency });
|
|
471
|
+
isRunning = true;
|
|
472
|
+
stopRequested = false;
|
|
473
|
+
analytics?.track("workflow.worker.started", {
|
|
474
|
+
concurrency
|
|
475
|
+
}).catch(() => {
|
|
476
|
+
});
|
|
477
|
+
const promises = [];
|
|
478
|
+
for (let i = 0; i < concurrency; i++) {
|
|
479
|
+
promises.push(workerLoop());
|
|
480
|
+
}
|
|
481
|
+
await Promise.all(promises);
|
|
482
|
+
},
|
|
483
|
+
async stop() {
|
|
484
|
+
if (!isRunning) {
|
|
485
|
+
return;
|
|
486
|
+
}
|
|
487
|
+
logger.info("Stopping workflow worker", {
|
|
488
|
+
runningJobsCount: runningJobs.size
|
|
489
|
+
});
|
|
490
|
+
stopRequested = true;
|
|
491
|
+
isRunning = false;
|
|
492
|
+
if (runningJobs.size > 0) {
|
|
493
|
+
logger.info("Waiting for in-flight jobs to complete", {
|
|
494
|
+
count: runningJobs.size
|
|
495
|
+
});
|
|
496
|
+
const shutdownTimeoutMs = parseInt(
|
|
497
|
+
process.env.WORKFLOW_SHUTDOWN_TIMEOUT_MS || "120000",
|
|
498
|
+
10
|
|
499
|
+
);
|
|
500
|
+
try {
|
|
501
|
+
await Promise.race([
|
|
502
|
+
Promise.all(Array.from(runningJobs.values())),
|
|
503
|
+
sleep(shutdownTimeoutMs)
|
|
504
|
+
]);
|
|
505
|
+
if (runningJobs.size > 0) {
|
|
506
|
+
logger.warn("Shutdown timeout reached, marking jobs as interrupted", {
|
|
507
|
+
count: runningJobs.size
|
|
508
|
+
});
|
|
509
|
+
await Promise.all(
|
|
510
|
+
Array.from(runningJobs.keys()).map(async (jobKey) => {
|
|
511
|
+
const [runId, jobId] = jobKey.split(":");
|
|
512
|
+
if (runId && jobId) {
|
|
513
|
+
await engine.markJobInterrupted(runId, jobId);
|
|
514
|
+
}
|
|
515
|
+
})
|
|
516
|
+
);
|
|
517
|
+
} else {
|
|
518
|
+
logger.info("All in-flight jobs completed gracefully");
|
|
519
|
+
}
|
|
520
|
+
} catch (error) {
|
|
521
|
+
logger.error("Error during graceful shutdown", error instanceof Error ? error : void 0, {
|
|
522
|
+
runningJobsCount: runningJobs.size
|
|
523
|
+
});
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
logger.info("Workflow worker stopped");
|
|
527
|
+
analytics?.track("workflow.worker.stopped", {
|
|
528
|
+
gracefulShutdown: runningJobs.size === 0,
|
|
529
|
+
interruptedJobs: runningJobs.size
|
|
530
|
+
}).catch(() => {
|
|
531
|
+
});
|
|
532
|
+
}
|
|
533
|
+
};
|
|
534
|
+
}
|
|
535
|
+
function sleep(ms) {
|
|
536
|
+
return new Promise((resolve) => {
|
|
537
|
+
setTimeout(resolve, ms);
|
|
538
|
+
});
|
|
539
|
+
}
|
|
540
|
+
function inferWorkspaceProvisionReasonCode(message) {
|
|
541
|
+
return /ETIMEDOUT|timeout/iu.test(message) ? "workspace_provision_timeout" : "workspace_provision_failed";
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
// src/job-broker.ts
|
|
545
|
+
var JobBroker = class {
|
|
546
|
+
constructor(engine, logger, platform2) {
|
|
547
|
+
this.engine = engine;
|
|
548
|
+
this.logger = logger;
|
|
549
|
+
this.platform = platform2;
|
|
550
|
+
}
|
|
551
|
+
/**
|
|
552
|
+
* Submit a background job for execution.
|
|
553
|
+
* Creates a WorkflowSpec with a single job and submits to WorkflowEngine.
|
|
554
|
+
*/
|
|
555
|
+
async submit(request) {
|
|
556
|
+
this.logger.info("Submitting background job", {
|
|
557
|
+
handler: request.handler,
|
|
558
|
+
priority: request.priority ?? "normal"
|
|
559
|
+
});
|
|
560
|
+
const uses = request.handler.includes(":") ? request.handler : `command:${request.handler}`;
|
|
561
|
+
const spec = {
|
|
562
|
+
name: `job-${request.handler}`,
|
|
563
|
+
version: "1.0.0",
|
|
564
|
+
on: { manual: true },
|
|
565
|
+
jobs: {
|
|
566
|
+
main: {
|
|
567
|
+
runsOn: "local",
|
|
568
|
+
steps: [
|
|
569
|
+
{
|
|
570
|
+
id: "execute",
|
|
571
|
+
name: "Execute handler",
|
|
572
|
+
uses,
|
|
573
|
+
// @ts-expect-error - WorkflowSpec step.with type mismatch
|
|
574
|
+
with: request.input ?? {}
|
|
575
|
+
}
|
|
576
|
+
]
|
|
577
|
+
}
|
|
578
|
+
}
|
|
579
|
+
};
|
|
580
|
+
const run = await this.engine.runFromInline(spec, {
|
|
581
|
+
env: {},
|
|
582
|
+
metadata: request.metadata
|
|
583
|
+
});
|
|
584
|
+
this.logger.info("Background job submitted", {
|
|
585
|
+
runId: run.id,
|
|
586
|
+
handler: request.handler,
|
|
587
|
+
status: run.status
|
|
588
|
+
});
|
|
589
|
+
return run;
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* Schedule a recurring job with cron expression.
|
|
593
|
+
* Registers job with CronScheduler (if available).
|
|
594
|
+
*
|
|
595
|
+
* NOTE: CronScheduler integration not yet implemented.
|
|
596
|
+
* This is a placeholder for future implementation.
|
|
597
|
+
*/
|
|
598
|
+
async schedule(request) {
|
|
599
|
+
this.logger.warn("CronScheduler not yet implemented", {
|
|
600
|
+
handler: request.handler,
|
|
601
|
+
cron: request.cron
|
|
602
|
+
});
|
|
603
|
+
throw new Error("CronScheduler not yet implemented");
|
|
604
|
+
}
|
|
605
|
+
/**
|
|
606
|
+
* Get job status by run ID.
|
|
607
|
+
*/
|
|
608
|
+
async getStatus(runId) {
|
|
609
|
+
return this.engine.getRun(runId);
|
|
610
|
+
}
|
|
611
|
+
/**
|
|
612
|
+
* Cancel a running job.
|
|
613
|
+
*/
|
|
614
|
+
async cancel(runId) {
|
|
615
|
+
await this.engine.cancelRun(runId);
|
|
616
|
+
this.logger.info("Job cancelled", { runId });
|
|
617
|
+
}
|
|
618
|
+
/**
|
|
619
|
+
* Get job logs by run ID.
|
|
620
|
+
* Returns execution logs with optional filtering by level and pagination.
|
|
621
|
+
* Uses platform.logs service to query logs by runId metadata.
|
|
622
|
+
*/
|
|
623
|
+
async getJobLogs(runId, options) {
|
|
624
|
+
const run = await this.engine.getRun(runId);
|
|
625
|
+
if (!run) {
|
|
626
|
+
return [];
|
|
627
|
+
}
|
|
628
|
+
const limit = options?.limit ?? 100;
|
|
629
|
+
const offset = options?.offset ?? 0;
|
|
630
|
+
const startTime = run.startedAt ? new Date(run.startedAt).getTime() : Date.now() - 36e5;
|
|
631
|
+
const endTime = run.finishedAt ? new Date(run.finishedAt).getTime() : Date.now();
|
|
632
|
+
const queryResult = await this.platform.logs.query(
|
|
633
|
+
{
|
|
634
|
+
from: startTime,
|
|
635
|
+
to: endTime,
|
|
636
|
+
level: options?.level && options.level !== "all" ? options.level : void 0
|
|
637
|
+
},
|
|
638
|
+
{
|
|
639
|
+
limit: 1e3,
|
|
640
|
+
// Query more logs, then filter in-memory
|
|
641
|
+
offset: 0
|
|
642
|
+
}
|
|
643
|
+
);
|
|
644
|
+
const filteredLogs = queryResult.logs.filter((log) => {
|
|
645
|
+
return log.fields.runId === runId || log.fields.executionId === runId || // Also check jobId and stepId for drill-down capability
|
|
646
|
+
log.fields.jobId?.toString().startsWith(runId);
|
|
647
|
+
});
|
|
648
|
+
const sortedLogs = filteredLogs.sort((a, b) => b.timestamp - a.timestamp);
|
|
649
|
+
const paginatedLogs = sortedLogs.slice(offset, offset + limit);
|
|
650
|
+
return paginatedLogs.map((log) => ({
|
|
651
|
+
timestamp: new Date(log.timestamp).toISOString(),
|
|
652
|
+
level: log.level,
|
|
653
|
+
message: log.message,
|
|
654
|
+
context: log.fields
|
|
655
|
+
}));
|
|
656
|
+
}
|
|
657
|
+
};
|
|
658
|
+
var CronScheduler = class {
|
|
659
|
+
jobBroker;
|
|
660
|
+
workflowEngine;
|
|
661
|
+
logger;
|
|
662
|
+
defaultTimezone;
|
|
663
|
+
analytics;
|
|
664
|
+
// eslint-disable-line @typescript-eslint/consistent-type-imports
|
|
665
|
+
registeredJobs = /* @__PURE__ */ new Map();
|
|
666
|
+
scheduledTasks = /* @__PURE__ */ new Map();
|
|
667
|
+
isRunning = false;
|
|
668
|
+
constructor(options) {
|
|
669
|
+
this.jobBroker = options.jobBroker;
|
|
670
|
+
this.workflowEngine = options.workflowEngine;
|
|
671
|
+
this.logger = options.logger;
|
|
672
|
+
this.defaultTimezone = options.timezone ?? "UTC";
|
|
673
|
+
this.analytics = options.analytics;
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* Register cron job from plugin manifest.
|
|
677
|
+
*/
|
|
678
|
+
registerPluginJob(pluginId, job) {
|
|
679
|
+
const cronJobId = `plugin:${pluginId}:${job.id}`;
|
|
680
|
+
if (this.registeredJobs.has(cronJobId)) {
|
|
681
|
+
this.logger.warn("Cron job already registered, skipping", { cronJobId });
|
|
682
|
+
return;
|
|
683
|
+
}
|
|
684
|
+
const registered = {
|
|
685
|
+
id: cronJobId,
|
|
686
|
+
source: "plugin",
|
|
687
|
+
schedule: job.schedule,
|
|
688
|
+
timezone: job.timezone ?? this.defaultTimezone,
|
|
689
|
+
priority: job.priority,
|
|
690
|
+
enabled: job.enabled,
|
|
691
|
+
handler: job.handler,
|
|
692
|
+
input: job.input,
|
|
693
|
+
metadata: job.metadata
|
|
694
|
+
};
|
|
695
|
+
this.registeredJobs.set(cronJobId, registered);
|
|
696
|
+
this.logger.debug("Plugin cron job registered", {
|
|
697
|
+
cronJobId,
|
|
698
|
+
schedule: job.schedule,
|
|
699
|
+
handler: job.handler
|
|
700
|
+
});
|
|
701
|
+
}
|
|
702
|
+
/**
|
|
703
|
+
* Register cron job from user YAML file.
|
|
704
|
+
*/
|
|
705
|
+
registerUserJob(fileName, job) {
|
|
706
|
+
const cronJobId = `user:${fileName}`;
|
|
707
|
+
if (this.registeredJobs.has(cronJobId)) {
|
|
708
|
+
this.logger.warn("Cron job already registered, skipping", { cronJobId });
|
|
709
|
+
return;
|
|
710
|
+
}
|
|
711
|
+
const registered = {
|
|
712
|
+
id: cronJobId,
|
|
713
|
+
source: "user",
|
|
714
|
+
schedule: job.schedule,
|
|
715
|
+
timezone: job.timezone ?? this.defaultTimezone,
|
|
716
|
+
priority: job.priority,
|
|
717
|
+
enabled: job.enabled,
|
|
718
|
+
workflowSpec: {
|
|
719
|
+
name: job.name,
|
|
720
|
+
jobs: job.jobs,
|
|
721
|
+
env: job.env
|
|
722
|
+
},
|
|
723
|
+
metadata: job.metadata
|
|
724
|
+
};
|
|
725
|
+
this.registeredJobs.set(cronJobId, registered);
|
|
726
|
+
this.logger.debug("User cron job registered", {
|
|
727
|
+
cronJobId,
|
|
728
|
+
schedule: job.schedule,
|
|
729
|
+
name: job.name
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
/**
|
|
733
|
+
* Start all registered cron jobs.
|
|
734
|
+
* Schedules enabled jobs using node-cron.
|
|
735
|
+
*/
|
|
736
|
+
async start() {
|
|
737
|
+
if (this.isRunning) {
|
|
738
|
+
this.logger.warn("CronScheduler already running");
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
this.logger.info("Starting CronScheduler", {
|
|
742
|
+
totalJobs: this.registeredJobs.size
|
|
743
|
+
});
|
|
744
|
+
for (const [cronJobId, job] of this.registeredJobs) {
|
|
745
|
+
if (!job.enabled) {
|
|
746
|
+
this.logger.debug("Skipping disabled cron job", { cronJobId });
|
|
747
|
+
continue;
|
|
748
|
+
}
|
|
749
|
+
if (!cron.validate(job.schedule)) {
|
|
750
|
+
this.logger.error("Invalid cron expression", void 0, {
|
|
751
|
+
cronJobId,
|
|
752
|
+
schedule: job.schedule
|
|
753
|
+
});
|
|
754
|
+
continue;
|
|
755
|
+
}
|
|
756
|
+
const task = cron.schedule(
|
|
757
|
+
job.schedule,
|
|
758
|
+
() => this.executeCronJob(cronJobId, job),
|
|
759
|
+
{
|
|
760
|
+
timezone: job.timezone
|
|
761
|
+
}
|
|
762
|
+
);
|
|
763
|
+
this.scheduledTasks.set(cronJobId, task);
|
|
764
|
+
this.logger.info("Cron job scheduled", {
|
|
765
|
+
cronJobId,
|
|
766
|
+
schedule: job.schedule,
|
|
767
|
+
timezone: job.timezone,
|
|
768
|
+
source: job.source
|
|
769
|
+
});
|
|
770
|
+
}
|
|
771
|
+
this.isRunning = true;
|
|
772
|
+
this.logger.info("CronScheduler started", {
|
|
773
|
+
scheduledJobs: this.scheduledTasks.size
|
|
774
|
+
});
|
|
775
|
+
this.analytics?.track("workflow.cron.scheduler.started", {
|
|
776
|
+
totalJobs: this.registeredJobs.size,
|
|
777
|
+
scheduledJobs: this.scheduledTasks.size
|
|
778
|
+
}).catch(() => {
|
|
779
|
+
});
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* Stop all scheduled cron jobs.
|
|
783
|
+
* Called during graceful shutdown.
|
|
784
|
+
*/
|
|
785
|
+
async stop() {
|
|
786
|
+
if (!this.isRunning) {
|
|
787
|
+
return;
|
|
788
|
+
}
|
|
789
|
+
this.logger.info("Stopping CronScheduler", {
|
|
790
|
+
scheduledJobs: this.scheduledTasks.size
|
|
791
|
+
});
|
|
792
|
+
for (const [cronJobId, task] of this.scheduledTasks) {
|
|
793
|
+
task.stop();
|
|
794
|
+
this.logger.debug("Cron job stopped", { cronJobId });
|
|
795
|
+
}
|
|
796
|
+
const stoppedCount = this.scheduledTasks.size;
|
|
797
|
+
this.scheduledTasks.clear();
|
|
798
|
+
this.isRunning = false;
|
|
799
|
+
this.logger.info("CronScheduler stopped");
|
|
800
|
+
this.analytics?.track("workflow.cron.scheduler.stopped", {
|
|
801
|
+
stoppedJobs: stoppedCount
|
|
802
|
+
}).catch(() => {
|
|
803
|
+
});
|
|
804
|
+
}
|
|
805
|
+
/**
|
|
806
|
+
* Execute cron job by submitting it to JobBroker.
|
|
807
|
+
*/
|
|
808
|
+
async executeCronJob(cronJobId, job) {
|
|
809
|
+
const startTime = Date.now();
|
|
810
|
+
this.logger.info("Executing cron job", { cronJobId });
|
|
811
|
+
this.analytics?.track("workflow.cron.job.started", {
|
|
812
|
+
cronJobId,
|
|
813
|
+
source: job.source,
|
|
814
|
+
schedule: job.schedule
|
|
815
|
+
}).catch(() => {
|
|
816
|
+
});
|
|
817
|
+
try {
|
|
818
|
+
if (job.source === "plugin" && job.handler) {
|
|
819
|
+
const result = await this.jobBroker.submit({
|
|
820
|
+
handler: job.handler,
|
|
821
|
+
input: job.input,
|
|
822
|
+
priority: job.priority,
|
|
823
|
+
metadata: {
|
|
824
|
+
...job.metadata,
|
|
825
|
+
cronJobId,
|
|
826
|
+
scheduledBy: "cron",
|
|
827
|
+
scheduledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
828
|
+
}
|
|
829
|
+
});
|
|
830
|
+
this.logger.info("Cron job submitted", {
|
|
831
|
+
cronJobId,
|
|
832
|
+
runId: result.id
|
|
833
|
+
});
|
|
834
|
+
} else if (job.source === "user" && job.workflowSpec) {
|
|
835
|
+
const spec = {
|
|
836
|
+
name: job.workflowSpec.name,
|
|
837
|
+
version: "1.0.0",
|
|
838
|
+
on: { manual: true },
|
|
839
|
+
// Cron-triggered workflows use manual trigger
|
|
840
|
+
jobs: job.workflowSpec.jobs,
|
|
841
|
+
env: job.workflowSpec.env
|
|
842
|
+
};
|
|
843
|
+
console.log("\u{1F50D} CRON SPEC:", JSON.stringify(spec, null, 2));
|
|
844
|
+
this.logger.debug("Running workflow from cron", {
|
|
845
|
+
cronJobId,
|
|
846
|
+
spec: JSON.stringify(spec, null, 2)
|
|
847
|
+
});
|
|
848
|
+
const result = await this.workflowEngine.runFromInline(spec, {
|
|
849
|
+
trigger: {
|
|
850
|
+
type: "schedule",
|
|
851
|
+
payload: {
|
|
852
|
+
cronJobId,
|
|
853
|
+
scheduledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
854
|
+
}
|
|
855
|
+
},
|
|
856
|
+
env: job.workflowSpec.env ?? {},
|
|
857
|
+
metadata: {
|
|
858
|
+
...job.metadata,
|
|
859
|
+
cronJobId,
|
|
860
|
+
scheduledBy: "cron",
|
|
861
|
+
scheduledAt: (/* @__PURE__ */ new Date()).toISOString()
|
|
862
|
+
}
|
|
863
|
+
});
|
|
864
|
+
this.logger.info("Cron workflow submitted", {
|
|
865
|
+
cronJobId,
|
|
866
|
+
runId: result.id,
|
|
867
|
+
workflowName: spec.name
|
|
868
|
+
});
|
|
869
|
+
} else {
|
|
870
|
+
throw new Error(`Invalid cron job configuration: ${cronJobId}`);
|
|
871
|
+
}
|
|
872
|
+
const duration = Date.now() - startTime;
|
|
873
|
+
this.analytics?.track("workflow.cron.job.completed", {
|
|
874
|
+
cronJobId,
|
|
875
|
+
source: job.source,
|
|
876
|
+
durationMs: duration
|
|
877
|
+
}).catch(() => {
|
|
878
|
+
});
|
|
879
|
+
} catch (error) {
|
|
880
|
+
const duration = Date.now() - startTime;
|
|
881
|
+
this.logger.error(
|
|
882
|
+
"Failed to execute cron job",
|
|
883
|
+
error instanceof Error ? error : void 0,
|
|
884
|
+
{ cronJobId }
|
|
885
|
+
);
|
|
886
|
+
this.analytics?.track("workflow.cron.job.failed", {
|
|
887
|
+
cronJobId,
|
|
888
|
+
source: job.source,
|
|
889
|
+
errorMessage: error instanceof Error ? error.message : "Unknown error",
|
|
890
|
+
durationMs: duration
|
|
891
|
+
}).catch(() => {
|
|
892
|
+
});
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
/**
|
|
896
|
+
* Get all registered cron jobs.
|
|
897
|
+
*/
|
|
898
|
+
getRegisteredJobs() {
|
|
899
|
+
return Array.from(this.registeredJobs.values());
|
|
900
|
+
}
|
|
901
|
+
/**
|
|
902
|
+
* Get cron job by ID.
|
|
903
|
+
*/
|
|
904
|
+
getJob(cronJobId) {
|
|
905
|
+
return this.registeredJobs.get(cronJobId);
|
|
906
|
+
}
|
|
907
|
+
/**
|
|
908
|
+
* Check if scheduler is running.
|
|
909
|
+
*/
|
|
910
|
+
isSchedulerRunning() {
|
|
911
|
+
return this.isRunning;
|
|
912
|
+
}
|
|
913
|
+
/**
|
|
914
|
+
* Register cron job (generic method for API usage).
|
|
915
|
+
* Supports both plugin and API-registered jobs.
|
|
916
|
+
*/
|
|
917
|
+
register(job) {
|
|
918
|
+
if (this.registeredJobs.has(job.id)) {
|
|
919
|
+
this.logger.warn("Cron job already registered, overwriting", { cronJobId: job.id });
|
|
920
|
+
}
|
|
921
|
+
this.registeredJobs.set(job.id, job);
|
|
922
|
+
this.logger.debug("Cron job registered", {
|
|
923
|
+
cronJobId: job.id,
|
|
924
|
+
schedule: job.schedule,
|
|
925
|
+
source: job.source
|
|
926
|
+
});
|
|
927
|
+
if (this.isRunning && job.enabled) {
|
|
928
|
+
this.scheduleJob(job.id, job);
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
/**
|
|
932
|
+
* Unregister cron job by ID.
|
|
933
|
+
* Stops the scheduled task if running.
|
|
934
|
+
*/
|
|
935
|
+
unregister(cronJobId) {
|
|
936
|
+
const job = this.registeredJobs.get(cronJobId);
|
|
937
|
+
if (!job) {
|
|
938
|
+
this.logger.warn("Cron job not found for unregister", { cronJobId });
|
|
939
|
+
return;
|
|
940
|
+
}
|
|
941
|
+
const task = this.scheduledTasks.get(cronJobId);
|
|
942
|
+
if (task) {
|
|
943
|
+
task.stop();
|
|
944
|
+
this.scheduledTasks.delete(cronJobId);
|
|
945
|
+
this.logger.debug("Cron task stopped", { cronJobId });
|
|
946
|
+
}
|
|
947
|
+
this.registeredJobs.delete(cronJobId);
|
|
948
|
+
this.logger.info("Cron job unregistered", { cronJobId });
|
|
949
|
+
}
|
|
950
|
+
/**
|
|
951
|
+
* Pause cron job (stop task but keep registration).
|
|
952
|
+
*/
|
|
953
|
+
pause(cronJobId) {
|
|
954
|
+
const job = this.registeredJobs.get(cronJobId);
|
|
955
|
+
if (!job) {
|
|
956
|
+
throw new Error(`Cron job not found: ${cronJobId}`);
|
|
957
|
+
}
|
|
958
|
+
const task = this.scheduledTasks.get(cronJobId);
|
|
959
|
+
if (!task) {
|
|
960
|
+
throw new Error(`Cron task not scheduled: ${cronJobId}`);
|
|
961
|
+
}
|
|
962
|
+
task.stop();
|
|
963
|
+
job.enabled = false;
|
|
964
|
+
this.logger.info("Cron job paused", { cronJobId });
|
|
965
|
+
}
|
|
966
|
+
/**
|
|
967
|
+
* Resume cron job (restart stopped task).
|
|
968
|
+
*/
|
|
969
|
+
resume(cronJobId) {
|
|
970
|
+
const job = this.registeredJobs.get(cronJobId);
|
|
971
|
+
if (!job) {
|
|
972
|
+
throw new Error(`Cron job not found: ${cronJobId}`);
|
|
973
|
+
}
|
|
974
|
+
const task = this.scheduledTasks.get(cronJobId);
|
|
975
|
+
if (!task) {
|
|
976
|
+
if (this.isRunning) {
|
|
977
|
+
this.scheduleJob(cronJobId, job);
|
|
978
|
+
}
|
|
979
|
+
} else {
|
|
980
|
+
task.start();
|
|
981
|
+
}
|
|
982
|
+
job.enabled = true;
|
|
983
|
+
this.logger.info("Cron job resumed", { cronJobId });
|
|
984
|
+
}
|
|
985
|
+
/**
|
|
986
|
+
* Trigger cron job immediately (manual execution).
|
|
987
|
+
*/
|
|
988
|
+
async triggerNow(cronJobId) {
|
|
989
|
+
const job = this.registeredJobs.get(cronJobId);
|
|
990
|
+
if (!job) {
|
|
991
|
+
throw new Error(`Cron job not found: ${cronJobId}`);
|
|
992
|
+
}
|
|
993
|
+
this.logger.info("Manually triggering cron job", { cronJobId });
|
|
994
|
+
await this.executeCronJob(cronJobId, job);
|
|
995
|
+
}
|
|
996
|
+
/**
|
|
997
|
+
* Schedule a single cron job (helper method).
|
|
998
|
+
*/
|
|
999
|
+
scheduleJob(cronJobId, job) {
|
|
1000
|
+
if (!cron.validate(job.schedule)) {
|
|
1001
|
+
this.logger.error("Invalid cron expression", void 0, {
|
|
1002
|
+
cronJobId,
|
|
1003
|
+
schedule: job.schedule
|
|
1004
|
+
});
|
|
1005
|
+
return;
|
|
1006
|
+
}
|
|
1007
|
+
const task = cron.schedule(
|
|
1008
|
+
job.schedule,
|
|
1009
|
+
() => this.executeCronJob(cronJobId, job),
|
|
1010
|
+
{
|
|
1011
|
+
timezone: job.timezone
|
|
1012
|
+
}
|
|
1013
|
+
);
|
|
1014
|
+
this.scheduledTasks.set(cronJobId, task);
|
|
1015
|
+
this.logger.info("Cron job scheduled", {
|
|
1016
|
+
cronJobId,
|
|
1017
|
+
schedule: job.schedule,
|
|
1018
|
+
timezone: job.timezone,
|
|
1019
|
+
source: job.source
|
|
1020
|
+
});
|
|
1021
|
+
}
|
|
1022
|
+
/**
|
|
1023
|
+
* Get next scheduled run time for a cron job.
|
|
1024
|
+
* Returns null if job doesn't exist or has invalid cron expression.
|
|
1025
|
+
*/
|
|
1026
|
+
getNextRunTime(cronJobId) {
|
|
1027
|
+
const job = this.registeredJobs.get(cronJobId);
|
|
1028
|
+
if (!job) {
|
|
1029
|
+
return null;
|
|
1030
|
+
}
|
|
1031
|
+
if (!job.schedule || job.schedule.trim() === "") {
|
|
1032
|
+
return null;
|
|
1033
|
+
}
|
|
1034
|
+
try {
|
|
1035
|
+
const interval = CronExpressionParser.parse(job.schedule, {
|
|
1036
|
+
tz: job.timezone,
|
|
1037
|
+
currentDate: /* @__PURE__ */ new Date()
|
|
1038
|
+
});
|
|
1039
|
+
return interval.next().toDate();
|
|
1040
|
+
} catch (error) {
|
|
1041
|
+
this.logger.error("Failed to parse cron expression", error instanceof Error ? error : void 0, {
|
|
1042
|
+
cronJobId,
|
|
1043
|
+
schedule: job.schedule
|
|
1044
|
+
});
|
|
1045
|
+
return null;
|
|
1046
|
+
}
|
|
1047
|
+
}
|
|
1048
|
+
/**
|
|
1049
|
+
* Clear all registered cron jobs.
|
|
1050
|
+
* IMPORTANT: This does NOT stop scheduled tasks - call stop() first if needed.
|
|
1051
|
+
*/
|
|
1052
|
+
clearAll() {
|
|
1053
|
+
this.logger.info("Clearing all registered cron jobs", {
|
|
1054
|
+
count: this.registeredJobs.size
|
|
1055
|
+
});
|
|
1056
|
+
this.registeredJobs.clear();
|
|
1057
|
+
}
|
|
1058
|
+
};
|
|
1059
|
+
var CronDiscovery = class {
|
|
1060
|
+
cliApi;
|
|
1061
|
+
scheduler;
|
|
1062
|
+
logger;
|
|
1063
|
+
workspaceRoot;
|
|
1064
|
+
constructor(options) {
|
|
1065
|
+
this.cliApi = options.cliApi;
|
|
1066
|
+
this.scheduler = options.scheduler;
|
|
1067
|
+
this.logger = options.logger;
|
|
1068
|
+
this.workspaceRoot = options.workspaceRoot;
|
|
1069
|
+
}
|
|
1070
|
+
/**
|
|
1071
|
+
* Discover and register all cron jobs.
|
|
1072
|
+
* Scans plugin manifests and user YAML files.
|
|
1073
|
+
*/
|
|
1074
|
+
async discoverAll() {
|
|
1075
|
+
this.logger.info("Starting cron job discovery");
|
|
1076
|
+
const pluginJobs = await this.discoverPluginJobs();
|
|
1077
|
+
const userJobs = await this.discoverUserJobs();
|
|
1078
|
+
this.logger.info("Cron job discovery complete", {
|
|
1079
|
+
pluginJobs,
|
|
1080
|
+
userJobs,
|
|
1081
|
+
total: pluginJobs + userJobs
|
|
1082
|
+
});
|
|
1083
|
+
return { plugins: pluginJobs, users: userJobs };
|
|
1084
|
+
}
|
|
1085
|
+
/**
|
|
1086
|
+
* Discover cron jobs from plugin manifests via entity registry.
|
|
1087
|
+
* Uses queryEntities({ kind: 'cron' }) — no manual manifest scanning.
|
|
1088
|
+
*/
|
|
1089
|
+
async discoverPluginJobs() {
|
|
1090
|
+
let count = 0;
|
|
1091
|
+
try {
|
|
1092
|
+
const cronEntities = this.cliApi.queryEntities({ kind: "cron" });
|
|
1093
|
+
for (const entity of cronEntities) {
|
|
1094
|
+
try {
|
|
1095
|
+
const validated = PluginCronJobSchema.parse(entity.declaration);
|
|
1096
|
+
this.scheduler.registerPluginJob(entity.ref.pluginId, validated);
|
|
1097
|
+
count++;
|
|
1098
|
+
} catch (error) {
|
|
1099
|
+
this.logger.warn("Invalid plugin cron job definition", {
|
|
1100
|
+
pluginId: entity.ref.pluginId,
|
|
1101
|
+
entityId: entity.ref.entityId,
|
|
1102
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1103
|
+
});
|
|
1104
|
+
}
|
|
1105
|
+
}
|
|
1106
|
+
if (cronEntities.length > 0) {
|
|
1107
|
+
this.logger.debug("Discovered plugin cron jobs via registry", {
|
|
1108
|
+
total: cronEntities.length,
|
|
1109
|
+
registered: count
|
|
1110
|
+
});
|
|
1111
|
+
}
|
|
1112
|
+
} catch (error) {
|
|
1113
|
+
this.logger.error(
|
|
1114
|
+
"Failed to discover plugin cron jobs",
|
|
1115
|
+
error instanceof Error ? error : void 0
|
|
1116
|
+
);
|
|
1117
|
+
}
|
|
1118
|
+
return count;
|
|
1119
|
+
}
|
|
1120
|
+
/**
|
|
1121
|
+
* Discover cron jobs from user YAML files in .kb/jobs/*.yml
|
|
1122
|
+
*/
|
|
1123
|
+
async discoverUserJobs() {
|
|
1124
|
+
let count = 0;
|
|
1125
|
+
const jobsDir = join(this.workspaceRoot, ".kb", "jobs");
|
|
1126
|
+
try {
|
|
1127
|
+
const dirStat = await stat(jobsDir).catch(() => null);
|
|
1128
|
+
if (!dirStat?.isDirectory()) {
|
|
1129
|
+
this.logger.debug("User jobs directory does not exist", { jobsDir });
|
|
1130
|
+
return 0;
|
|
1131
|
+
}
|
|
1132
|
+
const files = await readdir(jobsDir);
|
|
1133
|
+
const yamlFiles = files.filter((file) => {
|
|
1134
|
+
const ext = extname(file);
|
|
1135
|
+
return ext === ".yml" || ext === ".yaml";
|
|
1136
|
+
});
|
|
1137
|
+
const results = await Promise.allSettled(
|
|
1138
|
+
yamlFiles.map(async (file) => {
|
|
1139
|
+
const filePath = join(jobsDir, file);
|
|
1140
|
+
const ext = extname(file);
|
|
1141
|
+
const content = await readFile(filePath, "utf-8");
|
|
1142
|
+
const parsed = YAML.parse(content);
|
|
1143
|
+
const validated = UserCronJobSchema.parse(parsed);
|
|
1144
|
+
if (validated.autoStart) {
|
|
1145
|
+
const fileName = basename(file, ext);
|
|
1146
|
+
this.scheduler.registerUserJob(fileName, validated);
|
|
1147
|
+
return 1;
|
|
1148
|
+
} else {
|
|
1149
|
+
this.logger.debug("Skipping user cron job (autoStart: false)", {
|
|
1150
|
+
file
|
|
1151
|
+
});
|
|
1152
|
+
return 0;
|
|
1153
|
+
}
|
|
1154
|
+
})
|
|
1155
|
+
);
|
|
1156
|
+
for (const result of results) {
|
|
1157
|
+
if (result.status === "fulfilled") {
|
|
1158
|
+
count += result.value;
|
|
1159
|
+
} else {
|
|
1160
|
+
this.logger.warn("Failed to parse user cron job file", {
|
|
1161
|
+
error: result.reason instanceof Error ? result.reason.message : String(result.reason)
|
|
1162
|
+
});
|
|
1163
|
+
}
|
|
1164
|
+
}
|
|
1165
|
+
} catch (error) {
|
|
1166
|
+
this.logger.warn("Failed to discover user cron jobs", {
|
|
1167
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1168
|
+
});
|
|
1169
|
+
}
|
|
1170
|
+
return count;
|
|
1171
|
+
}
|
|
1172
|
+
};
|
|
1173
|
+
|
|
1174
|
+
// src/host/workflow-host-service.ts
|
|
1175
|
+
var TENANT_ID_PATTERN = /^[a-zA-Z0-9_-]+$/;
|
|
1176
|
+
var CRON_SCHEDULER_NOT_AVAILABLE = "Cron scheduler not available";
|
|
1177
|
+
var WorkflowHostService = class {
|
|
1178
|
+
constructor(options) {
|
|
1179
|
+
this.options = options;
|
|
1180
|
+
}
|
|
1181
|
+
getHealth() {
|
|
1182
|
+
return { ok: true, service: "workflow-daemon" };
|
|
1183
|
+
}
|
|
1184
|
+
async getMetrics() {
|
|
1185
|
+
return this.options.engine.getMetrics();
|
|
1186
|
+
}
|
|
1187
|
+
async submitJob(tenantId, request) {
|
|
1188
|
+
this.assertTenantId(tenantId);
|
|
1189
|
+
if (!request.type) {
|
|
1190
|
+
throw new Error("Missing required field: type");
|
|
1191
|
+
}
|
|
1192
|
+
if (request.priority !== void 0 && (request.priority < 1 || request.priority > 10)) {
|
|
1193
|
+
throw new Error("Priority must be between 1 and 10");
|
|
1194
|
+
}
|
|
1195
|
+
const run = await this.options.jobBroker.submit({
|
|
1196
|
+
handler: request.type,
|
|
1197
|
+
input: request.payload,
|
|
1198
|
+
priority: mapPriority(request.priority ?? 5)
|
|
1199
|
+
});
|
|
1200
|
+
this.options.logger.info("Job submitted", {
|
|
1201
|
+
jobId: run.id,
|
|
1202
|
+
type: request.type,
|
|
1203
|
+
tenantId
|
|
1204
|
+
});
|
|
1205
|
+
return { jobId: run.id };
|
|
1206
|
+
}
|
|
1207
|
+
async getJob(tenantId, jobId) {
|
|
1208
|
+
this.assertTenantId(tenantId);
|
|
1209
|
+
const run = await this.options.engine.getRun(jobId);
|
|
1210
|
+
if (!run) {
|
|
1211
|
+
throw new Error("Job not found");
|
|
1212
|
+
}
|
|
1213
|
+
return {
|
|
1214
|
+
id: run.id,
|
|
1215
|
+
type: run.name,
|
|
1216
|
+
status: mapRunStatusToJobStatus(run.status),
|
|
1217
|
+
tenantId: run.tenantId ?? tenantId,
|
|
1218
|
+
createdAt: run.createdAt,
|
|
1219
|
+
startedAt: run.startedAt,
|
|
1220
|
+
finishedAt: run.finishedAt,
|
|
1221
|
+
result: run.result,
|
|
1222
|
+
error: run.result?.error?.message,
|
|
1223
|
+
jobs: run.jobs?.map((job) => ({
|
|
1224
|
+
id: job.id,
|
|
1225
|
+
name: job.jobName,
|
|
1226
|
+
status: job.status,
|
|
1227
|
+
startedAt: job.startedAt,
|
|
1228
|
+
finishedAt: job.finishedAt,
|
|
1229
|
+
durationMs: job.durationMs,
|
|
1230
|
+
error: job.error?.message,
|
|
1231
|
+
steps: job.steps?.map((step) => ({
|
|
1232
|
+
id: step.id,
|
|
1233
|
+
name: step.name,
|
|
1234
|
+
status: step.status,
|
|
1235
|
+
handler: step.spec?.uses,
|
|
1236
|
+
startedAt: step.startedAt,
|
|
1237
|
+
finishedAt: step.finishedAt,
|
|
1238
|
+
durationMs: step.durationMs,
|
|
1239
|
+
outputs: step.outputs,
|
|
1240
|
+
error: step.error
|
|
1241
|
+
}))
|
|
1242
|
+
}))
|
|
1243
|
+
};
|
|
1244
|
+
}
|
|
1245
|
+
async getJobSteps(jobId) {
|
|
1246
|
+
const run = await this.options.engine.getRun(jobId);
|
|
1247
|
+
if (!run) {
|
|
1248
|
+
throw new Error("Job not found");
|
|
1249
|
+
}
|
|
1250
|
+
return run.jobs?.flatMap(
|
|
1251
|
+
(job) => job.steps?.map((step) => ({
|
|
1252
|
+
id: step.id,
|
|
1253
|
+
name: step.name,
|
|
1254
|
+
status: step.status,
|
|
1255
|
+
handler: step.spec?.uses,
|
|
1256
|
+
startedAt: step.startedAt,
|
|
1257
|
+
finishedAt: step.finishedAt,
|
|
1258
|
+
durationMs: step.durationMs,
|
|
1259
|
+
outputs: step.outputs,
|
|
1260
|
+
error: step.error,
|
|
1261
|
+
jobId: job.id,
|
|
1262
|
+
jobName: job.jobName
|
|
1263
|
+
})) ?? []
|
|
1264
|
+
) ?? [];
|
|
1265
|
+
}
|
|
1266
|
+
async getJobLogs(jobId, options) {
|
|
1267
|
+
const logs = await this.options.jobBroker.getJobLogs(jobId, options);
|
|
1268
|
+
return logs;
|
|
1269
|
+
}
|
|
1270
|
+
async cancelJob(tenantId, jobId) {
|
|
1271
|
+
await this.options.engine.cancelRun(jobId);
|
|
1272
|
+
this.options.logger.info("Job cancelled", { jobId, tenantId });
|
|
1273
|
+
return { cancelled: true };
|
|
1274
|
+
}
|
|
1275
|
+
async listJobs(tenantId, filter) {
|
|
1276
|
+
this.assertTenantId(tenantId);
|
|
1277
|
+
const { type, status, limit, offset } = filter;
|
|
1278
|
+
const allRuns = await this.options.engine.getAllRuns();
|
|
1279
|
+
let jobs = allRuns.map((run) => ({
|
|
1280
|
+
id: run.id,
|
|
1281
|
+
type: run.name,
|
|
1282
|
+
status: mapRunStatusToJobStatus(run.status),
|
|
1283
|
+
tenantId: run.tenantId ?? tenantId,
|
|
1284
|
+
createdAt: run.createdAt,
|
|
1285
|
+
startedAt: run.startedAt,
|
|
1286
|
+
finishedAt: run.finishedAt,
|
|
1287
|
+
result: run.result,
|
|
1288
|
+
error: run.result?.error?.message
|
|
1289
|
+
}));
|
|
1290
|
+
if (type) {
|
|
1291
|
+
const pattern = type.replace(/\*/g, ".*");
|
|
1292
|
+
const regex = new RegExp(`^${pattern}$`);
|
|
1293
|
+
jobs = jobs.filter((job) => regex.test(job.type));
|
|
1294
|
+
}
|
|
1295
|
+
if (status) {
|
|
1296
|
+
jobs = jobs.filter((job) => job.status === status);
|
|
1297
|
+
}
|
|
1298
|
+
const start = offset ?? 0;
|
|
1299
|
+
const end = limit ? start + limit : jobs.length;
|
|
1300
|
+
return { jobs: jobs.slice(start, end) };
|
|
1301
|
+
}
|
|
1302
|
+
async listActiveExecutions() {
|
|
1303
|
+
const runs = await this.options.engine.getActiveExecutions();
|
|
1304
|
+
return runs.map((run) => ({
|
|
1305
|
+
id: run.id,
|
|
1306
|
+
type: run.name,
|
|
1307
|
+
status: mapRunStatusToJobStatus(run.status),
|
|
1308
|
+
startedAt: run.startedAt,
|
|
1309
|
+
createdAt: run.createdAt
|
|
1310
|
+
}));
|
|
1311
|
+
}
|
|
1312
|
+
async listWorkflows(options) {
|
|
1313
|
+
const workflowService = this.requireWorkflowService();
|
|
1314
|
+
const workflows = await workflowService.listAll({
|
|
1315
|
+
source: options.source,
|
|
1316
|
+
status: options.status === "inactive" ? "disabled" : options.status,
|
|
1317
|
+
tags: options.tags ? options.tags.split(",") : void 0
|
|
1318
|
+
});
|
|
1319
|
+
const response = {
|
|
1320
|
+
workflows: workflows.map((w) => this.mapWorkflowInfo(w))
|
|
1321
|
+
};
|
|
1322
|
+
return response;
|
|
1323
|
+
}
|
|
1324
|
+
async getWorkflow(id) {
|
|
1325
|
+
const workflowService = this.requireWorkflowService();
|
|
1326
|
+
const workflow = await workflowService.get(id);
|
|
1327
|
+
if (!workflow) {
|
|
1328
|
+
return null;
|
|
1329
|
+
}
|
|
1330
|
+
return this.mapWorkflowInfo(workflow);
|
|
1331
|
+
}
|
|
1332
|
+
async runWorkflow(id, request) {
|
|
1333
|
+
const workflowService = this.requireWorkflowService();
|
|
1334
|
+
const workflow = await workflowService.get(id);
|
|
1335
|
+
if (!workflow) {
|
|
1336
|
+
throw new Error("Workflow not found");
|
|
1337
|
+
}
|
|
1338
|
+
const specInput = workflow.input;
|
|
1339
|
+
const spec = {
|
|
1340
|
+
...specInput,
|
|
1341
|
+
...request.target ? { target: request.target } : {},
|
|
1342
|
+
...request.isolation ? { isolation: request.isolation } : {}
|
|
1343
|
+
};
|
|
1344
|
+
const triggerType = request.trigger?.type === "cron" ? "schedule" : request.trigger?.type === "api" ? "webhook" : "manual";
|
|
1345
|
+
const run = await this.options.engine.runFromSpec(spec, {
|
|
1346
|
+
trigger: {
|
|
1347
|
+
type: triggerType,
|
|
1348
|
+
actor: request.trigger?.user,
|
|
1349
|
+
payload: request.input && typeof request.input === "object" ? request.input : void 0
|
|
1350
|
+
}
|
|
1351
|
+
});
|
|
1352
|
+
return {
|
|
1353
|
+
runId: run.id,
|
|
1354
|
+
status: run.status
|
|
1355
|
+
};
|
|
1356
|
+
}
|
|
1357
|
+
registerCron(tenantId, request) {
|
|
1358
|
+
this.assertTenantId(tenantId);
|
|
1359
|
+
const scheduler = this.requireCronScheduler();
|
|
1360
|
+
if (!request.id || !request.schedule || !request.jobType) {
|
|
1361
|
+
throw new Error("Missing required fields: id, schedule, jobType");
|
|
1362
|
+
}
|
|
1363
|
+
scheduler.register({
|
|
1364
|
+
id: request.id,
|
|
1365
|
+
source: "user",
|
|
1366
|
+
schedule: request.schedule,
|
|
1367
|
+
timezone: request.timezone ?? "UTC",
|
|
1368
|
+
priority: "normal",
|
|
1369
|
+
enabled: request.enabled ?? true,
|
|
1370
|
+
handler: request.jobType,
|
|
1371
|
+
metadata: {
|
|
1372
|
+
tenantId,
|
|
1373
|
+
payload: request.payload
|
|
1374
|
+
}
|
|
1375
|
+
});
|
|
1376
|
+
this.options.logger.info("Cron job registered", {
|
|
1377
|
+
id: request.id,
|
|
1378
|
+
schedule: request.schedule,
|
|
1379
|
+
jobType: request.jobType,
|
|
1380
|
+
tenantId
|
|
1381
|
+
});
|
|
1382
|
+
return { ok: true };
|
|
1383
|
+
}
|
|
1384
|
+
unregisterCron(tenantId, id) {
|
|
1385
|
+
const scheduler = this.requireCronScheduler();
|
|
1386
|
+
scheduler.unregister(id);
|
|
1387
|
+
this.options.logger.info("Cron job unregistered", { id, tenantId });
|
|
1388
|
+
return { ok: true };
|
|
1389
|
+
}
|
|
1390
|
+
triggerCron(tenantId, id) {
|
|
1391
|
+
const scheduler = this.requireCronScheduler();
|
|
1392
|
+
return scheduler.triggerNow(id).then(() => {
|
|
1393
|
+
this.options.logger.info("Cron job triggered manually", { id, tenantId });
|
|
1394
|
+
return { ok: true };
|
|
1395
|
+
});
|
|
1396
|
+
}
|
|
1397
|
+
pauseCron(tenantId, id) {
|
|
1398
|
+
const scheduler = this.requireCronScheduler();
|
|
1399
|
+
scheduler.pause(id);
|
|
1400
|
+
this.options.logger.info("Cron job paused", { id, tenantId });
|
|
1401
|
+
return { ok: true };
|
|
1402
|
+
}
|
|
1403
|
+
resumeCron(tenantId, id) {
|
|
1404
|
+
const scheduler = this.requireCronScheduler();
|
|
1405
|
+
scheduler.resume(id);
|
|
1406
|
+
this.options.logger.info("Cron job resumed", { id, tenantId });
|
|
1407
|
+
return { ok: true };
|
|
1408
|
+
}
|
|
1409
|
+
listCron() {
|
|
1410
|
+
const scheduler = this.requireCronScheduler();
|
|
1411
|
+
const crons = scheduler.getRegisteredJobs().map((job) => ({
|
|
1412
|
+
id: job.id,
|
|
1413
|
+
schedule: job.schedule,
|
|
1414
|
+
jobType: job.handler ?? "unknown",
|
|
1415
|
+
timezone: job.timezone ?? "UTC",
|
|
1416
|
+
enabled: job.enabled,
|
|
1417
|
+
lastRun: void 0,
|
|
1418
|
+
nextRun: scheduler.getNextRunTime(job.id) ?? void 0,
|
|
1419
|
+
pluginId: job.source === "plugin" ? job.id.split(":")[1] : void 0
|
|
1420
|
+
}));
|
|
1421
|
+
return { crons };
|
|
1422
|
+
}
|
|
1423
|
+
listLegacyCronJobs() {
|
|
1424
|
+
const scheduler = this.requireCronScheduler();
|
|
1425
|
+
const cronJobs = scheduler.getRegisteredJobs();
|
|
1426
|
+
return {
|
|
1427
|
+
cronJobs: cronJobs.map((job) => ({
|
|
1428
|
+
id: job.id,
|
|
1429
|
+
source: job.source,
|
|
1430
|
+
schedule: job.schedule,
|
|
1431
|
+
timezone: job.timezone,
|
|
1432
|
+
priority: job.priority,
|
|
1433
|
+
enabled: job.enabled,
|
|
1434
|
+
handler: job.handler,
|
|
1435
|
+
workflowName: job.workflowSpec?.name,
|
|
1436
|
+
metadata: job.metadata
|
|
1437
|
+
})),
|
|
1438
|
+
total: cronJobs.length,
|
|
1439
|
+
running: scheduler.isSchedulerRunning()
|
|
1440
|
+
};
|
|
1441
|
+
}
|
|
1442
|
+
assertTenantId(tenantId) {
|
|
1443
|
+
if (!TENANT_ID_PATTERN.test(tenantId) || tenantId.length > 64) {
|
|
1444
|
+
throw new Error("Invalid tenant ID format or length (max 64 chars)");
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
async listWorkflowRuns(workflowId, filters) {
|
|
1448
|
+
const allRuns = await this.options.engine.getAllRuns();
|
|
1449
|
+
let workflowName;
|
|
1450
|
+
if (this.options.workflowService) {
|
|
1451
|
+
const workflow = await this.options.workflowService.get(workflowId);
|
|
1452
|
+
workflowName = workflow?.name;
|
|
1453
|
+
}
|
|
1454
|
+
let runs = allRuns.filter(
|
|
1455
|
+
(run) => run.name === workflowId || workflowName && run.name === workflowName
|
|
1456
|
+
);
|
|
1457
|
+
if (filters?.status) {
|
|
1458
|
+
runs = runs.filter((run) => run.status === filters.status);
|
|
1459
|
+
}
|
|
1460
|
+
runs.sort((a, b) => new Date(b.createdAt ?? 0).getTime() - new Date(a.createdAt ?? 0).getTime());
|
|
1461
|
+
const start = filters?.offset ?? 0;
|
|
1462
|
+
const end = filters?.limit ? start + filters.limit : runs.length;
|
|
1463
|
+
const page = runs.slice(start, end);
|
|
1464
|
+
return {
|
|
1465
|
+
workflowId,
|
|
1466
|
+
runs: page.map((run) => ({
|
|
1467
|
+
id: run.id,
|
|
1468
|
+
workflowId,
|
|
1469
|
+
status: run.status,
|
|
1470
|
+
trigger: {
|
|
1471
|
+
type: run.trigger?.type ?? "manual",
|
|
1472
|
+
user: run.trigger?.actor
|
|
1473
|
+
},
|
|
1474
|
+
startedAt: run.startedAt ?? run.createdAt ?? (/* @__PURE__ */ new Date()).toISOString(),
|
|
1475
|
+
finishedAt: run.finishedAt,
|
|
1476
|
+
durationMs: run.durationMs,
|
|
1477
|
+
error: run.result?.error?.message
|
|
1478
|
+
})),
|
|
1479
|
+
total: runs.length
|
|
1480
|
+
};
|
|
1481
|
+
}
|
|
1482
|
+
mapWorkflowInfo(workflow) {
|
|
1483
|
+
return {
|
|
1484
|
+
id: workflow.id,
|
|
1485
|
+
name: workflow.name,
|
|
1486
|
+
description: workflow.description,
|
|
1487
|
+
source: workflow.source,
|
|
1488
|
+
pluginId: workflow.pluginId,
|
|
1489
|
+
status: workflow.status === "active" ? "active" : "inactive",
|
|
1490
|
+
tags: workflow.tags,
|
|
1491
|
+
inputs: workflow.inputSchema
|
|
1492
|
+
};
|
|
1493
|
+
}
|
|
1494
|
+
async getRun(runId) {
|
|
1495
|
+
return await this.options.engine.getRun(runId);
|
|
1496
|
+
}
|
|
1497
|
+
async listRuns(filters) {
|
|
1498
|
+
const allRuns = await this.options.engine.getAllRuns();
|
|
1499
|
+
let runs = allRuns;
|
|
1500
|
+
if (filters?.status) {
|
|
1501
|
+
runs = runs.filter((run) => run.status === filters.status);
|
|
1502
|
+
}
|
|
1503
|
+
runs.sort((a, b) => new Date(b.createdAt ?? 0).getTime() - new Date(a.createdAt ?? 0).getTime());
|
|
1504
|
+
const total = runs.length;
|
|
1505
|
+
const start = filters?.offset ?? 0;
|
|
1506
|
+
const end = filters?.limit ? start + filters.limit : runs.length;
|
|
1507
|
+
return { runs: runs.slice(start, end), total };
|
|
1508
|
+
}
|
|
1509
|
+
async cancelRun(runId) {
|
|
1510
|
+
const run = await this.options.engine.getRun(runId);
|
|
1511
|
+
if (!run) {
|
|
1512
|
+
throw new Error("Run not found");
|
|
1513
|
+
}
|
|
1514
|
+
if (run.status !== "running" && run.status !== "queued") {
|
|
1515
|
+
throw new Error(`Cannot cancel run with status "${run.status}"`);
|
|
1516
|
+
}
|
|
1517
|
+
await this.options.engine.cancelRun(runId);
|
|
1518
|
+
}
|
|
1519
|
+
requireWorkflowService() {
|
|
1520
|
+
if (!this.options.workflowService) {
|
|
1521
|
+
throw new Error("Workflow service not available");
|
|
1522
|
+
}
|
|
1523
|
+
return this.options.workflowService;
|
|
1524
|
+
}
|
|
1525
|
+
requireCronScheduler() {
|
|
1526
|
+
if (!this.options.cronScheduler) {
|
|
1527
|
+
throw new Error(CRON_SCHEDULER_NOT_AVAILABLE);
|
|
1528
|
+
}
|
|
1529
|
+
return this.options.cronScheduler;
|
|
1530
|
+
}
|
|
1531
|
+
};
|
|
1532
|
+
function mapRunStatusToJobStatus(status) {
|
|
1533
|
+
switch (status) {
|
|
1534
|
+
case "queued":
|
|
1535
|
+
return "pending";
|
|
1536
|
+
case "running":
|
|
1537
|
+
return "running";
|
|
1538
|
+
case "success":
|
|
1539
|
+
return "completed";
|
|
1540
|
+
case "failed":
|
|
1541
|
+
return "failed";
|
|
1542
|
+
case "cancelled":
|
|
1543
|
+
return "cancelled";
|
|
1544
|
+
case "skipped":
|
|
1545
|
+
return "cancelled";
|
|
1546
|
+
case "dlq":
|
|
1547
|
+
return "failed";
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
function mapPriority(priority) {
|
|
1551
|
+
if (priority <= 3) {
|
|
1552
|
+
return "low";
|
|
1553
|
+
}
|
|
1554
|
+
if (priority <= 7) {
|
|
1555
|
+
return "normal";
|
|
1556
|
+
}
|
|
1557
|
+
return "high";
|
|
1558
|
+
}
|
|
1559
|
+
|
|
1560
|
+
// src/api/response.ts
|
|
1561
|
+
function ok(data) {
|
|
1562
|
+
return { ok: true, data };
|
|
1563
|
+
}
|
|
1564
|
+
function fail(reply, statusCode, message) {
|
|
1565
|
+
reply.code(statusCode);
|
|
1566
|
+
return { ok: false, error: message };
|
|
1567
|
+
}
|
|
1568
|
+
|
|
1569
|
+
// src/api/jobs-api.ts
|
|
1570
|
+
function registerJobsAPI(options) {
|
|
1571
|
+
const { server, hostService, logger, observability } = options;
|
|
1572
|
+
server.post(
|
|
1573
|
+
"/api/v1/jobs",
|
|
1574
|
+
{ schema: { tags: ["Jobs"], summary: "Submit a new job" } },
|
|
1575
|
+
async (request, reply) => {
|
|
1576
|
+
const tenantId = request.headers["x-tenant-id"] ?? "default";
|
|
1577
|
+
try {
|
|
1578
|
+
const data = await observability.observeOperation("workflow.job.submit", () => hostService.submitJob(tenantId, request.body));
|
|
1579
|
+
return ok(data);
|
|
1580
|
+
} catch (error) {
|
|
1581
|
+
const message = error instanceof Error ? error.message : "Job submission failed";
|
|
1582
|
+
logger.error("Job submission failed", error instanceof Error ? error : void 0, { tenantId });
|
|
1583
|
+
if (message.startsWith("Invalid tenant") || message.startsWith("Missing required") || message.startsWith("Priority")) {
|
|
1584
|
+
return fail(reply, 400, message);
|
|
1585
|
+
} else {
|
|
1586
|
+
return fail(reply, 500, message);
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
);
|
|
1591
|
+
server.get(
|
|
1592
|
+
"/api/v1/jobs/:jobId",
|
|
1593
|
+
{ schema: { tags: ["Jobs"], summary: "Get job status" } },
|
|
1594
|
+
async (request, reply) => {
|
|
1595
|
+
const { jobId } = request.params;
|
|
1596
|
+
const tenantId = request.headers["x-tenant-id"] ?? "default";
|
|
1597
|
+
try {
|
|
1598
|
+
const data = await observability.observeOperation("workflow.job.get", () => hostService.getJob(tenantId, jobId));
|
|
1599
|
+
return ok(data);
|
|
1600
|
+
} catch (error) {
|
|
1601
|
+
const message = error instanceof Error ? error.message : "Failed to get job status";
|
|
1602
|
+
if (message === "Job not found") {
|
|
1603
|
+
return fail(reply, 404, message);
|
|
1604
|
+
}
|
|
1605
|
+
logger.error("Failed to get job status", error instanceof Error ? error : void 0, { jobId });
|
|
1606
|
+
return fail(reply, 500, message);
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
);
|
|
1610
|
+
server.post(
|
|
1611
|
+
"/api/v1/jobs/:jobId/cancel",
|
|
1612
|
+
{ schema: { tags: ["Jobs"], summary: "Cancel a job" } },
|
|
1613
|
+
async (request, reply) => {
|
|
1614
|
+
const { jobId } = request.params;
|
|
1615
|
+
const tenantId = request.headers["x-tenant-id"] ?? "default";
|
|
1616
|
+
try {
|
|
1617
|
+
const data = await observability.observeOperation("workflow.job.cancel", () => hostService.cancelJob(tenantId, jobId));
|
|
1618
|
+
return ok(data);
|
|
1619
|
+
} catch (error) {
|
|
1620
|
+
logger.error("Failed to cancel job", error instanceof Error ? error : void 0);
|
|
1621
|
+
return fail(reply, 500, error instanceof Error ? error.message : "Failed to cancel job");
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
);
|
|
1625
|
+
server.get(
|
|
1626
|
+
"/api/v1/jobs",
|
|
1627
|
+
{ schema: { tags: ["Jobs"], summary: "List jobs" } },
|
|
1628
|
+
async (request, reply) => {
|
|
1629
|
+
const tenantId = request.headers["x-tenant-id"] ?? "default";
|
|
1630
|
+
const filter = request.query;
|
|
1631
|
+
try {
|
|
1632
|
+
const data = await observability.observeOperation("workflow.job.list", () => hostService.listJobs(tenantId, filter));
|
|
1633
|
+
return ok(data);
|
|
1634
|
+
} catch (error) {
|
|
1635
|
+
logger.error("Failed to list jobs", error instanceof Error ? error : void 0);
|
|
1636
|
+
return fail(reply, 500, error instanceof Error ? error.message : "Failed to list jobs");
|
|
1637
|
+
}
|
|
1638
|
+
}
|
|
1639
|
+
);
|
|
1640
|
+
server.get(
|
|
1641
|
+
"/api/v1/jobs/:jobId/steps",
|
|
1642
|
+
{ schema: { tags: ["Jobs"], summary: "Get job steps" } },
|
|
1643
|
+
async (request, reply) => {
|
|
1644
|
+
const { jobId } = request.params;
|
|
1645
|
+
try {
|
|
1646
|
+
const steps = await observability.observeOperation("workflow.job.steps", () => hostService.getJobSteps(jobId));
|
|
1647
|
+
return ok(steps);
|
|
1648
|
+
} catch (error) {
|
|
1649
|
+
const message = error instanceof Error ? error.message : "Failed to get job steps";
|
|
1650
|
+
if (message === "Job not found") {
|
|
1651
|
+
return fail(reply, 404, message);
|
|
1652
|
+
}
|
|
1653
|
+
return fail(reply, 500, message);
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
);
|
|
1657
|
+
server.get(
|
|
1658
|
+
"/api/v1/jobs/:jobId/logs",
|
|
1659
|
+
{ schema: { tags: ["Jobs"], summary: "Get job logs" } },
|
|
1660
|
+
async (request, reply) => {
|
|
1661
|
+
const { jobId } = request.params;
|
|
1662
|
+
const limit = typeof request.query === "object" && request.query && "limit" in request.query ? Number(request.query.limit) : void 0;
|
|
1663
|
+
const offset = typeof request.query === "object" && request.query && "offset" in request.query ? Number(request.query.offset) : void 0;
|
|
1664
|
+
const level = typeof request.query === "object" && request.query && "level" in request.query ? String(request.query.level) : void 0;
|
|
1665
|
+
try {
|
|
1666
|
+
const logs = await observability.observeOperation(
|
|
1667
|
+
"workflow.job.logs",
|
|
1668
|
+
() => hostService.getJobLogs(jobId, { limit, offset, level })
|
|
1669
|
+
);
|
|
1670
|
+
return ok({ logs });
|
|
1671
|
+
} catch (error) {
|
|
1672
|
+
const message = error instanceof Error ? error.message : "Failed to get job logs";
|
|
1673
|
+
if (message === "Job not found") {
|
|
1674
|
+
return fail(reply, 404, message);
|
|
1675
|
+
}
|
|
1676
|
+
return fail(reply, 500, message);
|
|
1677
|
+
}
|
|
1678
|
+
}
|
|
1679
|
+
);
|
|
1680
|
+
}
|
|
1681
|
+
|
|
1682
|
+
// src/api/cron-api.ts
|
|
1683
|
+
function registerCronAPI(options) {
|
|
1684
|
+
const { server, hostService, logger, observability } = options;
|
|
1685
|
+
const registerCronHandler = async (request, reply) => {
|
|
1686
|
+
const tenantId = request.headers["x-tenant-id"] ?? "default";
|
|
1687
|
+
try {
|
|
1688
|
+
const data = await observability.observeOperation(
|
|
1689
|
+
"workflow.cron.register",
|
|
1690
|
+
() => Promise.resolve(hostService.registerCron(tenantId, request.body))
|
|
1691
|
+
);
|
|
1692
|
+
return ok(data);
|
|
1693
|
+
} catch (error) {
|
|
1694
|
+
const message = error instanceof Error ? error.message : "Failed to register cron job";
|
|
1695
|
+
if (message === "Cron scheduler not available") {
|
|
1696
|
+
return fail(reply, 503, message);
|
|
1697
|
+
}
|
|
1698
|
+
if (message.startsWith("Missing required fields")) {
|
|
1699
|
+
return fail(reply, 400, message);
|
|
1700
|
+
}
|
|
1701
|
+
logger.error("Failed to register cron job", error instanceof Error ? error : void 0);
|
|
1702
|
+
return fail(reply, 500, message);
|
|
1703
|
+
}
|
|
1704
|
+
};
|
|
1705
|
+
const listCronHandler = async (_request, reply) => {
|
|
1706
|
+
try {
|
|
1707
|
+
return ok(await observability.observeOperation("workflow.cron.list", () => Promise.resolve(hostService.listCron())));
|
|
1708
|
+
} catch (error) {
|
|
1709
|
+
const message = error instanceof Error ? error.message : "Failed to list cron jobs";
|
|
1710
|
+
if (message === "Cron scheduler not available") {
|
|
1711
|
+
return fail(reply, 503, message);
|
|
1712
|
+
}
|
|
1713
|
+
logger.error("Failed to list cron jobs", error instanceof Error ? error : void 0);
|
|
1714
|
+
return fail(reply, 500, message);
|
|
1715
|
+
}
|
|
1716
|
+
};
|
|
1717
|
+
const unregisterCronHandler = async (request, reply) => {
|
|
1718
|
+
const { id } = request.params;
|
|
1719
|
+
const tenantId = request.headers["x-tenant-id"] ?? "default";
|
|
1720
|
+
try {
|
|
1721
|
+
const data = await observability.observeOperation(
|
|
1722
|
+
"workflow.cron.unregister",
|
|
1723
|
+
() => Promise.resolve(hostService.unregisterCron(tenantId, id))
|
|
1724
|
+
);
|
|
1725
|
+
return ok(data);
|
|
1726
|
+
} catch (error) {
|
|
1727
|
+
const message = error instanceof Error ? error.message : "Failed to unregister cron job";
|
|
1728
|
+
if (message === "Cron scheduler not available") {
|
|
1729
|
+
return fail(reply, 503, message);
|
|
1730
|
+
}
|
|
1731
|
+
logger.error("Failed to unregister cron job", error instanceof Error ? error : void 0);
|
|
1732
|
+
return fail(reply, 500, message);
|
|
1733
|
+
}
|
|
1734
|
+
};
|
|
1735
|
+
const triggerCronHandler = async (request, reply) => {
|
|
1736
|
+
const { id } = request.params;
|
|
1737
|
+
const tenantId = request.headers["x-tenant-id"] ?? "default";
|
|
1738
|
+
try {
|
|
1739
|
+
const data = await observability.observeOperation("workflow.cron.trigger", () => hostService.triggerCron(tenantId, id));
|
|
1740
|
+
return ok(data);
|
|
1741
|
+
} catch (error) {
|
|
1742
|
+
const message = error instanceof Error ? error.message : "Failed to trigger cron job";
|
|
1743
|
+
if (message === "Cron scheduler not available") {
|
|
1744
|
+
return fail(reply, 503, message);
|
|
1745
|
+
}
|
|
1746
|
+
logger.error("Failed to trigger cron job", error instanceof Error ? error : void 0);
|
|
1747
|
+
return fail(reply, 500, message);
|
|
1748
|
+
}
|
|
1749
|
+
};
|
|
1750
|
+
const pauseCronHandler = async (request, reply) => {
|
|
1751
|
+
const { id } = request.params;
|
|
1752
|
+
const tenantId = request.headers["x-tenant-id"] ?? "default";
|
|
1753
|
+
try {
|
|
1754
|
+
const data = await observability.observeOperation(
|
|
1755
|
+
"workflow.cron.pause",
|
|
1756
|
+
() => Promise.resolve(hostService.pauseCron(tenantId, id))
|
|
1757
|
+
);
|
|
1758
|
+
return ok(data);
|
|
1759
|
+
} catch (error) {
|
|
1760
|
+
const message = error instanceof Error ? error.message : "Failed to pause cron job";
|
|
1761
|
+
if (message === "Cron scheduler not available") {
|
|
1762
|
+
return fail(reply, 503, message);
|
|
1763
|
+
}
|
|
1764
|
+
logger.error("Failed to pause cron job", error instanceof Error ? error : void 0);
|
|
1765
|
+
return fail(reply, 500, message);
|
|
1766
|
+
}
|
|
1767
|
+
};
|
|
1768
|
+
const resumeCronHandler = async (request, reply) => {
|
|
1769
|
+
const { id } = request.params;
|
|
1770
|
+
const tenantId = request.headers["x-tenant-id"] ?? "default";
|
|
1771
|
+
try {
|
|
1772
|
+
const data = await observability.observeOperation(
|
|
1773
|
+
"workflow.cron.resume",
|
|
1774
|
+
() => Promise.resolve(hostService.resumeCron(tenantId, id))
|
|
1775
|
+
);
|
|
1776
|
+
return ok(data);
|
|
1777
|
+
} catch (error) {
|
|
1778
|
+
const message = error instanceof Error ? error.message : "Failed to resume cron job";
|
|
1779
|
+
if (message === "Cron scheduler not available") {
|
|
1780
|
+
return fail(reply, 503, message);
|
|
1781
|
+
}
|
|
1782
|
+
logger.error("Failed to resume cron job", error instanceof Error ? error : void 0);
|
|
1783
|
+
return fail(reply, 500, message);
|
|
1784
|
+
}
|
|
1785
|
+
};
|
|
1786
|
+
server.post("/api/v1/crons", { schema: { tags: ["Cron"], summary: "Register a cron job" } }, registerCronHandler);
|
|
1787
|
+
server.get("/api/v1/crons", { schema: { tags: ["Cron"], summary: "List cron jobs" } }, listCronHandler);
|
|
1788
|
+
server.delete("/api/v1/crons/:id", { schema: { tags: ["Cron"], summary: "Unregister a cron job" } }, unregisterCronHandler);
|
|
1789
|
+
server.post("/api/v1/crons/:id/trigger", { schema: { tags: ["Cron"], summary: "Trigger a cron job immediately" } }, triggerCronHandler);
|
|
1790
|
+
server.post("/api/v1/crons/:id/pause", { schema: { tags: ["Cron"], summary: "Pause a cron job" } }, pauseCronHandler);
|
|
1791
|
+
server.post("/api/v1/crons/:id/resume", { schema: { tags: ["Cron"], summary: "Resume a cron job" } }, resumeCronHandler);
|
|
1792
|
+
}
|
|
1793
|
+
|
|
1794
|
+
// src/api/workflows-api.ts
|
|
1795
|
+
var TERMINAL_EVENTS = ["run.finished", "run.failed", "run.cancelled"];
|
|
1796
|
+
var TERMINAL_STATUSES = ["success", "failed", "cancelled", "skipped"];
|
|
1797
|
+
var KEEP_ALIVE_MS = 3e4;
|
|
1798
|
+
var IDLE_TIMEOUT_MS = 6e4;
|
|
1799
|
+
function registerWorkflowsAPI(options) {
|
|
1800
|
+
const { server, hostService, engine, workflowService, logger, observability } = options;
|
|
1801
|
+
server.post("/api/v1/workflows/refresh", { schema: { tags: ["Workflows"], summary: "Reload workflow definitions from disk" } }, async () => {
|
|
1802
|
+
try {
|
|
1803
|
+
logger.info("[workflows-api] Refreshing workflows from disk");
|
|
1804
|
+
if (workflowService) {
|
|
1805
|
+
await observability.observeOperation("workflow.catalog.refresh", () => workflowService.refreshManifests());
|
|
1806
|
+
}
|
|
1807
|
+
const workflows = workflowService ? await observability.observeOperation("workflow.catalog.list", () => workflowService.listAll()) : [];
|
|
1808
|
+
return ok({
|
|
1809
|
+
workflowsLoaded: workflows.length,
|
|
1810
|
+
workflowIds: workflows.map((w) => w.id)
|
|
1811
|
+
});
|
|
1812
|
+
} catch (error) {
|
|
1813
|
+
logger.error("[workflows-api] Failed to refresh workflows", error instanceof Error ? error : void 0);
|
|
1814
|
+
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
1815
|
+
}
|
|
1816
|
+
});
|
|
1817
|
+
server.get("/api/v1/workflows", { schema: { tags: ["Workflows"], summary: "List workflow definitions" } }, async (request, reply) => {
|
|
1818
|
+
try {
|
|
1819
|
+
const response = await observability.observeOperation("workflow.catalog.list", () => hostService.listWorkflows(request.query));
|
|
1820
|
+
return ok(response);
|
|
1821
|
+
} catch (error) {
|
|
1822
|
+
logger.error("[workflows-api] Error listing workflows", error instanceof Error ? error : void 0);
|
|
1823
|
+
return fail(reply, 500, error instanceof Error ? error.message : "Failed to list workflows");
|
|
1824
|
+
}
|
|
1825
|
+
});
|
|
1826
|
+
server.get("/api/v1/workflows/:id", { schema: { tags: ["Workflows"], summary: "Get workflow definition" } }, async (request, reply) => {
|
|
1827
|
+
try {
|
|
1828
|
+
const { id } = request.params;
|
|
1829
|
+
const workflow = await observability.observeOperation("workflow.catalog.get", () => hostService.getWorkflow(id));
|
|
1830
|
+
if (!workflow) {
|
|
1831
|
+
return fail(reply, 404, "Workflow not found");
|
|
1832
|
+
}
|
|
1833
|
+
return ok(workflow);
|
|
1834
|
+
} catch (error) {
|
|
1835
|
+
logger.error("[workflows-api] Error getting workflow", error instanceof Error ? error : void 0);
|
|
1836
|
+
return fail(reply, 500, error instanceof Error ? error.message : "Failed to get workflow");
|
|
1837
|
+
}
|
|
1838
|
+
});
|
|
1839
|
+
server.get("/api/v1/workflows/:id/runs", { schema: { tags: ["Workflows"], summary: "List runs for a workflow" } }, async (request, reply) => {
|
|
1840
|
+
try {
|
|
1841
|
+
const { id } = request.params;
|
|
1842
|
+
const { limit, offset, status } = request.query;
|
|
1843
|
+
const response = await observability.observeOperation(
|
|
1844
|
+
"workflow.run.list",
|
|
1845
|
+
() => hostService.listWorkflowRuns(id, {
|
|
1846
|
+
limit: limit ? parseInt(limit, 10) : 50,
|
|
1847
|
+
offset: offset ? parseInt(offset, 10) : 0,
|
|
1848
|
+
status
|
|
1849
|
+
})
|
|
1850
|
+
);
|
|
1851
|
+
return ok(response);
|
|
1852
|
+
} catch (error) {
|
|
1853
|
+
logger.error("[workflows-api] Error listing workflow runs", error instanceof Error ? error : void 0);
|
|
1854
|
+
return fail(reply, 500, error instanceof Error ? error.message : "Failed to list workflow runs");
|
|
1855
|
+
}
|
|
1856
|
+
});
|
|
1857
|
+
server.post("/api/v1/workflows/:id/runs", { schema: { tags: ["Workflows"], summary: "Run a workflow" } }, async (request, reply) => {
|
|
1858
|
+
try {
|
|
1859
|
+
const { id } = request.params;
|
|
1860
|
+
const response = await observability.observeOperation("workflow.run.start", () => hostService.runWorkflow(id, request.body || {}));
|
|
1861
|
+
return ok(response);
|
|
1862
|
+
} catch (error) {
|
|
1863
|
+
const message = error instanceof Error ? error.message : "Failed to run workflow";
|
|
1864
|
+
if (message === "Workflow not found") {
|
|
1865
|
+
return fail(reply, 404, message);
|
|
1866
|
+
}
|
|
1867
|
+
logger.error("[workflows-api] Error running workflow", error instanceof Error ? error : void 0);
|
|
1868
|
+
return fail(reply, 500, message);
|
|
1869
|
+
}
|
|
1870
|
+
});
|
|
1871
|
+
server.post("/api/v1/runs/:runId/cancel", { schema: { tags: ["Runs"], summary: "Cancel a workflow run" } }, async (request, reply) => {
|
|
1872
|
+
try {
|
|
1873
|
+
const { runId } = request.params;
|
|
1874
|
+
await observability.observeOperation("workflow.run.cancel", () => hostService.cancelRun(runId));
|
|
1875
|
+
return ok({ cancelled: true, runId });
|
|
1876
|
+
} catch (error) {
|
|
1877
|
+
const message = error instanceof Error ? error.message : "Failed to cancel run";
|
|
1878
|
+
if (message === "Run not found") {
|
|
1879
|
+
return fail(reply, 404, message);
|
|
1880
|
+
}
|
|
1881
|
+
if (message.startsWith("Cannot cancel run")) {
|
|
1882
|
+
return fail(reply, 409, message);
|
|
1883
|
+
}
|
|
1884
|
+
logger.error("[workflows-api] Error cancelling run", error instanceof Error ? error : void 0);
|
|
1885
|
+
return fail(reply, 500, message);
|
|
1886
|
+
}
|
|
1887
|
+
});
|
|
1888
|
+
server.get("/api/v1/runs", { schema: { tags: ["Runs"], summary: "List all workflow runs" } }, async (request, reply) => {
|
|
1889
|
+
try {
|
|
1890
|
+
const { status, limit, offset } = request.query;
|
|
1891
|
+
const response = await observability.observeOperation(
|
|
1892
|
+
"workflow.run.list",
|
|
1893
|
+
() => hostService.listRuns({
|
|
1894
|
+
status,
|
|
1895
|
+
limit: limit ? parseInt(limit, 10) : 50,
|
|
1896
|
+
offset: offset ? parseInt(offset, 10) : 0
|
|
1897
|
+
})
|
|
1898
|
+
);
|
|
1899
|
+
return ok(response);
|
|
1900
|
+
} catch (error) {
|
|
1901
|
+
logger.error("[workflows-api] Error listing runs", error instanceof Error ? error : void 0);
|
|
1902
|
+
return fail(reply, 500, error instanceof Error ? error.message : "Failed to list runs");
|
|
1903
|
+
}
|
|
1904
|
+
});
|
|
1905
|
+
server.get("/api/v1/runs/:runId", { schema: { tags: ["Runs"], summary: "Get a workflow run" } }, async (request, reply) => {
|
|
1906
|
+
try {
|
|
1907
|
+
const { runId } = request.params;
|
|
1908
|
+
const run = await observability.observeOperation("workflow.run.get", () => hostService.getRun(runId));
|
|
1909
|
+
if (!run) {
|
|
1910
|
+
return fail(reply, 404, "Run not found");
|
|
1911
|
+
}
|
|
1912
|
+
return ok({ run });
|
|
1913
|
+
} catch (error) {
|
|
1914
|
+
logger.error("[workflows-api] Error getting run", error instanceof Error ? error : void 0);
|
|
1915
|
+
return fail(reply, 500, error instanceof Error ? error.message : "Failed to get run");
|
|
1916
|
+
}
|
|
1917
|
+
});
|
|
1918
|
+
server.get("/api/v1/runs/:runId/events", { schema: { hide: true } }, async (request, reply) => {
|
|
1919
|
+
const { runId } = request.params;
|
|
1920
|
+
const run = await observability.observeOperation("workflow.run.events", () => engine.getRun(runId));
|
|
1921
|
+
if (!run) {
|
|
1922
|
+
return fail(reply, 404, "Run not found");
|
|
1923
|
+
}
|
|
1924
|
+
reply.hijack();
|
|
1925
|
+
const raw = reply.raw;
|
|
1926
|
+
const origin = request.headers.origin;
|
|
1927
|
+
if (typeof origin === "string" && origin.startsWith("http://localhost")) {
|
|
1928
|
+
raw.setHeader("Access-Control-Allow-Origin", origin);
|
|
1929
|
+
raw.setHeader("Access-Control-Allow-Credentials", "true");
|
|
1930
|
+
}
|
|
1931
|
+
raw.setHeader("Content-Type", "text/event-stream");
|
|
1932
|
+
raw.setHeader("Cache-Control", "no-cache, no-transform");
|
|
1933
|
+
raw.setHeader("Connection", "keep-alive");
|
|
1934
|
+
raw.flushHeaders?.();
|
|
1935
|
+
raw.write(": connected\n\n");
|
|
1936
|
+
const sendEvent = (type, payload) => {
|
|
1937
|
+
if (raw.writableEnded) return;
|
|
1938
|
+
raw.write(`event: workflow.event
|
|
1939
|
+
`);
|
|
1940
|
+
raw.write(`data: ${JSON.stringify({ type, runId, payload, timestamp: (/* @__PURE__ */ new Date()).toISOString() })}
|
|
1941
|
+
|
|
1942
|
+
`);
|
|
1943
|
+
};
|
|
1944
|
+
sendEvent("run.snapshot", run);
|
|
1945
|
+
if (TERMINAL_STATUSES.includes(run.status)) {
|
|
1946
|
+
raw.end();
|
|
1947
|
+
return;
|
|
1948
|
+
}
|
|
1949
|
+
let idleTimer = null;
|
|
1950
|
+
const resetIdle = () => {
|
|
1951
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
1952
|
+
idleTimer = setTimeout(() => {
|
|
1953
|
+
cleanup();
|
|
1954
|
+
}, IDLE_TIMEOUT_MS);
|
|
1955
|
+
};
|
|
1956
|
+
const keepAliveTimer = setInterval(() => {
|
|
1957
|
+
if (raw.writableEnded) return;
|
|
1958
|
+
raw.write(": keep-alive\n\n");
|
|
1959
|
+
}, KEEP_ALIVE_MS);
|
|
1960
|
+
const unsubscribe = engine.subscribeToRunEvents(runId, (event) => {
|
|
1961
|
+
sendEvent(event.type, event.payload);
|
|
1962
|
+
resetIdle();
|
|
1963
|
+
if (TERMINAL_EVENTS.includes(event.type)) {
|
|
1964
|
+
cleanup();
|
|
1965
|
+
}
|
|
1966
|
+
});
|
|
1967
|
+
const cleanup = () => {
|
|
1968
|
+
unsubscribe();
|
|
1969
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
1970
|
+
clearInterval(keepAliveTimer);
|
|
1971
|
+
if (!raw.writableEnded) raw.end();
|
|
1972
|
+
};
|
|
1973
|
+
resetIdle();
|
|
1974
|
+
request.raw.on("close", cleanup);
|
|
1975
|
+
});
|
|
1976
|
+
logger.info("[workflows-api] Workflows API endpoints registered");
|
|
1977
|
+
}
|
|
1978
|
+
|
|
1979
|
+
// src/api/approvals-api.ts
|
|
1980
|
+
function registerApprovalsAPI(options) {
|
|
1981
|
+
const { server, engine, logger, observability } = options;
|
|
1982
|
+
server.get("/api/v1/runs/:runId/approvals", { schema: { tags: ["Approvals"], summary: "List pending approvals for a run" } }, async (request, reply) => {
|
|
1983
|
+
try {
|
|
1984
|
+
const { runId } = request.params;
|
|
1985
|
+
const run = await observability.observeOperation("workflow.approval.list", () => engine.getRun(runId));
|
|
1986
|
+
if (!run) {
|
|
1987
|
+
return fail(reply, 404, "Run not found");
|
|
1988
|
+
}
|
|
1989
|
+
const pending = [];
|
|
1990
|
+
for (const job of run.jobs) {
|
|
1991
|
+
for (const step of job.steps) {
|
|
1992
|
+
if (step.status === "waiting_approval") {
|
|
1993
|
+
pending.push({
|
|
1994
|
+
jobId: job.id,
|
|
1995
|
+
stepId: step.id,
|
|
1996
|
+
stepName: step.name,
|
|
1997
|
+
specId: step.spec.id,
|
|
1998
|
+
context: step.spec.with ?? {},
|
|
1999
|
+
waitingSince: step.startedAt
|
|
2000
|
+
});
|
|
2001
|
+
}
|
|
2002
|
+
}
|
|
2003
|
+
}
|
|
2004
|
+
return ok({ runId, pending });
|
|
2005
|
+
} catch (error) {
|
|
2006
|
+
logger.error("[approvals-api] Error listing pending approvals", error instanceof Error ? error : void 0);
|
|
2007
|
+
return fail(reply, 500, error instanceof Error ? error.message : "Failed to list pending approvals");
|
|
2008
|
+
}
|
|
2009
|
+
});
|
|
2010
|
+
server.post("/api/v1/runs/:runId/approvals/resolve", { schema: { tags: ["Approvals"], summary: "Approve or reject a pending step" } }, async (request, reply) => {
|
|
2011
|
+
try {
|
|
2012
|
+
const { runId } = request.params;
|
|
2013
|
+
const { jobId, stepId, action, comment, data } = request.body;
|
|
2014
|
+
if (!jobId || !stepId || !action) {
|
|
2015
|
+
return fail(reply, 400, "Missing required fields: jobId, stepId, action");
|
|
2016
|
+
}
|
|
2017
|
+
if (action !== "approve" && action !== "reject") {
|
|
2018
|
+
return fail(reply, 400, 'action must be "approve" or "reject"');
|
|
2019
|
+
}
|
|
2020
|
+
const run = await observability.observeOperation("workflow.approval.get", () => engine.getRun(runId));
|
|
2021
|
+
if (!run) {
|
|
2022
|
+
return fail(reply, 404, "Run not found");
|
|
2023
|
+
}
|
|
2024
|
+
const job = run.jobs.find((j) => j.id === jobId);
|
|
2025
|
+
if (!job) {
|
|
2026
|
+
return fail(reply, 404, "Job not found");
|
|
2027
|
+
}
|
|
2028
|
+
const step = job.steps.find((s) => s.id === stepId);
|
|
2029
|
+
if (!step) {
|
|
2030
|
+
return fail(reply, 404, "Step not found");
|
|
2031
|
+
}
|
|
2032
|
+
if (step.status !== "waiting_approval") {
|
|
2033
|
+
return fail(reply, 409, `Step is not waiting for approval (current status: ${step.status})`);
|
|
2034
|
+
}
|
|
2035
|
+
await observability.observeOperation(
|
|
2036
|
+
"workflow.approval.resolve",
|
|
2037
|
+
() => engine.resolveApproval(runId, jobId, stepId, action, data, comment)
|
|
2038
|
+
);
|
|
2039
|
+
logger.info("[approvals-api] Approval resolved", {
|
|
2040
|
+
runId,
|
|
2041
|
+
jobId,
|
|
2042
|
+
stepId,
|
|
2043
|
+
action,
|
|
2044
|
+
comment
|
|
2045
|
+
});
|
|
2046
|
+
return ok({
|
|
2047
|
+
runId,
|
|
2048
|
+
jobId,
|
|
2049
|
+
stepId,
|
|
2050
|
+
action,
|
|
2051
|
+
resolved: true
|
|
2052
|
+
});
|
|
2053
|
+
} catch (error) {
|
|
2054
|
+
logger.error("[approvals-api] Error resolving approval", error instanceof Error ? error : void 0);
|
|
2055
|
+
return fail(reply, 500, error instanceof Error ? error.message : "Failed to resolve approval");
|
|
2056
|
+
}
|
|
2057
|
+
});
|
|
2058
|
+
}
|
|
2059
|
+
|
|
2060
|
+
// src/api/stats-api.ts
|
|
2061
|
+
function registerStatsAPI(options) {
|
|
2062
|
+
const { server, hostService, cronScheduler } = options;
|
|
2063
|
+
server.get("/api/v1/stats", { schema: { tags: ["Stats"], summary: "Get dashboard statistics" } }, async (_request, _reply) => {
|
|
2064
|
+
const tenantId = "default";
|
|
2065
|
+
const { jobs } = await hostService.listJobs(tenantId, {});
|
|
2066
|
+
const running = jobs.filter((j) => j.status === "running").length;
|
|
2067
|
+
const pending = jobs.filter((j) => j.status === "pending").length;
|
|
2068
|
+
const completed = jobs.filter((j) => j.status === "completed").length;
|
|
2069
|
+
const failed = jobs.filter((j) => j.status === "failed").length;
|
|
2070
|
+
const activeRaw = await hostService.listActiveExecutions();
|
|
2071
|
+
const now = Date.now();
|
|
2072
|
+
const activeExecutions = activeRaw.map((e) => {
|
|
2073
|
+
const startedAt = e["startedAt"];
|
|
2074
|
+
const startedAtStr = startedAt ? new Date(startedAt).toISOString() : (/* @__PURE__ */ new Date()).toISOString();
|
|
2075
|
+
const durationMs = startedAt ? now - new Date(startedAt).getTime() : void 0;
|
|
2076
|
+
return {
|
|
2077
|
+
id: String(e["id"] ?? ""),
|
|
2078
|
+
type: String(e["type"] ?? ""),
|
|
2079
|
+
status: "running",
|
|
2080
|
+
startedAt: startedAtStr,
|
|
2081
|
+
durationMs
|
|
2082
|
+
};
|
|
2083
|
+
});
|
|
2084
|
+
const finished = jobs.filter((j) => j.status === "completed" || j.status === "failed" || j.status === "cancelled").filter((j) => j.finishedAt != null).sort((a, b) => new Date(b.finishedAt).getTime() - new Date(a.finishedAt).getTime()).slice(0, 10);
|
|
2085
|
+
const recentActivity = finished.map((j) => {
|
|
2086
|
+
const finishedAt = new Date(j.finishedAt).toISOString();
|
|
2087
|
+
const durationMs = j.startedAt && j.finishedAt ? new Date(j.finishedAt).getTime() - new Date(j.startedAt).getTime() : void 0;
|
|
2088
|
+
return {
|
|
2089
|
+
id: j.id,
|
|
2090
|
+
type: j.type,
|
|
2091
|
+
status: j.status,
|
|
2092
|
+
finishedAt,
|
|
2093
|
+
durationMs,
|
|
2094
|
+
error: j.error
|
|
2095
|
+
};
|
|
2096
|
+
});
|
|
2097
|
+
let workflowStats = { total: 0, active: 0, inactive: 0 };
|
|
2098
|
+
try {
|
|
2099
|
+
const { workflows } = await hostService.listWorkflows({});
|
|
2100
|
+
const active = workflows.filter((w) => w.status !== "inactive").length;
|
|
2101
|
+
workflowStats = { total: workflows.length, active, inactive: workflows.length - active };
|
|
2102
|
+
} catch {
|
|
2103
|
+
}
|
|
2104
|
+
let cronStats = { total: 0, enabled: 0, disabled: 0 };
|
|
2105
|
+
if (cronScheduler) {
|
|
2106
|
+
try {
|
|
2107
|
+
const { crons } = hostService.listCron();
|
|
2108
|
+
const enabled = crons.filter((c) => c.enabled).length;
|
|
2109
|
+
cronStats = { total: crons.length, enabled, disabled: crons.length - enabled };
|
|
2110
|
+
} catch {
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
const stats = {
|
|
2114
|
+
workflows: workflowStats,
|
|
2115
|
+
jobs: { running, pending, completed, failed },
|
|
2116
|
+
crons: cronStats,
|
|
2117
|
+
activeExecutions,
|
|
2118
|
+
recentActivity
|
|
2119
|
+
};
|
|
2120
|
+
return ok(stats);
|
|
2121
|
+
});
|
|
2122
|
+
}
|
|
2123
|
+
|
|
2124
|
+
// src/server.ts
|
|
2125
|
+
async function createServer(options) {
|
|
2126
|
+
const { engine, jobBroker, workflowService, cronScheduler, cronDiscovery, logger } = options;
|
|
2127
|
+
const hostService = new WorkflowHostService({
|
|
2128
|
+
engine,
|
|
2129
|
+
jobBroker,
|
|
2130
|
+
workflowService,
|
|
2131
|
+
cronScheduler,
|
|
2132
|
+
logger
|
|
2133
|
+
});
|
|
2134
|
+
const server = Fastify({
|
|
2135
|
+
logger: false,
|
|
2136
|
+
// Use platform logger instead
|
|
2137
|
+
bodyLimit: 1048576
|
|
2138
|
+
// 1MB body limit (prevents parsing huge payloads)
|
|
2139
|
+
});
|
|
2140
|
+
const isProduction = process.env.NODE_ENV === "production";
|
|
2141
|
+
await registerOpenAPI(server, {
|
|
2142
|
+
title: "KB Labs Workflow Daemon",
|
|
2143
|
+
description: "Background job execution and workflow orchestration API",
|
|
2144
|
+
version: "1.0.0",
|
|
2145
|
+
servers: [{ url: "http://localhost:7778", description: "Local dev" }],
|
|
2146
|
+
ui: !isProduction
|
|
2147
|
+
});
|
|
2148
|
+
const requireAuth = process.env.KB_DAEMON_REQUIRE_AUTH === "true" || isProduction;
|
|
2149
|
+
const daemonApiKey = process.env.KB_DAEMON_API_KEY;
|
|
2150
|
+
const observability = new HttpObservabilityCollector({
|
|
2151
|
+
serviceId: "workflow",
|
|
2152
|
+
serviceType: "workflow-daemon",
|
|
2153
|
+
version: "1.0.0",
|
|
2154
|
+
logsSource: "workflow",
|
|
2155
|
+
dependencies: [
|
|
2156
|
+
{
|
|
2157
|
+
serviceId: "state-daemon",
|
|
2158
|
+
required: false,
|
|
2159
|
+
description: "Workflow run and job state storage"
|
|
2160
|
+
}
|
|
2161
|
+
]
|
|
2162
|
+
});
|
|
2163
|
+
if (requireAuth && !daemonApiKey) {
|
|
2164
|
+
throw new Error(
|
|
2165
|
+
"KB_DAEMON_API_KEY is required when daemon auth is enabled (KB_DAEMON_REQUIRE_AUTH=true or NODE_ENV=production)"
|
|
2166
|
+
);
|
|
2167
|
+
}
|
|
2168
|
+
server.addHook("onRequest", async (request, reply) => {
|
|
2169
|
+
if (!requireAuth) {
|
|
2170
|
+
return;
|
|
2171
|
+
}
|
|
2172
|
+
if (request.url === "/health") {
|
|
2173
|
+
return;
|
|
2174
|
+
}
|
|
2175
|
+
const apiKeyHeader = request.headers["x-api-key"];
|
|
2176
|
+
const authHeader = request.headers.authorization;
|
|
2177
|
+
const bearerToken = typeof authHeader === "string" && authHeader.startsWith("Bearer ") ? authHeader.slice("Bearer ".length).trim() : void 0;
|
|
2178
|
+
const token = (typeof apiKeyHeader === "string" ? apiKeyHeader : void 0) ?? bearerToken;
|
|
2179
|
+
if (!token || token !== daemonApiKey) {
|
|
2180
|
+
reply.code(401).send({ ok: false, error: "Unauthorized" });
|
|
2181
|
+
}
|
|
2182
|
+
});
|
|
2183
|
+
observability.register(server);
|
|
2184
|
+
const allowedOrigins = process.env.ALLOWED_ORIGINS?.split(",") || [
|
|
2185
|
+
"http://localhost:3000",
|
|
2186
|
+
"http://localhost:5173"
|
|
2187
|
+
// Vite dev server
|
|
2188
|
+
];
|
|
2189
|
+
await server.register(cors, {
|
|
2190
|
+
origin: (origin, callback) => {
|
|
2191
|
+
if (!origin) {
|
|
2192
|
+
const isDevelopment = process.env.NODE_ENV !== "production";
|
|
2193
|
+
if (isDevelopment) {
|
|
2194
|
+
callback(null, true);
|
|
2195
|
+
return;
|
|
2196
|
+
}
|
|
2197
|
+
callback(new Error("Origin header required in production"), false);
|
|
2198
|
+
return;
|
|
2199
|
+
}
|
|
2200
|
+
if (allowedOrigins.includes(origin)) {
|
|
2201
|
+
callback(null, true);
|
|
2202
|
+
} else {
|
|
2203
|
+
callback(new Error(`Origin ${origin} not allowed by CORS`), false);
|
|
2204
|
+
}
|
|
2205
|
+
}
|
|
2206
|
+
});
|
|
2207
|
+
registerJobsAPI({
|
|
2208
|
+
server,
|
|
2209
|
+
hostService,
|
|
2210
|
+
logger,
|
|
2211
|
+
observability
|
|
2212
|
+
});
|
|
2213
|
+
registerCronAPI({
|
|
2214
|
+
server,
|
|
2215
|
+
hostService,
|
|
2216
|
+
logger,
|
|
2217
|
+
observability
|
|
2218
|
+
});
|
|
2219
|
+
if (workflowService) {
|
|
2220
|
+
registerWorkflowsAPI({
|
|
2221
|
+
server,
|
|
2222
|
+
hostService,
|
|
2223
|
+
engine,
|
|
2224
|
+
workflowService,
|
|
2225
|
+
logger,
|
|
2226
|
+
observability
|
|
2227
|
+
});
|
|
2228
|
+
}
|
|
2229
|
+
registerApprovalsAPI({
|
|
2230
|
+
server,
|
|
2231
|
+
engine,
|
|
2232
|
+
logger,
|
|
2233
|
+
observability
|
|
2234
|
+
});
|
|
2235
|
+
registerStatsAPI({
|
|
2236
|
+
server,
|
|
2237
|
+
hostService,
|
|
2238
|
+
cronScheduler});
|
|
2239
|
+
server.get("/health", async () => {
|
|
2240
|
+
const metrics = await hostService.getMetrics();
|
|
2241
|
+
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2242
|
+
return {
|
|
2243
|
+
status: checks.some((entry) => entry.status === "error") ? "degraded" : "ok",
|
|
2244
|
+
service: "workflow",
|
|
2245
|
+
ts: Date.now()
|
|
2246
|
+
};
|
|
2247
|
+
});
|
|
2248
|
+
server.get("/ready", async () => {
|
|
2249
|
+
const metrics = await hostService.getMetrics();
|
|
2250
|
+
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2251
|
+
const hasErrors = checks.some((entry) => entry.status === "error");
|
|
2252
|
+
const hasWarnings = checks.some((entry) => entry.status === "warn");
|
|
2253
|
+
return createServiceReadyResponse({
|
|
2254
|
+
ready: !hasErrors,
|
|
2255
|
+
status: hasErrors ? "initializing" : hasWarnings ? "degraded" : "ready",
|
|
2256
|
+
reason: hasErrors ? "workflow_checks_failed" : "ready",
|
|
2257
|
+
components: {
|
|
2258
|
+
workflowEngine: {
|
|
2259
|
+
ready: true
|
|
2260
|
+
},
|
|
2261
|
+
workflowCatalog: {
|
|
2262
|
+
ready: Boolean(workflowService)
|
|
2263
|
+
},
|
|
2264
|
+
cronScheduler: {
|
|
2265
|
+
ready: Boolean(cronScheduler)
|
|
2266
|
+
}
|
|
2267
|
+
}
|
|
2268
|
+
});
|
|
2269
|
+
});
|
|
2270
|
+
server.get("/metrics", async (_request, reply) => {
|
|
2271
|
+
const metrics = await hostService.getMetrics();
|
|
2272
|
+
const healthStatus = resolveWorkflowHealthStatus();
|
|
2273
|
+
reply.header("Content-Type", "text/plain; version=0.0.4; charset=utf-8");
|
|
2274
|
+
return observability.renderPrometheusMetrics(
|
|
2275
|
+
healthStatus,
|
|
2276
|
+
buildWorkflowMetricLines(metrics)
|
|
2277
|
+
);
|
|
2278
|
+
});
|
|
2279
|
+
server.get("/observability/describe", async () => observability.buildDescribe());
|
|
2280
|
+
server.get("/observability/health", async () => {
|
|
2281
|
+
const metrics = await hostService.getMetrics();
|
|
2282
|
+
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2283
|
+
return observability.buildHealth({
|
|
2284
|
+
status: resolveWorkflowHealthStatus(),
|
|
2285
|
+
checks,
|
|
2286
|
+
topOperations: buildWorkflowTopOperations(metrics, observability.getTopOperations(3)),
|
|
2287
|
+
meta: {
|
|
2288
|
+
workflowServiceEnabled: Boolean(workflowService),
|
|
2289
|
+
cronSchedulerEnabled: Boolean(cronScheduler),
|
|
2290
|
+
cronDiscoveryEnabled: Boolean(cronDiscovery),
|
|
2291
|
+
runs: metrics.runs,
|
|
2292
|
+
jobs: metrics.jobs
|
|
2293
|
+
}
|
|
2294
|
+
});
|
|
2295
|
+
});
|
|
2296
|
+
return server;
|
|
2297
|
+
}
|
|
2298
|
+
function resolveWorkflowHealthStatus(metrics) {
|
|
2299
|
+
return "healthy";
|
|
2300
|
+
}
|
|
2301
|
+
function buildWorkflowChecks(input) {
|
|
2302
|
+
return [
|
|
2303
|
+
{
|
|
2304
|
+
id: "workflow-engine",
|
|
2305
|
+
status: "ok",
|
|
2306
|
+
message: `${input.metrics.runs.total} runs tracked`
|
|
2307
|
+
},
|
|
2308
|
+
{
|
|
2309
|
+
id: "workflow-catalog",
|
|
2310
|
+
status: input.workflowService ? "ok" : "warn",
|
|
2311
|
+
message: input.workflowService ? "Workflow service available" : "Workflow service not configured"
|
|
2312
|
+
},
|
|
2313
|
+
{
|
|
2314
|
+
id: "cron-scheduler",
|
|
2315
|
+
status: input.cronScheduler ? "ok" : "warn",
|
|
2316
|
+
message: input.cronScheduler ? "Cron scheduler available" : "Cron scheduler not configured"
|
|
2317
|
+
},
|
|
2318
|
+
{
|
|
2319
|
+
id: "workflow-failures",
|
|
2320
|
+
status: input.metrics.runs.failed > 0 || input.metrics.jobs.failed > 0 ? "warn" : "ok",
|
|
2321
|
+
message: input.metrics.runs.failed > 0 || input.metrics.jobs.failed > 0 ? `${input.metrics.runs.failed} failed runs, ${input.metrics.jobs.failed} failed jobs retained in history` : "No failed workflow runs or jobs in retained history"
|
|
2322
|
+
}
|
|
2323
|
+
];
|
|
2324
|
+
}
|
|
2325
|
+
function buildWorkflowTopOperations(metrics, httpOperations) {
|
|
2326
|
+
return [
|
|
2327
|
+
...httpOperations,
|
|
2328
|
+
{
|
|
2329
|
+
operation: "workflow.runs",
|
|
2330
|
+
count: metrics.runs.total,
|
|
2331
|
+
errorCount: metrics.runs.failed + metrics.runs.cancelled + metrics.runs.dlq
|
|
2332
|
+
},
|
|
2333
|
+
{
|
|
2334
|
+
operation: "workflow.jobs",
|
|
2335
|
+
count: metrics.jobs.total,
|
|
2336
|
+
errorCount: metrics.jobs.failed
|
|
2337
|
+
}
|
|
2338
|
+
].slice(0, 5);
|
|
2339
|
+
}
|
|
2340
|
+
function buildWorkflowMetricLines(metrics) {
|
|
2341
|
+
return [
|
|
2342
|
+
"# HELP workflow_runs_total Total workflow runs grouped by status",
|
|
2343
|
+
"# TYPE workflow_runs_total gauge",
|
|
2344
|
+
metricLine("workflow_runs_total", metrics.runs.total, { status: "total" }),
|
|
2345
|
+
metricLine("workflow_runs_total", metrics.runs.queued, { status: "queued" }),
|
|
2346
|
+
metricLine("workflow_runs_total", metrics.runs.running, { status: "running" }),
|
|
2347
|
+
metricLine("workflow_runs_total", metrics.runs.completed, { status: "completed" }),
|
|
2348
|
+
metricLine("workflow_runs_total", metrics.runs.failed, { status: "failed" }),
|
|
2349
|
+
metricLine("workflow_runs_total", metrics.runs.cancelled, { status: "cancelled" }),
|
|
2350
|
+
metricLine("workflow_runs_total", metrics.runs.dlq, { status: "dlq" }),
|
|
2351
|
+
"# HELP workflow_jobs_total Total workflow jobs grouped by status",
|
|
2352
|
+
"# TYPE workflow_jobs_total gauge",
|
|
2353
|
+
metricLine("workflow_jobs_total", metrics.jobs.total, { status: "total" }),
|
|
2354
|
+
metricLine("workflow_jobs_total", metrics.jobs.queued, { status: "queued" }),
|
|
2355
|
+
metricLine("workflow_jobs_total", metrics.jobs.running, { status: "running" }),
|
|
2356
|
+
metricLine("workflow_jobs_total", metrics.jobs.completed, { status: "completed" }),
|
|
2357
|
+
metricLine("workflow_jobs_total", metrics.jobs.failed, { status: "failed" }),
|
|
2358
|
+
metricLine("service_operation_total", metrics.runs.total, { operation: "workflow.runs", status: "ok" }),
|
|
2359
|
+
metricLine("service_operation_total", metrics.runs.failed + metrics.runs.cancelled + metrics.runs.dlq, { operation: "workflow.runs", status: "error" }),
|
|
2360
|
+
metricLine("service_operation_total", metrics.jobs.total, { operation: "workflow.jobs", status: "ok" }),
|
|
2361
|
+
metricLine("service_operation_total", metrics.jobs.failed, { operation: "workflow.jobs", status: "error" })
|
|
2362
|
+
];
|
|
2363
|
+
}
|
|
2364
|
+
var workerInstance = null;
|
|
2365
|
+
var serverInstance = null;
|
|
2366
|
+
var cronSchedulerInstance = null;
|
|
2367
|
+
async function bootstrap(cwd = process.cwd()) {
|
|
2368
|
+
const repoRoot = await findRepoRoot(cwd);
|
|
2369
|
+
await createServiceBootstrap({ appId: "workflow-daemon", repoRoot });
|
|
2370
|
+
if (!platform.isConfigured("workspace")) {
|
|
2371
|
+
process.stderr.write(
|
|
2372
|
+
'[workflow-daemon] WARNING: workspace adapter is not configured.\n[workflow-daemon] Workflows that use isolation: balanced (default) or isolation: strict will fail.\n[workflow-daemon] To fix: set platform.adapters.workspace in kb.config.json.\n[workflow-daemon] To run without a workspace: add "isolation: relaxed" to your workflow YAML.\n'
|
|
2373
|
+
);
|
|
2374
|
+
}
|
|
2375
|
+
if (!platform.isConfigured("environment") && platform.isConfigured("workspace")) {
|
|
2376
|
+
process.stderr.write(
|
|
2377
|
+
"[workflow-daemon] WARNING: environment adapter is not configured.\n[workflow-daemon] Workflows that use isolation: strict will fail.\n[workflow-daemon] To fix: set platform.adapters.environment in kb.config.json.\n"
|
|
2378
|
+
);
|
|
2379
|
+
}
|
|
2380
|
+
const startupRequestId = `workflow-startup-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
|
2381
|
+
const startupTraceId = randomUUID();
|
|
2382
|
+
const startupSpanId = randomUUID();
|
|
2383
|
+
const bootstrapLogger = createCorrelatedLogger(platform.logger, {
|
|
2384
|
+
serviceId: "workflow",
|
|
2385
|
+
logsSource: "workflow",
|
|
2386
|
+
layer: "workflow",
|
|
2387
|
+
service: "bootstrap",
|
|
2388
|
+
requestId: startupRequestId,
|
|
2389
|
+
traceId: startupTraceId,
|
|
2390
|
+
operation: "workflow.bootstrap",
|
|
2391
|
+
bindings: {
|
|
2392
|
+
spanId: startupSpanId,
|
|
2393
|
+
invocationId: startupSpanId,
|
|
2394
|
+
executionId: startupSpanId
|
|
2395
|
+
}
|
|
2396
|
+
});
|
|
2397
|
+
bootstrapLogger.info("Workflow daemon starting", { repoRoot });
|
|
2398
|
+
const createWorkflowLogger = (service, operation, bindings) => createCorrelatedLogger(platform.logger, {
|
|
2399
|
+
serviceId: "workflow",
|
|
2400
|
+
logsSource: "workflow",
|
|
2401
|
+
layer: "workflow",
|
|
2402
|
+
service,
|
|
2403
|
+
operation,
|
|
2404
|
+
bindings
|
|
2405
|
+
});
|
|
2406
|
+
bootstrapLogger.info("Loading plugin registry snapshot");
|
|
2407
|
+
const cliApi = await createRegistry({
|
|
2408
|
+
registry: {
|
|
2409
|
+
root: repoRoot
|
|
2410
|
+
},
|
|
2411
|
+
cache: {
|
|
2412
|
+
inMemory: true,
|
|
2413
|
+
ttlMs: 10 * 60 * 1e3
|
|
2414
|
+
// 10 minutes
|
|
2415
|
+
},
|
|
2416
|
+
logger: {
|
|
2417
|
+
level: "info"
|
|
2418
|
+
},
|
|
2419
|
+
snapshot: {
|
|
2420
|
+
mode: "consumer"
|
|
2421
|
+
}
|
|
2422
|
+
});
|
|
2423
|
+
await cliApi.initialize();
|
|
2424
|
+
const plugins = await cliApi.listPlugins();
|
|
2425
|
+
bootstrapLogger.info("Plugin registry snapshot loaded", {
|
|
2426
|
+
pluginsFound: plugins.length,
|
|
2427
|
+
pluginIds: plugins.map((p) => `${p.id}@${p.version}`)
|
|
2428
|
+
});
|
|
2429
|
+
bootstrapLogger.info("Creating WorkflowEngine");
|
|
2430
|
+
const engine = new WorkflowEngine({
|
|
2431
|
+
cache: platform.cache,
|
|
2432
|
+
events: platform.eventBus,
|
|
2433
|
+
logger: createWorkflowLogger("engine", "workflow.engine"),
|
|
2434
|
+
snapshotManager: platform.snapshotManager,
|
|
2435
|
+
workspaceRoot: repoRoot
|
|
2436
|
+
});
|
|
2437
|
+
bootstrapLogger.info("Cleaning up stale runs from previous daemon process");
|
|
2438
|
+
await engine.cleanupStaleRuns();
|
|
2439
|
+
bootstrapLogger.info("Resuming interrupted jobs");
|
|
2440
|
+
await engine.resumeInterruptedJobs();
|
|
2441
|
+
bootstrapLogger.info("Creating JobBroker");
|
|
2442
|
+
const jobBroker = new JobBroker(engine, createWorkflowLogger("job-broker", "workflow.job-broker"), platform);
|
|
2443
|
+
bootstrapLogger.info("Creating CronScheduler");
|
|
2444
|
+
const cronScheduler = new CronScheduler({
|
|
2445
|
+
jobBroker,
|
|
2446
|
+
workflowEngine: engine,
|
|
2447
|
+
logger: createWorkflowLogger("cron-scheduler", "workflow.cron-scheduler"),
|
|
2448
|
+
timezone: process.env.WORKFLOW_CRON_TIMEZONE
|
|
2449
|
+
});
|
|
2450
|
+
cronSchedulerInstance = cronScheduler;
|
|
2451
|
+
bootstrapLogger.info("Discovering cron jobs");
|
|
2452
|
+
const cronDiscovery = new CronDiscovery({
|
|
2453
|
+
cliApi,
|
|
2454
|
+
scheduler: cronScheduler,
|
|
2455
|
+
logger: createWorkflowLogger("cron-discovery", "workflow.cron-discovery"),
|
|
2456
|
+
workspaceRoot: repoRoot
|
|
2457
|
+
});
|
|
2458
|
+
const discovered = await cronDiscovery.discoverAll();
|
|
2459
|
+
bootstrapLogger.info("Cron job discovery complete", discovered);
|
|
2460
|
+
bootstrapLogger.info("Creating WorkflowService");
|
|
2461
|
+
const workflowService = new WorkflowService({
|
|
2462
|
+
cliApi,
|
|
2463
|
+
platform,
|
|
2464
|
+
workspaceRoot: repoRoot
|
|
2465
|
+
});
|
|
2466
|
+
bootstrapLogger.info("Creating HTTP server");
|
|
2467
|
+
const server = await createServer({
|
|
2468
|
+
engine,
|
|
2469
|
+
jobBroker,
|
|
2470
|
+
workflowService,
|
|
2471
|
+
cronScheduler,
|
|
2472
|
+
cronDiscovery,
|
|
2473
|
+
logger: createWorkflowLogger("api", "workflow.api")
|
|
2474
|
+
});
|
|
2475
|
+
const port = parseInt(process.env.WORKFLOW_PORT || "7778", 10);
|
|
2476
|
+
await server.listen({ port, host: "0.0.0.0" });
|
|
2477
|
+
bootstrapLogger.info("HTTP API listening", { port });
|
|
2478
|
+
serverInstance = server;
|
|
2479
|
+
bootstrapLogger.info("Creating WorkflowWorker");
|
|
2480
|
+
const worker = await createWorkflowWorker({
|
|
2481
|
+
engine,
|
|
2482
|
+
cliApi,
|
|
2483
|
+
logger: createWorkflowLogger("worker", "workflow.worker"),
|
|
2484
|
+
analytics: platform.analytics,
|
|
2485
|
+
platform,
|
|
2486
|
+
workspaceRoot: repoRoot,
|
|
2487
|
+
concurrency: parseInt(process.env.WORKFLOW_CONCURRENCY || "5", 10)
|
|
2488
|
+
});
|
|
2489
|
+
workerInstance = worker;
|
|
2490
|
+
bootstrapLogger.info("Starting WorkflowWorker");
|
|
2491
|
+
worker.start().catch((error) => {
|
|
2492
|
+
bootstrapLogger.error("Worker crashed - shutting down daemon", error instanceof Error ? error : void 0);
|
|
2493
|
+
process.kill(process.pid, "SIGTERM");
|
|
2494
|
+
});
|
|
2495
|
+
if (discovered.plugins + discovered.users > 0) {
|
|
2496
|
+
bootstrapLogger.info("Starting CronScheduler");
|
|
2497
|
+
await cronScheduler.start();
|
|
2498
|
+
} else {
|
|
2499
|
+
bootstrapLogger.info("No cron jobs found, skipping CronScheduler start");
|
|
2500
|
+
}
|
|
2501
|
+
bootstrapLogger.info("Workflow daemon started successfully", { port });
|
|
2502
|
+
const shutdown = async (signal) => {
|
|
2503
|
+
bootstrapLogger.warn("Received shutdown signal", { signal });
|
|
2504
|
+
if (cronSchedulerInstance) {
|
|
2505
|
+
await cronSchedulerInstance.stop();
|
|
2506
|
+
cronSchedulerInstance = null;
|
|
2507
|
+
}
|
|
2508
|
+
if (workerInstance) {
|
|
2509
|
+
await workerInstance.stop();
|
|
2510
|
+
workerInstance = null;
|
|
2511
|
+
}
|
|
2512
|
+
if (serverInstance) {
|
|
2513
|
+
await serverInstance.close();
|
|
2514
|
+
serverInstance = null;
|
|
2515
|
+
}
|
|
2516
|
+
await cliApi.dispose();
|
|
2517
|
+
await platform.shutdown();
|
|
2518
|
+
bootstrapLogger.info("Workflow daemon shutdown complete");
|
|
2519
|
+
process.exit(0);
|
|
2520
|
+
};
|
|
2521
|
+
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
2522
|
+
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
2523
|
+
}
|
|
2524
|
+
|
|
2525
|
+
// src/index.ts
|
|
2526
|
+
(async () => {
|
|
2527
|
+
try {
|
|
2528
|
+
await bootstrap(process.cwd());
|
|
2529
|
+
} catch (error) {
|
|
2530
|
+
console.error("Failed to start workflow daemon:", error);
|
|
2531
|
+
process.exit(1);
|
|
2532
|
+
}
|
|
2533
|
+
})();
|
|
2534
|
+
//# sourceMappingURL=index.js.map
|
|
2535
|
+
//# sourceMappingURL=index.js.map
|