@kb-labs/workflow-daemon 2.94.0 → 2.98.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +1040 -513
- package/dist/index.js.map +1 -1
- package/dist/manifest.js +3 -1
- package/dist/manifest.js.map +1 -1
- package/dist/manifest.json +27 -0
- package/package.json +22 -19
package/dist/index.js
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
2
|
+
import { platform, createServiceBootstrap } from '@kb-labs/core-runtime';
|
|
3
|
+
import { makeAssemblyHook } from '@kb-labs/plugin-runtime';
|
|
3
4
|
import { WorkflowEngine, WorkflowService } from '@kb-labs/workflow-engine';
|
|
4
|
-
import { createCorrelatedLogger,
|
|
5
|
+
import { createCorrelatedLogger, getListenOptions, HttpObservabilityCollector, createDaemonServer, createServiceReadyResponse, metricLine } from '@kb-labs/shared-http';
|
|
6
|
+
import { runDaemon } from '@kb-labs/shared-daemon';
|
|
5
7
|
import { logDiagnosticEvent } from '@kb-labs/core-platform';
|
|
6
|
-
import { PluginCronJobSchema, UserCronJobSchema, evaluateExpression, interpolateObject,
|
|
8
|
+
import { PluginCronJobSchema, UserCronJobSchema, evaluateExpression, interpolateObject, buildShellSafeCommand, coerceToString, interpolateString } from '@kb-labs/workflow-contracts';
|
|
9
|
+
import { GateHandler } from '@kb-labs/workflow-steps';
|
|
7
10
|
import { SandboxRunner } from '@kb-labs/workflow-runtime';
|
|
8
11
|
import * as cron from 'node-cron';
|
|
9
12
|
import { CronExpressionParser } from 'cron-parser';
|
|
10
|
-
import { stat, readdir, readFile } from 'fs/promises';
|
|
13
|
+
import { stat, readdir, readFile, access } from 'fs/promises';
|
|
11
14
|
import { join, extname, basename } from 'path';
|
|
12
15
|
import YAML from 'yaml';
|
|
13
|
-
import
|
|
14
|
-
import cors from '@fastify/cors';
|
|
16
|
+
import { watch } from 'fs';
|
|
15
17
|
import { createRegistry } from '@kb-labs/core-registry';
|
|
16
|
-
import { findRepoRoot } from '@kb-labs/core-sys';
|
|
17
18
|
import { randomUUID } from 'crypto';
|
|
18
19
|
|
|
19
20
|
async function createWorkflowWorker(options) {
|
|
@@ -28,6 +29,11 @@ async function createWorkflowWorker(options) {
|
|
|
28
29
|
analytics,
|
|
29
30
|
debugMode = process.env["WORKFLOW_DEBUG"] === "true"
|
|
30
31
|
} = options;
|
|
32
|
+
const wsProvider = platform2.getAdapter("workspace");
|
|
33
|
+
if (wsProvider && typeof wsProvider.startupCleanup === "function") {
|
|
34
|
+
wsProvider.startupCleanup().catch(() => {
|
|
35
|
+
});
|
|
36
|
+
}
|
|
31
37
|
let isRunning = false;
|
|
32
38
|
let stopRequested = false;
|
|
33
39
|
const runningJobs = /* @__PURE__ */ new Map();
|
|
@@ -67,6 +73,14 @@ async function createWorkflowWorker(options) {
|
|
|
67
73
|
return true;
|
|
68
74
|
}
|
|
69
75
|
claimedJobs.add(jobKey);
|
|
76
|
+
const waitMs = job.queuedAt ? Date.now() - new Date(job.queuedAt).getTime() : void 0;
|
|
77
|
+
logger.info("Job picked from queue", {
|
|
78
|
+
runId: run.id,
|
|
79
|
+
jobId: job.id,
|
|
80
|
+
jobName: job.jobName,
|
|
81
|
+
queuedAt: job.queuedAt,
|
|
82
|
+
waitMs
|
|
83
|
+
});
|
|
70
84
|
const jobStartTime = Date.now();
|
|
71
85
|
const jobLogger = createCorrelatedLogger(logger, {
|
|
72
86
|
serviceId: "workflow",
|
|
@@ -79,7 +93,8 @@ async function createWorkflowWorker(options) {
|
|
|
79
93
|
bindings: {
|
|
80
94
|
workflowId: run.id,
|
|
81
95
|
runId: run.id,
|
|
82
|
-
jobId: job.id
|
|
96
|
+
jobId: job.id,
|
|
97
|
+
jobName: job.jobName
|
|
83
98
|
}
|
|
84
99
|
});
|
|
85
100
|
jobLogger.info("Processing job", {
|
|
@@ -98,13 +113,13 @@ async function createWorkflowWorker(options) {
|
|
|
98
113
|
const runTarget = run.metadata?.target;
|
|
99
114
|
const jobTarget = job.target;
|
|
100
115
|
const target = jobTarget ?? runTarget;
|
|
101
|
-
const
|
|
102
|
-
let runWorkspace =
|
|
116
|
+
const wsProvider2 = platform2.getAdapter("workspace");
|
|
117
|
+
let runWorkspace = wsProvider2 ? workspaceRoot : process.env["KB_PROJECT_ROOT"] ?? workspaceRoot;
|
|
103
118
|
let provisionedWorkspaceId;
|
|
104
|
-
if (
|
|
119
|
+
if (wsProvider2) {
|
|
105
120
|
const wsId = `wt_${run.id.slice(0, 8)}`;
|
|
106
121
|
try {
|
|
107
|
-
const ws = await
|
|
122
|
+
const ws = await wsProvider2.materialize({
|
|
108
123
|
workspaceId: wsId,
|
|
109
124
|
sourceRef: "main",
|
|
110
125
|
metadata: { runId: run.id, jobId: job.id },
|
|
@@ -150,11 +165,20 @@ async function createWorkflowWorker(options) {
|
|
|
150
165
|
}
|
|
151
166
|
const jobPromise = (async () => {
|
|
152
167
|
try {
|
|
168
|
+
let wasCancelled = false;
|
|
153
169
|
for (const step of job.steps) {
|
|
154
170
|
if (step.status === "success") {
|
|
155
171
|
continue;
|
|
156
172
|
}
|
|
157
173
|
const freshRun = await engine.getRun(run.id);
|
|
174
|
+
if (freshRun?.status === "cancelled") {
|
|
175
|
+
jobLogger.info("[worker] Run cancelled \u2014 stopping step execution", {
|
|
176
|
+
runId: run.id,
|
|
177
|
+
jobId: job.id
|
|
178
|
+
});
|
|
179
|
+
wasCancelled = true;
|
|
180
|
+
break;
|
|
181
|
+
}
|
|
158
182
|
const exprCtx = {
|
|
159
183
|
env: freshRun?.env ?? {},
|
|
160
184
|
trigger: freshRun?.trigger ?? { type: "manual" },
|
|
@@ -181,11 +205,18 @@ async function createWorkflowWorker(options) {
|
|
|
181
205
|
const rawExpr = condition.trim().replace(/^\$\{\{\s*/, "").replace(/\s*\}\}$/, "");
|
|
182
206
|
const shouldRun = evaluateExpression(rawExpr, exprCtx);
|
|
183
207
|
if (!shouldRun) {
|
|
184
|
-
jobLogger.info("
|
|
208
|
+
jobLogger.info("[step] Skipped: condition evaluated to false", {
|
|
185
209
|
runId: run.id,
|
|
186
210
|
jobId: job.id,
|
|
187
211
|
stepId: step.id,
|
|
188
|
-
|
|
212
|
+
stepName: step.name,
|
|
213
|
+
stepIndex: step.index,
|
|
214
|
+
condition: rawExpr,
|
|
215
|
+
evaluatedContext: {
|
|
216
|
+
inputs: exprCtx.inputs,
|
|
217
|
+
steps: exprCtx.steps,
|
|
218
|
+
env: exprCtx.env
|
|
219
|
+
}
|
|
189
220
|
});
|
|
190
221
|
await engine.markStepCompleted(run.id, job.id, step.id, { skipped: true });
|
|
191
222
|
continue;
|
|
@@ -204,7 +235,25 @@ async function createWorkflowWorker(options) {
|
|
|
204
235
|
}
|
|
205
236
|
});
|
|
206
237
|
}
|
|
207
|
-
|
|
238
|
+
let interpolatedWith = step.spec.with ? interpolateObject(step.spec.with, exprCtx) : void 0;
|
|
239
|
+
if (step.spec.uses === "builtin:shell" && typeof step.spec.with?.["command"] === "string") {
|
|
240
|
+
const rawCommand = step.spec.with["command"];
|
|
241
|
+
const { command: safeCommand, shellEnvVars } = buildShellSafeCommand(rawCommand, exprCtx);
|
|
242
|
+
interpolatedWith = {
|
|
243
|
+
...interpolatedWith ?? {},
|
|
244
|
+
command: safeCommand,
|
|
245
|
+
env: {
|
|
246
|
+
...Object.fromEntries(
|
|
247
|
+
Object.entries(interpolatedWith?.["env"] ?? {}).map(([k, v]) => [k, coerceToString(v)])
|
|
248
|
+
),
|
|
249
|
+
...shellEnvVars
|
|
250
|
+
}
|
|
251
|
+
};
|
|
252
|
+
}
|
|
253
|
+
const interpolatedEnvRaw = step.spec.env ? interpolateObject(step.spec.env, exprCtx) : void 0;
|
|
254
|
+
const interpolatedEnv = interpolatedEnvRaw ? Object.fromEntries(
|
|
255
|
+
Object.entries(interpolatedEnvRaw).map(([k, v]) => [k, coerceToString(v)])
|
|
256
|
+
) : void 0;
|
|
208
257
|
if (debugMode && step.spec.with) {
|
|
209
258
|
jobLogger.info("[debug] Step input interpolation", {
|
|
210
259
|
runId: run.id,
|
|
@@ -218,7 +267,9 @@ async function createWorkflowWorker(options) {
|
|
|
218
267
|
const stepLogger = jobLogger.child({
|
|
219
268
|
operation: "workflow.step",
|
|
220
269
|
stepId: step.id,
|
|
221
|
-
|
|
270
|
+
stepName: step.name,
|
|
271
|
+
stepIndex: step.index,
|
|
272
|
+
attempt: job.attempt ?? 1,
|
|
222
273
|
executionId: stepExecutionId,
|
|
223
274
|
spanId: stepExecutionId,
|
|
224
275
|
invocationId: stepExecutionId
|
|
@@ -241,153 +292,88 @@ async function createWorkflowWorker(options) {
|
|
|
241
292
|
});
|
|
242
293
|
if (step.spec.uses === "builtin:approval") {
|
|
243
294
|
if (step.status === "failed") {
|
|
244
|
-
stepLogger.info("Approval already rejected \u2014 skipping to gate", {
|
|
245
|
-
runId: run.id,
|
|
246
|
-
stepId: step.id
|
|
247
|
-
});
|
|
295
|
+
stepLogger.info("Approval already rejected \u2014 skipping to gate", { runId: run.id, stepId: step.id });
|
|
248
296
|
continue;
|
|
249
297
|
}
|
|
250
298
|
if (step.status !== "waiting_approval") {
|
|
251
299
|
await engine.markStepWaitingApproval(run.id, job.id, step.id);
|
|
252
300
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
const currentStep = currentJob?.steps.find((s) => s.id === step.id);
|
|
264
|
-
if (!currentStep || currentStep.status === "success") {
|
|
265
|
-
stepLogger.info("Approval granted", { runId: run.id, stepId: step.id });
|
|
266
|
-
break;
|
|
267
|
-
}
|
|
268
|
-
if (currentStep.status === "failed") {
|
|
269
|
-
stepLogger.info("Approval rejected", { runId: run.id, stepId: step.id });
|
|
270
|
-
break;
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
if (stopRequested) {
|
|
274
|
-
stepLogger.info("Approval wait interrupted by shutdown", { stepId: step.id });
|
|
301
|
+
const approvalResult = await waitForApproval(
|
|
302
|
+
engine,
|
|
303
|
+
run.id,
|
|
304
|
+
job.id,
|
|
305
|
+
step,
|
|
306
|
+
interpolatedWith,
|
|
307
|
+
() => stopRequested,
|
|
308
|
+
stepLogger
|
|
309
|
+
);
|
|
310
|
+
if (approvalResult === "interrupted") {
|
|
275
311
|
return;
|
|
276
312
|
}
|
|
277
313
|
continue;
|
|
278
314
|
}
|
|
279
315
|
if (step.spec.uses === "builtin:gate") {
|
|
280
316
|
const gateInput = interpolatedWith ?? {};
|
|
281
|
-
const
|
|
282
|
-
const
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
const
|
|
286
|
-
const
|
|
287
|
-
|
|
317
|
+
const currentIteration = step.metadata?.["iterations"] ?? 0;
|
|
318
|
+
const sameJobStepIds = job.steps.flatMap(
|
|
319
|
+
(s) => [s.id, s.spec.id].filter((v) => typeof v === "string")
|
|
320
|
+
);
|
|
321
|
+
const jobNames = (freshRun?.jobs ?? run.jobs).map((j) => j.jobName);
|
|
322
|
+
const validRestartTargets = [...sameJobStepIds, ...jobNames];
|
|
323
|
+
const decision = new GateHandler().handle(gateInput, exprCtx, currentIteration, validRestartTargets);
|
|
324
|
+
stepLogger.info("[gate] Evaluating gate decision", {
|
|
288
325
|
runId: run.id,
|
|
289
326
|
stepId: step.id,
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
action:
|
|
327
|
+
stepName: step.name,
|
|
328
|
+
expression: gateInput.decision,
|
|
329
|
+
action: decision.action,
|
|
330
|
+
iteration: currentIteration
|
|
293
331
|
});
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
const currentIteration = metadata[iterationKey] ?? 0;
|
|
297
|
-
if (action === "continue") {
|
|
298
|
-
await engine.markStepCompleted(run.id, job.id, step.id, {
|
|
299
|
-
decisionValue,
|
|
300
|
-
action: "continue",
|
|
301
|
-
iteration: currentIteration
|
|
302
|
-
});
|
|
332
|
+
if (decision.action === "continue") {
|
|
333
|
+
await engine.markStepCompleted(run.id, job.id, step.id, decision.outputs);
|
|
303
334
|
continue;
|
|
304
335
|
}
|
|
305
|
-
if (action === "fail") {
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
iteration: currentIteration
|
|
311
|
-
});
|
|
312
|
-
throw error;
|
|
313
|
-
}
|
|
314
|
-
const restartAction = action;
|
|
315
|
-
const nextIteration = currentIteration + 1;
|
|
316
|
-
if (nextIteration >= maxIterations) {
|
|
317
|
-
const error = new Error(
|
|
318
|
-
`Gate max iterations reached (${maxIterations}) for step ${step.spec.id ?? step.id}`
|
|
319
|
-
);
|
|
320
|
-
await engine.markStepFailed(run.id, job.id, step.id, error, {
|
|
321
|
-
decisionValue,
|
|
322
|
-
action: "fail",
|
|
323
|
-
maxIterationsReached: true,
|
|
324
|
-
iteration: currentIteration,
|
|
325
|
-
maxIterations
|
|
336
|
+
if (decision.action === "fail") {
|
|
337
|
+
stepLogger.error("[gate] Gate condition failed, aborting job", decision.error, {
|
|
338
|
+
runId: run.id,
|
|
339
|
+
stepId: step.id,
|
|
340
|
+
stepName: step.name
|
|
326
341
|
});
|
|
327
|
-
|
|
342
|
+
await engine.markStepFailed(run.id, job.id, step.id, decision.error, decision.outputs);
|
|
343
|
+
throw decision.error;
|
|
328
344
|
}
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
maxIterations
|
|
333
|
-
});
|
|
334
|
-
await engine.markStepCompleted(run.id, job.id, step.id, {
|
|
335
|
-
decisionValue,
|
|
336
|
-
action: "restart",
|
|
337
|
-
restartFrom: restartAction.restartFrom,
|
|
338
|
-
iteration: nextIteration
|
|
339
|
-
});
|
|
340
|
-
await engine.updateRun(run.id, (draft) => {
|
|
341
|
-
const md = draft.metadata ?? {};
|
|
342
|
-
md[iterationKey] = nextIteration;
|
|
343
|
-
draft.metadata = md;
|
|
344
|
-
if (restartAction.context) {
|
|
345
|
-
const payload = draft.trigger.payload ?? {};
|
|
346
|
-
Object.assign(payload, restartAction.context);
|
|
347
|
-
draft.trigger.payload = payload;
|
|
348
|
-
}
|
|
349
|
-
return draft;
|
|
350
|
-
});
|
|
351
|
-
const stateStore = engine.getStateStore();
|
|
352
|
-
const scheduler = engine.getScheduler();
|
|
353
|
-
let foundTarget = false;
|
|
354
|
-
for (const s of job.steps) {
|
|
355
|
-
if (s.spec.id === restartAction.restartFrom || s.id === restartAction.restartFrom) {
|
|
356
|
-
foundTarget = true;
|
|
357
|
-
}
|
|
358
|
-
if (foundTarget) {
|
|
359
|
-
await stateStore.updateStep(run.id, job.id, s.id, (draft) => {
|
|
360
|
-
draft.status = "queued";
|
|
361
|
-
draft.startedAt = void 0;
|
|
362
|
-
draft.finishedAt = void 0;
|
|
363
|
-
draft.error = void 0;
|
|
364
|
-
draft.outputs = void 0;
|
|
365
|
-
});
|
|
366
|
-
}
|
|
367
|
-
}
|
|
368
|
-
await stateStore.updateJob(run.id, job.id, (draft) => {
|
|
369
|
-
draft.status = "queued";
|
|
370
|
-
draft.startedAt = void 0;
|
|
371
|
-
draft.finishedAt = void 0;
|
|
372
|
-
});
|
|
373
|
-
const updatedRun = await engine.getRun(run.id);
|
|
374
|
-
const updatedJob = updatedRun?.jobs.find((j) => j.id === job.id);
|
|
375
|
-
if (updatedJob) {
|
|
376
|
-
await scheduler.enqueueJob(run.id, updatedJob, updatedJob.priority ?? "normal");
|
|
345
|
+
if (decision.action === "skip") {
|
|
346
|
+
await applyGateSkip(engine, run, job, step, decision, stepLogger);
|
|
347
|
+
return;
|
|
377
348
|
}
|
|
349
|
+
await applyGateRestart(engine, run, job, step, decision, stepLogger);
|
|
378
350
|
return;
|
|
379
351
|
}
|
|
352
|
+
const stepStartTime = Date.now();
|
|
380
353
|
await engine.markStepStarted(run.id, job.id, step.id);
|
|
381
354
|
let baseSpec = step.spec;
|
|
382
355
|
if (baseSpec.run && !baseSpec.uses) {
|
|
383
356
|
const { run: rawRun, with: existingWith, ...rest } = baseSpec;
|
|
384
|
-
const command = typeof rawRun === "string" ?
|
|
385
|
-
baseSpec = {
|
|
357
|
+
const { command, shellEnvVars } = typeof rawRun === "string" ? buildShellSafeCommand(rawRun, exprCtx) : { command: rawRun, shellEnvVars: {} };
|
|
358
|
+
baseSpec = {
|
|
359
|
+
...rest,
|
|
360
|
+
uses: "builtin:shell",
|
|
361
|
+
with: { ...existingWith, command, env: { ...existingWith?.["env"] ?? {}, ...shellEnvVars } }
|
|
362
|
+
};
|
|
386
363
|
}
|
|
387
364
|
if (typeof baseSpec.summary === "string") {
|
|
388
365
|
baseSpec.summary = interpolateString(baseSpec.summary, exprCtx);
|
|
389
366
|
}
|
|
390
|
-
const
|
|
367
|
+
const specWithEnv = interpolatedEnv ? { ...baseSpec, with: { ...baseSpec.with ?? {}, env: { ...baseSpec.with?.["env"] ?? {}, ...interpolatedEnv } } } : baseSpec;
|
|
368
|
+
const interpolatedSpec = interpolatedWith ? {
|
|
369
|
+
...specWithEnv,
|
|
370
|
+
with: {
|
|
371
|
+
...specWithEnv.with ?? {},
|
|
372
|
+
...interpolatedWith,
|
|
373
|
+
// spec.env (interpolatedEnv) must survive the spread of interpolatedWith.env
|
|
374
|
+
...interpolatedEnv ? { env: { ...interpolatedWith["env"] ?? {}, ...interpolatedEnv } } : {}
|
|
375
|
+
}
|
|
376
|
+
} : specWithEnv;
|
|
391
377
|
const result = await runner.execute({
|
|
392
378
|
spec: interpolatedSpec,
|
|
393
379
|
context: {
|
|
@@ -395,9 +381,25 @@ async function createWorkflowWorker(options) {
|
|
|
395
381
|
jobId: job.id,
|
|
396
382
|
stepId: step.id,
|
|
397
383
|
attempt: 1,
|
|
398
|
-
env:
|
|
399
|
-
|
|
400
|
-
|
|
384
|
+
env: {
|
|
385
|
+
// KB_PLATFORM_ROOT: where platform code (dist/, node_modules) lives.
|
|
386
|
+
// Shell steps can use this to reference platform commands when the
|
|
387
|
+
// worktree doesn't have compiled dist/ directories.
|
|
388
|
+
KB_PLATFORM_ROOT: workspaceRoot,
|
|
389
|
+
// KB_WORKSPACE_ROOT: the worktree (or project dir when no worktree is used).
|
|
390
|
+
// Scripts cd into this path before invoking agents or running git commands.
|
|
391
|
+
KB_WORKSPACE_ROOT: runWorkspace,
|
|
392
|
+
...freshRun?.env || {}
|
|
393
|
+
},
|
|
394
|
+
// Secrets resolution not yet implemented: run.secrets contains names only.
|
|
395
|
+
// When a platform secrets store is available, resolve names → values here.
|
|
396
|
+
secrets: (() => {
|
|
397
|
+
const names = freshRun?.secrets ?? [];
|
|
398
|
+
if (names.length > 0) {
|
|
399
|
+
stepLogger.warn("Step declares secrets but secret resolution is not implemented", { secrets: names });
|
|
400
|
+
}
|
|
401
|
+
return {};
|
|
402
|
+
})(),
|
|
401
403
|
logger: {
|
|
402
404
|
debug: (message, meta) => stepLogger.debug(message, meta),
|
|
403
405
|
info: (message, meta) => stepLogger.info(message, meta),
|
|
@@ -409,21 +411,55 @@ async function createWorkflowWorker(options) {
|
|
|
409
411
|
spanId: stepExecutionId,
|
|
410
412
|
parentSpanId: job.id
|
|
411
413
|
},
|
|
414
|
+
// stepLogger as loggerOverride: ctx.logger.* in the plugin will use stepLogger
|
|
415
|
+
// as its base, writing to SQLite with runId/jobId/stepId context.
|
|
416
|
+
// See: plugins/workflow/docs/adr/0019-log-stream-separation.md
|
|
417
|
+
loggerOverride: stepLogger,
|
|
418
|
+
// ui/shell log entries: persist to SQLite with workflow context + publish for SSE.
|
|
419
|
+
// See: plugins/workflow/docs/adr/0019-log-stream-separation.md
|
|
412
420
|
onLog: (entry2) => {
|
|
413
|
-
|
|
421
|
+
stepLogger.info(entry2.message, {
|
|
422
|
+
stream: entry2.stream,
|
|
423
|
+
lineNo: entry2.lineNo,
|
|
424
|
+
logSource: entry2.stream === "stderr" ? "stderr" : "stdout"
|
|
425
|
+
});
|
|
426
|
+
void engine.publishLog(run.id, job.id, step.id, entry2, step.name);
|
|
427
|
+
},
|
|
428
|
+
// ctx.logger.* entries: stepLogger base already wrote to SQLite. Only publish for SSE.
|
|
429
|
+
// See: plugins/workflow/docs/adr/0019-log-stream-separation.md
|
|
430
|
+
onLoggerLog: (entry2) => {
|
|
431
|
+
void engine.publishLog(run.id, job.id, step.id, entry2, step.name);
|
|
414
432
|
}
|
|
415
433
|
},
|
|
416
|
-
|
|
434
|
+
// Scripts (.kb/workflows/scripts/*.sh) live in the project root, not the worktree.
|
|
435
|
+
// Use workspaceRoot as cwd so relative paths like `bash .kb/workflows/scripts/...`
|
|
436
|
+
// resolve correctly. Scripts that need to operate inside the worktree cd into
|
|
437
|
+
// KB_WORKSPACE_ROOT themselves (agent scripts, git operations).
|
|
438
|
+
workspace: workspaceRoot,
|
|
417
439
|
target
|
|
418
440
|
});
|
|
419
441
|
if (result.status === "failed") {
|
|
442
|
+
const stepDurationMs = Date.now() - stepStartTime;
|
|
420
443
|
const error = new Error(result.error?.message ?? "Step execution failed");
|
|
421
444
|
await engine.markStepFailed(run.id, job.id, step.id, error);
|
|
422
445
|
stepLogger.error("Step failed", error, {
|
|
423
446
|
runId: run.id,
|
|
424
447
|
jobId: job.id,
|
|
425
|
-
stepId: step.id
|
|
448
|
+
stepId: step.id,
|
|
449
|
+
stepName: step.name,
|
|
450
|
+
uses: step.spec.uses,
|
|
451
|
+
durationMs: stepDurationMs,
|
|
452
|
+
resolvedInputs: interpolatedWith,
|
|
453
|
+
errorCode: result.error?.code
|
|
426
454
|
});
|
|
455
|
+
if (step.continueOnError) {
|
|
456
|
+
stepLogger.warn("[step] continueOnError=true \u2014 continuing despite failure", {
|
|
457
|
+
runId: run.id,
|
|
458
|
+
jobId: job.id,
|
|
459
|
+
stepId: step.id
|
|
460
|
+
});
|
|
461
|
+
continue;
|
|
462
|
+
}
|
|
427
463
|
throw error;
|
|
428
464
|
}
|
|
429
465
|
const stepOutputs = result.status === "success" ? result.outputs : void 0;
|
|
@@ -442,6 +478,14 @@ async function createWorkflowWorker(options) {
|
|
|
442
478
|
});
|
|
443
479
|
}
|
|
444
480
|
}
|
|
481
|
+
if (wasCancelled) {
|
|
482
|
+
await engine.markJobInterrupted(run.id, job.id);
|
|
483
|
+
jobLogger.info("Job interrupted due to run cancellation", {
|
|
484
|
+
runId: run.id,
|
|
485
|
+
jobId: job.id
|
|
486
|
+
});
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
445
489
|
await engine.markJobCompleted(run.id, job.id);
|
|
446
490
|
const jobDuration = Date.now() - jobStartTime;
|
|
447
491
|
jobLogger.info("Job completed successfully", {
|
|
@@ -456,9 +500,9 @@ async function createWorkflowWorker(options) {
|
|
|
456
500
|
stepCount: job.steps.length
|
|
457
501
|
}).catch(() => {
|
|
458
502
|
});
|
|
459
|
-
if (provisionedWorkspaceId &&
|
|
503
|
+
if (provisionedWorkspaceId && wsProvider2) {
|
|
460
504
|
try {
|
|
461
|
-
await
|
|
505
|
+
await wsProvider2.release(provisionedWorkspaceId);
|
|
462
506
|
jobLogger.info("Workspace released", { workspaceId: provisionedWorkspaceId });
|
|
463
507
|
} catch (releaseErr) {
|
|
464
508
|
jobLogger.warn("Workspace release failed", {
|
|
@@ -597,6 +641,231 @@ function sleep(ms) {
|
|
|
597
641
|
setTimeout(resolve, ms);
|
|
598
642
|
});
|
|
599
643
|
}
|
|
644
|
+
async function waitForApproval(engine, runId, jobId, step, interpolatedWith, isStopRequested, stepLogger) {
|
|
645
|
+
const approvalTimeoutMs = interpolatedWith?.["timeoutMs"];
|
|
646
|
+
if (!approvalTimeoutMs) {
|
|
647
|
+
stepLogger.warn("[approval] No timeout configured \u2014 approval may wait indefinitely", {
|
|
648
|
+
runId,
|
|
649
|
+
jobId,
|
|
650
|
+
stepId: step.id,
|
|
651
|
+
stepName: step.name
|
|
652
|
+
});
|
|
653
|
+
}
|
|
654
|
+
stepLogger.info("[approval] Waiting for approval", {
|
|
655
|
+
runId,
|
|
656
|
+
jobId,
|
|
657
|
+
stepId: step.id,
|
|
658
|
+
stepName: step.name,
|
|
659
|
+
context: interpolatedWith
|
|
660
|
+
});
|
|
661
|
+
const approvalStartMs = Date.now();
|
|
662
|
+
let pollCount = 0;
|
|
663
|
+
while (!isStopRequested()) {
|
|
664
|
+
await sleep(2e3);
|
|
665
|
+
pollCount++;
|
|
666
|
+
const currentRun = await engine.getRun(runId);
|
|
667
|
+
if (currentRun?.status === "cancelled") {
|
|
668
|
+
stepLogger.info("[approval] Run cancelled \u2014 treating as interrupted, NOT approved", {
|
|
669
|
+
runId,
|
|
670
|
+
stepId: step.id,
|
|
671
|
+
stepName: step.name,
|
|
672
|
+
waitedMs: Date.now() - approvalStartMs
|
|
673
|
+
});
|
|
674
|
+
return "interrupted";
|
|
675
|
+
}
|
|
676
|
+
const currentJob = currentRun?.jobs.find((j) => j.id === jobId);
|
|
677
|
+
const currentStep = currentJob?.steps.find((s) => s.id === step.id);
|
|
678
|
+
if (!currentStep || currentStep.status === "success") {
|
|
679
|
+
stepLogger.info("[approval] Approval granted", {
|
|
680
|
+
runId,
|
|
681
|
+
stepId: step.id,
|
|
682
|
+
stepName: step.name,
|
|
683
|
+
waitedMs: Date.now() - approvalStartMs
|
|
684
|
+
});
|
|
685
|
+
break;
|
|
686
|
+
}
|
|
687
|
+
if (currentStep.status === "failed") {
|
|
688
|
+
stepLogger.info("[approval] Approval rejected", {
|
|
689
|
+
runId,
|
|
690
|
+
stepId: step.id,
|
|
691
|
+
stepName: step.name,
|
|
692
|
+
waitedMs: Date.now() - approvalStartMs
|
|
693
|
+
});
|
|
694
|
+
break;
|
|
695
|
+
}
|
|
696
|
+
if (pollCount % 10 === 0) {
|
|
697
|
+
stepLogger.info("[approval] Still waiting for approval", {
|
|
698
|
+
runId,
|
|
699
|
+
stepId: step.id,
|
|
700
|
+
stepName: step.name,
|
|
701
|
+
waitedMs: Date.now() - approvalStartMs,
|
|
702
|
+
pollCount
|
|
703
|
+
});
|
|
704
|
+
}
|
|
705
|
+
}
|
|
706
|
+
if (isStopRequested()) {
|
|
707
|
+
stepLogger.info("Approval wait interrupted by shutdown", { stepId: step.id });
|
|
708
|
+
return "interrupted";
|
|
709
|
+
}
|
|
710
|
+
return "done";
|
|
711
|
+
}
|
|
712
|
+
async function applyGateSkip(engine, run, job, step, decision, stepLogger) {
|
|
713
|
+
const { skipTo, outputs } = decision;
|
|
714
|
+
stepLogger.info("[gate] Gate triggered skip-forward", {
|
|
715
|
+
runId: run.id,
|
|
716
|
+
stepId: step.id,
|
|
717
|
+
stepName: step.name,
|
|
718
|
+
skipTo
|
|
719
|
+
});
|
|
720
|
+
const stateStore = engine.getStateStore();
|
|
721
|
+
const scheduler = engine.getScheduler();
|
|
722
|
+
await engine.markStepCompleted(run.id, job.id, step.id, outputs);
|
|
723
|
+
let pastGate = false;
|
|
724
|
+
for (const s of job.steps) {
|
|
725
|
+
if (s.id === step.id) {
|
|
726
|
+
pastGate = true;
|
|
727
|
+
continue;
|
|
728
|
+
}
|
|
729
|
+
if (!pastGate) {
|
|
730
|
+
continue;
|
|
731
|
+
}
|
|
732
|
+
if (s.spec.id === skipTo || s.id === skipTo) {
|
|
733
|
+
break;
|
|
734
|
+
}
|
|
735
|
+
await stateStore.updateStep(run.id, job.id, s.id, (draft) => {
|
|
736
|
+
draft.status = "success";
|
|
737
|
+
draft.outputs = { skipped: true };
|
|
738
|
+
draft.startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
739
|
+
draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
740
|
+
});
|
|
741
|
+
}
|
|
742
|
+
const freshRun = await engine.getRun(run.id);
|
|
743
|
+
const freshJob = freshRun?.jobs.find((j) => j.id === job.id);
|
|
744
|
+
if (freshJob) {
|
|
745
|
+
await scheduler.enqueueJob(run.id, freshJob, freshJob.priority ?? "normal");
|
|
746
|
+
}
|
|
747
|
+
stepLogger.info("[gate] Skip applied \u2014 re-enqueued job at skipTo target", {
|
|
748
|
+
runId: run.id,
|
|
749
|
+
skipTo
|
|
750
|
+
});
|
|
751
|
+
}
|
|
752
|
+
async function applyGateRestart(engine, run, job, step, decision, stepLogger) {
|
|
753
|
+
const { restartFrom, context, outputs, nextIteration } = decision;
|
|
754
|
+
const stepsToReset = [];
|
|
755
|
+
for (const s of job.steps) {
|
|
756
|
+
if (s.spec.id === restartFrom || s.id === restartFrom) {
|
|
757
|
+
stepsToReset.push(s.name ?? s.id);
|
|
758
|
+
} else if (stepsToReset.length > 0) {
|
|
759
|
+
stepsToReset.push(s.name ?? s.id);
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
stepLogger.warn("[gate] Gate triggered restart", {
|
|
763
|
+
runId: run.id,
|
|
764
|
+
stepId: step.id,
|
|
765
|
+
stepName: step.name,
|
|
766
|
+
restartFrom,
|
|
767
|
+
iteration: nextIteration,
|
|
768
|
+
stepsToReset
|
|
769
|
+
});
|
|
770
|
+
const stateStore = engine.getStateStore();
|
|
771
|
+
const scheduler = engine.getScheduler();
|
|
772
|
+
await stateStore.updateStep(run.id, job.id, step.id, (draft) => {
|
|
773
|
+
draft.metadata = { ...draft.metadata ?? {}, iterations: nextIteration };
|
|
774
|
+
});
|
|
775
|
+
if (context) {
|
|
776
|
+
await engine.updateRun(run.id, (draft) => {
|
|
777
|
+
const payload = draft.trigger.payload ?? {};
|
|
778
|
+
Object.assign(payload, context);
|
|
779
|
+
draft.trigger.payload = payload;
|
|
780
|
+
return draft;
|
|
781
|
+
});
|
|
782
|
+
}
|
|
783
|
+
const sameJobTarget = job.steps.some((s) => s.spec.id === restartFrom || s.id === restartFrom);
|
|
784
|
+
if (!sameJobTarget) {
|
|
785
|
+
const fresh = await engine.getRun(run.id) ?? run;
|
|
786
|
+
const resetNames = /* @__PURE__ */ new Set([restartFrom]);
|
|
787
|
+
let changed = true;
|
|
788
|
+
while (changed) {
|
|
789
|
+
changed = false;
|
|
790
|
+
for (const j of fresh.jobs) {
|
|
791
|
+
if (resetNames.has(j.jobName)) {
|
|
792
|
+
continue;
|
|
793
|
+
}
|
|
794
|
+
if ((j.needs ?? []).some((n) => resetNames.has(n))) {
|
|
795
|
+
resetNames.add(j.jobName);
|
|
796
|
+
changed = true;
|
|
797
|
+
}
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
for (const rj of fresh.jobs) {
|
|
801
|
+
if (!resetNames.has(rj.jobName)) {
|
|
802
|
+
continue;
|
|
803
|
+
}
|
|
804
|
+
const pending = (rj.needs ?? []).filter((n) => resetNames.has(n));
|
|
805
|
+
for (const s of rj.steps) {
|
|
806
|
+
await stateStore.updateStep(run.id, rj.id, s.id, (draft) => {
|
|
807
|
+
draft.status = "queued";
|
|
808
|
+
draft.startedAt = void 0;
|
|
809
|
+
draft.finishedAt = void 0;
|
|
810
|
+
draft.error = void 0;
|
|
811
|
+
draft.outputs = void 0;
|
|
812
|
+
});
|
|
813
|
+
}
|
|
814
|
+
await stateStore.updateJob(run.id, rj.id, (draft) => {
|
|
815
|
+
draft.status = "queued";
|
|
816
|
+
draft.startedAt = void 0;
|
|
817
|
+
draft.finishedAt = void 0;
|
|
818
|
+
draft.pendingDependencies = [...pending];
|
|
819
|
+
draft.blocked = pending.length > 0;
|
|
820
|
+
});
|
|
821
|
+
}
|
|
822
|
+
const refreshed = await engine.getRun(run.id);
|
|
823
|
+
const targetJob = refreshed?.jobs.find((j) => j.jobName === restartFrom);
|
|
824
|
+
if (targetJob && !targetJob.blocked) {
|
|
825
|
+
await scheduler.enqueueJob(run.id, targetJob, targetJob.priority ?? "normal");
|
|
826
|
+
}
|
|
827
|
+
stepLogger.warn("[gate] Cross-job restart re-enqueued target", {
|
|
828
|
+
runId: run.id,
|
|
829
|
+
restartFrom,
|
|
830
|
+
iteration: nextIteration,
|
|
831
|
+
resetJobs: [...resetNames]
|
|
832
|
+
});
|
|
833
|
+
return;
|
|
834
|
+
}
|
|
835
|
+
await engine.markStepCompleted(run.id, job.id, step.id, outputs);
|
|
836
|
+
let foundTarget = false;
|
|
837
|
+
for (const s of job.steps) {
|
|
838
|
+
if (s.spec.id === restartFrom || s.id === restartFrom) {
|
|
839
|
+
foundTarget = true;
|
|
840
|
+
}
|
|
841
|
+
if (foundTarget) {
|
|
842
|
+
await stateStore.updateStep(run.id, job.id, s.id, (draft) => {
|
|
843
|
+
draft.status = "queued";
|
|
844
|
+
draft.startedAt = void 0;
|
|
845
|
+
draft.finishedAt = void 0;
|
|
846
|
+
draft.error = void 0;
|
|
847
|
+
draft.outputs = void 0;
|
|
848
|
+
});
|
|
849
|
+
}
|
|
850
|
+
}
|
|
851
|
+
await stateStore.updateJob(run.id, job.id, (draft) => {
|
|
852
|
+
draft.status = "queued";
|
|
853
|
+
draft.startedAt = void 0;
|
|
854
|
+
draft.finishedAt = void 0;
|
|
855
|
+
});
|
|
856
|
+
const updatedRun = await engine.getRun(run.id);
|
|
857
|
+
const updatedJob = updatedRun?.jobs.find((j) => j.id === job.id);
|
|
858
|
+
if (updatedJob) {
|
|
859
|
+
await scheduler.enqueueJob(run.id, updatedJob, updatedJob.priority ?? "normal");
|
|
860
|
+
stepLogger.info("[gate] Job re-enqueued for restart", {
|
|
861
|
+
runId: run.id,
|
|
862
|
+
jobId: job.id,
|
|
863
|
+
jobName: job.jobName,
|
|
864
|
+
iteration: nextIteration,
|
|
865
|
+
restartFrom
|
|
866
|
+
});
|
|
867
|
+
}
|
|
868
|
+
}
|
|
600
869
|
function inferWorkspaceProvisionReasonCode(message) {
|
|
601
870
|
return /ETIMEDOUT|timeout/iu.test(message) ? "workspace_provision_timeout" : "workspace_provision_failed";
|
|
602
871
|
}
|
|
@@ -633,7 +902,6 @@ var JobBroker = class {
|
|
|
633
902
|
id: "execute",
|
|
634
903
|
name: "Execute handler",
|
|
635
904
|
uses,
|
|
636
|
-
// @ts-expect-error - WorkflowSpec step.with type mismatch
|
|
637
905
|
with: request.input ?? {}
|
|
638
906
|
}
|
|
639
907
|
]
|
|
@@ -641,6 +909,7 @@ var JobBroker = class {
|
|
|
641
909
|
}
|
|
642
910
|
};
|
|
643
911
|
const run = await this.engine.runFromInline(spec, {
|
|
912
|
+
trigger: { type: "manual" },
|
|
644
913
|
env: {},
|
|
645
914
|
metadata: request.metadata
|
|
646
915
|
});
|
|
@@ -651,20 +920,6 @@ var JobBroker = class {
|
|
|
651
920
|
});
|
|
652
921
|
return run;
|
|
653
922
|
}
|
|
654
|
-
/**
|
|
655
|
-
* Schedule a recurring job with cron expression.
|
|
656
|
-
* Registers job with CronScheduler (if available).
|
|
657
|
-
*
|
|
658
|
-
* NOTE: CronScheduler integration not yet implemented.
|
|
659
|
-
* This is a placeholder for future implementation.
|
|
660
|
-
*/
|
|
661
|
-
async schedule(request) {
|
|
662
|
-
this.logger.warn("CronScheduler not yet implemented", {
|
|
663
|
-
handler: request.handler,
|
|
664
|
-
cron: request.cron
|
|
665
|
-
});
|
|
666
|
-
throw new Error("CronScheduler not yet implemented");
|
|
667
|
-
}
|
|
668
923
|
/**
|
|
669
924
|
* Get job status by run ID.
|
|
670
925
|
*/
|
|
@@ -701,22 +956,38 @@ var JobBroker = class {
|
|
|
701
956
|
if (!run) {
|
|
702
957
|
return [];
|
|
703
958
|
}
|
|
959
|
+
const stepNameMap = /* @__PURE__ */ new Map();
|
|
960
|
+
for (const job of run.jobs ?? []) {
|
|
961
|
+
for (const step of job.steps ?? []) {
|
|
962
|
+
if (step.id && step.name) {
|
|
963
|
+
stepNameMap.set(step.id, step.name);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
}
|
|
704
967
|
const limit = options?.limit ?? 100;
|
|
705
968
|
const offset = options?.offset ?? 0;
|
|
706
969
|
const startTime = run.startedAt ? new Date(run.startedAt).getTime() : Date.now() - 36e5;
|
|
707
970
|
const endTime = run.finishedAt ? new Date(run.finishedAt).getTime() : Date.now();
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
971
|
+
let queryResult = { logs: [] };
|
|
972
|
+
try {
|
|
973
|
+
queryResult = await this.platform.logs.query(
|
|
974
|
+
{
|
|
975
|
+
from: startTime,
|
|
976
|
+
to: endTime,
|
|
977
|
+
level: options?.level && options.level !== "all" ? options.level : void 0
|
|
978
|
+
},
|
|
979
|
+
{
|
|
980
|
+
limit: 2e3,
|
|
981
|
+
offset: 0
|
|
982
|
+
}
|
|
983
|
+
);
|
|
984
|
+
} catch {
|
|
985
|
+
this.logger.warn("Log backend unavailable \u2014 returning empty log list for run", { runId });
|
|
986
|
+
}
|
|
987
|
+
const filtered = (queryResult.logs ?? []).filter((log) => {
|
|
988
|
+
if (!log.fields) {
|
|
989
|
+
return false;
|
|
717
990
|
}
|
|
718
|
-
);
|
|
719
|
-
const filtered = queryResult.logs.filter((log) => {
|
|
720
991
|
if (log.fields["runId"] !== runId) {
|
|
721
992
|
return false;
|
|
722
993
|
}
|
|
@@ -727,12 +998,18 @@ var JobBroker = class {
|
|
|
727
998
|
});
|
|
728
999
|
filtered.sort((a, b) => a.timestamp - b.timestamp);
|
|
729
1000
|
const page = filtered.slice(offset, offset + limit);
|
|
730
|
-
return page.map((log) =>
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
1001
|
+
return page.map((log) => {
|
|
1002
|
+
const stepId = log.fields["stepId"];
|
|
1003
|
+
return {
|
|
1004
|
+
timestamp: new Date(log.timestamp).toISOString(),
|
|
1005
|
+
level: log.level,
|
|
1006
|
+
message: log.message,
|
|
1007
|
+
stepId,
|
|
1008
|
+
stepName: stepId ? stepNameMap.get(stepId) : void 0,
|
|
1009
|
+
stream: log.fields["logSource"],
|
|
1010
|
+
context: log.fields
|
|
1011
|
+
};
|
|
1012
|
+
});
|
|
736
1013
|
}
|
|
737
1014
|
};
|
|
738
1015
|
var CronScheduler = class {
|
|
@@ -920,7 +1197,6 @@ var CronScheduler = class {
|
|
|
920
1197
|
jobs: job.workflowSpec.jobs,
|
|
921
1198
|
env: job.workflowSpec.env
|
|
922
1199
|
};
|
|
923
|
-
console.log("\u{1F50D} CRON SPEC:", JSON.stringify(spec, null, 2));
|
|
924
1200
|
this.logger.debug("Running workflow from cron", {
|
|
925
1201
|
cronJobId,
|
|
926
1202
|
spec: JSON.stringify(spec, null, 2)
|
|
@@ -1135,6 +1411,23 @@ var CronScheduler = class {
|
|
|
1135
1411
|
});
|
|
1136
1412
|
this.registeredJobs.clear();
|
|
1137
1413
|
}
|
|
1414
|
+
/**
|
|
1415
|
+
* Unregister all user-sourced cron jobs (source === 'user'), stopping their
|
|
1416
|
+
* scheduled tasks. Safe to call while the scheduler is running — jobs are
|
|
1417
|
+
* stopped and removed so they can be re-registered by a fresh discovery pass.
|
|
1418
|
+
*/
|
|
1419
|
+
clearUserJobs() {
|
|
1420
|
+
const userJobIds = Array.from(this.registeredJobs.keys()).filter((id) => id.startsWith("user:"));
|
|
1421
|
+
for (const cronJobId of userJobIds) {
|
|
1422
|
+
const task = this.scheduledTasks.get(cronJobId);
|
|
1423
|
+
if (task) {
|
|
1424
|
+
task.stop();
|
|
1425
|
+
this.scheduledTasks.delete(cronJobId);
|
|
1426
|
+
}
|
|
1427
|
+
this.registeredJobs.delete(cronJobId);
|
|
1428
|
+
}
|
|
1429
|
+
this.logger.info("Cleared user cron jobs", { count: userJobIds.length });
|
|
1430
|
+
}
|
|
1138
1431
|
};
|
|
1139
1432
|
var CronDiscovery = class {
|
|
1140
1433
|
cliApi;
|
|
@@ -1250,6 +1543,92 @@ var CronDiscovery = class {
|
|
|
1250
1543
|
return count;
|
|
1251
1544
|
}
|
|
1252
1545
|
};
|
|
1546
|
+
var WorkflowFileWatcher = class {
|
|
1547
|
+
watchers = [];
|
|
1548
|
+
workflowService;
|
|
1549
|
+
cronDiscovery;
|
|
1550
|
+
cronScheduler;
|
|
1551
|
+
logger;
|
|
1552
|
+
debounceMs;
|
|
1553
|
+
debounceTimer = null;
|
|
1554
|
+
constructor(options) {
|
|
1555
|
+
this.workflowService = options.workflowService;
|
|
1556
|
+
this.cronDiscovery = options.cronDiscovery;
|
|
1557
|
+
this.cronScheduler = options.cronScheduler;
|
|
1558
|
+
this.logger = options.logger;
|
|
1559
|
+
this.debounceMs = options.debounceMs ?? 300;
|
|
1560
|
+
for (const dir of options.watchDirs) {
|
|
1561
|
+
this.startWatcher(dir);
|
|
1562
|
+
}
|
|
1563
|
+
}
|
|
1564
|
+
startWatcher(dir) {
|
|
1565
|
+
access(dir).then(() => {
|
|
1566
|
+
try {
|
|
1567
|
+
const watcher = watch(dir, { persistent: false }, (eventType, filename) => {
|
|
1568
|
+
if (!filename) {
|
|
1569
|
+
return;
|
|
1570
|
+
}
|
|
1571
|
+
if (!filename.endsWith(".yml") && !filename.endsWith(".yaml")) {
|
|
1572
|
+
return;
|
|
1573
|
+
}
|
|
1574
|
+
this.scheduleReload(dir, filename);
|
|
1575
|
+
});
|
|
1576
|
+
watcher.on("error", (err) => {
|
|
1577
|
+
this.logger.warn("WorkflowFileWatcher: watcher error", {
|
|
1578
|
+
dir,
|
|
1579
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1580
|
+
});
|
|
1581
|
+
});
|
|
1582
|
+
this.watchers.push(watcher);
|
|
1583
|
+
this.logger.info("WorkflowFileWatcher: watching directory", { dir });
|
|
1584
|
+
} catch (err) {
|
|
1585
|
+
this.logger.warn("WorkflowFileWatcher: could not start watcher", {
|
|
1586
|
+
dir,
|
|
1587
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1588
|
+
});
|
|
1589
|
+
}
|
|
1590
|
+
}).catch(() => {
|
|
1591
|
+
this.logger.debug("WorkflowFileWatcher: directory does not exist, skipping", { dir });
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1594
|
+
scheduleReload(dir, filename) {
|
|
1595
|
+
if (this.debounceTimer) {
|
|
1596
|
+
clearTimeout(this.debounceTimer);
|
|
1597
|
+
}
|
|
1598
|
+
this.debounceTimer = setTimeout(() => {
|
|
1599
|
+
this.debounceTimer = null;
|
|
1600
|
+
this.reload(dir, filename).catch((err) => {
|
|
1601
|
+
this.logger.error(
|
|
1602
|
+
"WorkflowFileWatcher: reload failed",
|
|
1603
|
+
err instanceof Error ? err : void 0,
|
|
1604
|
+
{ dir, filename }
|
|
1605
|
+
);
|
|
1606
|
+
});
|
|
1607
|
+
}, this.debounceMs);
|
|
1608
|
+
}
|
|
1609
|
+
async reload(dir, filename) {
|
|
1610
|
+
this.logger.info("WorkflowFileWatcher: YAML change detected, reloading", { dir, filename });
|
|
1611
|
+
await this.workflowService.refreshManifests();
|
|
1612
|
+
this.cronScheduler.clearUserJobs();
|
|
1613
|
+
const discovered = await this.cronDiscovery.discoverAll();
|
|
1614
|
+
this.logger.info("WorkflowFileWatcher: reload complete", {
|
|
1615
|
+
filename,
|
|
1616
|
+
cronJobs: discovered
|
|
1617
|
+
});
|
|
1618
|
+
}
|
|
1619
|
+
/** Stop all watchers and cancel any pending debounce timer. */
|
|
1620
|
+
close() {
|
|
1621
|
+
if (this.debounceTimer) {
|
|
1622
|
+
clearTimeout(this.debounceTimer);
|
|
1623
|
+
this.debounceTimer = null;
|
|
1624
|
+
}
|
|
1625
|
+
for (const watcher of this.watchers) {
|
|
1626
|
+
watcher.close();
|
|
1627
|
+
}
|
|
1628
|
+
this.watchers.length = 0;
|
|
1629
|
+
this.logger.info("WorkflowFileWatcher: stopped");
|
|
1630
|
+
}
|
|
1631
|
+
};
|
|
1253
1632
|
|
|
1254
1633
|
// src/host/workflow-host-service.ts
|
|
1255
1634
|
var TENANT_ID_PATTERN = /^[a-zA-Z0-9_-]+$/;
|
|
@@ -1349,7 +1728,11 @@ var WorkflowHostService = class {
|
|
|
1349
1728
|
return logs;
|
|
1350
1729
|
}
|
|
1351
1730
|
async getRunLogs(runId, options) {
|
|
1352
|
-
const
|
|
1731
|
+
const resolved = await this.resolveRunId(runId);
|
|
1732
|
+
if (!resolved) {
|
|
1733
|
+
throw new Error("Run not found");
|
|
1734
|
+
}
|
|
1735
|
+
const logs = await this.options.jobBroker.getRunLogs(resolved, options);
|
|
1353
1736
|
return logs;
|
|
1354
1737
|
}
|
|
1355
1738
|
async cancelJob(tenantId, jobId) {
|
|
@@ -1373,8 +1756,8 @@ var WorkflowHostService = class {
|
|
|
1373
1756
|
error: run.result?.error?.message
|
|
1374
1757
|
}));
|
|
1375
1758
|
if (type) {
|
|
1376
|
-
const
|
|
1377
|
-
const regex = new RegExp(`^${
|
|
1759
|
+
const escaped = type.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
1760
|
+
const regex = new RegExp(`^${escaped}$`);
|
|
1378
1761
|
jobs = jobs.filter((job) => regex.test(job.type));
|
|
1379
1762
|
}
|
|
1380
1763
|
if (status) {
|
|
@@ -1429,6 +1812,15 @@ var WorkflowHostService = class {
|
|
|
1429
1812
|
const triggerType = request.trigger?.type === "cron" ? "schedule" : request.trigger?.type === "api" ? "webhook" : "manual";
|
|
1430
1813
|
const specInputDefs = specInput["inputs"] ?? {};
|
|
1431
1814
|
const userInputs = request.inputs ?? (request.input && typeof request.input === "object" ? request.input : {});
|
|
1815
|
+
const missingRequired = [];
|
|
1816
|
+
for (const [key, def] of Object.entries(specInputDefs)) {
|
|
1817
|
+
if (def.required && !(key in userInputs) && def.default === void 0) {
|
|
1818
|
+
missingRequired.push(key);
|
|
1819
|
+
}
|
|
1820
|
+
}
|
|
1821
|
+
if (missingRequired.length > 0) {
|
|
1822
|
+
throw new Error(`Missing required input(s): ${missingRequired.join(", ")}`);
|
|
1823
|
+
}
|
|
1432
1824
|
const resolvedInputs = {};
|
|
1433
1825
|
for (const [key, def] of Object.entries(specInputDefs)) {
|
|
1434
1826
|
resolvedInputs[key] = key in userInputs ? userInputs[key] : def.default;
|
|
@@ -1446,6 +1838,109 @@ var WorkflowHostService = class {
|
|
|
1446
1838
|
status: run.status
|
|
1447
1839
|
};
|
|
1448
1840
|
}
|
|
1841
|
+
async rerunWorkflow(runId, request) {
|
|
1842
|
+
const resolved = await this.resolveRunId(runId);
|
|
1843
|
+
const sourceRun = resolved ? await this.options.engine.getRun(resolved) : null;
|
|
1844
|
+
if (!sourceRun) {
|
|
1845
|
+
throw new Error("Run not found");
|
|
1846
|
+
}
|
|
1847
|
+
if (request.failedOnly) {
|
|
1848
|
+
return this.rerunFailedOnly(sourceRun.id, sourceRun);
|
|
1849
|
+
}
|
|
1850
|
+
const workflowId = sourceRun.metadata?.["workflowId"] ?? sourceRun.name;
|
|
1851
|
+
const workflowService = this.requireWorkflowService();
|
|
1852
|
+
const workflow = await workflowService.get(workflowId);
|
|
1853
|
+
if (!workflow) {
|
|
1854
|
+
throw new Error("Workflow not found");
|
|
1855
|
+
}
|
|
1856
|
+
const specInput = workflow.input;
|
|
1857
|
+
const spec = {
|
|
1858
|
+
...specInput
|
|
1859
|
+
};
|
|
1860
|
+
const resolvedInputs = sourceRun.inputs ?? {};
|
|
1861
|
+
const run = await this.options.engine.runFromSpec(spec, {
|
|
1862
|
+
trigger: {
|
|
1863
|
+
type: "manual",
|
|
1864
|
+
actor: "cli-rerun",
|
|
1865
|
+
payload: resolvedInputs
|
|
1866
|
+
},
|
|
1867
|
+
inputs: resolvedInputs
|
|
1868
|
+
});
|
|
1869
|
+
return {
|
|
1870
|
+
runId: run.id,
|
|
1871
|
+
status: run.status
|
|
1872
|
+
};
|
|
1873
|
+
}
|
|
1874
|
+
/**
|
|
1875
|
+
* failedOnly reruns resume from the source run's snapshot instead of
|
|
1876
|
+
* starting a brand-new run (via runFromSpec + a job-filtered spec), so
|
|
1877
|
+
* completed step outputs from jobs that are not being rerun are carried
|
|
1878
|
+
* forward into the interpolation context instead of being discarded
|
|
1879
|
+
* (see issue #263: `rerun --failed-only` used to redo the whole run).
|
|
1880
|
+
*/
|
|
1881
|
+
async rerunFailedOnly(resolvedRunId, sourceRun) {
|
|
1882
|
+
const failedJobNames = new Set(
|
|
1883
|
+
(sourceRun.jobs ?? []).filter((job) => job.status === "failed" || job.status === "interrupted").map((job) => job.jobName)
|
|
1884
|
+
);
|
|
1885
|
+
if (failedJobNames.size === 0) {
|
|
1886
|
+
throw new Error("No failed jobs to rerun");
|
|
1887
|
+
}
|
|
1888
|
+
let fromStepId;
|
|
1889
|
+
for (const job of sourceRun.jobs ?? []) {
|
|
1890
|
+
if (!failedJobNames.has(job.jobName)) {
|
|
1891
|
+
continue;
|
|
1892
|
+
}
|
|
1893
|
+
const unfinishedStep = (job.steps ?? []).find(
|
|
1894
|
+
(s) => s.status !== "success" && s.status !== "skipped"
|
|
1895
|
+
);
|
|
1896
|
+
if (unfinishedStep) {
|
|
1897
|
+
fromStepId = unfinishedStep.id;
|
|
1898
|
+
break;
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
if (!fromStepId) {
|
|
1902
|
+
throw new Error(`Cannot determine resume point: no failed step found in run ${sourceRun.id}`);
|
|
1903
|
+
}
|
|
1904
|
+
const run = await this.options.engine.replayRun(resolvedRunId, { fromStepId });
|
|
1905
|
+
if (!run) {
|
|
1906
|
+
throw new Error(
|
|
1907
|
+
`No replay snapshot for run ${resolvedRunId} \u2014 cannot rerun failed steps only. Snapshots are created when a run finishes; check the workflow daemon logs for "Failed to create run snapshot" around that time.`
|
|
1908
|
+
);
|
|
1909
|
+
}
|
|
1910
|
+
return {
|
|
1911
|
+
runId: run.id,
|
|
1912
|
+
status: run.status
|
|
1913
|
+
};
|
|
1914
|
+
}
|
|
1915
|
+
async restartRun(runId, request) {
|
|
1916
|
+
const resolved = await this.resolveRunId(runId);
|
|
1917
|
+
if (!resolved) {
|
|
1918
|
+
throw new Error(`Run not found: ${runId}`);
|
|
1919
|
+
}
|
|
1920
|
+
const currentRun = await this.options.engine.getRun(resolved);
|
|
1921
|
+
if (!currentRun) {
|
|
1922
|
+
throw new Error(`Run not found: ${resolved}`);
|
|
1923
|
+
}
|
|
1924
|
+
if (currentRun.status === "running" || currentRun.status === "queued") {
|
|
1925
|
+
throw new Error(
|
|
1926
|
+
`Cannot restart an active run (status: ${currentRun.status}); cancel it first`
|
|
1927
|
+
);
|
|
1928
|
+
}
|
|
1929
|
+
const run = await this.options.engine.replayRun(resolved, {
|
|
1930
|
+
fromStepId: request.fromStepId,
|
|
1931
|
+
env: request.env
|
|
1932
|
+
});
|
|
1933
|
+
if (!run) {
|
|
1934
|
+
throw new Error(
|
|
1935
|
+
`No replay snapshot for run ${resolved} (status: ${currentRun.status}, finished: ${currentRun.finishedAt ?? "n/a"}). Snapshots are created when a run finishes; if this run finished but has no snapshot, check the workflow daemon logs for "Failed to create run snapshot" around that time.`
|
|
1936
|
+
);
|
|
1937
|
+
}
|
|
1938
|
+
return {
|
|
1939
|
+
runId: run.id,
|
|
1940
|
+
status: run.status,
|
|
1941
|
+
fromStepId: request.fromStepId
|
|
1942
|
+
};
|
|
1943
|
+
}
|
|
1449
1944
|
registerCron(tenantId, request) {
|
|
1450
1945
|
this.assertTenantId(tenantId);
|
|
1451
1946
|
const scheduler = this.requireCronScheduler();
|
|
@@ -1580,11 +2075,28 @@ var WorkflowHostService = class {
|
|
|
1580
2075
|
pluginId: workflow.pluginId,
|
|
1581
2076
|
status: workflow.status === "active" ? "active" : "inactive",
|
|
1582
2077
|
tags: workflow.tags,
|
|
1583
|
-
inputs: workflow.inputSchema
|
|
2078
|
+
inputs: workflow.inputSchema,
|
|
2079
|
+
version: workflow.version,
|
|
2080
|
+
updatedAt: workflow.updatedAt
|
|
1584
2081
|
};
|
|
1585
2082
|
}
|
|
2083
|
+
async resolveRunId(idOrPrefix) {
|
|
2084
|
+
const run = await this.options.engine.getRun(idOrPrefix);
|
|
2085
|
+
if (run) {
|
|
2086
|
+
return run.id;
|
|
2087
|
+
}
|
|
2088
|
+
if (idOrPrefix.length <= 8 && !idOrPrefix.includes("-")) {
|
|
2089
|
+
const all = await this.options.engine.getAllRuns();
|
|
2090
|
+
return all.find((r) => r.id.startsWith(idOrPrefix))?.id ?? null;
|
|
2091
|
+
}
|
|
2092
|
+
return null;
|
|
2093
|
+
}
|
|
1586
2094
|
async getRun(runId) {
|
|
1587
|
-
|
|
2095
|
+
const resolved = await this.resolveRunId(runId);
|
|
2096
|
+
if (!resolved) {
|
|
2097
|
+
return null;
|
|
2098
|
+
}
|
|
2099
|
+
return await this.options.engine.getRun(resolved);
|
|
1588
2100
|
}
|
|
1589
2101
|
async listRuns(filters) {
|
|
1590
2102
|
const allRuns = await this.options.engine.getAllRuns();
|
|
@@ -1592,21 +2104,40 @@ var WorkflowHostService = class {
|
|
|
1592
2104
|
if (filters?.status) {
|
|
1593
2105
|
runs = runs.filter((run) => run.status === filters.status);
|
|
1594
2106
|
}
|
|
2107
|
+
if (filters?.workflowId) {
|
|
2108
|
+
runs = runs.filter((run) => run.name === filters.workflowId);
|
|
2109
|
+
}
|
|
1595
2110
|
runs.sort((a, b) => new Date(b.createdAt ?? 0).getTime() - new Date(a.createdAt ?? 0).getTime());
|
|
1596
2111
|
const total = runs.length;
|
|
1597
2112
|
const start = filters?.offset ?? 0;
|
|
1598
2113
|
const end = filters?.limit ? start + filters.limit : runs.length;
|
|
1599
|
-
|
|
2114
|
+
const page = runs.slice(start, end).map((run) => {
|
|
2115
|
+
const allSteps = (run.jobs ?? []).flatMap((j) => j.steps ?? []);
|
|
2116
|
+
const activeSteps = allSteps.filter(
|
|
2117
|
+
(s) => s.status === "running" || s.status === "waiting_approval"
|
|
2118
|
+
);
|
|
2119
|
+
let currentStepName;
|
|
2120
|
+
if (run.status === "running" && activeSteps.length > 0) {
|
|
2121
|
+
currentStepName = activeSteps.length === 1 ? activeSteps[0].name : `${activeSteps[0].name} (+${activeSteps.length - 1})`;
|
|
2122
|
+
}
|
|
2123
|
+
return {
|
|
2124
|
+
...run,
|
|
2125
|
+
hasPendingApproval: run.status === "running" && allSteps.some((s) => s.status === "waiting_approval"),
|
|
2126
|
+
currentStepName
|
|
2127
|
+
};
|
|
2128
|
+
});
|
|
2129
|
+
return { runs: page, total };
|
|
1600
2130
|
}
|
|
1601
2131
|
async cancelRun(runId) {
|
|
1602
|
-
const
|
|
2132
|
+
const resolved = await this.resolveRunId(runId);
|
|
2133
|
+
const run = resolved ? await this.options.engine.getRun(resolved) : null;
|
|
1603
2134
|
if (!run) {
|
|
1604
2135
|
throw new Error("Run not found");
|
|
1605
2136
|
}
|
|
1606
2137
|
if (run.status !== "running" && run.status !== "queued") {
|
|
1607
2138
|
throw new Error(`Cannot cancel run with status "${run.status}"`);
|
|
1608
2139
|
}
|
|
1609
|
-
await this.options.engine.cancelRun(
|
|
2140
|
+
await this.options.engine.cancelRun(resolved);
|
|
1610
2141
|
}
|
|
1611
2142
|
requireWorkflowService() {
|
|
1612
2143
|
if (!this.options.workflowService) {
|
|
@@ -1908,7 +2439,7 @@ var KEEP_ALIVE_MS = 3e4;
|
|
|
1908
2439
|
var IDLE_TIMEOUT_MS = 6e4;
|
|
1909
2440
|
function registerWorkflowsAPI(options) {
|
|
1910
2441
|
const { server, hostService, engine, workflowService, logger, observability } = options;
|
|
1911
|
-
|
|
2442
|
+
const reloadHandler = async () => {
|
|
1912
2443
|
try {
|
|
1913
2444
|
logger.info("[workflows-api] Refreshing workflows from disk");
|
|
1914
2445
|
if (workflowService) {
|
|
@@ -1923,7 +2454,9 @@ function registerWorkflowsAPI(options) {
|
|
|
1923
2454
|
logger.error("[workflows-api] Failed to refresh workflows", error instanceof Error ? error : void 0);
|
|
1924
2455
|
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
1925
2456
|
}
|
|
1926
|
-
}
|
|
2457
|
+
};
|
|
2458
|
+
server.post("/api/v1/workflows/reload", { schema: { tags: ["Workflows"], summary: "Reload workflow definitions from disk" } }, reloadHandler);
|
|
2459
|
+
server.post("/api/v1/workflows/refresh", { schema: { tags: ["Workflows"], summary: "Reload workflow definitions from disk (alias for /reload)" } }, reloadHandler);
|
|
1927
2460
|
server.get("/api/v1/workflows", { schema: { tags: ["Workflows"], summary: "List workflow definitions" } }, async (request, reply) => {
|
|
1928
2461
|
try {
|
|
1929
2462
|
const response = await observability.observeOperation("workflow.catalog.list", () => hostService.listWorkflows(request.query));
|
|
@@ -1974,6 +2507,9 @@ function registerWorkflowsAPI(options) {
|
|
|
1974
2507
|
if (message === "Workflow not found") {
|
|
1975
2508
|
return fail(reply, 404, message);
|
|
1976
2509
|
}
|
|
2510
|
+
if (message.startsWith("Missing required input")) {
|
|
2511
|
+
return fail(reply, 400, message);
|
|
2512
|
+
}
|
|
1977
2513
|
logger.error("[workflows-api] Error running workflow", error instanceof Error ? error : void 0);
|
|
1978
2514
|
return fail(reply, 500, message);
|
|
1979
2515
|
}
|
|
@@ -1995,13 +2531,54 @@ function registerWorkflowsAPI(options) {
|
|
|
1995
2531
|
return fail(reply, 500, message);
|
|
1996
2532
|
}
|
|
1997
2533
|
});
|
|
2534
|
+
server.post("/api/v1/runs/:runId/rerun", { schema: { tags: ["Runs"], summary: "Rerun a workflow run" } }, async (request, reply) => {
|
|
2535
|
+
try {
|
|
2536
|
+
const { runId } = request.params;
|
|
2537
|
+
const response = await observability.observeOperation(
|
|
2538
|
+
"workflow.run.rerun",
|
|
2539
|
+
() => hostService.rerunWorkflow(runId, request.body ?? {})
|
|
2540
|
+
);
|
|
2541
|
+
return ok(response);
|
|
2542
|
+
} catch (error) {
|
|
2543
|
+
const message = error instanceof Error ? error.message : "Failed to rerun workflow";
|
|
2544
|
+
if (message === "Run not found" || message === "Workflow not found") {
|
|
2545
|
+
return fail(reply, 404, message);
|
|
2546
|
+
}
|
|
2547
|
+
if (message === "No failed jobs to rerun") {
|
|
2548
|
+
return fail(reply, 400, message);
|
|
2549
|
+
}
|
|
2550
|
+
logger.error("[workflows-api] Error rerunning workflow", error instanceof Error ? error : void 0);
|
|
2551
|
+
return fail(reply, 500, message);
|
|
2552
|
+
}
|
|
2553
|
+
});
|
|
2554
|
+
server.post("/api/v1/runs/:runId/restart", { schema: { tags: ["Runs"], summary: "Restart a run from a specific step (snapshot-based)" } }, async (request, reply) => {
|
|
2555
|
+
try {
|
|
2556
|
+
const { runId } = request.params;
|
|
2557
|
+
const response = await observability.observeOperation(
|
|
2558
|
+
"workflow.run.restart",
|
|
2559
|
+
() => hostService.restartRun(runId, request.body ?? {})
|
|
2560
|
+
);
|
|
2561
|
+
return ok(response);
|
|
2562
|
+
} catch (error) {
|
|
2563
|
+
const message = error instanceof Error ? error.message : "Failed to restart run";
|
|
2564
|
+
if (message.includes("not found") || message.includes("snapshot not available")) {
|
|
2565
|
+
return fail(reply, 404, message);
|
|
2566
|
+
}
|
|
2567
|
+
if (message.startsWith("Cannot restart an active run")) {
|
|
2568
|
+
return fail(reply, 409, message);
|
|
2569
|
+
}
|
|
2570
|
+
logger.error("[workflows-api] Error restarting run", error instanceof Error ? error : void 0);
|
|
2571
|
+
return fail(reply, 500, message);
|
|
2572
|
+
}
|
|
2573
|
+
});
|
|
1998
2574
|
server.get("/api/v1/runs", { schema: { tags: ["Runs"], summary: "List all workflow runs" } }, async (request, reply) => {
|
|
1999
2575
|
try {
|
|
2000
|
-
const { status, limit, offset } = request.query;
|
|
2576
|
+
const { status, workflowId, limit, offset } = request.query;
|
|
2001
2577
|
const response = await observability.observeOperation(
|
|
2002
2578
|
"workflow.run.list",
|
|
2003
2579
|
() => hostService.listRuns({
|
|
2004
2580
|
status,
|
|
2581
|
+
workflowId,
|
|
2005
2582
|
limit: limit ? parseInt(limit, 10) : 50,
|
|
2006
2583
|
offset: offset ? parseInt(offset, 10) : 0
|
|
2007
2584
|
})
|
|
@@ -2053,7 +2630,8 @@ function registerWorkflowsAPI(options) {
|
|
|
2053
2630
|
}
|
|
2054
2631
|
);
|
|
2055
2632
|
server.get("/api/v1/runs/:runId/events", { schema: { hide: true } }, async (request, reply) => {
|
|
2056
|
-
const { runId } = request.params;
|
|
2633
|
+
const { runId: rawId } = request.params;
|
|
2634
|
+
const runId = await hostService.resolveRunId(rawId) ?? rawId;
|
|
2057
2635
|
const run = await observability.observeOperation("workflow.run.events", () => engine.getRun(runId));
|
|
2058
2636
|
if (!run) {
|
|
2059
2637
|
return fail(reply, 404, "Run not found");
|
|
@@ -2074,7 +2652,7 @@ function registerWorkflowsAPI(options) {
|
|
|
2074
2652
|
if (raw.writableEnded) {
|
|
2075
2653
|
return;
|
|
2076
2654
|
}
|
|
2077
|
-
raw.write(`event:
|
|
2655
|
+
raw.write(`event: ${type}
|
|
2078
2656
|
`);
|
|
2079
2657
|
raw.write(`data: ${JSON.stringify({ type, runId, payload, timestamp: (/* @__PURE__ */ new Date()).toISOString() })}
|
|
2080
2658
|
|
|
@@ -2082,6 +2660,8 @@ function registerWorkflowsAPI(options) {
|
|
|
2082
2660
|
};
|
|
2083
2661
|
sendEvent("run.snapshot", run);
|
|
2084
2662
|
if (TERMINAL_STATUSES.includes(run.status)) {
|
|
2663
|
+
const terminalType = run.status === "success" ? "run.finished" : run.status === "failed" ? "run.failed" : "run.cancelled";
|
|
2664
|
+
sendEvent(terminalType, run);
|
|
2085
2665
|
raw.end();
|
|
2086
2666
|
return;
|
|
2087
2667
|
}
|
|
@@ -2125,11 +2705,11 @@ function registerWorkflowsAPI(options) {
|
|
|
2125
2705
|
|
|
2126
2706
|
// src/api/approvals-api.ts
|
|
2127
2707
|
function registerApprovalsAPI(options) {
|
|
2128
|
-
const { server, engine, logger, observability } = options;
|
|
2708
|
+
const { server, hostService, engine, logger, observability } = options;
|
|
2129
2709
|
server.get("/api/v1/runs/:runId/approvals", { schema: { tags: ["Approvals"], summary: "List pending approvals for a run" } }, async (request, reply) => {
|
|
2130
2710
|
try {
|
|
2131
|
-
const { runId } = request.params;
|
|
2132
|
-
const run = await observability.observeOperation("workflow.approval.list", () =>
|
|
2711
|
+
const { runId: rawId } = request.params;
|
|
2712
|
+
const run = await observability.observeOperation("workflow.approval.list", () => hostService.getRun(rawId));
|
|
2133
2713
|
if (!run) {
|
|
2134
2714
|
return fail(reply, 404, "Run not found");
|
|
2135
2715
|
}
|
|
@@ -2142,13 +2722,13 @@ function registerApprovalsAPI(options) {
|
|
|
2142
2722
|
stepId: step.id,
|
|
2143
2723
|
stepName: step.name,
|
|
2144
2724
|
specId: step.spec.id,
|
|
2145
|
-
context: step.spec.with ?? {},
|
|
2725
|
+
context: step.resolvedInputs ?? step.spec.with ?? {},
|
|
2146
2726
|
waitingSince: step.startedAt
|
|
2147
2727
|
});
|
|
2148
2728
|
}
|
|
2149
2729
|
}
|
|
2150
2730
|
}
|
|
2151
|
-
return ok({ runId, pending });
|
|
2731
|
+
return ok({ runId: run.id, pending });
|
|
2152
2732
|
} catch (error) {
|
|
2153
2733
|
logger.error("[approvals-api] Error listing pending approvals", error instanceof Error ? error : void 0);
|
|
2154
2734
|
return fail(reply, 500, error instanceof Error ? error.message : "Failed to list pending approvals");
|
|
@@ -2156,7 +2736,7 @@ function registerApprovalsAPI(options) {
|
|
|
2156
2736
|
});
|
|
2157
2737
|
server.post("/api/v1/runs/:runId/approvals/resolve", { schema: { tags: ["Approvals"], summary: "Approve or reject a pending step" } }, async (request, reply) => {
|
|
2158
2738
|
try {
|
|
2159
|
-
const { runId } = request.params;
|
|
2739
|
+
const { runId: rawId } = request.params;
|
|
2160
2740
|
const { jobId, stepId, action, comment, data } = request.body;
|
|
2161
2741
|
if (!jobId || !stepId || !action) {
|
|
2162
2742
|
return fail(reply, 400, "Missing required fields: jobId, stepId, action");
|
|
@@ -2164,7 +2744,7 @@ function registerApprovalsAPI(options) {
|
|
|
2164
2744
|
if (action !== "approve" && action !== "reject") {
|
|
2165
2745
|
return fail(reply, 400, 'action must be "approve" or "reject"');
|
|
2166
2746
|
}
|
|
2167
|
-
const run = await observability.observeOperation("workflow.approval.get", () =>
|
|
2747
|
+
const run = await observability.observeOperation("workflow.approval.get", () => hostService.getRun(rawId));
|
|
2168
2748
|
if (!run) {
|
|
2169
2749
|
return fail(reply, 404, "Run not found");
|
|
2170
2750
|
}
|
|
@@ -2181,17 +2761,17 @@ function registerApprovalsAPI(options) {
|
|
|
2181
2761
|
}
|
|
2182
2762
|
await observability.observeOperation(
|
|
2183
2763
|
"workflow.approval.resolve",
|
|
2184
|
-
() => engine.resolveApproval(
|
|
2764
|
+
() => engine.resolveApproval(run.id, jobId, stepId, action, data, comment)
|
|
2185
2765
|
);
|
|
2186
2766
|
logger.info("[approvals-api] Approval resolved", {
|
|
2187
|
-
runId,
|
|
2767
|
+
runId: run.id,
|
|
2188
2768
|
jobId,
|
|
2189
2769
|
stepId,
|
|
2190
2770
|
action,
|
|
2191
2771
|
comment
|
|
2192
2772
|
});
|
|
2193
2773
|
return ok({
|
|
2194
|
-
runId,
|
|
2774
|
+
runId: run.id,
|
|
2195
2775
|
jobId,
|
|
2196
2776
|
stepId,
|
|
2197
2777
|
action,
|
|
@@ -2278,169 +2858,128 @@ async function createServer(options) {
|
|
|
2278
2858
|
cronScheduler,
|
|
2279
2859
|
logger
|
|
2280
2860
|
});
|
|
2281
|
-
const server = Fastify({
|
|
2282
|
-
logger: false,
|
|
2283
|
-
// Use platform logger instead
|
|
2284
|
-
bodyLimit: 1048576
|
|
2285
|
-
// 1MB body limit (prevents parsing huge payloads)
|
|
2286
|
-
});
|
|
2287
2861
|
const isProduction = process.env.NODE_ENV === "production";
|
|
2288
|
-
|
|
2289
|
-
title: "KB Labs Workflow Daemon",
|
|
2290
|
-
description: "Background job execution and workflow orchestration API",
|
|
2291
|
-
version: "1.0.0",
|
|
2292
|
-
servers: [{ url: "http://localhost:7778", description: "Local dev" }],
|
|
2293
|
-
ui: !isProduction
|
|
2294
|
-
});
|
|
2295
|
-
const requireAuth = process.env.KB_DAEMON_REQUIRE_AUTH === "true" || isProduction;
|
|
2862
|
+
const requireAuth = process.env.KB_DAEMON_REQUIRE_AUTH === "false" ? false : process.env.KB_DAEMON_REQUIRE_AUTH === "true" || isProduction;
|
|
2296
2863
|
const daemonApiKey = process.env.KB_DAEMON_API_KEY;
|
|
2864
|
+
const allowedOrigins = process.env.ALLOWED_ORIGINS?.split(",") ?? [
|
|
2865
|
+
"http://localhost:3000",
|
|
2866
|
+
"http://localhost:5173"
|
|
2867
|
+
];
|
|
2868
|
+
if (requireAuth && !daemonApiKey) {
|
|
2869
|
+
throw new Error(
|
|
2870
|
+
"KB_DAEMON_API_KEY is required when daemon auth is enabled (KB_DAEMON_REQUIRE_AUTH=true or NODE_ENV=production)"
|
|
2871
|
+
);
|
|
2872
|
+
}
|
|
2297
2873
|
const observability = new HttpObservabilityCollector({
|
|
2298
2874
|
serviceId: "workflow",
|
|
2299
2875
|
serviceType: "workflow-daemon",
|
|
2300
2876
|
version: "1.0.0",
|
|
2301
2877
|
logsSource: "workflow",
|
|
2302
2878
|
dependencies: [
|
|
2303
|
-
{
|
|
2304
|
-
serviceId: "state-daemon",
|
|
2305
|
-
required: false,
|
|
2306
|
-
description: "Workflow run and job state storage"
|
|
2307
|
-
}
|
|
2879
|
+
{ serviceId: "state-daemon", required: false, description: "Workflow run and job state storage" }
|
|
2308
2880
|
]
|
|
2309
2881
|
});
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
});
|
|
2330
|
-
observability.register(server);
|
|
2331
|
-
const allowedOrigins = process.env.ALLOWED_ORIGINS?.split(",") || [
|
|
2332
|
-
"http://localhost:3000",
|
|
2333
|
-
"http://localhost:5173"
|
|
2334
|
-
// Vite dev server
|
|
2335
|
-
];
|
|
2336
|
-
await server.register(cors, {
|
|
2337
|
-
origin: (origin, callback) => {
|
|
2338
|
-
if (!origin) {
|
|
2339
|
-
const isDevelopment = process.env.NODE_ENV !== "production";
|
|
2340
|
-
if (isDevelopment) {
|
|
2341
|
-
callback(null, true);
|
|
2882
|
+
return createDaemonServer({
|
|
2883
|
+
serviceId: "workflow",
|
|
2884
|
+
logger,
|
|
2885
|
+
observability,
|
|
2886
|
+
bodyLimit: 1048576,
|
|
2887
|
+
skipStandardRoutes: true,
|
|
2888
|
+
openapi: {
|
|
2889
|
+
title: "KB Labs Workflow Daemon",
|
|
2890
|
+
description: "Background job execution and workflow orchestration API"
|
|
2891
|
+
},
|
|
2892
|
+
cors: {
|
|
2893
|
+
origin: (origin, callback) => {
|
|
2894
|
+
if (!origin) {
|
|
2895
|
+
const isDevelopment = process.env.KB_DAEMON_REQUIRE_AUTH === "false" || process.env.NODE_ENV !== "production";
|
|
2896
|
+
if (isDevelopment) {
|
|
2897
|
+
callback(null, true);
|
|
2898
|
+
} else {
|
|
2899
|
+
callback(new Error("Origin header required in production"), false);
|
|
2900
|
+
}
|
|
2342
2901
|
return;
|
|
2343
2902
|
}
|
|
2344
|
-
|
|
2345
|
-
|
|
2903
|
+
if (allowedOrigins.includes(origin)) {
|
|
2904
|
+
callback(null, true);
|
|
2905
|
+
} else {
|
|
2906
|
+
callback(new Error(`Origin ${origin} not allowed by CORS`), false);
|
|
2907
|
+
}
|
|
2346
2908
|
}
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2909
|
+
},
|
|
2910
|
+
onRequest: async (request, reply) => {
|
|
2911
|
+
if (!requireAuth || request.url === "/health") {
|
|
2912
|
+
return;
|
|
2351
2913
|
}
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
observability
|
|
2359
|
-
});
|
|
2360
|
-
registerCronAPI({
|
|
2361
|
-
server,
|
|
2362
|
-
hostService,
|
|
2363
|
-
logger,
|
|
2364
|
-
observability
|
|
2365
|
-
});
|
|
2366
|
-
if (workflowService) {
|
|
2367
|
-
registerWorkflowsAPI({
|
|
2368
|
-
server,
|
|
2369
|
-
hostService,
|
|
2370
|
-
engine,
|
|
2371
|
-
workflowService,
|
|
2372
|
-
logger,
|
|
2373
|
-
observability
|
|
2374
|
-
});
|
|
2375
|
-
}
|
|
2376
|
-
registerApprovalsAPI({
|
|
2377
|
-
server,
|
|
2378
|
-
engine,
|
|
2379
|
-
logger,
|
|
2380
|
-
observability
|
|
2381
|
-
});
|
|
2382
|
-
registerStatsAPI({
|
|
2383
|
-
server,
|
|
2384
|
-
hostService,
|
|
2385
|
-
cronScheduler});
|
|
2386
|
-
server.get("/health", async () => {
|
|
2387
|
-
const metrics = await hostService.getMetrics();
|
|
2388
|
-
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2389
|
-
return {
|
|
2390
|
-
status: checks.some((entry) => entry.status === "error") ? "degraded" : "ok",
|
|
2391
|
-
service: "workflow",
|
|
2392
|
-
ts: Date.now()
|
|
2393
|
-
};
|
|
2394
|
-
});
|
|
2395
|
-
server.get("/ready", async () => {
|
|
2396
|
-
const metrics = await hostService.getMetrics();
|
|
2397
|
-
const checks = buildWorkflowReadinessChecks({ workflowService, cronScheduler, metrics });
|
|
2398
|
-
const hasErrors = checks.some((entry) => entry.status === "error");
|
|
2399
|
-
const hasWarnings = checks.some((entry) => entry.status === "warn");
|
|
2400
|
-
return createServiceReadyResponse({
|
|
2401
|
-
ready: !hasErrors,
|
|
2402
|
-
status: hasErrors ? "initializing" : hasWarnings ? "degraded" : "ready",
|
|
2403
|
-
reason: hasErrors ? "workflow_checks_failed" : "ready",
|
|
2404
|
-
components: {
|
|
2405
|
-
workflowEngine: {
|
|
2406
|
-
ready: true
|
|
2407
|
-
},
|
|
2408
|
-
workflowCatalog: {
|
|
2409
|
-
ready: Boolean(workflowService)
|
|
2410
|
-
},
|
|
2411
|
-
cronScheduler: {
|
|
2412
|
-
ready: Boolean(cronScheduler)
|
|
2413
|
-
}
|
|
2914
|
+
const apiKeyHeader = request.headers["x-api-key"];
|
|
2915
|
+
const authHeader = request.headers.authorization;
|
|
2916
|
+
const bearerToken = typeof authHeader === "string" && authHeader.startsWith("Bearer ") ? authHeader.slice("Bearer ".length).trim() : void 0;
|
|
2917
|
+
const token = (typeof apiKeyHeader === "string" ? apiKeyHeader : void 0) ?? bearerToken;
|
|
2918
|
+
if (!token || token !== daemonApiKey) {
|
|
2919
|
+
reply.code(401).send({ ok: false, error: "Unauthorized" });
|
|
2414
2920
|
}
|
|
2415
|
-
}
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
return observability.renderPrometheusMetrics(
|
|
2422
|
-
healthStatus,
|
|
2423
|
-
buildWorkflowMetricLines(metrics)
|
|
2424
|
-
);
|
|
2425
|
-
});
|
|
2426
|
-
server.get("/observability/describe", async () => observability.buildDescribe());
|
|
2427
|
-
server.get("/observability/health", async () => {
|
|
2428
|
-
const metrics = await hostService.getMetrics();
|
|
2429
|
-
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2430
|
-
return observability.buildHealth({
|
|
2431
|
-
status: resolveWorkflowHealthStatus(),
|
|
2432
|
-
checks,
|
|
2433
|
-
topOperations: buildWorkflowTopOperations(metrics, observability.getTopOperations(3)),
|
|
2434
|
-
meta: {
|
|
2435
|
-
workflowServiceEnabled: Boolean(workflowService),
|
|
2436
|
-
cronSchedulerEnabled: Boolean(cronScheduler),
|
|
2437
|
-
cronDiscoveryEnabled: Boolean(cronDiscovery),
|
|
2438
|
-
runs: metrics.runs,
|
|
2439
|
-
jobs: metrics.jobs
|
|
2921
|
+
},
|
|
2922
|
+
registerRoutes: async (server) => {
|
|
2923
|
+
registerJobsAPI({ server, hostService, logger, observability });
|
|
2924
|
+
registerCronAPI({ server, hostService, logger, observability });
|
|
2925
|
+
if (workflowService) {
|
|
2926
|
+
registerWorkflowsAPI({ server, hostService, engine, workflowService, logger, observability });
|
|
2440
2927
|
}
|
|
2441
|
-
|
|
2928
|
+
registerApprovalsAPI({ server, hostService, engine, logger, observability });
|
|
2929
|
+
registerStatsAPI({ server, hostService, cronScheduler});
|
|
2930
|
+
server.get("/health", async () => {
|
|
2931
|
+
const metrics = await hostService.getMetrics();
|
|
2932
|
+
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2933
|
+
return {
|
|
2934
|
+
status: checks.some((entry) => entry.status === "error") ? "degraded" : "ok",
|
|
2935
|
+
service: "workflow",
|
|
2936
|
+
ts: Date.now()
|
|
2937
|
+
};
|
|
2938
|
+
});
|
|
2939
|
+
server.get("/ready", async () => {
|
|
2940
|
+
const metrics = await hostService.getMetrics();
|
|
2941
|
+
const checks = buildWorkflowReadinessChecks({ workflowService, cronScheduler, metrics });
|
|
2942
|
+
const hasErrors = checks.some((entry) => entry.status === "error");
|
|
2943
|
+
const hasWarnings = checks.some((entry) => entry.status === "warn");
|
|
2944
|
+
return createServiceReadyResponse({
|
|
2945
|
+
ready: !hasErrors,
|
|
2946
|
+
status: hasErrors ? "initializing" : hasWarnings ? "degraded" : "ready",
|
|
2947
|
+
reason: hasErrors ? "workflow_checks_failed" : "ready",
|
|
2948
|
+
components: {
|
|
2949
|
+
workflowEngine: { ready: true },
|
|
2950
|
+
workflowCatalog: { ready: Boolean(workflowService) },
|
|
2951
|
+
cronScheduler: { ready: Boolean(cronScheduler) }
|
|
2952
|
+
}
|
|
2953
|
+
});
|
|
2954
|
+
});
|
|
2955
|
+
server.get("/metrics", async (_request, reply) => {
|
|
2956
|
+
const metrics = await hostService.getMetrics();
|
|
2957
|
+
const healthStatus = resolveWorkflowHealthStatus();
|
|
2958
|
+
reply.header("Content-Type", "text/plain; version=0.0.4; charset=utf-8");
|
|
2959
|
+
return observability.renderPrometheusMetrics(
|
|
2960
|
+
healthStatus,
|
|
2961
|
+
buildWorkflowMetricLines(metrics)
|
|
2962
|
+
);
|
|
2963
|
+
});
|
|
2964
|
+
server.get("/observability/describe", async () => observability.buildDescribe());
|
|
2965
|
+
server.get("/observability/health", async () => {
|
|
2966
|
+
const metrics = await hostService.getMetrics();
|
|
2967
|
+
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2968
|
+
return observability.buildHealth({
|
|
2969
|
+
status: resolveWorkflowHealthStatus(),
|
|
2970
|
+
checks,
|
|
2971
|
+
topOperations: buildWorkflowTopOperations(metrics, observability.getTopOperations(3)),
|
|
2972
|
+
meta: {
|
|
2973
|
+
workflowServiceEnabled: Boolean(workflowService),
|
|
2974
|
+
cronSchedulerEnabled: Boolean(cronScheduler),
|
|
2975
|
+
cronDiscoveryEnabled: Boolean(cronDiscovery),
|
|
2976
|
+
runs: metrics.runs,
|
|
2977
|
+
jobs: metrics.jobs
|
|
2978
|
+
}
|
|
2979
|
+
});
|
|
2980
|
+
});
|
|
2981
|
+
}
|
|
2442
2982
|
});
|
|
2443
|
-
return server;
|
|
2444
2983
|
}
|
|
2445
2984
|
function resolveWorkflowHealthStatus(metrics) {
|
|
2446
2985
|
return "healthy";
|
|
@@ -2477,16 +3016,8 @@ function buildWorkflowReadinessChecks(input) {
|
|
|
2477
3016
|
function buildWorkflowTopOperations(metrics, httpOperations) {
|
|
2478
3017
|
return [
|
|
2479
3018
|
...httpOperations,
|
|
2480
|
-
{
|
|
2481
|
-
|
|
2482
|
-
count: metrics.runs.total,
|
|
2483
|
-
errorCount: metrics.runs.failed + metrics.runs.cancelled + metrics.runs.dlq
|
|
2484
|
-
},
|
|
2485
|
-
{
|
|
2486
|
-
operation: "workflow.jobs",
|
|
2487
|
-
count: metrics.jobs.total,
|
|
2488
|
-
errorCount: metrics.jobs.failed
|
|
2489
|
-
}
|
|
3019
|
+
{ operation: "workflow.runs", count: metrics.runs.total, errorCount: metrics.runs.failed + metrics.runs.cancelled + metrics.runs.dlq },
|
|
3020
|
+
{ operation: "workflow.jobs", count: metrics.jobs.total, errorCount: metrics.jobs.failed }
|
|
2490
3021
|
].slice(0, 5);
|
|
2491
3022
|
}
|
|
2492
3023
|
function buildWorkflowMetricLines(metrics) {
|
|
@@ -2513,164 +3044,159 @@ function buildWorkflowMetricLines(metrics) {
|
|
|
2513
3044
|
metricLine("service_operation_total", metrics.jobs.failed, { operation: "workflow.jobs", status: "error" })
|
|
2514
3045
|
];
|
|
2515
3046
|
}
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
}
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
bootstrapLogger.info("Workflow daemon shutdown complete");
|
|
2670
|
-
process.exit(0);
|
|
2671
|
-
};
|
|
2672
|
-
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
2673
|
-
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
3047
|
+
async function bootstrap(_cwd = process.cwd()) {
|
|
3048
|
+
await runDaemon(
|
|
3049
|
+
{
|
|
3050
|
+
appId: "workflow-daemon",
|
|
3051
|
+
// serviceId in the transport map / devservices is 'workflow' (≠ appId).
|
|
3052
|
+
serviceId: "workflow",
|
|
3053
|
+
defaultPort: 7778,
|
|
3054
|
+
portEnvVar: "WORKFLOW_PORT",
|
|
3055
|
+
defaultHost: "0.0.0.0",
|
|
3056
|
+
hostEnvVar: "WORKFLOW_HOST",
|
|
3057
|
+
async setup({ platform: _p, logger: _l, port, host, repoRoot }) {
|
|
3058
|
+
const projectRoot = process.env["KB_PROJECT_ROOT"] ?? repoRoot;
|
|
3059
|
+
if (!platform.isConfigured("workspace")) {
|
|
3060
|
+
process.stderr.write(
|
|
3061
|
+
'[workflow-daemon] WARNING: workspace adapter is not configured.\n[workflow-daemon] Workflows that use isolation: balanced (default) or isolation: strict will fail.\n[workflow-daemon] To fix: set platform.adapters.workspace in kb.config.json.\n[workflow-daemon] To run without a workspace: add "isolation: relaxed" to your workflow YAML.\n'
|
|
3062
|
+
);
|
|
3063
|
+
}
|
|
3064
|
+
if (!platform.isConfigured("environment") && platform.isConfigured("workspace")) {
|
|
3065
|
+
process.stderr.write(
|
|
3066
|
+
"[workflow-daemon] WARNING: environment adapter is not configured.\n[workflow-daemon] Workflows that use isolation: strict will fail.\n[workflow-daemon] To fix: set platform.adapters.environment in kb.config.json.\n"
|
|
3067
|
+
);
|
|
3068
|
+
}
|
|
3069
|
+
const startupRequestId = `workflow-startup-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
|
3070
|
+
const startupTraceId = randomUUID();
|
|
3071
|
+
const startupSpanId = randomUUID();
|
|
3072
|
+
const bootstrapLogger = createCorrelatedLogger(platform.logger, {
|
|
3073
|
+
serviceId: "workflow",
|
|
3074
|
+
logsSource: "workflow",
|
|
3075
|
+
layer: "workflow",
|
|
3076
|
+
service: "bootstrap",
|
|
3077
|
+
requestId: startupRequestId,
|
|
3078
|
+
traceId: startupTraceId,
|
|
3079
|
+
operation: "workflow.bootstrap",
|
|
3080
|
+
bindings: { spanId: startupSpanId, invocationId: startupSpanId, executionId: startupSpanId }
|
|
3081
|
+
});
|
|
3082
|
+
const debugMode = process.env["WORKFLOW_DEBUG"] === "true";
|
|
3083
|
+
bootstrapLogger.info("Workflow daemon starting", { projectRoot, debugMode });
|
|
3084
|
+
if (debugMode) {
|
|
3085
|
+
bootstrapLogger.warn(
|
|
3086
|
+
"[WORKFLOW_DEBUG=true] Verbose debug logging is ON \u2014 step inputs, outputs, and expr contexts will be logged. Disable in production (unset WORKFLOW_DEBUG or set to false)."
|
|
3087
|
+
);
|
|
3088
|
+
}
|
|
3089
|
+
const createWorkflowLogger = (service, operation, bindings) => createCorrelatedLogger(platform.logger, {
|
|
3090
|
+
serviceId: "workflow",
|
|
3091
|
+
logsSource: "workflow",
|
|
3092
|
+
layer: "workflow",
|
|
3093
|
+
service,
|
|
3094
|
+
operation,
|
|
3095
|
+
bindings
|
|
3096
|
+
});
|
|
3097
|
+
bootstrapLogger.info("Loading plugin registry snapshot");
|
|
3098
|
+
const cliApi = await createRegistry({ root: repoRoot, cache: { ttlMs: 10 * 60 * 1e3 } });
|
|
3099
|
+
await cliApi.initialize();
|
|
3100
|
+
const plugins = await cliApi.listPlugins();
|
|
3101
|
+
bootstrapLogger.info("Plugin registry snapshot loaded", {
|
|
3102
|
+
pluginsFound: plugins.length,
|
|
3103
|
+
pluginIds: plugins.map((p) => `${p.id}@${p.version}`)
|
|
3104
|
+
});
|
|
3105
|
+
bootstrapLogger.info("Creating WorkflowEngine");
|
|
3106
|
+
const engine = new WorkflowEngine({
|
|
3107
|
+
cache: platform.cache,
|
|
3108
|
+
events: platform.eventBus,
|
|
3109
|
+
logger: createWorkflowLogger("engine", "workflow.engine"),
|
|
3110
|
+
snapshotManager: platform.snapshotManager,
|
|
3111
|
+
workspaceRoot: projectRoot
|
|
3112
|
+
});
|
|
3113
|
+
bootstrapLogger.info("Cleaning up stale runs from previous daemon process");
|
|
3114
|
+
await engine.cleanupStaleRuns();
|
|
3115
|
+
bootstrapLogger.info("Resuming interrupted jobs");
|
|
3116
|
+
await engine.resumeInterruptedJobs();
|
|
3117
|
+
bootstrapLogger.info("Creating JobBroker");
|
|
3118
|
+
const jobBroker = new JobBroker(
|
|
3119
|
+
engine,
|
|
3120
|
+
createWorkflowLogger("job-broker", "workflow.job-broker"),
|
|
3121
|
+
platform
|
|
3122
|
+
);
|
|
3123
|
+
bootstrapLogger.info("Creating CronScheduler");
|
|
3124
|
+
const cronScheduler = new CronScheduler({
|
|
3125
|
+
jobBroker,
|
|
3126
|
+
workflowEngine: engine,
|
|
3127
|
+
logger: createWorkflowLogger("cron-scheduler", "workflow.cron-scheduler"),
|
|
3128
|
+
timezone: process.env.WORKFLOW_CRON_TIMEZONE
|
|
3129
|
+
});
|
|
3130
|
+
bootstrapLogger.info("Discovering cron jobs");
|
|
3131
|
+
const cronDiscovery = new CronDiscovery({
|
|
3132
|
+
cliApi,
|
|
3133
|
+
scheduler: cronScheduler,
|
|
3134
|
+
logger: createWorkflowLogger("cron-discovery", "workflow.cron-discovery"),
|
|
3135
|
+
workspaceRoot: projectRoot
|
|
3136
|
+
});
|
|
3137
|
+
const discovered = await cronDiscovery.discoverAll();
|
|
3138
|
+
bootstrapLogger.info("Cron job discovery complete", discovered);
|
|
3139
|
+
bootstrapLogger.info("Creating WorkflowService");
|
|
3140
|
+
const workflowService = new WorkflowService({ cliApi, platform, workspaceRoot: projectRoot });
|
|
3141
|
+
workflowService.listAll().catch(
|
|
3142
|
+
(err) => bootstrapLogger.warn("Manifest scanner warmup failed", { err })
|
|
3143
|
+
);
|
|
3144
|
+
bootstrapLogger.info("Starting WorkflowFileWatcher");
|
|
3145
|
+
const fileWatcher = new WorkflowFileWatcher({
|
|
3146
|
+
watchDirs: [
|
|
3147
|
+
join(projectRoot, ".kb", "workflows"),
|
|
3148
|
+
join(projectRoot, ".kb", "jobs")
|
|
3149
|
+
],
|
|
3150
|
+
workflowService,
|
|
3151
|
+
cronDiscovery,
|
|
3152
|
+
cronScheduler,
|
|
3153
|
+
logger: createWorkflowLogger("file-watcher", "workflow.file-watcher")
|
|
3154
|
+
});
|
|
3155
|
+
bootstrapLogger.info("Creating HTTP server");
|
|
3156
|
+
const server = await createServer({
|
|
3157
|
+
engine,
|
|
3158
|
+
jobBroker,
|
|
3159
|
+
workflowService,
|
|
3160
|
+
cronScheduler,
|
|
3161
|
+
cronDiscovery,
|
|
3162
|
+
logger: createWorkflowLogger("api", "workflow.api")
|
|
3163
|
+
});
|
|
3164
|
+
await server.listen(getListenOptions(port, host));
|
|
3165
|
+
bootstrapLogger.info("HTTP API listening", { port });
|
|
3166
|
+
bootstrapLogger.info("Creating WorkflowWorker");
|
|
3167
|
+
const worker = await createWorkflowWorker({
|
|
3168
|
+
engine,
|
|
3169
|
+
cliApi,
|
|
3170
|
+
logger: createWorkflowLogger("worker", "workflow.worker"),
|
|
3171
|
+
analytics: platform.analytics,
|
|
3172
|
+
platform,
|
|
3173
|
+
workspaceRoot: projectRoot,
|
|
3174
|
+
concurrency: parseInt(process.env.WORKFLOW_CONCURRENCY ?? "5", 10),
|
|
3175
|
+
debugMode
|
|
3176
|
+
});
|
|
3177
|
+
bootstrapLogger.info("Starting WorkflowWorker");
|
|
3178
|
+
worker.start().catch((error) => {
|
|
3179
|
+
bootstrapLogger.error(
|
|
3180
|
+
"Worker crashed - shutting down daemon",
|
|
3181
|
+
error instanceof Error ? error : void 0
|
|
3182
|
+
);
|
|
3183
|
+
process.kill(process.pid, "SIGTERM");
|
|
3184
|
+
});
|
|
3185
|
+
bootstrapLogger.info("Starting CronScheduler");
|
|
3186
|
+
await cronScheduler.start();
|
|
3187
|
+
bootstrapLogger.info("Workflow daemon started successfully", { port });
|
|
3188
|
+
return async () => {
|
|
3189
|
+
bootstrapLogger.warn("Stopping workflow daemon components");
|
|
3190
|
+
fileWatcher.close();
|
|
3191
|
+
await cronScheduler.stop();
|
|
3192
|
+
await worker.stop();
|
|
3193
|
+
await server.close();
|
|
3194
|
+
await cliApi.dispose();
|
|
3195
|
+
};
|
|
3196
|
+
}
|
|
3197
|
+
},
|
|
3198
|
+
(appId, repoRoot) => createServiceBootstrap({ appId, repoRoot, assemblyHook: makeAssemblyHook() })
|
|
3199
|
+
);
|
|
2674
3200
|
}
|
|
2675
3201
|
|
|
2676
3202
|
// src/index.ts
|
|
@@ -2678,7 +3204,8 @@ async function bootstrap(cwd = process.cwd()) {
|
|
|
2678
3204
|
try {
|
|
2679
3205
|
await bootstrap(process.cwd());
|
|
2680
3206
|
} catch (error) {
|
|
2681
|
-
|
|
3207
|
+
process.stderr.write(`[workflow-daemon] FATAL: ${error instanceof Error ? error.message : String(error)}
|
|
3208
|
+
`);
|
|
2682
3209
|
process.exit(1);
|
|
2683
3210
|
}
|
|
2684
3211
|
})();
|