@kb-labs/workflow-daemon 2.94.0 → 2.96.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +997 -513
- package/dist/index.js.map +1 -1
- package/dist/manifest.js +3 -1
- package/dist/manifest.js.map +1 -1
- package/dist/manifest.json +27 -0
- package/package.json +22 -19
package/dist/index.js
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import {
|
|
2
|
+
import { platform, createServiceBootstrap } from '@kb-labs/core-runtime';
|
|
3
|
+
import { makeAssemblyHook } from '@kb-labs/plugin-runtime';
|
|
3
4
|
import { WorkflowEngine, WorkflowService } from '@kb-labs/workflow-engine';
|
|
4
|
-
import { createCorrelatedLogger,
|
|
5
|
+
import { createCorrelatedLogger, getListenOptions, HttpObservabilityCollector, createDaemonServer, createServiceReadyResponse, metricLine } from '@kb-labs/shared-http';
|
|
6
|
+
import { runDaemon } from '@kb-labs/shared-daemon';
|
|
5
7
|
import { logDiagnosticEvent } from '@kb-labs/core-platform';
|
|
6
|
-
import { PluginCronJobSchema, UserCronJobSchema, evaluateExpression, interpolateObject,
|
|
8
|
+
import { PluginCronJobSchema, UserCronJobSchema, evaluateExpression, interpolateObject, buildShellSafeCommand, coerceToString, interpolateString } from '@kb-labs/workflow-contracts';
|
|
9
|
+
import { GateHandler } from '@kb-labs/workflow-steps';
|
|
7
10
|
import { SandboxRunner } from '@kb-labs/workflow-runtime';
|
|
8
11
|
import * as cron from 'node-cron';
|
|
9
12
|
import { CronExpressionParser } from 'cron-parser';
|
|
10
|
-
import { stat, readdir, readFile } from 'fs/promises';
|
|
13
|
+
import { stat, readdir, readFile, access } from 'fs/promises';
|
|
11
14
|
import { join, extname, basename } from 'path';
|
|
12
15
|
import YAML from 'yaml';
|
|
13
|
-
import
|
|
14
|
-
import cors from '@fastify/cors';
|
|
16
|
+
import { watch } from 'fs';
|
|
15
17
|
import { createRegistry } from '@kb-labs/core-registry';
|
|
16
|
-
import { findRepoRoot } from '@kb-labs/core-sys';
|
|
17
18
|
import { randomUUID } from 'crypto';
|
|
18
19
|
|
|
19
20
|
async function createWorkflowWorker(options) {
|
|
@@ -28,6 +29,11 @@ async function createWorkflowWorker(options) {
|
|
|
28
29
|
analytics,
|
|
29
30
|
debugMode = process.env["WORKFLOW_DEBUG"] === "true"
|
|
30
31
|
} = options;
|
|
32
|
+
const wsProvider = platform2.getAdapter("workspace");
|
|
33
|
+
if (wsProvider && typeof wsProvider.startupCleanup === "function") {
|
|
34
|
+
wsProvider.startupCleanup().catch(() => {
|
|
35
|
+
});
|
|
36
|
+
}
|
|
31
37
|
let isRunning = false;
|
|
32
38
|
let stopRequested = false;
|
|
33
39
|
const runningJobs = /* @__PURE__ */ new Map();
|
|
@@ -67,6 +73,14 @@ async function createWorkflowWorker(options) {
|
|
|
67
73
|
return true;
|
|
68
74
|
}
|
|
69
75
|
claimedJobs.add(jobKey);
|
|
76
|
+
const waitMs = job.queuedAt ? Date.now() - new Date(job.queuedAt).getTime() : void 0;
|
|
77
|
+
logger.info("Job picked from queue", {
|
|
78
|
+
runId: run.id,
|
|
79
|
+
jobId: job.id,
|
|
80
|
+
jobName: job.jobName,
|
|
81
|
+
queuedAt: job.queuedAt,
|
|
82
|
+
waitMs
|
|
83
|
+
});
|
|
70
84
|
const jobStartTime = Date.now();
|
|
71
85
|
const jobLogger = createCorrelatedLogger(logger, {
|
|
72
86
|
serviceId: "workflow",
|
|
@@ -79,7 +93,8 @@ async function createWorkflowWorker(options) {
|
|
|
79
93
|
bindings: {
|
|
80
94
|
workflowId: run.id,
|
|
81
95
|
runId: run.id,
|
|
82
|
-
jobId: job.id
|
|
96
|
+
jobId: job.id,
|
|
97
|
+
jobName: job.jobName
|
|
83
98
|
}
|
|
84
99
|
});
|
|
85
100
|
jobLogger.info("Processing job", {
|
|
@@ -98,13 +113,13 @@ async function createWorkflowWorker(options) {
|
|
|
98
113
|
const runTarget = run.metadata?.target;
|
|
99
114
|
const jobTarget = job.target;
|
|
100
115
|
const target = jobTarget ?? runTarget;
|
|
101
|
-
const
|
|
102
|
-
let runWorkspace =
|
|
116
|
+
const wsProvider2 = platform2.getAdapter("workspace");
|
|
117
|
+
let runWorkspace = wsProvider2 ? workspaceRoot : process.env["KB_PROJECT_ROOT"] ?? workspaceRoot;
|
|
103
118
|
let provisionedWorkspaceId;
|
|
104
|
-
if (
|
|
119
|
+
if (wsProvider2) {
|
|
105
120
|
const wsId = `wt_${run.id.slice(0, 8)}`;
|
|
106
121
|
try {
|
|
107
|
-
const ws = await
|
|
122
|
+
const ws = await wsProvider2.materialize({
|
|
108
123
|
workspaceId: wsId,
|
|
109
124
|
sourceRef: "main",
|
|
110
125
|
metadata: { runId: run.id, jobId: job.id },
|
|
@@ -155,6 +170,13 @@ async function createWorkflowWorker(options) {
|
|
|
155
170
|
continue;
|
|
156
171
|
}
|
|
157
172
|
const freshRun = await engine.getRun(run.id);
|
|
173
|
+
if (freshRun?.status === "cancelled") {
|
|
174
|
+
jobLogger.info("[worker] Run cancelled \u2014 stopping step execution", {
|
|
175
|
+
runId: run.id,
|
|
176
|
+
jobId: job.id
|
|
177
|
+
});
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
158
180
|
const exprCtx = {
|
|
159
181
|
env: freshRun?.env ?? {},
|
|
160
182
|
trigger: freshRun?.trigger ?? { type: "manual" },
|
|
@@ -181,11 +203,18 @@ async function createWorkflowWorker(options) {
|
|
|
181
203
|
const rawExpr = condition.trim().replace(/^\$\{\{\s*/, "").replace(/\s*\}\}$/, "");
|
|
182
204
|
const shouldRun = evaluateExpression(rawExpr, exprCtx);
|
|
183
205
|
if (!shouldRun) {
|
|
184
|
-
jobLogger.info("
|
|
206
|
+
jobLogger.info("[step] Skipped: condition evaluated to false", {
|
|
185
207
|
runId: run.id,
|
|
186
208
|
jobId: job.id,
|
|
187
209
|
stepId: step.id,
|
|
188
|
-
|
|
210
|
+
stepName: step.name,
|
|
211
|
+
stepIndex: step.index,
|
|
212
|
+
condition: rawExpr,
|
|
213
|
+
evaluatedContext: {
|
|
214
|
+
inputs: exprCtx.inputs,
|
|
215
|
+
steps: exprCtx.steps,
|
|
216
|
+
env: exprCtx.env
|
|
217
|
+
}
|
|
189
218
|
});
|
|
190
219
|
await engine.markStepCompleted(run.id, job.id, step.id, { skipped: true });
|
|
191
220
|
continue;
|
|
@@ -204,7 +233,25 @@ async function createWorkflowWorker(options) {
|
|
|
204
233
|
}
|
|
205
234
|
});
|
|
206
235
|
}
|
|
207
|
-
|
|
236
|
+
let interpolatedWith = step.spec.with ? interpolateObject(step.spec.with, exprCtx) : void 0;
|
|
237
|
+
if (step.spec.uses === "builtin:shell" && typeof step.spec.with?.["command"] === "string") {
|
|
238
|
+
const rawCommand = step.spec.with["command"];
|
|
239
|
+
const { command: safeCommand, shellEnvVars } = buildShellSafeCommand(rawCommand, exprCtx);
|
|
240
|
+
interpolatedWith = {
|
|
241
|
+
...interpolatedWith ?? {},
|
|
242
|
+
command: safeCommand,
|
|
243
|
+
env: {
|
|
244
|
+
...Object.fromEntries(
|
|
245
|
+
Object.entries(interpolatedWith?.["env"] ?? {}).map(([k, v]) => [k, coerceToString(v)])
|
|
246
|
+
),
|
|
247
|
+
...shellEnvVars
|
|
248
|
+
}
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
const interpolatedEnvRaw = step.spec.env ? interpolateObject(step.spec.env, exprCtx) : void 0;
|
|
252
|
+
const interpolatedEnv = interpolatedEnvRaw ? Object.fromEntries(
|
|
253
|
+
Object.entries(interpolatedEnvRaw).map(([k, v]) => [k, coerceToString(v)])
|
|
254
|
+
) : void 0;
|
|
208
255
|
if (debugMode && step.spec.with) {
|
|
209
256
|
jobLogger.info("[debug] Step input interpolation", {
|
|
210
257
|
runId: run.id,
|
|
@@ -218,7 +265,9 @@ async function createWorkflowWorker(options) {
|
|
|
218
265
|
const stepLogger = jobLogger.child({
|
|
219
266
|
operation: "workflow.step",
|
|
220
267
|
stepId: step.id,
|
|
221
|
-
|
|
268
|
+
stepName: step.name,
|
|
269
|
+
stepIndex: step.index,
|
|
270
|
+
attempt: job.attempt ?? 1,
|
|
222
271
|
executionId: stepExecutionId,
|
|
223
272
|
spanId: stepExecutionId,
|
|
224
273
|
invocationId: stepExecutionId
|
|
@@ -241,153 +290,88 @@ async function createWorkflowWorker(options) {
|
|
|
241
290
|
});
|
|
242
291
|
if (step.spec.uses === "builtin:approval") {
|
|
243
292
|
if (step.status === "failed") {
|
|
244
|
-
stepLogger.info("Approval already rejected \u2014 skipping to gate", {
|
|
245
|
-
runId: run.id,
|
|
246
|
-
stepId: step.id
|
|
247
|
-
});
|
|
293
|
+
stepLogger.info("Approval already rejected \u2014 skipping to gate", { runId: run.id, stepId: step.id });
|
|
248
294
|
continue;
|
|
249
295
|
}
|
|
250
296
|
if (step.status !== "waiting_approval") {
|
|
251
297
|
await engine.markStepWaitingApproval(run.id, job.id, step.id);
|
|
252
298
|
}
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
const currentStep = currentJob?.steps.find((s) => s.id === step.id);
|
|
264
|
-
if (!currentStep || currentStep.status === "success") {
|
|
265
|
-
stepLogger.info("Approval granted", { runId: run.id, stepId: step.id });
|
|
266
|
-
break;
|
|
267
|
-
}
|
|
268
|
-
if (currentStep.status === "failed") {
|
|
269
|
-
stepLogger.info("Approval rejected", { runId: run.id, stepId: step.id });
|
|
270
|
-
break;
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
|
-
if (stopRequested) {
|
|
274
|
-
stepLogger.info("Approval wait interrupted by shutdown", { stepId: step.id });
|
|
299
|
+
const approvalResult = await waitForApproval(
|
|
300
|
+
engine,
|
|
301
|
+
run.id,
|
|
302
|
+
job.id,
|
|
303
|
+
step,
|
|
304
|
+
interpolatedWith,
|
|
305
|
+
() => stopRequested,
|
|
306
|
+
stepLogger
|
|
307
|
+
);
|
|
308
|
+
if (approvalResult === "interrupted") {
|
|
275
309
|
return;
|
|
276
310
|
}
|
|
277
311
|
continue;
|
|
278
312
|
}
|
|
279
313
|
if (step.spec.uses === "builtin:gate") {
|
|
280
314
|
const gateInput = interpolatedWith ?? {};
|
|
281
|
-
const
|
|
282
|
-
const
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
const
|
|
286
|
-
const
|
|
287
|
-
|
|
315
|
+
const currentIteration = step.metadata?.["iterations"] ?? 0;
|
|
316
|
+
const sameJobStepIds = job.steps.flatMap(
|
|
317
|
+
(s) => [s.id, s.spec.id].filter((v) => typeof v === "string")
|
|
318
|
+
);
|
|
319
|
+
const jobNames = (freshRun?.jobs ?? run.jobs).map((j) => j.jobName);
|
|
320
|
+
const validRestartTargets = [...sameJobStepIds, ...jobNames];
|
|
321
|
+
const decision = new GateHandler().handle(gateInput, exprCtx, currentIteration, validRestartTargets);
|
|
322
|
+
stepLogger.info("[gate] Evaluating gate decision", {
|
|
288
323
|
runId: run.id,
|
|
289
324
|
stepId: step.id,
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
action:
|
|
325
|
+
stepName: step.name,
|
|
326
|
+
expression: gateInput.decision,
|
|
327
|
+
action: decision.action,
|
|
328
|
+
iteration: currentIteration
|
|
293
329
|
});
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
const currentIteration = metadata[iterationKey] ?? 0;
|
|
297
|
-
if (action === "continue") {
|
|
298
|
-
await engine.markStepCompleted(run.id, job.id, step.id, {
|
|
299
|
-
decisionValue,
|
|
300
|
-
action: "continue",
|
|
301
|
-
iteration: currentIteration
|
|
302
|
-
});
|
|
330
|
+
if (decision.action === "continue") {
|
|
331
|
+
await engine.markStepCompleted(run.id, job.id, step.id, decision.outputs);
|
|
303
332
|
continue;
|
|
304
333
|
}
|
|
305
|
-
if (action === "fail") {
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
iteration: currentIteration
|
|
311
|
-
});
|
|
312
|
-
throw error;
|
|
313
|
-
}
|
|
314
|
-
const restartAction = action;
|
|
315
|
-
const nextIteration = currentIteration + 1;
|
|
316
|
-
if (nextIteration >= maxIterations) {
|
|
317
|
-
const error = new Error(
|
|
318
|
-
`Gate max iterations reached (${maxIterations}) for step ${step.spec.id ?? step.id}`
|
|
319
|
-
);
|
|
320
|
-
await engine.markStepFailed(run.id, job.id, step.id, error, {
|
|
321
|
-
decisionValue,
|
|
322
|
-
action: "fail",
|
|
323
|
-
maxIterationsReached: true,
|
|
324
|
-
iteration: currentIteration,
|
|
325
|
-
maxIterations
|
|
334
|
+
if (decision.action === "fail") {
|
|
335
|
+
stepLogger.error("[gate] Gate condition failed, aborting job", decision.error, {
|
|
336
|
+
runId: run.id,
|
|
337
|
+
stepId: step.id,
|
|
338
|
+
stepName: step.name
|
|
326
339
|
});
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
stepLogger.info("Gate triggering restart", {
|
|
330
|
-
restartFrom: restartAction.restartFrom,
|
|
331
|
-
iteration: nextIteration,
|
|
332
|
-
maxIterations
|
|
333
|
-
});
|
|
334
|
-
await engine.markStepCompleted(run.id, job.id, step.id, {
|
|
335
|
-
decisionValue,
|
|
336
|
-
action: "restart",
|
|
337
|
-
restartFrom: restartAction.restartFrom,
|
|
338
|
-
iteration: nextIteration
|
|
339
|
-
});
|
|
340
|
-
await engine.updateRun(run.id, (draft) => {
|
|
341
|
-
const md = draft.metadata ?? {};
|
|
342
|
-
md[iterationKey] = nextIteration;
|
|
343
|
-
draft.metadata = md;
|
|
344
|
-
if (restartAction.context) {
|
|
345
|
-
const payload = draft.trigger.payload ?? {};
|
|
346
|
-
Object.assign(payload, restartAction.context);
|
|
347
|
-
draft.trigger.payload = payload;
|
|
348
|
-
}
|
|
349
|
-
return draft;
|
|
350
|
-
});
|
|
351
|
-
const stateStore = engine.getStateStore();
|
|
352
|
-
const scheduler = engine.getScheduler();
|
|
353
|
-
let foundTarget = false;
|
|
354
|
-
for (const s of job.steps) {
|
|
355
|
-
if (s.spec.id === restartAction.restartFrom || s.id === restartAction.restartFrom) {
|
|
356
|
-
foundTarget = true;
|
|
357
|
-
}
|
|
358
|
-
if (foundTarget) {
|
|
359
|
-
await stateStore.updateStep(run.id, job.id, s.id, (draft) => {
|
|
360
|
-
draft.status = "queued";
|
|
361
|
-
draft.startedAt = void 0;
|
|
362
|
-
draft.finishedAt = void 0;
|
|
363
|
-
draft.error = void 0;
|
|
364
|
-
draft.outputs = void 0;
|
|
365
|
-
});
|
|
366
|
-
}
|
|
340
|
+
await engine.markStepFailed(run.id, job.id, step.id, decision.error, decision.outputs);
|
|
341
|
+
throw decision.error;
|
|
367
342
|
}
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
draft.finishedAt = void 0;
|
|
372
|
-
});
|
|
373
|
-
const updatedRun = await engine.getRun(run.id);
|
|
374
|
-
const updatedJob = updatedRun?.jobs.find((j) => j.id === job.id);
|
|
375
|
-
if (updatedJob) {
|
|
376
|
-
await scheduler.enqueueJob(run.id, updatedJob, updatedJob.priority ?? "normal");
|
|
343
|
+
if (decision.action === "skip") {
|
|
344
|
+
await applyGateSkip(engine, run, job, step, decision, stepLogger);
|
|
345
|
+
return;
|
|
377
346
|
}
|
|
347
|
+
await applyGateRestart(engine, run, job, step, decision, stepLogger);
|
|
378
348
|
return;
|
|
379
349
|
}
|
|
350
|
+
const stepStartTime = Date.now();
|
|
380
351
|
await engine.markStepStarted(run.id, job.id, step.id);
|
|
381
352
|
let baseSpec = step.spec;
|
|
382
353
|
if (baseSpec.run && !baseSpec.uses) {
|
|
383
354
|
const { run: rawRun, with: existingWith, ...rest } = baseSpec;
|
|
384
|
-
const command = typeof rawRun === "string" ?
|
|
385
|
-
baseSpec = {
|
|
355
|
+
const { command, shellEnvVars } = typeof rawRun === "string" ? buildShellSafeCommand(rawRun, exprCtx) : { command: rawRun, shellEnvVars: {} };
|
|
356
|
+
baseSpec = {
|
|
357
|
+
...rest,
|
|
358
|
+
uses: "builtin:shell",
|
|
359
|
+
with: { ...existingWith, command, env: { ...existingWith?.["env"] ?? {}, ...shellEnvVars } }
|
|
360
|
+
};
|
|
386
361
|
}
|
|
387
362
|
if (typeof baseSpec.summary === "string") {
|
|
388
363
|
baseSpec.summary = interpolateString(baseSpec.summary, exprCtx);
|
|
389
364
|
}
|
|
390
|
-
const
|
|
365
|
+
const specWithEnv = interpolatedEnv ? { ...baseSpec, with: { ...baseSpec.with ?? {}, env: { ...baseSpec.with?.["env"] ?? {}, ...interpolatedEnv } } } : baseSpec;
|
|
366
|
+
const interpolatedSpec = interpolatedWith ? {
|
|
367
|
+
...specWithEnv,
|
|
368
|
+
with: {
|
|
369
|
+
...specWithEnv.with ?? {},
|
|
370
|
+
...interpolatedWith,
|
|
371
|
+
// spec.env (interpolatedEnv) must survive the spread of interpolatedWith.env
|
|
372
|
+
...interpolatedEnv ? { env: { ...interpolatedWith["env"] ?? {}, ...interpolatedEnv } } : {}
|
|
373
|
+
}
|
|
374
|
+
} : specWithEnv;
|
|
391
375
|
const result = await runner.execute({
|
|
392
376
|
spec: interpolatedSpec,
|
|
393
377
|
context: {
|
|
@@ -395,9 +379,25 @@ async function createWorkflowWorker(options) {
|
|
|
395
379
|
jobId: job.id,
|
|
396
380
|
stepId: step.id,
|
|
397
381
|
attempt: 1,
|
|
398
|
-
env:
|
|
399
|
-
|
|
400
|
-
|
|
382
|
+
env: {
|
|
383
|
+
// KB_PLATFORM_ROOT: where platform code (dist/, node_modules) lives.
|
|
384
|
+
// Shell steps can use this to reference platform commands when the
|
|
385
|
+
// worktree doesn't have compiled dist/ directories.
|
|
386
|
+
KB_PLATFORM_ROOT: workspaceRoot,
|
|
387
|
+
// KB_WORKSPACE_ROOT: the worktree (or project dir when no worktree is used).
|
|
388
|
+
// Scripts cd into this path before invoking agents or running git commands.
|
|
389
|
+
KB_WORKSPACE_ROOT: runWorkspace,
|
|
390
|
+
...freshRun?.env || {}
|
|
391
|
+
},
|
|
392
|
+
// Secrets resolution not yet implemented: run.secrets contains names only.
|
|
393
|
+
// When a platform secrets store is available, resolve names → values here.
|
|
394
|
+
secrets: (() => {
|
|
395
|
+
const names = freshRun?.secrets ?? [];
|
|
396
|
+
if (names.length > 0) {
|
|
397
|
+
stepLogger.warn("Step declares secrets but secret resolution is not implemented", { secrets: names });
|
|
398
|
+
}
|
|
399
|
+
return {};
|
|
400
|
+
})(),
|
|
401
401
|
logger: {
|
|
402
402
|
debug: (message, meta) => stepLogger.debug(message, meta),
|
|
403
403
|
info: (message, meta) => stepLogger.info(message, meta),
|
|
@@ -409,21 +409,55 @@ async function createWorkflowWorker(options) {
|
|
|
409
409
|
spanId: stepExecutionId,
|
|
410
410
|
parentSpanId: job.id
|
|
411
411
|
},
|
|
412
|
+
// stepLogger as loggerOverride: ctx.logger.* in the plugin will use stepLogger
|
|
413
|
+
// as its base, writing to SQLite with runId/jobId/stepId context.
|
|
414
|
+
// See: plugins/workflow/docs/adr/0019-log-stream-separation.md
|
|
415
|
+
loggerOverride: stepLogger,
|
|
416
|
+
// ui/shell log entries: persist to SQLite with workflow context + publish for SSE.
|
|
417
|
+
// See: plugins/workflow/docs/adr/0019-log-stream-separation.md
|
|
412
418
|
onLog: (entry2) => {
|
|
413
|
-
|
|
419
|
+
stepLogger.info(entry2.message, {
|
|
420
|
+
stream: entry2.stream,
|
|
421
|
+
lineNo: entry2.lineNo,
|
|
422
|
+
logSource: entry2.stream === "stderr" ? "stderr" : "stdout"
|
|
423
|
+
});
|
|
424
|
+
void engine.publishLog(run.id, job.id, step.id, entry2, step.name);
|
|
425
|
+
},
|
|
426
|
+
// ctx.logger.* entries: stepLogger base already wrote to SQLite. Only publish for SSE.
|
|
427
|
+
// See: plugins/workflow/docs/adr/0019-log-stream-separation.md
|
|
428
|
+
onLoggerLog: (entry2) => {
|
|
429
|
+
void engine.publishLog(run.id, job.id, step.id, entry2, step.name);
|
|
414
430
|
}
|
|
415
431
|
},
|
|
416
|
-
|
|
432
|
+
// Scripts (.kb/workflows/scripts/*.sh) live in the project root, not the worktree.
|
|
433
|
+
// Use workspaceRoot as cwd so relative paths like `bash .kb/workflows/scripts/...`
|
|
434
|
+
// resolve correctly. Scripts that need to operate inside the worktree cd into
|
|
435
|
+
// KB_WORKSPACE_ROOT themselves (agent scripts, git operations).
|
|
436
|
+
workspace: workspaceRoot,
|
|
417
437
|
target
|
|
418
438
|
});
|
|
419
439
|
if (result.status === "failed") {
|
|
440
|
+
const stepDurationMs = Date.now() - stepStartTime;
|
|
420
441
|
const error = new Error(result.error?.message ?? "Step execution failed");
|
|
421
442
|
await engine.markStepFailed(run.id, job.id, step.id, error);
|
|
422
443
|
stepLogger.error("Step failed", error, {
|
|
423
444
|
runId: run.id,
|
|
424
445
|
jobId: job.id,
|
|
425
|
-
stepId: step.id
|
|
446
|
+
stepId: step.id,
|
|
447
|
+
stepName: step.name,
|
|
448
|
+
uses: step.spec.uses,
|
|
449
|
+
durationMs: stepDurationMs,
|
|
450
|
+
resolvedInputs: interpolatedWith,
|
|
451
|
+
errorCode: result.error?.code
|
|
426
452
|
});
|
|
453
|
+
if (step.continueOnError) {
|
|
454
|
+
stepLogger.warn("[step] continueOnError=true \u2014 continuing despite failure", {
|
|
455
|
+
runId: run.id,
|
|
456
|
+
jobId: job.id,
|
|
457
|
+
stepId: step.id
|
|
458
|
+
});
|
|
459
|
+
continue;
|
|
460
|
+
}
|
|
427
461
|
throw error;
|
|
428
462
|
}
|
|
429
463
|
const stepOutputs = result.status === "success" ? result.outputs : void 0;
|
|
@@ -456,9 +490,9 @@ async function createWorkflowWorker(options) {
|
|
|
456
490
|
stepCount: job.steps.length
|
|
457
491
|
}).catch(() => {
|
|
458
492
|
});
|
|
459
|
-
if (provisionedWorkspaceId &&
|
|
493
|
+
if (provisionedWorkspaceId && wsProvider2) {
|
|
460
494
|
try {
|
|
461
|
-
await
|
|
495
|
+
await wsProvider2.release(provisionedWorkspaceId);
|
|
462
496
|
jobLogger.info("Workspace released", { workspaceId: provisionedWorkspaceId });
|
|
463
497
|
} catch (releaseErr) {
|
|
464
498
|
jobLogger.warn("Workspace release failed", {
|
|
@@ -597,6 +631,227 @@ function sleep(ms) {
|
|
|
597
631
|
setTimeout(resolve, ms);
|
|
598
632
|
});
|
|
599
633
|
}
|
|
634
|
+
async function waitForApproval(engine, runId, jobId, step, interpolatedWith, isStopRequested, stepLogger) {
|
|
635
|
+
const approvalTimeoutMs = interpolatedWith?.["timeoutMs"];
|
|
636
|
+
if (!approvalTimeoutMs) {
|
|
637
|
+
stepLogger.warn("[approval] No timeout configured \u2014 approval may wait indefinitely", {
|
|
638
|
+
runId,
|
|
639
|
+
jobId,
|
|
640
|
+
stepId: step.id,
|
|
641
|
+
stepName: step.name
|
|
642
|
+
});
|
|
643
|
+
}
|
|
644
|
+
stepLogger.info("[approval] Waiting for approval", {
|
|
645
|
+
runId,
|
|
646
|
+
jobId,
|
|
647
|
+
stepId: step.id,
|
|
648
|
+
stepName: step.name,
|
|
649
|
+
context: interpolatedWith
|
|
650
|
+
});
|
|
651
|
+
const approvalStartMs = Date.now();
|
|
652
|
+
let pollCount = 0;
|
|
653
|
+
while (!isStopRequested()) {
|
|
654
|
+
await sleep(2e3);
|
|
655
|
+
pollCount++;
|
|
656
|
+
const currentRun = await engine.getRun(runId);
|
|
657
|
+
if (currentRun?.status === "cancelled") {
|
|
658
|
+
stepLogger.info("[approval] Run cancelled \u2014 treating as interrupted, NOT approved", {
|
|
659
|
+
runId,
|
|
660
|
+
stepId: step.id,
|
|
661
|
+
stepName: step.name,
|
|
662
|
+
waitedMs: Date.now() - approvalStartMs
|
|
663
|
+
});
|
|
664
|
+
return "interrupted";
|
|
665
|
+
}
|
|
666
|
+
const currentJob = currentRun?.jobs.find((j) => j.id === jobId);
|
|
667
|
+
const currentStep = currentJob?.steps.find((s) => s.id === step.id);
|
|
668
|
+
if (!currentStep || currentStep.status === "success") {
|
|
669
|
+
stepLogger.info("[approval] Approval granted", {
|
|
670
|
+
runId,
|
|
671
|
+
stepId: step.id,
|
|
672
|
+
stepName: step.name,
|
|
673
|
+
waitedMs: Date.now() - approvalStartMs
|
|
674
|
+
});
|
|
675
|
+
break;
|
|
676
|
+
}
|
|
677
|
+
if (currentStep.status === "failed") {
|
|
678
|
+
stepLogger.info("[approval] Approval rejected", {
|
|
679
|
+
runId,
|
|
680
|
+
stepId: step.id,
|
|
681
|
+
stepName: step.name,
|
|
682
|
+
waitedMs: Date.now() - approvalStartMs
|
|
683
|
+
});
|
|
684
|
+
break;
|
|
685
|
+
}
|
|
686
|
+
if (pollCount % 10 === 0) {
|
|
687
|
+
stepLogger.info("[approval] Still waiting for approval", {
|
|
688
|
+
runId,
|
|
689
|
+
stepId: step.id,
|
|
690
|
+
stepName: step.name,
|
|
691
|
+
waitedMs: Date.now() - approvalStartMs,
|
|
692
|
+
pollCount
|
|
693
|
+
});
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
if (isStopRequested()) {
|
|
697
|
+
stepLogger.info("Approval wait interrupted by shutdown", { stepId: step.id });
|
|
698
|
+
return "interrupted";
|
|
699
|
+
}
|
|
700
|
+
return "done";
|
|
701
|
+
}
|
|
702
|
+
async function applyGateSkip(engine, run, job, step, decision, stepLogger) {
|
|
703
|
+
const { skipTo, outputs } = decision;
|
|
704
|
+
stepLogger.info("[gate] Gate triggered skip-forward", {
|
|
705
|
+
runId: run.id,
|
|
706
|
+
stepId: step.id,
|
|
707
|
+
stepName: step.name,
|
|
708
|
+
skipTo
|
|
709
|
+
});
|
|
710
|
+
const stateStore = engine.getStateStore();
|
|
711
|
+
const scheduler = engine.getScheduler();
|
|
712
|
+
await engine.markStepCompleted(run.id, job.id, step.id, outputs);
|
|
713
|
+
let pastGate = false;
|
|
714
|
+
for (const s of job.steps) {
|
|
715
|
+
if (s.id === step.id) {
|
|
716
|
+
pastGate = true;
|
|
717
|
+
continue;
|
|
718
|
+
}
|
|
719
|
+
if (!pastGate) continue;
|
|
720
|
+
if (s.spec.id === skipTo || s.id === skipTo) break;
|
|
721
|
+
await stateStore.updateStep(run.id, job.id, s.id, (draft) => {
|
|
722
|
+
draft.status = "success";
|
|
723
|
+
draft.outputs = { skipped: true };
|
|
724
|
+
draft.startedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
725
|
+
draft.finishedAt = (/* @__PURE__ */ new Date()).toISOString();
|
|
726
|
+
});
|
|
727
|
+
}
|
|
728
|
+
const freshRun = await engine.getRun(run.id);
|
|
729
|
+
const freshJob = freshRun?.jobs.find((j) => j.id === job.id);
|
|
730
|
+
if (freshJob) {
|
|
731
|
+
await scheduler.enqueueJob(run.id, freshJob, freshJob.priority ?? "normal");
|
|
732
|
+
}
|
|
733
|
+
stepLogger.info("[gate] Skip applied \u2014 re-enqueued job at skipTo target", {
|
|
734
|
+
runId: run.id,
|
|
735
|
+
skipTo
|
|
736
|
+
});
|
|
737
|
+
}
|
|
738
|
+
async function applyGateRestart(engine, run, job, step, decision, stepLogger) {
|
|
739
|
+
const { restartFrom, context, outputs, nextIteration } = decision;
|
|
740
|
+
const stepsToReset = [];
|
|
741
|
+
for (const s of job.steps) {
|
|
742
|
+
if (s.spec.id === restartFrom || s.id === restartFrom) {
|
|
743
|
+
stepsToReset.push(s.name ?? s.id);
|
|
744
|
+
} else if (stepsToReset.length > 0) {
|
|
745
|
+
stepsToReset.push(s.name ?? s.id);
|
|
746
|
+
}
|
|
747
|
+
}
|
|
748
|
+
stepLogger.warn("[gate] Gate triggered restart", {
|
|
749
|
+
runId: run.id,
|
|
750
|
+
stepId: step.id,
|
|
751
|
+
stepName: step.name,
|
|
752
|
+
restartFrom,
|
|
753
|
+
iteration: nextIteration,
|
|
754
|
+
stepsToReset
|
|
755
|
+
});
|
|
756
|
+
const stateStore = engine.getStateStore();
|
|
757
|
+
const scheduler = engine.getScheduler();
|
|
758
|
+
await stateStore.updateStep(run.id, job.id, step.id, (draft) => {
|
|
759
|
+
draft.metadata = { ...draft.metadata ?? {}, iterations: nextIteration };
|
|
760
|
+
});
|
|
761
|
+
if (context) {
|
|
762
|
+
await engine.updateRun(run.id, (draft) => {
|
|
763
|
+
const payload = draft.trigger.payload ?? {};
|
|
764
|
+
Object.assign(payload, context);
|
|
765
|
+
draft.trigger.payload = payload;
|
|
766
|
+
return draft;
|
|
767
|
+
});
|
|
768
|
+
}
|
|
769
|
+
const sameJobTarget = job.steps.some((s) => s.spec.id === restartFrom || s.id === restartFrom);
|
|
770
|
+
if (!sameJobTarget) {
|
|
771
|
+
const fresh = await engine.getRun(run.id) ?? run;
|
|
772
|
+
const resetNames = /* @__PURE__ */ new Set([restartFrom]);
|
|
773
|
+
let changed = true;
|
|
774
|
+
while (changed) {
|
|
775
|
+
changed = false;
|
|
776
|
+
for (const j of fresh.jobs) {
|
|
777
|
+
if (resetNames.has(j.jobName)) {
|
|
778
|
+
continue;
|
|
779
|
+
}
|
|
780
|
+
if ((j.needs ?? []).some((n) => resetNames.has(n))) {
|
|
781
|
+
resetNames.add(j.jobName);
|
|
782
|
+
changed = true;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
for (const rj of fresh.jobs) {
|
|
787
|
+
if (!resetNames.has(rj.jobName)) {
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
790
|
+
const pending = (rj.needs ?? []).filter((n) => resetNames.has(n));
|
|
791
|
+
for (const s of rj.steps) {
|
|
792
|
+
await stateStore.updateStep(run.id, rj.id, s.id, (draft) => {
|
|
793
|
+
draft.status = "queued";
|
|
794
|
+
draft.startedAt = void 0;
|
|
795
|
+
draft.finishedAt = void 0;
|
|
796
|
+
draft.error = void 0;
|
|
797
|
+
draft.outputs = void 0;
|
|
798
|
+
});
|
|
799
|
+
}
|
|
800
|
+
await stateStore.updateJob(run.id, rj.id, (draft) => {
|
|
801
|
+
draft.status = "queued";
|
|
802
|
+
draft.startedAt = void 0;
|
|
803
|
+
draft.finishedAt = void 0;
|
|
804
|
+
draft.pendingDependencies = [...pending];
|
|
805
|
+
draft.blocked = pending.length > 0;
|
|
806
|
+
});
|
|
807
|
+
}
|
|
808
|
+
const refreshed = await engine.getRun(run.id);
|
|
809
|
+
const targetJob = refreshed?.jobs.find((j) => j.jobName === restartFrom);
|
|
810
|
+
if (targetJob && !targetJob.blocked) {
|
|
811
|
+
await scheduler.enqueueJob(run.id, targetJob, targetJob.priority ?? "normal");
|
|
812
|
+
}
|
|
813
|
+
stepLogger.warn("[gate] Cross-job restart re-enqueued target", {
|
|
814
|
+
runId: run.id,
|
|
815
|
+
restartFrom,
|
|
816
|
+
iteration: nextIteration,
|
|
817
|
+
resetJobs: [...resetNames]
|
|
818
|
+
});
|
|
819
|
+
return;
|
|
820
|
+
}
|
|
821
|
+
await engine.markStepCompleted(run.id, job.id, step.id, outputs);
|
|
822
|
+
let foundTarget = false;
|
|
823
|
+
for (const s of job.steps) {
|
|
824
|
+
if (s.spec.id === restartFrom || s.id === restartFrom) {
|
|
825
|
+
foundTarget = true;
|
|
826
|
+
}
|
|
827
|
+
if (foundTarget) {
|
|
828
|
+
await stateStore.updateStep(run.id, job.id, s.id, (draft) => {
|
|
829
|
+
draft.status = "queued";
|
|
830
|
+
draft.startedAt = void 0;
|
|
831
|
+
draft.finishedAt = void 0;
|
|
832
|
+
draft.error = void 0;
|
|
833
|
+
draft.outputs = void 0;
|
|
834
|
+
});
|
|
835
|
+
}
|
|
836
|
+
}
|
|
837
|
+
await stateStore.updateJob(run.id, job.id, (draft) => {
|
|
838
|
+
draft.status = "queued";
|
|
839
|
+
draft.startedAt = void 0;
|
|
840
|
+
draft.finishedAt = void 0;
|
|
841
|
+
});
|
|
842
|
+
const updatedRun = await engine.getRun(run.id);
|
|
843
|
+
const updatedJob = updatedRun?.jobs.find((j) => j.id === job.id);
|
|
844
|
+
if (updatedJob) {
|
|
845
|
+
await scheduler.enqueueJob(run.id, updatedJob, updatedJob.priority ?? "normal");
|
|
846
|
+
stepLogger.info("[gate] Job re-enqueued for restart", {
|
|
847
|
+
runId: run.id,
|
|
848
|
+
jobId: job.id,
|
|
849
|
+
jobName: job.jobName,
|
|
850
|
+
iteration: nextIteration,
|
|
851
|
+
restartFrom
|
|
852
|
+
});
|
|
853
|
+
}
|
|
854
|
+
}
|
|
600
855
|
function inferWorkspaceProvisionReasonCode(message) {
|
|
601
856
|
return /ETIMEDOUT|timeout/iu.test(message) ? "workspace_provision_timeout" : "workspace_provision_failed";
|
|
602
857
|
}
|
|
@@ -633,7 +888,6 @@ var JobBroker = class {
|
|
|
633
888
|
id: "execute",
|
|
634
889
|
name: "Execute handler",
|
|
635
890
|
uses,
|
|
636
|
-
// @ts-expect-error - WorkflowSpec step.with type mismatch
|
|
637
891
|
with: request.input ?? {}
|
|
638
892
|
}
|
|
639
893
|
]
|
|
@@ -641,6 +895,7 @@ var JobBroker = class {
|
|
|
641
895
|
}
|
|
642
896
|
};
|
|
643
897
|
const run = await this.engine.runFromInline(spec, {
|
|
898
|
+
trigger: { type: "manual" },
|
|
644
899
|
env: {},
|
|
645
900
|
metadata: request.metadata
|
|
646
901
|
});
|
|
@@ -651,20 +906,6 @@ var JobBroker = class {
|
|
|
651
906
|
});
|
|
652
907
|
return run;
|
|
653
908
|
}
|
|
654
|
-
/**
|
|
655
|
-
* Schedule a recurring job with cron expression.
|
|
656
|
-
* Registers job with CronScheduler (if available).
|
|
657
|
-
*
|
|
658
|
-
* NOTE: CronScheduler integration not yet implemented.
|
|
659
|
-
* This is a placeholder for future implementation.
|
|
660
|
-
*/
|
|
661
|
-
async schedule(request) {
|
|
662
|
-
this.logger.warn("CronScheduler not yet implemented", {
|
|
663
|
-
handler: request.handler,
|
|
664
|
-
cron: request.cron
|
|
665
|
-
});
|
|
666
|
-
throw new Error("CronScheduler not yet implemented");
|
|
667
|
-
}
|
|
668
909
|
/**
|
|
669
910
|
* Get job status by run ID.
|
|
670
911
|
*/
|
|
@@ -701,22 +942,38 @@ var JobBroker = class {
|
|
|
701
942
|
if (!run) {
|
|
702
943
|
return [];
|
|
703
944
|
}
|
|
945
|
+
const stepNameMap = /* @__PURE__ */ new Map();
|
|
946
|
+
for (const job of run.jobs ?? []) {
|
|
947
|
+
for (const step of job.steps ?? []) {
|
|
948
|
+
if (step.id && step.name) {
|
|
949
|
+
stepNameMap.set(step.id, step.name);
|
|
950
|
+
}
|
|
951
|
+
}
|
|
952
|
+
}
|
|
704
953
|
const limit = options?.limit ?? 100;
|
|
705
954
|
const offset = options?.offset ?? 0;
|
|
706
955
|
const startTime = run.startedAt ? new Date(run.startedAt).getTime() : Date.now() - 36e5;
|
|
707
956
|
const endTime = run.finishedAt ? new Date(run.finishedAt).getTime() : Date.now();
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
957
|
+
let queryResult = { logs: [] };
|
|
958
|
+
try {
|
|
959
|
+
queryResult = await this.platform.logs.query(
|
|
960
|
+
{
|
|
961
|
+
from: startTime,
|
|
962
|
+
to: endTime,
|
|
963
|
+
level: options?.level && options.level !== "all" ? options.level : void 0
|
|
964
|
+
},
|
|
965
|
+
{
|
|
966
|
+
limit: 2e3,
|
|
967
|
+
offset: 0
|
|
968
|
+
}
|
|
969
|
+
);
|
|
970
|
+
} catch {
|
|
971
|
+
this.logger.warn("Log backend unavailable \u2014 returning empty log list for run", { runId });
|
|
972
|
+
}
|
|
973
|
+
const filtered = (queryResult.logs ?? []).filter((log) => {
|
|
974
|
+
if (!log.fields) {
|
|
975
|
+
return false;
|
|
717
976
|
}
|
|
718
|
-
);
|
|
719
|
-
const filtered = queryResult.logs.filter((log) => {
|
|
720
977
|
if (log.fields["runId"] !== runId) {
|
|
721
978
|
return false;
|
|
722
979
|
}
|
|
@@ -727,12 +984,18 @@ var JobBroker = class {
|
|
|
727
984
|
});
|
|
728
985
|
filtered.sort((a, b) => a.timestamp - b.timestamp);
|
|
729
986
|
const page = filtered.slice(offset, offset + limit);
|
|
730
|
-
return page.map((log) =>
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
987
|
+
return page.map((log) => {
|
|
988
|
+
const stepId = log.fields["stepId"];
|
|
989
|
+
return {
|
|
990
|
+
timestamp: new Date(log.timestamp).toISOString(),
|
|
991
|
+
level: log.level,
|
|
992
|
+
message: log.message,
|
|
993
|
+
stepId,
|
|
994
|
+
stepName: stepId ? stepNameMap.get(stepId) : void 0,
|
|
995
|
+
stream: log.fields["logSource"],
|
|
996
|
+
context: log.fields
|
|
997
|
+
};
|
|
998
|
+
});
|
|
736
999
|
}
|
|
737
1000
|
};
|
|
738
1001
|
var CronScheduler = class {
|
|
@@ -920,7 +1183,6 @@ var CronScheduler = class {
|
|
|
920
1183
|
jobs: job.workflowSpec.jobs,
|
|
921
1184
|
env: job.workflowSpec.env
|
|
922
1185
|
};
|
|
923
|
-
console.log("\u{1F50D} CRON SPEC:", JSON.stringify(spec, null, 2));
|
|
924
1186
|
this.logger.debug("Running workflow from cron", {
|
|
925
1187
|
cronJobId,
|
|
926
1188
|
spec: JSON.stringify(spec, null, 2)
|
|
@@ -1135,6 +1397,23 @@ var CronScheduler = class {
|
|
|
1135
1397
|
});
|
|
1136
1398
|
this.registeredJobs.clear();
|
|
1137
1399
|
}
|
|
1400
|
+
/**
|
|
1401
|
+
* Unregister all user-sourced cron jobs (source === 'user'), stopping their
|
|
1402
|
+
* scheduled tasks. Safe to call while the scheduler is running — jobs are
|
|
1403
|
+
* stopped and removed so they can be re-registered by a fresh discovery pass.
|
|
1404
|
+
*/
|
|
1405
|
+
clearUserJobs() {
|
|
1406
|
+
const userJobIds = Array.from(this.registeredJobs.keys()).filter((id) => id.startsWith("user:"));
|
|
1407
|
+
for (const cronJobId of userJobIds) {
|
|
1408
|
+
const task = this.scheduledTasks.get(cronJobId);
|
|
1409
|
+
if (task) {
|
|
1410
|
+
task.stop();
|
|
1411
|
+
this.scheduledTasks.delete(cronJobId);
|
|
1412
|
+
}
|
|
1413
|
+
this.registeredJobs.delete(cronJobId);
|
|
1414
|
+
}
|
|
1415
|
+
this.logger.info("Cleared user cron jobs", { count: userJobIds.length });
|
|
1416
|
+
}
|
|
1138
1417
|
};
|
|
1139
1418
|
var CronDiscovery = class {
|
|
1140
1419
|
cliApi;
|
|
@@ -1250,6 +1529,92 @@ var CronDiscovery = class {
|
|
|
1250
1529
|
return count;
|
|
1251
1530
|
}
|
|
1252
1531
|
};
|
|
1532
|
+
var WorkflowFileWatcher = class {
|
|
1533
|
+
watchers = [];
|
|
1534
|
+
workflowService;
|
|
1535
|
+
cronDiscovery;
|
|
1536
|
+
cronScheduler;
|
|
1537
|
+
logger;
|
|
1538
|
+
debounceMs;
|
|
1539
|
+
debounceTimer = null;
|
|
1540
|
+
constructor(options) {
|
|
1541
|
+
this.workflowService = options.workflowService;
|
|
1542
|
+
this.cronDiscovery = options.cronDiscovery;
|
|
1543
|
+
this.cronScheduler = options.cronScheduler;
|
|
1544
|
+
this.logger = options.logger;
|
|
1545
|
+
this.debounceMs = options.debounceMs ?? 300;
|
|
1546
|
+
for (const dir of options.watchDirs) {
|
|
1547
|
+
this.startWatcher(dir);
|
|
1548
|
+
}
|
|
1549
|
+
}
|
|
1550
|
+
startWatcher(dir) {
|
|
1551
|
+
access(dir).then(() => {
|
|
1552
|
+
try {
|
|
1553
|
+
const watcher = watch(dir, { persistent: false }, (eventType, filename) => {
|
|
1554
|
+
if (!filename) {
|
|
1555
|
+
return;
|
|
1556
|
+
}
|
|
1557
|
+
if (!filename.endsWith(".yml") && !filename.endsWith(".yaml")) {
|
|
1558
|
+
return;
|
|
1559
|
+
}
|
|
1560
|
+
this.scheduleReload(dir, filename);
|
|
1561
|
+
});
|
|
1562
|
+
watcher.on("error", (err) => {
|
|
1563
|
+
this.logger.warn("WorkflowFileWatcher: watcher error", {
|
|
1564
|
+
dir,
|
|
1565
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1566
|
+
});
|
|
1567
|
+
});
|
|
1568
|
+
this.watchers.push(watcher);
|
|
1569
|
+
this.logger.info("WorkflowFileWatcher: watching directory", { dir });
|
|
1570
|
+
} catch (err) {
|
|
1571
|
+
this.logger.warn("WorkflowFileWatcher: could not start watcher", {
|
|
1572
|
+
dir,
|
|
1573
|
+
error: err instanceof Error ? err.message : String(err)
|
|
1574
|
+
});
|
|
1575
|
+
}
|
|
1576
|
+
}).catch(() => {
|
|
1577
|
+
this.logger.debug("WorkflowFileWatcher: directory does not exist, skipping", { dir });
|
|
1578
|
+
});
|
|
1579
|
+
}
|
|
1580
|
+
scheduleReload(dir, filename) {
|
|
1581
|
+
if (this.debounceTimer) {
|
|
1582
|
+
clearTimeout(this.debounceTimer);
|
|
1583
|
+
}
|
|
1584
|
+
this.debounceTimer = setTimeout(() => {
|
|
1585
|
+
this.debounceTimer = null;
|
|
1586
|
+
this.reload(dir, filename).catch((err) => {
|
|
1587
|
+
this.logger.error(
|
|
1588
|
+
"WorkflowFileWatcher: reload failed",
|
|
1589
|
+
err instanceof Error ? err : void 0,
|
|
1590
|
+
{ dir, filename }
|
|
1591
|
+
);
|
|
1592
|
+
});
|
|
1593
|
+
}, this.debounceMs);
|
|
1594
|
+
}
|
|
1595
|
+
async reload(dir, filename) {
|
|
1596
|
+
this.logger.info("WorkflowFileWatcher: YAML change detected, reloading", { dir, filename });
|
|
1597
|
+
await this.workflowService.refreshManifests();
|
|
1598
|
+
this.cronScheduler.clearUserJobs();
|
|
1599
|
+
const discovered = await this.cronDiscovery.discoverAll();
|
|
1600
|
+
this.logger.info("WorkflowFileWatcher: reload complete", {
|
|
1601
|
+
filename,
|
|
1602
|
+
cronJobs: discovered
|
|
1603
|
+
});
|
|
1604
|
+
}
|
|
1605
|
+
/** Stop all watchers and cancel any pending debounce timer. */
|
|
1606
|
+
close() {
|
|
1607
|
+
if (this.debounceTimer) {
|
|
1608
|
+
clearTimeout(this.debounceTimer);
|
|
1609
|
+
this.debounceTimer = null;
|
|
1610
|
+
}
|
|
1611
|
+
for (const watcher of this.watchers) {
|
|
1612
|
+
watcher.close();
|
|
1613
|
+
}
|
|
1614
|
+
this.watchers.length = 0;
|
|
1615
|
+
this.logger.info("WorkflowFileWatcher: stopped");
|
|
1616
|
+
}
|
|
1617
|
+
};
|
|
1253
1618
|
|
|
1254
1619
|
// src/host/workflow-host-service.ts
|
|
1255
1620
|
var TENANT_ID_PATTERN = /^[a-zA-Z0-9_-]+$/;
|
|
@@ -1349,7 +1714,11 @@ var WorkflowHostService = class {
|
|
|
1349
1714
|
return logs;
|
|
1350
1715
|
}
|
|
1351
1716
|
async getRunLogs(runId, options) {
|
|
1352
|
-
const
|
|
1717
|
+
const resolved = await this.resolveRunId(runId);
|
|
1718
|
+
if (!resolved) {
|
|
1719
|
+
throw new Error("Run not found");
|
|
1720
|
+
}
|
|
1721
|
+
const logs = await this.options.jobBroker.getRunLogs(resolved, options);
|
|
1353
1722
|
return logs;
|
|
1354
1723
|
}
|
|
1355
1724
|
async cancelJob(tenantId, jobId) {
|
|
@@ -1373,8 +1742,8 @@ var WorkflowHostService = class {
|
|
|
1373
1742
|
error: run.result?.error?.message
|
|
1374
1743
|
}));
|
|
1375
1744
|
if (type) {
|
|
1376
|
-
const
|
|
1377
|
-
const regex = new RegExp(`^${
|
|
1745
|
+
const escaped = type.replace(/[.+?^${}()|[\]\\]/g, "\\$&").replace(/\*/g, ".*");
|
|
1746
|
+
const regex = new RegExp(`^${escaped}$`);
|
|
1378
1747
|
jobs = jobs.filter((job) => regex.test(job.type));
|
|
1379
1748
|
}
|
|
1380
1749
|
if (status) {
|
|
@@ -1429,6 +1798,15 @@ var WorkflowHostService = class {
|
|
|
1429
1798
|
const triggerType = request.trigger?.type === "cron" ? "schedule" : request.trigger?.type === "api" ? "webhook" : "manual";
|
|
1430
1799
|
const specInputDefs = specInput["inputs"] ?? {};
|
|
1431
1800
|
const userInputs = request.inputs ?? (request.input && typeof request.input === "object" ? request.input : {});
|
|
1801
|
+
const missingRequired = [];
|
|
1802
|
+
for (const [key, def] of Object.entries(specInputDefs)) {
|
|
1803
|
+
if (def.required && !(key in userInputs) && def.default === void 0) {
|
|
1804
|
+
missingRequired.push(key);
|
|
1805
|
+
}
|
|
1806
|
+
}
|
|
1807
|
+
if (missingRequired.length > 0) {
|
|
1808
|
+
throw new Error(`Missing required input(s): ${missingRequired.join(", ")}`);
|
|
1809
|
+
}
|
|
1432
1810
|
const resolvedInputs = {};
|
|
1433
1811
|
for (const [key, def] of Object.entries(specInputDefs)) {
|
|
1434
1812
|
resolvedInputs[key] = key in userInputs ? userInputs[key] : def.default;
|
|
@@ -1446,6 +1824,80 @@ var WorkflowHostService = class {
|
|
|
1446
1824
|
status: run.status
|
|
1447
1825
|
};
|
|
1448
1826
|
}
|
|
1827
|
+
async rerunWorkflow(runId, request) {
|
|
1828
|
+
const resolved = await this.resolveRunId(runId);
|
|
1829
|
+
const sourceRun = resolved ? await this.options.engine.getRun(resolved) : null;
|
|
1830
|
+
if (!sourceRun) {
|
|
1831
|
+
throw new Error("Run not found");
|
|
1832
|
+
}
|
|
1833
|
+
const workflowId = sourceRun.metadata?.["workflowId"] ?? sourceRun.name;
|
|
1834
|
+
const workflowService = this.requireWorkflowService();
|
|
1835
|
+
const workflow = await workflowService.get(workflowId);
|
|
1836
|
+
if (!workflow) {
|
|
1837
|
+
throw new Error("Workflow not found");
|
|
1838
|
+
}
|
|
1839
|
+
const specInput = workflow.input;
|
|
1840
|
+
let spec = {
|
|
1841
|
+
...specInput
|
|
1842
|
+
};
|
|
1843
|
+
if (request.failedOnly) {
|
|
1844
|
+
const failedJobNames = new Set(
|
|
1845
|
+
(sourceRun.jobs ?? []).filter((job) => job.status === "failed" || job.status === "interrupted").map((job) => job.jobName)
|
|
1846
|
+
);
|
|
1847
|
+
if (failedJobNames.size === 0) {
|
|
1848
|
+
throw new Error("No failed jobs to rerun");
|
|
1849
|
+
}
|
|
1850
|
+
const filteredJobs = {};
|
|
1851
|
+
for (const [name, jobSpec] of Object.entries(spec.jobs)) {
|
|
1852
|
+
if (failedJobNames.has(name)) {
|
|
1853
|
+
const js = jobSpec;
|
|
1854
|
+
const needs = Array.isArray(js["needs"]) ? js["needs"].filter((dep) => failedJobNames.has(dep)) : void 0;
|
|
1855
|
+
filteredJobs[name] = needs !== void 0 && needs.length < js["needs"].length ? { ...js, needs } : js;
|
|
1856
|
+
}
|
|
1857
|
+
}
|
|
1858
|
+
spec = { ...spec, jobs: filteredJobs };
|
|
1859
|
+
}
|
|
1860
|
+
const resolvedInputs = sourceRun.inputs ?? {};
|
|
1861
|
+
const run = await this.options.engine.runFromSpec(spec, {
|
|
1862
|
+
trigger: {
|
|
1863
|
+
type: "manual",
|
|
1864
|
+
actor: "cli-rerun",
|
|
1865
|
+
payload: resolvedInputs
|
|
1866
|
+
},
|
|
1867
|
+
inputs: resolvedInputs
|
|
1868
|
+
});
|
|
1869
|
+
return {
|
|
1870
|
+
runId: run.id,
|
|
1871
|
+
status: run.status
|
|
1872
|
+
};
|
|
1873
|
+
}
|
|
1874
|
+
async restartRun(runId, request) {
|
|
1875
|
+
const resolved = await this.resolveRunId(runId);
|
|
1876
|
+
if (!resolved) {
|
|
1877
|
+
throw new Error("Run not found or snapshot not available");
|
|
1878
|
+
}
|
|
1879
|
+
const currentRun = await this.options.engine.getRun(resolved);
|
|
1880
|
+
if (!currentRun) {
|
|
1881
|
+
throw new Error("Run not found or snapshot not available");
|
|
1882
|
+
}
|
|
1883
|
+
if (currentRun.status === "running" || currentRun.status === "queued") {
|
|
1884
|
+
throw new Error(
|
|
1885
|
+
`Cannot restart an active run (status: ${currentRun.status}); cancel it first`
|
|
1886
|
+
);
|
|
1887
|
+
}
|
|
1888
|
+
const run = await this.options.engine.replayRun(resolved, {
|
|
1889
|
+
fromStepId: request.fromStepId,
|
|
1890
|
+
env: request.env
|
|
1891
|
+
});
|
|
1892
|
+
if (!run) {
|
|
1893
|
+
throw new Error("Run not found or snapshot not available");
|
|
1894
|
+
}
|
|
1895
|
+
return {
|
|
1896
|
+
runId: run.id,
|
|
1897
|
+
status: run.status,
|
|
1898
|
+
fromStepId: request.fromStepId
|
|
1899
|
+
};
|
|
1900
|
+
}
|
|
1449
1901
|
registerCron(tenantId, request) {
|
|
1450
1902
|
this.assertTenantId(tenantId);
|
|
1451
1903
|
const scheduler = this.requireCronScheduler();
|
|
@@ -1580,11 +2032,28 @@ var WorkflowHostService = class {
|
|
|
1580
2032
|
pluginId: workflow.pluginId,
|
|
1581
2033
|
status: workflow.status === "active" ? "active" : "inactive",
|
|
1582
2034
|
tags: workflow.tags,
|
|
1583
|
-
inputs: workflow.inputSchema
|
|
2035
|
+
inputs: workflow.inputSchema,
|
|
2036
|
+
version: workflow.version,
|
|
2037
|
+
updatedAt: workflow.updatedAt
|
|
1584
2038
|
};
|
|
1585
2039
|
}
|
|
2040
|
+
async resolveRunId(idOrPrefix) {
|
|
2041
|
+
const run = await this.options.engine.getRun(idOrPrefix);
|
|
2042
|
+
if (run) {
|
|
2043
|
+
return run.id;
|
|
2044
|
+
}
|
|
2045
|
+
if (idOrPrefix.length <= 8 && !idOrPrefix.includes("-")) {
|
|
2046
|
+
const all = await this.options.engine.getAllRuns();
|
|
2047
|
+
return all.find((r) => r.id.startsWith(idOrPrefix))?.id ?? null;
|
|
2048
|
+
}
|
|
2049
|
+
return null;
|
|
2050
|
+
}
|
|
1586
2051
|
async getRun(runId) {
|
|
1587
|
-
|
|
2052
|
+
const resolved = await this.resolveRunId(runId);
|
|
2053
|
+
if (!resolved) {
|
|
2054
|
+
return null;
|
|
2055
|
+
}
|
|
2056
|
+
return await this.options.engine.getRun(resolved);
|
|
1588
2057
|
}
|
|
1589
2058
|
async listRuns(filters) {
|
|
1590
2059
|
const allRuns = await this.options.engine.getAllRuns();
|
|
@@ -1592,21 +2061,40 @@ var WorkflowHostService = class {
|
|
|
1592
2061
|
if (filters?.status) {
|
|
1593
2062
|
runs = runs.filter((run) => run.status === filters.status);
|
|
1594
2063
|
}
|
|
2064
|
+
if (filters?.workflowId) {
|
|
2065
|
+
runs = runs.filter((run) => run.name === filters.workflowId);
|
|
2066
|
+
}
|
|
1595
2067
|
runs.sort((a, b) => new Date(b.createdAt ?? 0).getTime() - new Date(a.createdAt ?? 0).getTime());
|
|
1596
2068
|
const total = runs.length;
|
|
1597
2069
|
const start = filters?.offset ?? 0;
|
|
1598
2070
|
const end = filters?.limit ? start + filters.limit : runs.length;
|
|
1599
|
-
|
|
2071
|
+
const page = runs.slice(start, end).map((run) => {
|
|
2072
|
+
const allSteps = (run.jobs ?? []).flatMap((j) => j.steps ?? []);
|
|
2073
|
+
const activeSteps = allSteps.filter(
|
|
2074
|
+
(s) => s.status === "running" || s.status === "waiting_approval"
|
|
2075
|
+
);
|
|
2076
|
+
let currentStepName;
|
|
2077
|
+
if (run.status === "running" && activeSteps.length > 0) {
|
|
2078
|
+
currentStepName = activeSteps.length === 1 ? activeSteps[0].name : `${activeSteps[0].name} (+${activeSteps.length - 1})`;
|
|
2079
|
+
}
|
|
2080
|
+
return {
|
|
2081
|
+
...run,
|
|
2082
|
+
hasPendingApproval: run.status === "running" && allSteps.some((s) => s.status === "waiting_approval"),
|
|
2083
|
+
currentStepName
|
|
2084
|
+
};
|
|
2085
|
+
});
|
|
2086
|
+
return { runs: page, total };
|
|
1600
2087
|
}
|
|
1601
2088
|
async cancelRun(runId) {
|
|
1602
|
-
const
|
|
2089
|
+
const resolved = await this.resolveRunId(runId);
|
|
2090
|
+
const run = resolved ? await this.options.engine.getRun(resolved) : null;
|
|
1603
2091
|
if (!run) {
|
|
1604
2092
|
throw new Error("Run not found");
|
|
1605
2093
|
}
|
|
1606
2094
|
if (run.status !== "running" && run.status !== "queued") {
|
|
1607
2095
|
throw new Error(`Cannot cancel run with status "${run.status}"`);
|
|
1608
2096
|
}
|
|
1609
|
-
await this.options.engine.cancelRun(
|
|
2097
|
+
await this.options.engine.cancelRun(resolved);
|
|
1610
2098
|
}
|
|
1611
2099
|
requireWorkflowService() {
|
|
1612
2100
|
if (!this.options.workflowService) {
|
|
@@ -1908,7 +2396,7 @@ var KEEP_ALIVE_MS = 3e4;
|
|
|
1908
2396
|
var IDLE_TIMEOUT_MS = 6e4;
|
|
1909
2397
|
function registerWorkflowsAPI(options) {
|
|
1910
2398
|
const { server, hostService, engine, workflowService, logger, observability } = options;
|
|
1911
|
-
|
|
2399
|
+
const reloadHandler = async () => {
|
|
1912
2400
|
try {
|
|
1913
2401
|
logger.info("[workflows-api] Refreshing workflows from disk");
|
|
1914
2402
|
if (workflowService) {
|
|
@@ -1923,7 +2411,9 @@ function registerWorkflowsAPI(options) {
|
|
|
1923
2411
|
logger.error("[workflows-api] Failed to refresh workflows", error instanceof Error ? error : void 0);
|
|
1924
2412
|
return { ok: false, error: error instanceof Error ? error.message : String(error) };
|
|
1925
2413
|
}
|
|
1926
|
-
}
|
|
2414
|
+
};
|
|
2415
|
+
server.post("/api/v1/workflows/reload", { schema: { tags: ["Workflows"], summary: "Reload workflow definitions from disk" } }, reloadHandler);
|
|
2416
|
+
server.post("/api/v1/workflows/refresh", { schema: { tags: ["Workflows"], summary: "Reload workflow definitions from disk (alias for /reload)" } }, reloadHandler);
|
|
1927
2417
|
server.get("/api/v1/workflows", { schema: { tags: ["Workflows"], summary: "List workflow definitions" } }, async (request, reply) => {
|
|
1928
2418
|
try {
|
|
1929
2419
|
const response = await observability.observeOperation("workflow.catalog.list", () => hostService.listWorkflows(request.query));
|
|
@@ -1974,6 +2464,9 @@ function registerWorkflowsAPI(options) {
|
|
|
1974
2464
|
if (message === "Workflow not found") {
|
|
1975
2465
|
return fail(reply, 404, message);
|
|
1976
2466
|
}
|
|
2467
|
+
if (message.startsWith("Missing required input")) {
|
|
2468
|
+
return fail(reply, 400, message);
|
|
2469
|
+
}
|
|
1977
2470
|
logger.error("[workflows-api] Error running workflow", error instanceof Error ? error : void 0);
|
|
1978
2471
|
return fail(reply, 500, message);
|
|
1979
2472
|
}
|
|
@@ -1995,13 +2488,54 @@ function registerWorkflowsAPI(options) {
|
|
|
1995
2488
|
return fail(reply, 500, message);
|
|
1996
2489
|
}
|
|
1997
2490
|
});
|
|
2491
|
+
server.post("/api/v1/runs/:runId/rerun", { schema: { tags: ["Runs"], summary: "Rerun a workflow run" } }, async (request, reply) => {
|
|
2492
|
+
try {
|
|
2493
|
+
const { runId } = request.params;
|
|
2494
|
+
const response = await observability.observeOperation(
|
|
2495
|
+
"workflow.run.rerun",
|
|
2496
|
+
() => hostService.rerunWorkflow(runId, request.body ?? {})
|
|
2497
|
+
);
|
|
2498
|
+
return ok(response);
|
|
2499
|
+
} catch (error) {
|
|
2500
|
+
const message = error instanceof Error ? error.message : "Failed to rerun workflow";
|
|
2501
|
+
if (message === "Run not found" || message === "Workflow not found") {
|
|
2502
|
+
return fail(reply, 404, message);
|
|
2503
|
+
}
|
|
2504
|
+
if (message === "No failed jobs to rerun") {
|
|
2505
|
+
return fail(reply, 400, message);
|
|
2506
|
+
}
|
|
2507
|
+
logger.error("[workflows-api] Error rerunning workflow", error instanceof Error ? error : void 0);
|
|
2508
|
+
return fail(reply, 500, message);
|
|
2509
|
+
}
|
|
2510
|
+
});
|
|
2511
|
+
server.post("/api/v1/runs/:runId/restart", { schema: { tags: ["Runs"], summary: "Restart a run from a specific step (snapshot-based)" } }, async (request, reply) => {
|
|
2512
|
+
try {
|
|
2513
|
+
const { runId } = request.params;
|
|
2514
|
+
const response = await observability.observeOperation(
|
|
2515
|
+
"workflow.run.restart",
|
|
2516
|
+
() => hostService.restartRun(runId, request.body ?? {})
|
|
2517
|
+
);
|
|
2518
|
+
return ok(response);
|
|
2519
|
+
} catch (error) {
|
|
2520
|
+
const message = error instanceof Error ? error.message : "Failed to restart run";
|
|
2521
|
+
if (message.includes("not found") || message.includes("snapshot not available")) {
|
|
2522
|
+
return fail(reply, 404, message);
|
|
2523
|
+
}
|
|
2524
|
+
if (message.startsWith("Cannot restart an active run")) {
|
|
2525
|
+
return fail(reply, 409, message);
|
|
2526
|
+
}
|
|
2527
|
+
logger.error("[workflows-api] Error restarting run", error instanceof Error ? error : void 0);
|
|
2528
|
+
return fail(reply, 500, message);
|
|
2529
|
+
}
|
|
2530
|
+
});
|
|
1998
2531
|
server.get("/api/v1/runs", { schema: { tags: ["Runs"], summary: "List all workflow runs" } }, async (request, reply) => {
|
|
1999
2532
|
try {
|
|
2000
|
-
const { status, limit, offset } = request.query;
|
|
2533
|
+
const { status, workflowId, limit, offset } = request.query;
|
|
2001
2534
|
const response = await observability.observeOperation(
|
|
2002
2535
|
"workflow.run.list",
|
|
2003
2536
|
() => hostService.listRuns({
|
|
2004
2537
|
status,
|
|
2538
|
+
workflowId,
|
|
2005
2539
|
limit: limit ? parseInt(limit, 10) : 50,
|
|
2006
2540
|
offset: offset ? parseInt(offset, 10) : 0
|
|
2007
2541
|
})
|
|
@@ -2053,7 +2587,8 @@ function registerWorkflowsAPI(options) {
|
|
|
2053
2587
|
}
|
|
2054
2588
|
);
|
|
2055
2589
|
server.get("/api/v1/runs/:runId/events", { schema: { hide: true } }, async (request, reply) => {
|
|
2056
|
-
const { runId } = request.params;
|
|
2590
|
+
const { runId: rawId } = request.params;
|
|
2591
|
+
const runId = await hostService.resolveRunId(rawId) ?? rawId;
|
|
2057
2592
|
const run = await observability.observeOperation("workflow.run.events", () => engine.getRun(runId));
|
|
2058
2593
|
if (!run) {
|
|
2059
2594
|
return fail(reply, 404, "Run not found");
|
|
@@ -2074,7 +2609,7 @@ function registerWorkflowsAPI(options) {
|
|
|
2074
2609
|
if (raw.writableEnded) {
|
|
2075
2610
|
return;
|
|
2076
2611
|
}
|
|
2077
|
-
raw.write(`event:
|
|
2612
|
+
raw.write(`event: ${type}
|
|
2078
2613
|
`);
|
|
2079
2614
|
raw.write(`data: ${JSON.stringify({ type, runId, payload, timestamp: (/* @__PURE__ */ new Date()).toISOString() })}
|
|
2080
2615
|
|
|
@@ -2082,6 +2617,8 @@ function registerWorkflowsAPI(options) {
|
|
|
2082
2617
|
};
|
|
2083
2618
|
sendEvent("run.snapshot", run);
|
|
2084
2619
|
if (TERMINAL_STATUSES.includes(run.status)) {
|
|
2620
|
+
const terminalType = run.status === "success" ? "run.finished" : run.status === "failed" ? "run.failed" : "run.cancelled";
|
|
2621
|
+
sendEvent(terminalType, run);
|
|
2085
2622
|
raw.end();
|
|
2086
2623
|
return;
|
|
2087
2624
|
}
|
|
@@ -2125,11 +2662,11 @@ function registerWorkflowsAPI(options) {
|
|
|
2125
2662
|
|
|
2126
2663
|
// src/api/approvals-api.ts
|
|
2127
2664
|
function registerApprovalsAPI(options) {
|
|
2128
|
-
const { server, engine, logger, observability } = options;
|
|
2665
|
+
const { server, hostService, engine, logger, observability } = options;
|
|
2129
2666
|
server.get("/api/v1/runs/:runId/approvals", { schema: { tags: ["Approvals"], summary: "List pending approvals for a run" } }, async (request, reply) => {
|
|
2130
2667
|
try {
|
|
2131
|
-
const { runId } = request.params;
|
|
2132
|
-
const run = await observability.observeOperation("workflow.approval.list", () =>
|
|
2668
|
+
const { runId: rawId } = request.params;
|
|
2669
|
+
const run = await observability.observeOperation("workflow.approval.list", () => hostService.getRun(rawId));
|
|
2133
2670
|
if (!run) {
|
|
2134
2671
|
return fail(reply, 404, "Run not found");
|
|
2135
2672
|
}
|
|
@@ -2142,13 +2679,13 @@ function registerApprovalsAPI(options) {
|
|
|
2142
2679
|
stepId: step.id,
|
|
2143
2680
|
stepName: step.name,
|
|
2144
2681
|
specId: step.spec.id,
|
|
2145
|
-
context: step.spec.with ?? {},
|
|
2682
|
+
context: step.resolvedInputs ?? step.spec.with ?? {},
|
|
2146
2683
|
waitingSince: step.startedAt
|
|
2147
2684
|
});
|
|
2148
2685
|
}
|
|
2149
2686
|
}
|
|
2150
2687
|
}
|
|
2151
|
-
return ok({ runId, pending });
|
|
2688
|
+
return ok({ runId: run.id, pending });
|
|
2152
2689
|
} catch (error) {
|
|
2153
2690
|
logger.error("[approvals-api] Error listing pending approvals", error instanceof Error ? error : void 0);
|
|
2154
2691
|
return fail(reply, 500, error instanceof Error ? error.message : "Failed to list pending approvals");
|
|
@@ -2156,7 +2693,7 @@ function registerApprovalsAPI(options) {
|
|
|
2156
2693
|
});
|
|
2157
2694
|
server.post("/api/v1/runs/:runId/approvals/resolve", { schema: { tags: ["Approvals"], summary: "Approve or reject a pending step" } }, async (request, reply) => {
|
|
2158
2695
|
try {
|
|
2159
|
-
const { runId } = request.params;
|
|
2696
|
+
const { runId: rawId } = request.params;
|
|
2160
2697
|
const { jobId, stepId, action, comment, data } = request.body;
|
|
2161
2698
|
if (!jobId || !stepId || !action) {
|
|
2162
2699
|
return fail(reply, 400, "Missing required fields: jobId, stepId, action");
|
|
@@ -2164,7 +2701,7 @@ function registerApprovalsAPI(options) {
|
|
|
2164
2701
|
if (action !== "approve" && action !== "reject") {
|
|
2165
2702
|
return fail(reply, 400, 'action must be "approve" or "reject"');
|
|
2166
2703
|
}
|
|
2167
|
-
const run = await observability.observeOperation("workflow.approval.get", () =>
|
|
2704
|
+
const run = await observability.observeOperation("workflow.approval.get", () => hostService.getRun(rawId));
|
|
2168
2705
|
if (!run) {
|
|
2169
2706
|
return fail(reply, 404, "Run not found");
|
|
2170
2707
|
}
|
|
@@ -2181,17 +2718,17 @@ function registerApprovalsAPI(options) {
|
|
|
2181
2718
|
}
|
|
2182
2719
|
await observability.observeOperation(
|
|
2183
2720
|
"workflow.approval.resolve",
|
|
2184
|
-
() => engine.resolveApproval(
|
|
2721
|
+
() => engine.resolveApproval(run.id, jobId, stepId, action, data, comment)
|
|
2185
2722
|
);
|
|
2186
2723
|
logger.info("[approvals-api] Approval resolved", {
|
|
2187
|
-
runId,
|
|
2724
|
+
runId: run.id,
|
|
2188
2725
|
jobId,
|
|
2189
2726
|
stepId,
|
|
2190
2727
|
action,
|
|
2191
2728
|
comment
|
|
2192
2729
|
});
|
|
2193
2730
|
return ok({
|
|
2194
|
-
runId,
|
|
2731
|
+
runId: run.id,
|
|
2195
2732
|
jobId,
|
|
2196
2733
|
stepId,
|
|
2197
2734
|
action,
|
|
@@ -2278,169 +2815,128 @@ async function createServer(options) {
|
|
|
2278
2815
|
cronScheduler,
|
|
2279
2816
|
logger
|
|
2280
2817
|
});
|
|
2281
|
-
const server = Fastify({
|
|
2282
|
-
logger: false,
|
|
2283
|
-
// Use platform logger instead
|
|
2284
|
-
bodyLimit: 1048576
|
|
2285
|
-
// 1MB body limit (prevents parsing huge payloads)
|
|
2286
|
-
});
|
|
2287
2818
|
const isProduction = process.env.NODE_ENV === "production";
|
|
2288
|
-
|
|
2289
|
-
title: "KB Labs Workflow Daemon",
|
|
2290
|
-
description: "Background job execution and workflow orchestration API",
|
|
2291
|
-
version: "1.0.0",
|
|
2292
|
-
servers: [{ url: "http://localhost:7778", description: "Local dev" }],
|
|
2293
|
-
ui: !isProduction
|
|
2294
|
-
});
|
|
2295
|
-
const requireAuth = process.env.KB_DAEMON_REQUIRE_AUTH === "true" || isProduction;
|
|
2819
|
+
const requireAuth = process.env.KB_DAEMON_REQUIRE_AUTH === "false" ? false : process.env.KB_DAEMON_REQUIRE_AUTH === "true" || isProduction;
|
|
2296
2820
|
const daemonApiKey = process.env.KB_DAEMON_API_KEY;
|
|
2821
|
+
const allowedOrigins = process.env.ALLOWED_ORIGINS?.split(",") ?? [
|
|
2822
|
+
"http://localhost:3000",
|
|
2823
|
+
"http://localhost:5173"
|
|
2824
|
+
];
|
|
2825
|
+
if (requireAuth && !daemonApiKey) {
|
|
2826
|
+
throw new Error(
|
|
2827
|
+
"KB_DAEMON_API_KEY is required when daemon auth is enabled (KB_DAEMON_REQUIRE_AUTH=true or NODE_ENV=production)"
|
|
2828
|
+
);
|
|
2829
|
+
}
|
|
2297
2830
|
const observability = new HttpObservabilityCollector({
|
|
2298
2831
|
serviceId: "workflow",
|
|
2299
2832
|
serviceType: "workflow-daemon",
|
|
2300
2833
|
version: "1.0.0",
|
|
2301
2834
|
logsSource: "workflow",
|
|
2302
2835
|
dependencies: [
|
|
2303
|
-
{
|
|
2304
|
-
serviceId: "state-daemon",
|
|
2305
|
-
required: false,
|
|
2306
|
-
description: "Workflow run and job state storage"
|
|
2307
|
-
}
|
|
2836
|
+
{ serviceId: "state-daemon", required: false, description: "Workflow run and job state storage" }
|
|
2308
2837
|
]
|
|
2309
2838
|
});
|
|
2310
|
-
|
|
2311
|
-
|
|
2312
|
-
|
|
2313
|
-
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
});
|
|
2330
|
-
observability.register(server);
|
|
2331
|
-
const allowedOrigins = process.env.ALLOWED_ORIGINS?.split(",") || [
|
|
2332
|
-
"http://localhost:3000",
|
|
2333
|
-
"http://localhost:5173"
|
|
2334
|
-
// Vite dev server
|
|
2335
|
-
];
|
|
2336
|
-
await server.register(cors, {
|
|
2337
|
-
origin: (origin, callback) => {
|
|
2338
|
-
if (!origin) {
|
|
2339
|
-
const isDevelopment = process.env.NODE_ENV !== "production";
|
|
2340
|
-
if (isDevelopment) {
|
|
2341
|
-
callback(null, true);
|
|
2839
|
+
return createDaemonServer({
|
|
2840
|
+
serviceId: "workflow",
|
|
2841
|
+
logger,
|
|
2842
|
+
observability,
|
|
2843
|
+
bodyLimit: 1048576,
|
|
2844
|
+
skipStandardRoutes: true,
|
|
2845
|
+
openapi: {
|
|
2846
|
+
title: "KB Labs Workflow Daemon",
|
|
2847
|
+
description: "Background job execution and workflow orchestration API"
|
|
2848
|
+
},
|
|
2849
|
+
cors: {
|
|
2850
|
+
origin: (origin, callback) => {
|
|
2851
|
+
if (!origin) {
|
|
2852
|
+
const isDevelopment = process.env.KB_DAEMON_REQUIRE_AUTH === "false" || process.env.NODE_ENV !== "production";
|
|
2853
|
+
if (isDevelopment) {
|
|
2854
|
+
callback(null, true);
|
|
2855
|
+
} else {
|
|
2856
|
+
callback(new Error("Origin header required in production"), false);
|
|
2857
|
+
}
|
|
2342
2858
|
return;
|
|
2343
2859
|
}
|
|
2344
|
-
|
|
2345
|
-
|
|
2860
|
+
if (allowedOrigins.includes(origin)) {
|
|
2861
|
+
callback(null, true);
|
|
2862
|
+
} else {
|
|
2863
|
+
callback(new Error(`Origin ${origin} not allowed by CORS`), false);
|
|
2864
|
+
}
|
|
2346
2865
|
}
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2866
|
+
},
|
|
2867
|
+
onRequest: async (request, reply) => {
|
|
2868
|
+
if (!requireAuth || request.url === "/health") {
|
|
2869
|
+
return;
|
|
2351
2870
|
}
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
observability
|
|
2359
|
-
});
|
|
2360
|
-
registerCronAPI({
|
|
2361
|
-
server,
|
|
2362
|
-
hostService,
|
|
2363
|
-
logger,
|
|
2364
|
-
observability
|
|
2365
|
-
});
|
|
2366
|
-
if (workflowService) {
|
|
2367
|
-
registerWorkflowsAPI({
|
|
2368
|
-
server,
|
|
2369
|
-
hostService,
|
|
2370
|
-
engine,
|
|
2371
|
-
workflowService,
|
|
2372
|
-
logger,
|
|
2373
|
-
observability
|
|
2374
|
-
});
|
|
2375
|
-
}
|
|
2376
|
-
registerApprovalsAPI({
|
|
2377
|
-
server,
|
|
2378
|
-
engine,
|
|
2379
|
-
logger,
|
|
2380
|
-
observability
|
|
2381
|
-
});
|
|
2382
|
-
registerStatsAPI({
|
|
2383
|
-
server,
|
|
2384
|
-
hostService,
|
|
2385
|
-
cronScheduler});
|
|
2386
|
-
server.get("/health", async () => {
|
|
2387
|
-
const metrics = await hostService.getMetrics();
|
|
2388
|
-
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2389
|
-
return {
|
|
2390
|
-
status: checks.some((entry) => entry.status === "error") ? "degraded" : "ok",
|
|
2391
|
-
service: "workflow",
|
|
2392
|
-
ts: Date.now()
|
|
2393
|
-
};
|
|
2394
|
-
});
|
|
2395
|
-
server.get("/ready", async () => {
|
|
2396
|
-
const metrics = await hostService.getMetrics();
|
|
2397
|
-
const checks = buildWorkflowReadinessChecks({ workflowService, cronScheduler, metrics });
|
|
2398
|
-
const hasErrors = checks.some((entry) => entry.status === "error");
|
|
2399
|
-
const hasWarnings = checks.some((entry) => entry.status === "warn");
|
|
2400
|
-
return createServiceReadyResponse({
|
|
2401
|
-
ready: !hasErrors,
|
|
2402
|
-
status: hasErrors ? "initializing" : hasWarnings ? "degraded" : "ready",
|
|
2403
|
-
reason: hasErrors ? "workflow_checks_failed" : "ready",
|
|
2404
|
-
components: {
|
|
2405
|
-
workflowEngine: {
|
|
2406
|
-
ready: true
|
|
2407
|
-
},
|
|
2408
|
-
workflowCatalog: {
|
|
2409
|
-
ready: Boolean(workflowService)
|
|
2410
|
-
},
|
|
2411
|
-
cronScheduler: {
|
|
2412
|
-
ready: Boolean(cronScheduler)
|
|
2413
|
-
}
|
|
2871
|
+
const apiKeyHeader = request.headers["x-api-key"];
|
|
2872
|
+
const authHeader = request.headers.authorization;
|
|
2873
|
+
const bearerToken = typeof authHeader === "string" && authHeader.startsWith("Bearer ") ? authHeader.slice("Bearer ".length).trim() : void 0;
|
|
2874
|
+
const token = (typeof apiKeyHeader === "string" ? apiKeyHeader : void 0) ?? bearerToken;
|
|
2875
|
+
if (!token || token !== daemonApiKey) {
|
|
2876
|
+
reply.code(401).send({ ok: false, error: "Unauthorized" });
|
|
2414
2877
|
}
|
|
2415
|
-
}
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2421
|
-
return observability.renderPrometheusMetrics(
|
|
2422
|
-
healthStatus,
|
|
2423
|
-
buildWorkflowMetricLines(metrics)
|
|
2424
|
-
);
|
|
2425
|
-
});
|
|
2426
|
-
server.get("/observability/describe", async () => observability.buildDescribe());
|
|
2427
|
-
server.get("/observability/health", async () => {
|
|
2428
|
-
const metrics = await hostService.getMetrics();
|
|
2429
|
-
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2430
|
-
return observability.buildHealth({
|
|
2431
|
-
status: resolveWorkflowHealthStatus(),
|
|
2432
|
-
checks,
|
|
2433
|
-
topOperations: buildWorkflowTopOperations(metrics, observability.getTopOperations(3)),
|
|
2434
|
-
meta: {
|
|
2435
|
-
workflowServiceEnabled: Boolean(workflowService),
|
|
2436
|
-
cronSchedulerEnabled: Boolean(cronScheduler),
|
|
2437
|
-
cronDiscoveryEnabled: Boolean(cronDiscovery),
|
|
2438
|
-
runs: metrics.runs,
|
|
2439
|
-
jobs: metrics.jobs
|
|
2878
|
+
},
|
|
2879
|
+
registerRoutes: async (server) => {
|
|
2880
|
+
registerJobsAPI({ server, hostService, logger, observability });
|
|
2881
|
+
registerCronAPI({ server, hostService, logger, observability });
|
|
2882
|
+
if (workflowService) {
|
|
2883
|
+
registerWorkflowsAPI({ server, hostService, engine, workflowService, logger, observability });
|
|
2440
2884
|
}
|
|
2441
|
-
|
|
2885
|
+
registerApprovalsAPI({ server, hostService, engine, logger, observability });
|
|
2886
|
+
registerStatsAPI({ server, hostService, cronScheduler});
|
|
2887
|
+
server.get("/health", async () => {
|
|
2888
|
+
const metrics = await hostService.getMetrics();
|
|
2889
|
+
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2890
|
+
return {
|
|
2891
|
+
status: checks.some((entry) => entry.status === "error") ? "degraded" : "ok",
|
|
2892
|
+
service: "workflow",
|
|
2893
|
+
ts: Date.now()
|
|
2894
|
+
};
|
|
2895
|
+
});
|
|
2896
|
+
server.get("/ready", async () => {
|
|
2897
|
+
const metrics = await hostService.getMetrics();
|
|
2898
|
+
const checks = buildWorkflowReadinessChecks({ workflowService, cronScheduler, metrics });
|
|
2899
|
+
const hasErrors = checks.some((entry) => entry.status === "error");
|
|
2900
|
+
const hasWarnings = checks.some((entry) => entry.status === "warn");
|
|
2901
|
+
return createServiceReadyResponse({
|
|
2902
|
+
ready: !hasErrors,
|
|
2903
|
+
status: hasErrors ? "initializing" : hasWarnings ? "degraded" : "ready",
|
|
2904
|
+
reason: hasErrors ? "workflow_checks_failed" : "ready",
|
|
2905
|
+
components: {
|
|
2906
|
+
workflowEngine: { ready: true },
|
|
2907
|
+
workflowCatalog: { ready: Boolean(workflowService) },
|
|
2908
|
+
cronScheduler: { ready: Boolean(cronScheduler) }
|
|
2909
|
+
}
|
|
2910
|
+
});
|
|
2911
|
+
});
|
|
2912
|
+
server.get("/metrics", async (_request, reply) => {
|
|
2913
|
+
const metrics = await hostService.getMetrics();
|
|
2914
|
+
const healthStatus = resolveWorkflowHealthStatus();
|
|
2915
|
+
reply.header("Content-Type", "text/plain; version=0.0.4; charset=utf-8");
|
|
2916
|
+
return observability.renderPrometheusMetrics(
|
|
2917
|
+
healthStatus,
|
|
2918
|
+
buildWorkflowMetricLines(metrics)
|
|
2919
|
+
);
|
|
2920
|
+
});
|
|
2921
|
+
server.get("/observability/describe", async () => observability.buildDescribe());
|
|
2922
|
+
server.get("/observability/health", async () => {
|
|
2923
|
+
const metrics = await hostService.getMetrics();
|
|
2924
|
+
const checks = buildWorkflowChecks({ workflowService, cronScheduler, metrics });
|
|
2925
|
+
return observability.buildHealth({
|
|
2926
|
+
status: resolveWorkflowHealthStatus(),
|
|
2927
|
+
checks,
|
|
2928
|
+
topOperations: buildWorkflowTopOperations(metrics, observability.getTopOperations(3)),
|
|
2929
|
+
meta: {
|
|
2930
|
+
workflowServiceEnabled: Boolean(workflowService),
|
|
2931
|
+
cronSchedulerEnabled: Boolean(cronScheduler),
|
|
2932
|
+
cronDiscoveryEnabled: Boolean(cronDiscovery),
|
|
2933
|
+
runs: metrics.runs,
|
|
2934
|
+
jobs: metrics.jobs
|
|
2935
|
+
}
|
|
2936
|
+
});
|
|
2937
|
+
});
|
|
2938
|
+
}
|
|
2442
2939
|
});
|
|
2443
|
-
return server;
|
|
2444
2940
|
}
|
|
2445
2941
|
function resolveWorkflowHealthStatus(metrics) {
|
|
2446
2942
|
return "healthy";
|
|
@@ -2477,16 +2973,8 @@ function buildWorkflowReadinessChecks(input) {
|
|
|
2477
2973
|
function buildWorkflowTopOperations(metrics, httpOperations) {
|
|
2478
2974
|
return [
|
|
2479
2975
|
...httpOperations,
|
|
2480
|
-
{
|
|
2481
|
-
|
|
2482
|
-
count: metrics.runs.total,
|
|
2483
|
-
errorCount: metrics.runs.failed + metrics.runs.cancelled + metrics.runs.dlq
|
|
2484
|
-
},
|
|
2485
|
-
{
|
|
2486
|
-
operation: "workflow.jobs",
|
|
2487
|
-
count: metrics.jobs.total,
|
|
2488
|
-
errorCount: metrics.jobs.failed
|
|
2489
|
-
}
|
|
2976
|
+
{ operation: "workflow.runs", count: metrics.runs.total, errorCount: metrics.runs.failed + metrics.runs.cancelled + metrics.runs.dlq },
|
|
2977
|
+
{ operation: "workflow.jobs", count: metrics.jobs.total, errorCount: metrics.jobs.failed }
|
|
2490
2978
|
].slice(0, 5);
|
|
2491
2979
|
}
|
|
2492
2980
|
function buildWorkflowMetricLines(metrics) {
|
|
@@ -2513,164 +3001,159 @@ function buildWorkflowMetricLines(metrics) {
|
|
|
2513
3001
|
metricLine("service_operation_total", metrics.jobs.failed, { operation: "workflow.jobs", status: "error" })
|
|
2514
3002
|
];
|
|
2515
3003
|
}
|
|
2516
|
-
|
|
2517
|
-
|
|
2518
|
-
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
|
|
2529
|
-
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2538
|
-
|
|
2539
|
-
|
|
2540
|
-
|
|
2541
|
-
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
|
|
2555
|
-
|
|
2556
|
-
|
|
2557
|
-
|
|
2558
|
-
|
|
2559
|
-
|
|
2560
|
-
|
|
2561
|
-
|
|
2562
|
-
|
|
2563
|
-
|
|
2564
|
-
|
|
2565
|
-
|
|
2566
|
-
|
|
2567
|
-
|
|
2568
|
-
|
|
2569
|
-
|
|
2570
|
-
|
|
2571
|
-
|
|
2572
|
-
|
|
2573
|
-
|
|
2574
|
-
|
|
2575
|
-
|
|
2576
|
-
|
|
2577
|
-
|
|
2578
|
-
|
|
2579
|
-
|
|
2580
|
-
|
|
2581
|
-
|
|
2582
|
-
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
|
|
2586
|
-
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2590
|
-
|
|
2591
|
-
|
|
2592
|
-
|
|
2593
|
-
|
|
2594
|
-
|
|
2595
|
-
|
|
2596
|
-
|
|
2597
|
-
|
|
2598
|
-
|
|
2599
|
-
|
|
2600
|
-
|
|
2601
|
-
|
|
2602
|
-
|
|
2603
|
-
|
|
2604
|
-
|
|
2605
|
-
|
|
2606
|
-
|
|
2607
|
-
|
|
2608
|
-
|
|
2609
|
-
|
|
2610
|
-
|
|
2611
|
-
|
|
2612
|
-
|
|
2613
|
-
|
|
2614
|
-
|
|
2615
|
-
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
|
|
2620
|
-
|
|
2621
|
-
|
|
2622
|
-
|
|
2623
|
-
|
|
2624
|
-
|
|
2625
|
-
|
|
2626
|
-
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2631
|
-
|
|
2632
|
-
|
|
2633
|
-
|
|
2634
|
-
|
|
2635
|
-
|
|
2636
|
-
|
|
2637
|
-
|
|
2638
|
-
|
|
2639
|
-
|
|
2640
|
-
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2648
|
-
|
|
2649
|
-
|
|
2650
|
-
|
|
2651
|
-
|
|
2652
|
-
|
|
2653
|
-
|
|
2654
|
-
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
|
|
2662
|
-
|
|
2663
|
-
|
|
2664
|
-
|
|
2665
|
-
|
|
2666
|
-
}
|
|
2667
|
-
|
|
2668
|
-
|
|
2669
|
-
bootstrapLogger.info("Workflow daemon shutdown complete");
|
|
2670
|
-
process.exit(0);
|
|
2671
|
-
};
|
|
2672
|
-
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
2673
|
-
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
3004
|
+
async function bootstrap(_cwd = process.cwd()) {
|
|
3005
|
+
await runDaemon(
|
|
3006
|
+
{
|
|
3007
|
+
appId: "workflow-daemon",
|
|
3008
|
+
// serviceId in the transport map / devservices is 'workflow' (≠ appId).
|
|
3009
|
+
serviceId: "workflow",
|
|
3010
|
+
defaultPort: 7778,
|
|
3011
|
+
portEnvVar: "WORKFLOW_PORT",
|
|
3012
|
+
defaultHost: "0.0.0.0",
|
|
3013
|
+
hostEnvVar: "WORKFLOW_HOST",
|
|
3014
|
+
async setup({ platform: _p, logger: _l, port, host, repoRoot }) {
|
|
3015
|
+
const projectRoot = process.env["KB_PROJECT_ROOT"] ?? repoRoot;
|
|
3016
|
+
if (!platform.isConfigured("workspace")) {
|
|
3017
|
+
process.stderr.write(
|
|
3018
|
+
'[workflow-daemon] WARNING: workspace adapter is not configured.\n[workflow-daemon] Workflows that use isolation: balanced (default) or isolation: strict will fail.\n[workflow-daemon] To fix: set platform.adapters.workspace in kb.config.json.\n[workflow-daemon] To run without a workspace: add "isolation: relaxed" to your workflow YAML.\n'
|
|
3019
|
+
);
|
|
3020
|
+
}
|
|
3021
|
+
if (!platform.isConfigured("environment") && platform.isConfigured("workspace")) {
|
|
3022
|
+
process.stderr.write(
|
|
3023
|
+
"[workflow-daemon] WARNING: environment adapter is not configured.\n[workflow-daemon] Workflows that use isolation: strict will fail.\n[workflow-daemon] To fix: set platform.adapters.environment in kb.config.json.\n"
|
|
3024
|
+
);
|
|
3025
|
+
}
|
|
3026
|
+
const startupRequestId = `workflow-startup-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
|
3027
|
+
const startupTraceId = randomUUID();
|
|
3028
|
+
const startupSpanId = randomUUID();
|
|
3029
|
+
const bootstrapLogger = createCorrelatedLogger(platform.logger, {
|
|
3030
|
+
serviceId: "workflow",
|
|
3031
|
+
logsSource: "workflow",
|
|
3032
|
+
layer: "workflow",
|
|
3033
|
+
service: "bootstrap",
|
|
3034
|
+
requestId: startupRequestId,
|
|
3035
|
+
traceId: startupTraceId,
|
|
3036
|
+
operation: "workflow.bootstrap",
|
|
3037
|
+
bindings: { spanId: startupSpanId, invocationId: startupSpanId, executionId: startupSpanId }
|
|
3038
|
+
});
|
|
3039
|
+
const debugMode = process.env["WORKFLOW_DEBUG"] === "true";
|
|
3040
|
+
bootstrapLogger.info("Workflow daemon starting", { projectRoot, debugMode });
|
|
3041
|
+
if (debugMode) {
|
|
3042
|
+
bootstrapLogger.warn(
|
|
3043
|
+
"[WORKFLOW_DEBUG=true] Verbose debug logging is ON \u2014 step inputs, outputs, and expr contexts will be logged. Disable in production (unset WORKFLOW_DEBUG or set to false)."
|
|
3044
|
+
);
|
|
3045
|
+
}
|
|
3046
|
+
const createWorkflowLogger = (service, operation, bindings) => createCorrelatedLogger(platform.logger, {
|
|
3047
|
+
serviceId: "workflow",
|
|
3048
|
+
logsSource: "workflow",
|
|
3049
|
+
layer: "workflow",
|
|
3050
|
+
service,
|
|
3051
|
+
operation,
|
|
3052
|
+
bindings
|
|
3053
|
+
});
|
|
3054
|
+
bootstrapLogger.info("Loading plugin registry snapshot");
|
|
3055
|
+
const cliApi = await createRegistry({ root: repoRoot, cache: { ttlMs: 10 * 60 * 1e3 } });
|
|
3056
|
+
await cliApi.initialize();
|
|
3057
|
+
const plugins = await cliApi.listPlugins();
|
|
3058
|
+
bootstrapLogger.info("Plugin registry snapshot loaded", {
|
|
3059
|
+
pluginsFound: plugins.length,
|
|
3060
|
+
pluginIds: plugins.map((p) => `${p.id}@${p.version}`)
|
|
3061
|
+
});
|
|
3062
|
+
bootstrapLogger.info("Creating WorkflowEngine");
|
|
3063
|
+
const engine = new WorkflowEngine({
|
|
3064
|
+
cache: platform.cache,
|
|
3065
|
+
events: platform.eventBus,
|
|
3066
|
+
logger: createWorkflowLogger("engine", "workflow.engine"),
|
|
3067
|
+
snapshotManager: platform.snapshotManager,
|
|
3068
|
+
workspaceRoot: projectRoot
|
|
3069
|
+
});
|
|
3070
|
+
bootstrapLogger.info("Cleaning up stale runs from previous daemon process");
|
|
3071
|
+
await engine.cleanupStaleRuns();
|
|
3072
|
+
bootstrapLogger.info("Resuming interrupted jobs");
|
|
3073
|
+
await engine.resumeInterruptedJobs();
|
|
3074
|
+
bootstrapLogger.info("Creating JobBroker");
|
|
3075
|
+
const jobBroker = new JobBroker(
|
|
3076
|
+
engine,
|
|
3077
|
+
createWorkflowLogger("job-broker", "workflow.job-broker"),
|
|
3078
|
+
platform
|
|
3079
|
+
);
|
|
3080
|
+
bootstrapLogger.info("Creating CronScheduler");
|
|
3081
|
+
const cronScheduler = new CronScheduler({
|
|
3082
|
+
jobBroker,
|
|
3083
|
+
workflowEngine: engine,
|
|
3084
|
+
logger: createWorkflowLogger("cron-scheduler", "workflow.cron-scheduler"),
|
|
3085
|
+
timezone: process.env.WORKFLOW_CRON_TIMEZONE
|
|
3086
|
+
});
|
|
3087
|
+
bootstrapLogger.info("Discovering cron jobs");
|
|
3088
|
+
const cronDiscovery = new CronDiscovery({
|
|
3089
|
+
cliApi,
|
|
3090
|
+
scheduler: cronScheduler,
|
|
3091
|
+
logger: createWorkflowLogger("cron-discovery", "workflow.cron-discovery"),
|
|
3092
|
+
workspaceRoot: projectRoot
|
|
3093
|
+
});
|
|
3094
|
+
const discovered = await cronDiscovery.discoverAll();
|
|
3095
|
+
bootstrapLogger.info("Cron job discovery complete", discovered);
|
|
3096
|
+
bootstrapLogger.info("Creating WorkflowService");
|
|
3097
|
+
const workflowService = new WorkflowService({ cliApi, platform, workspaceRoot: projectRoot });
|
|
3098
|
+
workflowService.listAll().catch(
|
|
3099
|
+
(err) => bootstrapLogger.warn("Manifest scanner warmup failed", { err })
|
|
3100
|
+
);
|
|
3101
|
+
bootstrapLogger.info("Starting WorkflowFileWatcher");
|
|
3102
|
+
const fileWatcher = new WorkflowFileWatcher({
|
|
3103
|
+
watchDirs: [
|
|
3104
|
+
join(projectRoot, ".kb", "workflows"),
|
|
3105
|
+
join(projectRoot, ".kb", "jobs")
|
|
3106
|
+
],
|
|
3107
|
+
workflowService,
|
|
3108
|
+
cronDiscovery,
|
|
3109
|
+
cronScheduler,
|
|
3110
|
+
logger: createWorkflowLogger("file-watcher", "workflow.file-watcher")
|
|
3111
|
+
});
|
|
3112
|
+
bootstrapLogger.info("Creating HTTP server");
|
|
3113
|
+
const server = await createServer({
|
|
3114
|
+
engine,
|
|
3115
|
+
jobBroker,
|
|
3116
|
+
workflowService,
|
|
3117
|
+
cronScheduler,
|
|
3118
|
+
cronDiscovery,
|
|
3119
|
+
logger: createWorkflowLogger("api", "workflow.api")
|
|
3120
|
+
});
|
|
3121
|
+
await server.listen(getListenOptions(port, host));
|
|
3122
|
+
bootstrapLogger.info("HTTP API listening", { port });
|
|
3123
|
+
bootstrapLogger.info("Creating WorkflowWorker");
|
|
3124
|
+
const worker = await createWorkflowWorker({
|
|
3125
|
+
engine,
|
|
3126
|
+
cliApi,
|
|
3127
|
+
logger: createWorkflowLogger("worker", "workflow.worker"),
|
|
3128
|
+
analytics: platform.analytics,
|
|
3129
|
+
platform,
|
|
3130
|
+
workspaceRoot: projectRoot,
|
|
3131
|
+
concurrency: parseInt(process.env.WORKFLOW_CONCURRENCY ?? "5", 10),
|
|
3132
|
+
debugMode
|
|
3133
|
+
});
|
|
3134
|
+
bootstrapLogger.info("Starting WorkflowWorker");
|
|
3135
|
+
worker.start().catch((error) => {
|
|
3136
|
+
bootstrapLogger.error(
|
|
3137
|
+
"Worker crashed - shutting down daemon",
|
|
3138
|
+
error instanceof Error ? error : void 0
|
|
3139
|
+
);
|
|
3140
|
+
process.kill(process.pid, "SIGTERM");
|
|
3141
|
+
});
|
|
3142
|
+
bootstrapLogger.info("Starting CronScheduler");
|
|
3143
|
+
await cronScheduler.start();
|
|
3144
|
+
bootstrapLogger.info("Workflow daemon started successfully", { port });
|
|
3145
|
+
return async () => {
|
|
3146
|
+
bootstrapLogger.warn("Stopping workflow daemon components");
|
|
3147
|
+
fileWatcher.close();
|
|
3148
|
+
await cronScheduler.stop();
|
|
3149
|
+
await worker.stop();
|
|
3150
|
+
await server.close();
|
|
3151
|
+
await cliApi.dispose();
|
|
3152
|
+
};
|
|
3153
|
+
}
|
|
3154
|
+
},
|
|
3155
|
+
(appId, repoRoot) => createServiceBootstrap({ appId, repoRoot, assemblyHook: makeAssemblyHook() })
|
|
3156
|
+
);
|
|
2674
3157
|
}
|
|
2675
3158
|
|
|
2676
3159
|
// src/index.ts
|
|
@@ -2678,7 +3161,8 @@ async function bootstrap(cwd = process.cwd()) {
|
|
|
2678
3161
|
try {
|
|
2679
3162
|
await bootstrap(process.cwd());
|
|
2680
3163
|
} catch (error) {
|
|
2681
|
-
|
|
3164
|
+
process.stderr.write(`[workflow-daemon] FATAL: ${error instanceof Error ? error.message : String(error)}
|
|
3165
|
+
`);
|
|
2682
3166
|
process.exit(1);
|
|
2683
3167
|
}
|
|
2684
3168
|
})();
|