@kb-labs/workflow-daemon 2.112.0 → 2.116.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.js +367 -235
- package/dist/index.js.map +1 -1
- package/package.json +35 -35
package/dist/index.js
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
import { platform, createServiceBootstrap } from '@kb-labs/core-runtime';
|
|
3
2
|
import { makeAssemblyHook } from '@kb-labs/plugin-runtime';
|
|
4
3
|
import { WorkflowEngine, WorkflowService } from '@kb-labs/workflow-engine';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { logDiagnosticEvent } from '@kb-labs/core-platform';
|
|
4
|
+
import { getListenOptions, HttpObservabilityCollector, createDaemonServer, createServiceReadyResponse, metricLine } from '@kb-labs/shared-http';
|
|
5
|
+
import { runService } from '@kb-labs/shared-daemon';
|
|
6
|
+
import { logDiagnosticEvent, createContextLogger } from '@kb-labs/core-platform';
|
|
8
7
|
import { PluginCronJobSchema, UserCronJobSchema, evaluateExpression, interpolateObject, buildShellSafeCommand, coerceToString, interpolateString } from '@kb-labs/workflow-contracts';
|
|
9
8
|
import { GateHandler } from '@kb-labs/workflow-steps';
|
|
10
9
|
import { SandboxRunner } from '@kb-labs/workflow-runtime';
|
|
@@ -22,14 +21,14 @@ async function createWorkflowWorker(options) {
|
|
|
22
21
|
engine,
|
|
23
22
|
cliApi,
|
|
24
23
|
logger,
|
|
25
|
-
platform
|
|
24
|
+
platform,
|
|
26
25
|
workspaceRoot,
|
|
27
26
|
concurrency = 5,
|
|
28
27
|
defaultTimeout = 12e4,
|
|
29
28
|
analytics,
|
|
30
29
|
debugMode = process.env["WORKFLOW_DEBUG"] === "true"
|
|
31
30
|
} = options;
|
|
32
|
-
const wsProvider =
|
|
31
|
+
const wsProvider = platform.getAdapter("workspace");
|
|
33
32
|
if (wsProvider && typeof wsProvider.startupCleanup === "function") {
|
|
34
33
|
wsProvider.startupCleanup().catch(() => {
|
|
35
34
|
});
|
|
@@ -38,7 +37,7 @@ async function createWorkflowWorker(options) {
|
|
|
38
37
|
let stopRequested = false;
|
|
39
38
|
const runningJobs = /* @__PURE__ */ new Map();
|
|
40
39
|
const claimedJobs = /* @__PURE__ */ new Set();
|
|
41
|
-
const executionBackend =
|
|
40
|
+
const executionBackend = platform.executionBackend;
|
|
42
41
|
const runner = new SandboxRunner({
|
|
43
42
|
// IExecutionBackend<unknown> (core-contracts) is structurally compatible with
|
|
44
43
|
// ExecutionBackend (plugin-execution) — both expose the same execute/health/stats/shutdown API.
|
|
@@ -54,10 +53,14 @@ async function createWorkflowWorker(options) {
|
|
|
54
53
|
}
|
|
55
54
|
const run = await engine.getRun(entry.runId);
|
|
56
55
|
if (!run) {
|
|
57
|
-
logger.error(
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
56
|
+
logger.error(
|
|
57
|
+
"Data inconsistency: Run not found for job entry",
|
|
58
|
+
void 0,
|
|
59
|
+
{
|
|
60
|
+
runId: entry.runId,
|
|
61
|
+
jobId: entry.jobId
|
|
62
|
+
}
|
|
63
|
+
);
|
|
61
64
|
return true;
|
|
62
65
|
}
|
|
63
66
|
const job = run.jobs.find((j) => j.id === entry.jobId);
|
|
@@ -82,20 +85,19 @@ async function createWorkflowWorker(options) {
|
|
|
82
85
|
waitMs
|
|
83
86
|
});
|
|
84
87
|
const jobStartTime = Date.now();
|
|
85
|
-
const jobLogger =
|
|
88
|
+
const jobLogger = createContextLogger(logger, {
|
|
89
|
+
applicationId: "workflow-daemon",
|
|
86
90
|
serviceId: "workflow",
|
|
87
|
-
|
|
88
|
-
layer: "
|
|
89
|
-
|
|
91
|
+
instanceId: `${process.pid}`,
|
|
92
|
+
layer: "service"
|
|
93
|
+
}).forComponent("workflow-worker").forOperation("workflow.job", {
|
|
90
94
|
requestId: run.id,
|
|
91
|
-
traceId: run.id
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
jobName: job.jobName
|
|
98
|
-
}
|
|
95
|
+
traceId: run.id
|
|
96
|
+
}).with({
|
|
97
|
+
"workflow.id": run.id,
|
|
98
|
+
"workflow.run_id": run.id,
|
|
99
|
+
"workflow.job_id": job.id,
|
|
100
|
+
"workflow.job_name": job.jobName
|
|
99
101
|
});
|
|
100
102
|
jobLogger.info("Processing job", {
|
|
101
103
|
runId: run.id,
|
|
@@ -113,7 +115,7 @@ async function createWorkflowWorker(options) {
|
|
|
113
115
|
const runTarget = run.metadata?.target;
|
|
114
116
|
const jobTarget = job.target;
|
|
115
117
|
const target = jobTarget ?? runTarget;
|
|
116
|
-
const wsProvider2 =
|
|
118
|
+
const wsProvider2 = platform.getAdapter("workspace");
|
|
117
119
|
let runWorkspace = wsProvider2 ? workspaceRoot : process.env["KB_PROJECT_ROOT"] ?? workspaceRoot;
|
|
118
120
|
let provisionedWorkspaceId;
|
|
119
121
|
if (wsProvider2) {
|
|
@@ -220,7 +222,9 @@ async function createWorkflowWorker(options) {
|
|
|
220
222
|
env: exprCtx.env
|
|
221
223
|
}
|
|
222
224
|
});
|
|
223
|
-
await engine.markStepCompleted(run.id, job.id, step.id, {
|
|
225
|
+
await engine.markStepCompleted(run.id, job.id, step.id, {
|
|
226
|
+
skipped: true
|
|
227
|
+
});
|
|
224
228
|
continue;
|
|
225
229
|
}
|
|
226
230
|
}
|
|
@@ -237,7 +241,10 @@ async function createWorkflowWorker(options) {
|
|
|
237
241
|
}
|
|
238
242
|
});
|
|
239
243
|
}
|
|
240
|
-
let interpolatedWith = step.spec.with ? interpolateObject(
|
|
244
|
+
let interpolatedWith = step.spec.with ? interpolateObject(
|
|
245
|
+
step.spec.with,
|
|
246
|
+
exprCtx
|
|
247
|
+
) : void 0;
|
|
241
248
|
if (step.spec.uses === "builtin:shell" && typeof step.spec.with?.["command"] === "string") {
|
|
242
249
|
const rawCommand = step.spec.with["command"];
|
|
243
250
|
const { command: safeCommand, shellEnvVars } = buildShellSafeCommand(rawCommand, exprCtx);
|
|
@@ -246,15 +253,23 @@ async function createWorkflowWorker(options) {
|
|
|
246
253
|
command: safeCommand,
|
|
247
254
|
env: {
|
|
248
255
|
...Object.fromEntries(
|
|
249
|
-
Object.entries(
|
|
256
|
+
Object.entries(
|
|
257
|
+
interpolatedWith?.["env"] ?? {}
|
|
258
|
+
).map(([k, v]) => [k, coerceToString(v)])
|
|
250
259
|
),
|
|
251
260
|
...shellEnvVars
|
|
252
261
|
}
|
|
253
262
|
};
|
|
254
263
|
}
|
|
255
|
-
const interpolatedEnvRaw = step.spec.env ? interpolateObject(
|
|
264
|
+
const interpolatedEnvRaw = step.spec.env ? interpolateObject(
|
|
265
|
+
step.spec.env,
|
|
266
|
+
exprCtx
|
|
267
|
+
) : void 0;
|
|
256
268
|
const interpolatedEnv = interpolatedEnvRaw ? Object.fromEntries(
|
|
257
|
-
Object.entries(interpolatedEnvRaw).map(([k, v]) => [
|
|
269
|
+
Object.entries(interpolatedEnvRaw).map(([k, v]) => [
|
|
270
|
+
k,
|
|
271
|
+
coerceToString(v)
|
|
272
|
+
])
|
|
258
273
|
) : void 0;
|
|
259
274
|
if (debugMode && step.spec.with) {
|
|
260
275
|
jobLogger.info("[debug] Step input interpolation", {
|
|
@@ -294,7 +309,10 @@ async function createWorkflowWorker(options) {
|
|
|
294
309
|
});
|
|
295
310
|
if (step.spec.uses === "builtin:approval") {
|
|
296
311
|
if (step.status === "failed") {
|
|
297
|
-
stepLogger.info("Approval already rejected \u2014 skipping to gate", {
|
|
312
|
+
stepLogger.info("Approval already rejected \u2014 skipping to gate", {
|
|
313
|
+
runId: run.id,
|
|
314
|
+
stepId: step.id
|
|
315
|
+
});
|
|
298
316
|
continue;
|
|
299
317
|
}
|
|
300
318
|
if (step.status !== "waiting_approval") {
|
|
@@ -318,11 +336,18 @@ async function createWorkflowWorker(options) {
|
|
|
318
336
|
const gateInput = interpolatedWith ?? {};
|
|
319
337
|
const currentIteration = step.metadata?.["iterations"] ?? 0;
|
|
320
338
|
const sameJobStepIds = job.steps.flatMap(
|
|
321
|
-
(s) => [s.id, s.spec.id].filter(
|
|
339
|
+
(s) => [s.id, s.spec.id].filter(
|
|
340
|
+
(v) => typeof v === "string"
|
|
341
|
+
)
|
|
322
342
|
);
|
|
323
343
|
const jobNames = (freshRun?.jobs ?? run.jobs).map((j) => j.jobName);
|
|
324
344
|
const validRestartTargets = [...sameJobStepIds, ...jobNames];
|
|
325
|
-
const decision = new GateHandler().handle(
|
|
345
|
+
const decision = new GateHandler().handle(
|
|
346
|
+
gateInput,
|
|
347
|
+
exprCtx,
|
|
348
|
+
currentIteration,
|
|
349
|
+
validRestartTargets
|
|
350
|
+
);
|
|
326
351
|
stepLogger.info("[gate] Evaluating gate decision", {
|
|
327
352
|
runId: run.id,
|
|
328
353
|
stepId: step.id,
|
|
@@ -332,23 +357,45 @@ async function createWorkflowWorker(options) {
|
|
|
332
357
|
iteration: currentIteration
|
|
333
358
|
});
|
|
334
359
|
if (decision.action === "continue") {
|
|
335
|
-
await engine.markStepCompleted(
|
|
360
|
+
await engine.markStepCompleted(
|
|
361
|
+
run.id,
|
|
362
|
+
job.id,
|
|
363
|
+
step.id,
|
|
364
|
+
decision.outputs
|
|
365
|
+
);
|
|
336
366
|
continue;
|
|
337
367
|
}
|
|
338
368
|
if (decision.action === "fail") {
|
|
339
|
-
stepLogger.error(
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
369
|
+
stepLogger.error(
|
|
370
|
+
"[gate] Gate condition failed, aborting job",
|
|
371
|
+
decision.error,
|
|
372
|
+
{
|
|
373
|
+
runId: run.id,
|
|
374
|
+
stepId: step.id,
|
|
375
|
+
stepName: step.name
|
|
376
|
+
}
|
|
377
|
+
);
|
|
378
|
+
await engine.markStepFailed(
|
|
379
|
+
run.id,
|
|
380
|
+
job.id,
|
|
381
|
+
step.id,
|
|
382
|
+
decision.error,
|
|
383
|
+
decision.outputs
|
|
384
|
+
);
|
|
345
385
|
throw decision.error;
|
|
346
386
|
}
|
|
347
387
|
if (decision.action === "skip") {
|
|
348
388
|
await applyGateSkip(engine, run, job, step, decision, stepLogger);
|
|
349
389
|
return;
|
|
350
390
|
}
|
|
351
|
-
await applyGateRestart(
|
|
391
|
+
await applyGateRestart(
|
|
392
|
+
engine,
|
|
393
|
+
run,
|
|
394
|
+
job,
|
|
395
|
+
step,
|
|
396
|
+
decision,
|
|
397
|
+
stepLogger
|
|
398
|
+
);
|
|
352
399
|
return;
|
|
353
400
|
}
|
|
354
401
|
const stepStartTime = Date.now();
|
|
@@ -360,20 +407,41 @@ async function createWorkflowWorker(options) {
|
|
|
360
407
|
baseSpec = {
|
|
361
408
|
...rest,
|
|
362
409
|
uses: "builtin:shell",
|
|
363
|
-
with: {
|
|
410
|
+
with: {
|
|
411
|
+
...existingWith,
|
|
412
|
+
command,
|
|
413
|
+
env: {
|
|
414
|
+
...existingWith?.["env"] ?? {},
|
|
415
|
+
...shellEnvVars
|
|
416
|
+
}
|
|
417
|
+
}
|
|
364
418
|
};
|
|
365
419
|
}
|
|
366
420
|
if (typeof baseSpec.summary === "string") {
|
|
367
421
|
baseSpec.summary = interpolateString(baseSpec.summary, exprCtx);
|
|
368
422
|
}
|
|
369
|
-
const specWithEnv = interpolatedEnv ? {
|
|
423
|
+
const specWithEnv = interpolatedEnv ? {
|
|
424
|
+
...baseSpec,
|
|
425
|
+
with: {
|
|
426
|
+
...baseSpec.with ?? {},
|
|
427
|
+
env: {
|
|
428
|
+
...baseSpec.with?.["env"] ?? {},
|
|
429
|
+
...interpolatedEnv
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
} : baseSpec;
|
|
370
433
|
const interpolatedSpec = interpolatedWith ? {
|
|
371
434
|
...specWithEnv,
|
|
372
435
|
with: {
|
|
373
436
|
...specWithEnv.with ?? {},
|
|
374
437
|
...interpolatedWith,
|
|
375
438
|
// spec.env (interpolatedEnv) must survive the spread of interpolatedWith.env
|
|
376
|
-
...interpolatedEnv ? {
|
|
439
|
+
...interpolatedEnv ? {
|
|
440
|
+
env: {
|
|
441
|
+
...interpolatedWith["env"] ?? {},
|
|
442
|
+
...interpolatedEnv
|
|
443
|
+
}
|
|
444
|
+
} : {}
|
|
377
445
|
}
|
|
378
446
|
} : specWithEnv;
|
|
379
447
|
const result = await runner.execute({
|
|
@@ -398,7 +466,10 @@ async function createWorkflowWorker(options) {
|
|
|
398
466
|
secrets: (() => {
|
|
399
467
|
const names = freshRun?.secrets ?? [];
|
|
400
468
|
if (names.length > 0) {
|
|
401
|
-
stepLogger.warn(
|
|
469
|
+
stepLogger.warn(
|
|
470
|
+
"Step declares secrets but secret resolution is not implemented",
|
|
471
|
+
{ secrets: names }
|
|
472
|
+
);
|
|
402
473
|
}
|
|
403
474
|
return {};
|
|
404
475
|
})(),
|
|
@@ -425,12 +496,24 @@ async function createWorkflowWorker(options) {
|
|
|
425
496
|
lineNo: entry2.lineNo,
|
|
426
497
|
logSource: entry2.stream === "stderr" ? "stderr" : "stdout"
|
|
427
498
|
});
|
|
428
|
-
void engine.publishLog(
|
|
499
|
+
void engine.publishLog(
|
|
500
|
+
run.id,
|
|
501
|
+
job.id,
|
|
502
|
+
step.id,
|
|
503
|
+
entry2,
|
|
504
|
+
step.name
|
|
505
|
+
);
|
|
429
506
|
},
|
|
430
507
|
// ctx.logger.* entries: stepLogger base already wrote to SQLite. Only publish for SSE.
|
|
431
508
|
// See: plugins/workflow/docs/adr/0019-log-stream-separation.md
|
|
432
509
|
onLoggerLog: (entry2) => {
|
|
433
|
-
void engine.publishLog(
|
|
510
|
+
void engine.publishLog(
|
|
511
|
+
run.id,
|
|
512
|
+
job.id,
|
|
513
|
+
step.id,
|
|
514
|
+
entry2,
|
|
515
|
+
step.name
|
|
516
|
+
);
|
|
434
517
|
}
|
|
435
518
|
},
|
|
436
519
|
// Scripts (.kb/workflows/scripts/*.sh) live in the project root, not the worktree.
|
|
@@ -442,7 +525,9 @@ async function createWorkflowWorker(options) {
|
|
|
442
525
|
});
|
|
443
526
|
if (result.status === "failed") {
|
|
444
527
|
const stepDurationMs = Date.now() - stepStartTime;
|
|
445
|
-
const error = new Error(
|
|
528
|
+
const error = new Error(
|
|
529
|
+
result.error?.message ?? "Step execution failed"
|
|
530
|
+
);
|
|
446
531
|
await engine.markStepFailed(run.id, job.id, step.id, error);
|
|
447
532
|
failedStepFailure = result.failure;
|
|
448
533
|
failedStepRetry = step.spec.retry;
|
|
@@ -457,11 +542,14 @@ async function createWorkflowWorker(options) {
|
|
|
457
542
|
errorCode: result.error?.code
|
|
458
543
|
});
|
|
459
544
|
if (step.continueOnError) {
|
|
460
|
-
stepLogger.warn(
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
545
|
+
stepLogger.warn(
|
|
546
|
+
"[step] continueOnError=true \u2014 continuing despite failure",
|
|
547
|
+
{
|
|
548
|
+
runId: run.id,
|
|
549
|
+
jobId: job.id,
|
|
550
|
+
stepId: step.id
|
|
551
|
+
}
|
|
552
|
+
);
|
|
465
553
|
continue;
|
|
466
554
|
}
|
|
467
555
|
throw error;
|
|
@@ -507,7 +595,9 @@ async function createWorkflowWorker(options) {
|
|
|
507
595
|
if (provisionedWorkspaceId && wsProvider2) {
|
|
508
596
|
try {
|
|
509
597
|
await wsProvider2.release(provisionedWorkspaceId);
|
|
510
|
-
jobLogger.info("Workspace released", {
|
|
598
|
+
jobLogger.info("Workspace released", {
|
|
599
|
+
workspaceId: provisionedWorkspaceId
|
|
600
|
+
});
|
|
511
601
|
} catch (releaseErr) {
|
|
512
602
|
jobLogger.warn("Workspace release failed", {
|
|
513
603
|
workspaceId: provisionedWorkspaceId,
|
|
@@ -519,7 +609,13 @@ async function createWorkflowWorker(options) {
|
|
|
519
609
|
const err = error instanceof Error ? error : new Error(String(error));
|
|
520
610
|
const jobDuration = Date.now() - jobStartTime;
|
|
521
611
|
const retryPolicy = failedStepRetry ?? job.retries;
|
|
522
|
-
await engine.markJobFailed(
|
|
612
|
+
await engine.markJobFailed(
|
|
613
|
+
run.id,
|
|
614
|
+
job.id,
|
|
615
|
+
err,
|
|
616
|
+
failedStepFailure,
|
|
617
|
+
retryPolicy
|
|
618
|
+
);
|
|
523
619
|
analytics?.track("workflow.worker.job.failed", {
|
|
524
620
|
runId: run.id,
|
|
525
621
|
jobId: job.id,
|
|
@@ -612,9 +708,12 @@ async function createWorkflowWorker(options) {
|
|
|
612
708
|
sleep(shutdownTimeoutMs)
|
|
613
709
|
]);
|
|
614
710
|
if (runningJobs.size > 0) {
|
|
615
|
-
logger.warn(
|
|
616
|
-
|
|
617
|
-
|
|
711
|
+
logger.warn(
|
|
712
|
+
"Shutdown timeout reached, marking jobs as interrupted",
|
|
713
|
+
{
|
|
714
|
+
count: runningJobs.size
|
|
715
|
+
}
|
|
716
|
+
);
|
|
618
717
|
await Promise.all(
|
|
619
718
|
Array.from(runningJobs.keys()).map(async (jobKey) => {
|
|
620
719
|
const [runId, jobId] = jobKey.split(":");
|
|
@@ -627,9 +726,13 @@ async function createWorkflowWorker(options) {
|
|
|
627
726
|
logger.info("All in-flight jobs completed gracefully");
|
|
628
727
|
}
|
|
629
728
|
} catch (error) {
|
|
630
|
-
logger.error(
|
|
631
|
-
|
|
632
|
-
|
|
729
|
+
logger.error(
|
|
730
|
+
"Error during graceful shutdown",
|
|
731
|
+
error instanceof Error ? error : void 0,
|
|
732
|
+
{
|
|
733
|
+
runningJobsCount: runningJobs.size
|
|
734
|
+
}
|
|
735
|
+
);
|
|
633
736
|
}
|
|
634
737
|
}
|
|
635
738
|
logger.info("Workflow worker stopped");
|
|
@@ -649,12 +752,15 @@ function sleep(ms) {
|
|
|
649
752
|
async function waitForApproval(engine, runId, jobId, step, interpolatedWith, isStopRequested, stepLogger) {
|
|
650
753
|
const approvalTimeoutMs = interpolatedWith?.["timeoutMs"];
|
|
651
754
|
if (!approvalTimeoutMs) {
|
|
652
|
-
stepLogger.warn(
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
755
|
+
stepLogger.warn(
|
|
756
|
+
"[approval] No timeout configured \u2014 approval may wait indefinitely",
|
|
757
|
+
{
|
|
758
|
+
runId,
|
|
759
|
+
jobId,
|
|
760
|
+
stepId: step.id,
|
|
761
|
+
stepName: step.name
|
|
762
|
+
}
|
|
763
|
+
);
|
|
658
764
|
}
|
|
659
765
|
stepLogger.info("[approval] Waiting for approval", {
|
|
660
766
|
runId,
|
|
@@ -670,12 +776,15 @@ async function waitForApproval(engine, runId, jobId, step, interpolatedWith, isS
|
|
|
670
776
|
pollCount++;
|
|
671
777
|
const currentRun = await engine.getRun(runId);
|
|
672
778
|
if (currentRun?.status === "cancelled") {
|
|
673
|
-
stepLogger.info(
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
779
|
+
stepLogger.info(
|
|
780
|
+
"[approval] Run cancelled \u2014 treating as interrupted, NOT approved",
|
|
781
|
+
{
|
|
782
|
+
runId,
|
|
783
|
+
stepId: step.id,
|
|
784
|
+
stepName: step.name,
|
|
785
|
+
waitedMs: Date.now() - approvalStartMs
|
|
786
|
+
}
|
|
787
|
+
);
|
|
679
788
|
return "interrupted";
|
|
680
789
|
}
|
|
681
790
|
const currentJob = currentRun?.jobs.find((j) => j.id === jobId);
|
|
@@ -709,7 +818,9 @@ async function waitForApproval(engine, runId, jobId, step, interpolatedWith, isS
|
|
|
709
818
|
}
|
|
710
819
|
}
|
|
711
820
|
if (isStopRequested()) {
|
|
712
|
-
stepLogger.info("Approval wait interrupted by shutdown", {
|
|
821
|
+
stepLogger.info("Approval wait interrupted by shutdown", {
|
|
822
|
+
stepId: step.id
|
|
823
|
+
});
|
|
713
824
|
return "interrupted";
|
|
714
825
|
}
|
|
715
826
|
return "done";
|
|
@@ -785,7 +896,9 @@ async function applyGateRestart(engine, run, job, step, decision, stepLogger) {
|
|
|
785
896
|
return draft;
|
|
786
897
|
});
|
|
787
898
|
}
|
|
788
|
-
const sameJobTarget = job.steps.some(
|
|
899
|
+
const sameJobTarget = job.steps.some(
|
|
900
|
+
(s) => s.spec.id === restartFrom || s.id === restartFrom
|
|
901
|
+
);
|
|
789
902
|
if (!sameJobTarget) {
|
|
790
903
|
const fresh = await engine.getRun(run.id) ?? run;
|
|
791
904
|
const resetNames = /* @__PURE__ */ new Set([restartFrom]);
|
|
@@ -827,7 +940,11 @@ async function applyGateRestart(engine, run, job, step, decision, stepLogger) {
|
|
|
827
940
|
const refreshed = await engine.getRun(run.id);
|
|
828
941
|
const targetJob = refreshed?.jobs.find((j) => j.jobName === restartFrom);
|
|
829
942
|
if (targetJob && !targetJob.blocked) {
|
|
830
|
-
await scheduler.enqueueJob(
|
|
943
|
+
await scheduler.enqueueJob(
|
|
944
|
+
run.id,
|
|
945
|
+
targetJob,
|
|
946
|
+
targetJob.priority ?? "normal"
|
|
947
|
+
);
|
|
831
948
|
}
|
|
832
949
|
stepLogger.warn("[gate] Cross-job restart re-enqueued target", {
|
|
833
950
|
runId: run.id,
|
|
@@ -861,7 +978,11 @@ async function applyGateRestart(engine, run, job, step, decision, stepLogger) {
|
|
|
861
978
|
const updatedRun = await engine.getRun(run.id);
|
|
862
979
|
const updatedJob = updatedRun?.jobs.find((j) => j.id === job.id);
|
|
863
980
|
if (updatedJob) {
|
|
864
|
-
await scheduler.enqueueJob(
|
|
981
|
+
await scheduler.enqueueJob(
|
|
982
|
+
run.id,
|
|
983
|
+
updatedJob,
|
|
984
|
+
updatedJob.priority ?? "normal"
|
|
985
|
+
);
|
|
865
986
|
stepLogger.info("[gate] Job re-enqueued for restart", {
|
|
866
987
|
runId: run.id,
|
|
867
988
|
jobId: job.id,
|
|
@@ -877,10 +998,10 @@ function inferWorkspaceProvisionReasonCode(message) {
|
|
|
877
998
|
|
|
878
999
|
// src/job-broker.ts
|
|
879
1000
|
var JobBroker = class {
|
|
880
|
-
constructor(engine, logger,
|
|
1001
|
+
constructor(engine, logger, platform) {
|
|
881
1002
|
this.engine = engine;
|
|
882
1003
|
this.logger = logger;
|
|
883
|
-
this.platform =
|
|
1004
|
+
this.platform = platform;
|
|
884
1005
|
}
|
|
885
1006
|
engine;
|
|
886
1007
|
logger;
|
|
@@ -3050,169 +3171,180 @@ function buildWorkflowMetricLines(metrics) {
|
|
|
3050
3171
|
];
|
|
3051
3172
|
}
|
|
3052
3173
|
async function bootstrap(_cwd = process.cwd()) {
|
|
3053
|
-
await
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
|
|
3059
|
-
|
|
3060
|
-
|
|
3061
|
-
|
|
3062
|
-
|
|
3063
|
-
|
|
3064
|
-
|
|
3065
|
-
|
|
3066
|
-
|
|
3067
|
-
|
|
3068
|
-
|
|
3069
|
-
|
|
3070
|
-
|
|
3071
|
-
|
|
3072
|
-
|
|
3073
|
-
|
|
3074
|
-
|
|
3075
|
-
|
|
3076
|
-
|
|
3077
|
-
|
|
3078
|
-
|
|
3079
|
-
|
|
3080
|
-
|
|
3081
|
-
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
});
|
|
3087
|
-
const debugMode = process.env["WORKFLOW_DEBUG"] === "true";
|
|
3088
|
-
bootstrapLogger.info("Workflow daemon starting", { projectRoot, debugMode });
|
|
3089
|
-
if (debugMode) {
|
|
3090
|
-
bootstrapLogger.warn(
|
|
3091
|
-
"[WORKFLOW_DEBUG=true] Verbose debug logging is ON \u2014 step inputs, outputs, and expr contexts will be logged. Disable in production (unset WORKFLOW_DEBUG or set to false)."
|
|
3092
|
-
);
|
|
3093
|
-
}
|
|
3094
|
-
const createWorkflowLogger = (service, operation, bindings) => createCorrelatedLogger(platform.logger, {
|
|
3095
|
-
serviceId: "workflow",
|
|
3096
|
-
logsSource: "workflow",
|
|
3097
|
-
layer: "workflow",
|
|
3098
|
-
service,
|
|
3099
|
-
operation,
|
|
3100
|
-
bindings
|
|
3101
|
-
});
|
|
3102
|
-
bootstrapLogger.info("Loading plugin registry snapshot");
|
|
3103
|
-
const cliApi = await createRegistry({ root: repoRoot, cache: { ttlMs: 10 * 60 * 1e3 } });
|
|
3104
|
-
await cliApi.initialize();
|
|
3105
|
-
const plugins = await cliApi.listPlugins();
|
|
3106
|
-
bootstrapLogger.info("Plugin registry snapshot loaded", {
|
|
3107
|
-
pluginsFound: plugins.length,
|
|
3108
|
-
pluginIds: plugins.map((p) => `${p.id}@${p.version}`)
|
|
3109
|
-
});
|
|
3110
|
-
bootstrapLogger.info("Creating WorkflowEngine");
|
|
3111
|
-
const engine = new WorkflowEngine({
|
|
3112
|
-
cache: platform.cache,
|
|
3113
|
-
events: platform.eventBus,
|
|
3114
|
-
logger: createWorkflowLogger("engine", "workflow.engine"),
|
|
3115
|
-
snapshotManager: platform.snapshotManager,
|
|
3116
|
-
workspaceRoot: projectRoot
|
|
3117
|
-
});
|
|
3118
|
-
bootstrapLogger.info("Cleaning up stale runs from previous daemon process");
|
|
3119
|
-
await engine.cleanupStaleRuns();
|
|
3120
|
-
bootstrapLogger.info("Resuming interrupted jobs");
|
|
3121
|
-
await engine.resumeInterruptedJobs();
|
|
3122
|
-
bootstrapLogger.info("Creating JobBroker");
|
|
3123
|
-
const jobBroker = new JobBroker(
|
|
3124
|
-
engine,
|
|
3125
|
-
createWorkflowLogger("job-broker", "workflow.job-broker"),
|
|
3126
|
-
platform
|
|
3127
|
-
);
|
|
3128
|
-
bootstrapLogger.info("Creating CronScheduler");
|
|
3129
|
-
const cronScheduler = new CronScheduler({
|
|
3130
|
-
jobBroker,
|
|
3131
|
-
workflowEngine: engine,
|
|
3132
|
-
logger: createWorkflowLogger("cron-scheduler", "workflow.cron-scheduler"),
|
|
3133
|
-
timezone: process.env.WORKFLOW_CRON_TIMEZONE
|
|
3174
|
+
await runService({
|
|
3175
|
+
appId: "workflow-daemon",
|
|
3176
|
+
startDir: _cwd,
|
|
3177
|
+
// serviceId in the transport map / devservices is 'workflow' (≠ appId).
|
|
3178
|
+
serviceId: "workflow",
|
|
3179
|
+
defaultPort: 7778,
|
|
3180
|
+
portEnvVar: "WORKFLOW_PORT",
|
|
3181
|
+
defaultHost: "0.0.0.0",
|
|
3182
|
+
hostEnvVar: "WORKFLOW_HOST",
|
|
3183
|
+
platform: {
|
|
3184
|
+
assemblyHook: makeAssemblyHook()
|
|
3185
|
+
},
|
|
3186
|
+
async setup({
|
|
3187
|
+
platform,
|
|
3188
|
+
port,
|
|
3189
|
+
host,
|
|
3190
|
+
projectRoot,
|
|
3191
|
+
platformRoot,
|
|
3192
|
+
logger: serviceLogger
|
|
3193
|
+
}) {
|
|
3194
|
+
const workspaceRoot = process.env["KB_PROJECT_ROOT"] ?? projectRoot;
|
|
3195
|
+
const startupRequestId = `workflow-startup-${Date.now()}-${Math.random().toString(36).slice(2, 9)}`;
|
|
3196
|
+
const startupTraceId = randomUUID();
|
|
3197
|
+
const startupSpanId = randomUUID();
|
|
3198
|
+
const bootstrapLogger = serviceLogger.forComponent("workflow-bootstrap").forOperation("workflow.bootstrap", {
|
|
3199
|
+
requestId: startupRequestId,
|
|
3200
|
+
traceId: startupTraceId,
|
|
3201
|
+
spanId: startupSpanId
|
|
3202
|
+
}).with({ invocationId: startupSpanId, executionId: startupSpanId });
|
|
3203
|
+
if (!platform.isConfigured("workspace")) {
|
|
3204
|
+
bootstrapLogger.warn("Workspace adapter is not configured", {
|
|
3205
|
+
impact: "Workflows using balanced or strict isolation will fail.",
|
|
3206
|
+
remediation: "Configure platform.adapters.workspace or use relaxed isolation."
|
|
3134
3207
|
});
|
|
3135
|
-
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
workspaceRoot: projectRoot
|
|
3208
|
+
}
|
|
3209
|
+
if (!platform.isConfigured("environment") && platform.isConfigured("workspace")) {
|
|
3210
|
+
bootstrapLogger.warn("Environment adapter is not configured", {
|
|
3211
|
+
impact: "Workflows using strict isolation will fail.",
|
|
3212
|
+
remediation: "Configure platform.adapters.environment."
|
|
3141
3213
|
});
|
|
3142
|
-
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
|
|
3146
|
-
|
|
3147
|
-
|
|
3214
|
+
}
|
|
3215
|
+
const debugMode = process.env["WORKFLOW_DEBUG"] === "true";
|
|
3216
|
+
bootstrapLogger.info("Workflow daemon starting", {
|
|
3217
|
+
projectRoot: workspaceRoot,
|
|
3218
|
+
debugMode
|
|
3219
|
+
});
|
|
3220
|
+
if (debugMode) {
|
|
3221
|
+
bootstrapLogger.warn(
|
|
3222
|
+
"[WORKFLOW_DEBUG=true] Verbose debug logging is ON \u2014 step inputs, outputs, and expr contexts will be logged. Disable in production (unset WORKFLOW_DEBUG or set to false)."
|
|
3148
3223
|
);
|
|
3149
|
-
bootstrapLogger.info("Starting WorkflowFileWatcher");
|
|
3150
|
-
const fileWatcher = new WorkflowFileWatcher({
|
|
3151
|
-
watchDirs: [
|
|
3152
|
-
join(projectRoot, ".kb", "workflows"),
|
|
3153
|
-
join(projectRoot, ".kb", "jobs")
|
|
3154
|
-
],
|
|
3155
|
-
workflowService,
|
|
3156
|
-
cronDiscovery,
|
|
3157
|
-
cronScheduler,
|
|
3158
|
-
logger: createWorkflowLogger("file-watcher", "workflow.file-watcher")
|
|
3159
|
-
});
|
|
3160
|
-
bootstrapLogger.info("Creating HTTP server");
|
|
3161
|
-
const server = await createServer({
|
|
3162
|
-
engine,
|
|
3163
|
-
jobBroker,
|
|
3164
|
-
workflowService,
|
|
3165
|
-
cronScheduler,
|
|
3166
|
-
cronDiscovery,
|
|
3167
|
-
logger: createWorkflowLogger("api", "workflow.api")
|
|
3168
|
-
});
|
|
3169
|
-
await server.listen(getListenOptions(port, host));
|
|
3170
|
-
bootstrapLogger.info("HTTP API listening", { port });
|
|
3171
|
-
bootstrapLogger.info("Creating WorkflowWorker");
|
|
3172
|
-
const worker = await createWorkflowWorker({
|
|
3173
|
-
engine,
|
|
3174
|
-
cliApi,
|
|
3175
|
-
logger: createWorkflowLogger("worker", "workflow.worker"),
|
|
3176
|
-
analytics: platform.analytics,
|
|
3177
|
-
platform,
|
|
3178
|
-
workspaceRoot: projectRoot,
|
|
3179
|
-
concurrency: parseInt(process.env.WORKFLOW_CONCURRENCY ?? "5", 10),
|
|
3180
|
-
debugMode
|
|
3181
|
-
});
|
|
3182
|
-
bootstrapLogger.info("Starting WorkflowWorker");
|
|
3183
|
-
worker.start().catch((error) => {
|
|
3184
|
-
bootstrapLogger.error(
|
|
3185
|
-
"Worker crashed - shutting down daemon",
|
|
3186
|
-
error instanceof Error ? error : void 0
|
|
3187
|
-
);
|
|
3188
|
-
process.kill(process.pid, "SIGTERM");
|
|
3189
|
-
});
|
|
3190
|
-
bootstrapLogger.info("Starting CronScheduler");
|
|
3191
|
-
await cronScheduler.start();
|
|
3192
|
-
bootstrapLogger.info("Workflow daemon started successfully", { port });
|
|
3193
|
-
return async () => {
|
|
3194
|
-
bootstrapLogger.warn("Stopping workflow daemon components");
|
|
3195
|
-
fileWatcher.close();
|
|
3196
|
-
await cronScheduler.stop();
|
|
3197
|
-
await worker.stop();
|
|
3198
|
-
await server.close();
|
|
3199
|
-
await cliApi.dispose();
|
|
3200
|
-
};
|
|
3201
3224
|
}
|
|
3202
|
-
|
|
3203
|
-
|
|
3204
|
-
|
|
3225
|
+
const createWorkflowLogger = (service, operation, bindings) => serviceLogger.forComponent(`workflow-${service}`).forOperation(operation).with({});
|
|
3226
|
+
bootstrapLogger.info("Loading plugin registry snapshot");
|
|
3227
|
+
const cliApi = await createRegistry({
|
|
3228
|
+
root: workspaceRoot,
|
|
3229
|
+
platformRoot,
|
|
3230
|
+
cache: { ttlMs: 10 * 60 * 1e3 }
|
|
3231
|
+
});
|
|
3232
|
+
await cliApi.initialize();
|
|
3233
|
+
const plugins = await cliApi.listPlugins();
|
|
3234
|
+
bootstrapLogger.info("Plugin registry snapshot loaded", {
|
|
3235
|
+
pluginsFound: plugins.length,
|
|
3236
|
+
pluginIds: plugins.map((p) => `${p.id}@${p.version}`)
|
|
3237
|
+
});
|
|
3238
|
+
bootstrapLogger.info("Creating WorkflowEngine");
|
|
3239
|
+
const engine = new WorkflowEngine({
|
|
3240
|
+
cache: platform.cache,
|
|
3241
|
+
events: platform.eventBus,
|
|
3242
|
+
logger: createWorkflowLogger("engine", "workflow.engine"),
|
|
3243
|
+
snapshotManager: platform.snapshotManager,
|
|
3244
|
+
workspaceRoot
|
|
3245
|
+
});
|
|
3246
|
+
bootstrapLogger.info(
|
|
3247
|
+
"Cleaning up stale runs from previous daemon process"
|
|
3248
|
+
);
|
|
3249
|
+
await engine.cleanupStaleRuns();
|
|
3250
|
+
bootstrapLogger.info("Resuming interrupted jobs");
|
|
3251
|
+
await engine.resumeInterruptedJobs();
|
|
3252
|
+
bootstrapLogger.info("Creating JobBroker");
|
|
3253
|
+
const jobBroker = new JobBroker(
|
|
3254
|
+
engine,
|
|
3255
|
+
createWorkflowLogger("job-broker", "workflow.job-broker"),
|
|
3256
|
+
platform
|
|
3257
|
+
);
|
|
3258
|
+
bootstrapLogger.info("Creating CronScheduler");
|
|
3259
|
+
const cronScheduler = new CronScheduler({
|
|
3260
|
+
jobBroker,
|
|
3261
|
+
workflowEngine: engine,
|
|
3262
|
+
logger: createWorkflowLogger(
|
|
3263
|
+
"cron-scheduler",
|
|
3264
|
+
"workflow.cron-scheduler"
|
|
3265
|
+
),
|
|
3266
|
+
timezone: process.env.WORKFLOW_CRON_TIMEZONE
|
|
3267
|
+
});
|
|
3268
|
+
bootstrapLogger.info("Discovering cron jobs");
|
|
3269
|
+
const cronDiscovery = new CronDiscovery({
|
|
3270
|
+
cliApi,
|
|
3271
|
+
scheduler: cronScheduler,
|
|
3272
|
+
logger: createWorkflowLogger(
|
|
3273
|
+
"cron-discovery",
|
|
3274
|
+
"workflow.cron-discovery"
|
|
3275
|
+
),
|
|
3276
|
+
workspaceRoot
|
|
3277
|
+
});
|
|
3278
|
+
const discovered = await cronDiscovery.discoverAll();
|
|
3279
|
+
bootstrapLogger.info("Cron job discovery complete", discovered);
|
|
3280
|
+
bootstrapLogger.info("Creating WorkflowService");
|
|
3281
|
+
const workflowService = new WorkflowService({
|
|
3282
|
+
cliApi,
|
|
3283
|
+
platform,
|
|
3284
|
+
workspaceRoot
|
|
3285
|
+
});
|
|
3286
|
+
workflowService.listAll().catch(
|
|
3287
|
+
(err) => bootstrapLogger.warn("Manifest scanner warmup failed", { err })
|
|
3288
|
+
);
|
|
3289
|
+
bootstrapLogger.info("Starting WorkflowFileWatcher");
|
|
3290
|
+
const fileWatcher = new WorkflowFileWatcher({
|
|
3291
|
+
watchDirs: [
|
|
3292
|
+
join(workspaceRoot, ".kb", "workflows"),
|
|
3293
|
+
join(workspaceRoot, ".kb", "jobs")
|
|
3294
|
+
],
|
|
3295
|
+
workflowService,
|
|
3296
|
+
cronDiscovery,
|
|
3297
|
+
cronScheduler,
|
|
3298
|
+
logger: createWorkflowLogger("file-watcher", "workflow.file-watcher")
|
|
3299
|
+
});
|
|
3300
|
+
bootstrapLogger.info("Creating HTTP server");
|
|
3301
|
+
const server = await createServer({
|
|
3302
|
+
engine,
|
|
3303
|
+
jobBroker,
|
|
3304
|
+
workflowService,
|
|
3305
|
+
cronScheduler,
|
|
3306
|
+
cronDiscovery,
|
|
3307
|
+
logger: createWorkflowLogger("api", "workflow.api")
|
|
3308
|
+
});
|
|
3309
|
+
await server.listen(getListenOptions(port, host));
|
|
3310
|
+
bootstrapLogger.info("HTTP API listening", { port });
|
|
3311
|
+
bootstrapLogger.info("Creating WorkflowWorker");
|
|
3312
|
+
const worker = await createWorkflowWorker({
|
|
3313
|
+
engine,
|
|
3314
|
+
cliApi,
|
|
3315
|
+
logger: createWorkflowLogger("worker", "workflow.worker"),
|
|
3316
|
+
analytics: platform.analytics,
|
|
3317
|
+
platform,
|
|
3318
|
+
workspaceRoot,
|
|
3319
|
+
concurrency: parseInt(process.env.WORKFLOW_CONCURRENCY ?? "5", 10),
|
|
3320
|
+
debugMode
|
|
3321
|
+
});
|
|
3322
|
+
bootstrapLogger.info("Starting WorkflowWorker");
|
|
3323
|
+
worker.start().catch((error) => {
|
|
3324
|
+
bootstrapLogger.error(
|
|
3325
|
+
"Worker crashed - shutting down daemon",
|
|
3326
|
+
error instanceof Error ? error : void 0
|
|
3327
|
+
);
|
|
3328
|
+
process.kill(process.pid, "SIGTERM");
|
|
3329
|
+
});
|
|
3330
|
+
bootstrapLogger.info("Starting CronScheduler");
|
|
3331
|
+
await cronScheduler.start();
|
|
3332
|
+
bootstrapLogger.info("Workflow daemon started successfully", { port });
|
|
3333
|
+
return async () => {
|
|
3334
|
+
bootstrapLogger.warn("Stopping workflow daemon components");
|
|
3335
|
+
fileWatcher.close();
|
|
3336
|
+
await cronScheduler.stop();
|
|
3337
|
+
await worker.stop();
|
|
3338
|
+
await server.close();
|
|
3339
|
+
await cliApi.dispose();
|
|
3340
|
+
};
|
|
3341
|
+
}
|
|
3342
|
+
});
|
|
3205
3343
|
}
|
|
3206
3344
|
|
|
3207
3345
|
// src/index.ts
|
|
3208
|
-
(
|
|
3209
|
-
|
|
3210
|
-
|
|
3211
|
-
} catch (error) {
|
|
3212
|
-
process.stderr.write(`[workflow-daemon] FATAL: ${error instanceof Error ? error.message : String(error)}
|
|
3213
|
-
`);
|
|
3214
|
-
process.exit(1);
|
|
3215
|
-
}
|
|
3216
|
-
})();
|
|
3346
|
+
bootstrap(process.cwd()).catch(() => {
|
|
3347
|
+
process.exitCode = 1;
|
|
3348
|
+
});
|
|
3217
3349
|
//# sourceMappingURL=index.js.map
|
|
3218
3350
|
//# sourceMappingURL=index.js.map
|