nvent 0.5.4 → 0.5.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/module.json +1 -1
- package/dist/module.mjs +110 -24
- package/dist/runtime/adapters/factory.js +8 -7
- package/dist/runtime/adapters/interfaces/queue.d.ts +5 -0
- package/dist/runtime/config/index.js +14 -1
- package/dist/runtime/config/types.d.ts +42 -0
- package/dist/runtime/events/types.d.ts +0 -1
- package/dist/runtime/events/utils/stallDetector.d.ts +13 -77
- package/dist/runtime/events/utils/stallDetector.js +8 -192
- package/dist/runtime/events/wiring/flowWiring.js +311 -107
- package/dist/runtime/events/wiring/triggerWiring.js +3 -1
- package/dist/runtime/nitro/plugins/02.workers.js +31 -2
- package/dist/runtime/nitro/routes/webhook.await.js +28 -6
- package/dist/runtime/nitro/utils/awaitPatterns/event.js +58 -50
- package/dist/runtime/nitro/utils/awaitPatterns/schedule.js +6 -1
- package/dist/runtime/nitro/utils/awaitPatterns/time.d.ts +1 -1
- package/dist/runtime/nitro/utils/awaitPatterns/time.js +6 -2
- package/dist/runtime/nitro/utils/awaitPatterns/webhook.js +53 -45
- package/dist/runtime/nitro/utils/defineFunction.d.ts +2 -9
- package/dist/runtime/nitro/utils/defineFunction.js +1 -14
- package/dist/runtime/nitro/utils/defineFunctionConfig.d.ts +84 -16
- package/dist/runtime/nitro/utils/defineHooks.d.ts +64 -10
- package/dist/runtime/nitro/utils/defineHooks.js +3 -0
- package/dist/runtime/nitro/utils/useAwait.d.ts +12 -0
- package/dist/runtime/nitro/utils/useAwait.js +34 -4
- package/dist/runtime/nitro/utils/useFlow.js +13 -2
- package/dist/runtime/nitro/utils/useHookRegistry.d.ts +10 -4
- package/dist/runtime/scheduler/scheduler.d.ts +19 -0
- package/dist/runtime/scheduler/scheduler.js +184 -8
- package/dist/runtime/scheduler/types.d.ts +6 -0
- package/dist/runtime/worker/node/runner.js +32 -91
- package/dist/runtime/worker/system/awaitHandlers.d.ts +27 -0
- package/dist/runtime/worker/system/awaitHandlers.js +230 -0
- package/dist/runtime/worker/system/index.d.ts +24 -0
- package/dist/runtime/worker/system/index.js +39 -0
- package/package.json +1 -1
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { getEventBus } from "../eventBus.js";
|
|
2
2
|
import { useNventLogger, useStoreAdapter, useQueueAdapter, $useAnalyzedFlows, $useFunctionRegistry, useStreamTopics, useRuntimeConfig, useScheduler } from "#imports";
|
|
3
3
|
import { createStallDetector } from "../utils/stallDetector.js";
|
|
4
|
+
import { SYSTEM_HANDLERS } from "../../worker/system/index.js";
|
|
4
5
|
export function checkPendingStepTriggers(step, emittedEvents, completedSteps) {
|
|
5
6
|
if (!step.subscribes || step.subscribes.length === 0) {
|
|
6
7
|
return true;
|
|
@@ -70,49 +71,22 @@ export async function checkAndTriggerPendingSteps(flowName, runId, store) {
|
|
|
70
71
|
for (const [stepName, stepDef] of Object.entries(flowDef.steps)) {
|
|
71
72
|
const step = stepDef;
|
|
72
73
|
if (!step.subscribes || completedSteps.has(stepName)) continue;
|
|
73
|
-
const
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
runId,
|
|
78
|
-
stepName,
|
|
79
|
-
awaitType: awaitState.awaitType,
|
|
80
|
-
position: awaitState.position,
|
|
81
|
-
status: awaitState.status
|
|
82
|
-
});
|
|
83
|
-
continue;
|
|
84
|
-
}
|
|
85
|
-
if (awaitState && awaitState.status === "timeout") {
|
|
86
|
-
logger.debug("Step await timed out, skipping trigger", {
|
|
87
|
-
flowName,
|
|
88
|
-
runId,
|
|
89
|
-
stepName,
|
|
90
|
-
awaitType: awaitState.awaitType
|
|
91
|
-
});
|
|
92
|
-
continue;
|
|
93
|
-
}
|
|
94
|
-
if (awaitState?.status === "resolved") {
|
|
95
|
-
logger.debug("Step await is resolved, will proceed", {
|
|
96
|
-
flowName,
|
|
97
|
-
runId,
|
|
98
|
-
stepName,
|
|
99
|
-
awaitType: awaitState.awaitType,
|
|
100
|
-
position: awaitState.position
|
|
101
|
-
});
|
|
102
|
-
}
|
|
74
|
+
const awaitBeforeKey = `${stepName}:before`;
|
|
75
|
+
const awaitState = flowEntry?.metadata?.awaitingSteps?.[awaitBeforeKey];
|
|
76
|
+
if (awaitState && awaitState.status === "awaiting") continue;
|
|
77
|
+
if (awaitState && awaitState.status === "timeout") continue;
|
|
103
78
|
const isDependencyAwaiting = step.subscribes.some((sub) => {
|
|
104
79
|
const emitEvent = allEvents.find(
|
|
105
80
|
(evt) => evt.type === "emit" && evt.data?.name === sub
|
|
106
81
|
);
|
|
107
|
-
if (!emitEvent)
|
|
108
|
-
return false;
|
|
109
|
-
}
|
|
82
|
+
if (!emitEvent) return false;
|
|
110
83
|
const emitStepName = emitEvent.stepName;
|
|
111
84
|
if (!emitStepName) {
|
|
112
85
|
return false;
|
|
113
86
|
}
|
|
114
|
-
const
|
|
115
|
-
|
|
87
|
+
const awaitAfterKey = `${emitStepName}:after`;
|
|
88
|
+
const awaitState2 = awaitingSteps[awaitAfterKey];
|
|
89
|
+
if (awaitState2?.status === "awaiting") {
|
|
116
90
|
return true;
|
|
117
91
|
}
|
|
118
92
|
if (awaitState2?.status === "resolved") {
|
|
@@ -128,20 +102,80 @@ export async function checkAndTriggerPendingSteps(flowName, runId, store) {
|
|
|
128
102
|
);
|
|
129
103
|
if (stepCompleted) {
|
|
130
104
|
const awaitResolved = allEvents.some(
|
|
131
|
-
(evt) => evt.type === "await.resolved" && evt.stepName === emitStepName
|
|
105
|
+
(evt) => evt.type === "await.resolved" && evt.stepName === emitStepName && evt.position === "after"
|
|
132
106
|
);
|
|
133
|
-
if (!awaitResolved)
|
|
134
|
-
return true;
|
|
135
|
-
}
|
|
107
|
+
if (!awaitResolved) return true;
|
|
136
108
|
}
|
|
137
109
|
}
|
|
138
110
|
return false;
|
|
139
111
|
});
|
|
140
|
-
if (isDependencyAwaiting)
|
|
141
|
-
continue;
|
|
142
|
-
}
|
|
112
|
+
if (isDependencyAwaiting) continue;
|
|
143
113
|
const canTrigger = checkPendingStepTriggers(step, emittedEvents, completedSteps);
|
|
144
|
-
if (canTrigger) {
|
|
114
|
+
if (canTrigger && step.awaitBefore) {
|
|
115
|
+
if (awaitState?.status === "awaiting") continue;
|
|
116
|
+
if (awaitState?.status !== "resolved") {
|
|
117
|
+
try {
|
|
118
|
+
const emitData = {};
|
|
119
|
+
const subscribes = step.subscribes || [];
|
|
120
|
+
for (const sub of subscribes) {
|
|
121
|
+
const emitEvent = allEvents.find(
|
|
122
|
+
(evt) => evt.type === "emit" && evt.data?.name === sub
|
|
123
|
+
);
|
|
124
|
+
if (emitEvent && emitEvent.data?.payload !== void 0) {
|
|
125
|
+
emitData[sub] = emitEvent.data.payload;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
const payload = {
|
|
129
|
+
flowId: runId,
|
|
130
|
+
flowName,
|
|
131
|
+
stepName,
|
|
132
|
+
position: "before",
|
|
133
|
+
awaitConfig: step.awaitBefore,
|
|
134
|
+
input: emitData
|
|
135
|
+
};
|
|
136
|
+
const jobId = `${runId}__${stepName}__await-register-before`;
|
|
137
|
+
const flowRegistry = (registry?.flows || {})[flowName];
|
|
138
|
+
const stepMeta = flowRegistry?.steps?.[stepName];
|
|
139
|
+
let stepQueue = stepMeta?.queue;
|
|
140
|
+
if (!stepQueue && flowRegistry?.entry?.step === stepName) {
|
|
141
|
+
stepQueue = flowRegistry.entry.queue;
|
|
142
|
+
}
|
|
143
|
+
if (!stepQueue && registry?.workers) {
|
|
144
|
+
const worker = registry.workers.find((w) => {
|
|
145
|
+
const flowNames = w?.flow?.names || (w?.flow?.name ? [w?.flow?.name] : []);
|
|
146
|
+
const stepMatch = w?.flow?.step === stepName || Array.isArray(w?.flow?.step) && w?.flow?.step.includes(stepName);
|
|
147
|
+
return flowNames.includes(flowName) && stepMatch;
|
|
148
|
+
});
|
|
149
|
+
stepQueue = worker?.queue?.name;
|
|
150
|
+
}
|
|
151
|
+
if (!stepQueue) {
|
|
152
|
+
logger.error("Cannot find queue for step", {
|
|
153
|
+
stepName,
|
|
154
|
+
flowName,
|
|
155
|
+
availableSteps: Object.keys(flowRegistry?.steps || {}),
|
|
156
|
+
entryStep: flowRegistry?.entry?.step
|
|
157
|
+
});
|
|
158
|
+
throw new Error(`Cannot register await: queue not found for step ${stepName} in flow ${flowName}`);
|
|
159
|
+
}
|
|
160
|
+
const analyzedAwaitStep = (flowDef.analyzed?.steps || {})[stepName];
|
|
161
|
+
const awaitStepTimeout = analyzedAwaitStep?.stepTimeout;
|
|
162
|
+
await queue.enqueue(stepQueue, {
|
|
163
|
+
name: SYSTEM_HANDLERS.AWAIT_REGISTER,
|
|
164
|
+
data: payload,
|
|
165
|
+
opts: { jobId, timeout: awaitStepTimeout }
|
|
166
|
+
});
|
|
167
|
+
} catch (err) {
|
|
168
|
+
logger.error("Failed to register awaitBefore pattern", {
|
|
169
|
+
flowName,
|
|
170
|
+
stepName,
|
|
171
|
+
error: err.message
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
const shouldEnqueueStep = canTrigger && (!step.awaitBefore || awaitState?.status === "resolved");
|
|
178
|
+
if (shouldEnqueueStep) {
|
|
145
179
|
const flowRegistry = (registry?.flows || {})[flowName];
|
|
146
180
|
const stepMeta = flowRegistry?.steps?.[stepName];
|
|
147
181
|
if (stepMeta?.queue) {
|
|
@@ -161,16 +195,20 @@ export async function checkAndTriggerPendingSteps(flowName, runId, store) {
|
|
|
161
195
|
input: emitData
|
|
162
196
|
// Keyed by event name
|
|
163
197
|
};
|
|
164
|
-
|
|
198
|
+
const isAwaitResuming = awaitState?.status === "resolved" && awaitState?.position === "before";
|
|
199
|
+
if (isAwaitResuming) {
|
|
165
200
|
payload.awaitResolved = true;
|
|
166
201
|
payload.awaitData = awaitState.triggerData;
|
|
202
|
+
payload.awaitPosition = "before";
|
|
167
203
|
}
|
|
168
|
-
const jobId = `${runId}__${stepName}`;
|
|
204
|
+
const jobId = isAwaitResuming ? `${runId}__${stepName}__resumed` : `${runId}__${stepName}`;
|
|
169
205
|
const worker = registry?.workers?.find(
|
|
170
206
|
(w) => w?.flow?.step === stepName && w?.queue?.name === stepMeta.queue
|
|
171
207
|
);
|
|
172
208
|
const defaultOpts = worker?.queue?.defaultJobOptions || {};
|
|
173
|
-
const
|
|
209
|
+
const analyzedStep = (flowDef.analyzed?.steps || {})[stepName];
|
|
210
|
+
const stepTimeout = analyzedStep?.stepTimeout;
|
|
211
|
+
const opts = { ...defaultOpts, jobId, timeout: stepTimeout };
|
|
174
212
|
try {
|
|
175
213
|
await queue.enqueue(stepMeta.queue, { name: stepName, data: payload, opts });
|
|
176
214
|
} catch {
|
|
@@ -402,6 +440,21 @@ export function createFlowWiring() {
|
|
|
402
440
|
const persistedEvent = await store.stream.append(streamName, eventData);
|
|
403
441
|
await bus.publish(persistedEvent);
|
|
404
442
|
if (e.type === "flow.completed" || e.type === "flow.failed") {
|
|
443
|
+
try {
|
|
444
|
+
const scheduler = useScheduler();
|
|
445
|
+
const flowJobs = await scheduler.getJobsByPattern(runId);
|
|
446
|
+
for (const job of flowJobs) {
|
|
447
|
+
await scheduler.unschedule(job.id);
|
|
448
|
+
logger.debug(`Unscheduled job for ${e.type} flow: ${job.id}`);
|
|
449
|
+
}
|
|
450
|
+
logger.debug(`Unscheduled ${flowJobs.length} scheduled jobs for ${e.type} flow runId '${runId}'`, {
|
|
451
|
+
jobs: flowJobs.map((j) => j.id)
|
|
452
|
+
});
|
|
453
|
+
} catch (error) {
|
|
454
|
+
logger.debug(`Could not unschedule jobs for runId '${runId}'`, {
|
|
455
|
+
error: error.message
|
|
456
|
+
});
|
|
457
|
+
}
|
|
405
458
|
const publishKey = `${runId}:terminal`;
|
|
406
459
|
setTimeout(() => {
|
|
407
460
|
try {
|
|
@@ -528,11 +581,19 @@ export function createFlowWiring() {
|
|
|
528
581
|
});
|
|
529
582
|
}
|
|
530
583
|
} catch (err) {
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
584
|
+
const errorMsg = err?.message || "";
|
|
585
|
+
if (!errorMsg.includes("Entry not found")) {
|
|
586
|
+
logger.warn("Failed to update flow stats", {
|
|
587
|
+
type: e.type,
|
|
588
|
+
flowName: e.flowName,
|
|
589
|
+
error: errorMsg
|
|
590
|
+
});
|
|
591
|
+
} else {
|
|
592
|
+
logger.debug("Flow entry not found (will be created)", {
|
|
593
|
+
type: e.type,
|
|
594
|
+
flowName: e.flowName
|
|
595
|
+
});
|
|
596
|
+
}
|
|
536
597
|
}
|
|
537
598
|
};
|
|
538
599
|
const handleOrchestration = async (e) => {
|
|
@@ -561,6 +622,35 @@ export function createFlowWiring() {
|
|
|
561
622
|
emittedEvents: {}
|
|
562
623
|
// Object for atomic updates
|
|
563
624
|
});
|
|
625
|
+
try {
|
|
626
|
+
const analyzedFlows = $useAnalyzedFlows();
|
|
627
|
+
const flowMeta = analyzedFlows.find((f) => f.id === flowName);
|
|
628
|
+
const stallTimeout = flowMeta?.analyzed?.stallTimeout || 30 * 60 * 1e3;
|
|
629
|
+
const scheduler = useScheduler();
|
|
630
|
+
const stallJobId = `stall-timeout:${runId}`;
|
|
631
|
+
await scheduler.schedule({
|
|
632
|
+
id: stallJobId,
|
|
633
|
+
name: `Stall Timeout - ${flowName}`,
|
|
634
|
+
type: "one-time",
|
|
635
|
+
executeAt: timestamp + stallTimeout,
|
|
636
|
+
handler: async () => {
|
|
637
|
+
if (stallDetector) {
|
|
638
|
+
logger.info(`Per-flow stall timeout fired for '${flowName}' runId '${runId}'`);
|
|
639
|
+
await stallDetector.markAsStalled(flowName, runId, "Stall timeout reached");
|
|
640
|
+
}
|
|
641
|
+
},
|
|
642
|
+
metadata: {
|
|
643
|
+
component: "stall-detector",
|
|
644
|
+
flowName,
|
|
645
|
+
runId
|
|
646
|
+
}
|
|
647
|
+
});
|
|
648
|
+
logger.debug(`Scheduled stall timeout for flow '${flowName}' runId '${runId}' in ${stallTimeout / 1e3}s`, { jobId: stallJobId });
|
|
649
|
+
} catch (error) {
|
|
650
|
+
logger.warn(`Failed to schedule stall timeout for flow '${flowName}' runId '${runId}'`, {
|
|
651
|
+
error: error.message
|
|
652
|
+
});
|
|
653
|
+
}
|
|
564
654
|
}
|
|
565
655
|
if (e.type === "flow.cancel") {
|
|
566
656
|
try {
|
|
@@ -571,8 +661,17 @@ export function createFlowWiring() {
|
|
|
571
661
|
});
|
|
572
662
|
logger.info("Marked flow as canceled", { flowName, runId });
|
|
573
663
|
}
|
|
664
|
+
const scheduler = useScheduler();
|
|
665
|
+
const flowJobs = await scheduler.getJobsByPattern(runId);
|
|
666
|
+
for (const job of flowJobs) {
|
|
667
|
+
await scheduler.unschedule(job.id);
|
|
668
|
+
logger.debug(`Unscheduled job for canceled flow: ${job.id}`);
|
|
669
|
+
}
|
|
670
|
+
logger.debug(`Unscheduled ${flowJobs.length} scheduled jobs for canceled flow runId '${runId}'`, {
|
|
671
|
+
jobs: flowJobs.map((j) => j.id)
|
|
672
|
+
});
|
|
574
673
|
} catch (err) {
|
|
575
|
-
logger.warn("Failed to update canceled status", {
|
|
674
|
+
logger.warn("Failed to update canceled status or unschedule flow jobs", {
|
|
576
675
|
flowName,
|
|
577
676
|
runId,
|
|
578
677
|
error: err?.message
|
|
@@ -580,8 +679,36 @@ export function createFlowWiring() {
|
|
|
580
679
|
}
|
|
581
680
|
}
|
|
582
681
|
if (e.type === "step.started" || e.type === "step.completed" || e.type === "step.failed" || e.type === "step.retry") {
|
|
583
|
-
|
|
584
|
-
|
|
682
|
+
try {
|
|
683
|
+
const scheduler = useScheduler();
|
|
684
|
+
const stallJobId = `stall-timeout:${runId}`;
|
|
685
|
+
const analyzedFlows = $useAnalyzedFlows();
|
|
686
|
+
const flowMeta = analyzedFlows.find((f) => f.id === flowName);
|
|
687
|
+
const stallTimeout = flowMeta?.analyzed?.stallTimeout || 30 * 60 * 1e3;
|
|
688
|
+
await scheduler.unschedule(stallJobId);
|
|
689
|
+
await scheduler.schedule({
|
|
690
|
+
id: stallJobId,
|
|
691
|
+
name: `Stall Timeout - ${flowName}`,
|
|
692
|
+
type: "one-time",
|
|
693
|
+
executeAt: Date.now() + stallTimeout,
|
|
694
|
+
handler: async () => {
|
|
695
|
+
if (stallDetector) {
|
|
696
|
+
logger.info(`Per-flow stall timeout fired for '${flowName}' runId '${runId}'`);
|
|
697
|
+
await stallDetector.markAsStalled(flowName, runId, "Stall timeout reached");
|
|
698
|
+
}
|
|
699
|
+
},
|
|
700
|
+
metadata: {
|
|
701
|
+
component: "stall-detector",
|
|
702
|
+
flowName,
|
|
703
|
+
runId,
|
|
704
|
+
stallTimeout
|
|
705
|
+
}
|
|
706
|
+
});
|
|
707
|
+
logger.debug(`Rescheduled stall timeout for flow '${flowName}' runId '${runId}' (activity: ${e.type})`);
|
|
708
|
+
} catch (error) {
|
|
709
|
+
logger.debug(`Could not reschedule stall timeout for flow '${flowName}' runId '${runId}'`, {
|
|
710
|
+
error: error.message
|
|
711
|
+
});
|
|
585
712
|
}
|
|
586
713
|
}
|
|
587
714
|
if (e.type === "step.completed") {
|
|
@@ -620,10 +747,15 @@ export function createFlowWiring() {
|
|
|
620
747
|
if (!timeoutAt) {
|
|
621
748
|
timeoutAt = now + 24 * 60 * 60 * 1e3;
|
|
622
749
|
}
|
|
750
|
+
const awaitKey = `${stepName}:${position}`;
|
|
623
751
|
const updatePayload = {
|
|
752
|
+
status: "awaiting",
|
|
753
|
+
// Set flow status to awaiting
|
|
624
754
|
awaitingSteps: {
|
|
625
|
-
[
|
|
755
|
+
[awaitKey]: {
|
|
626
756
|
status: "awaiting",
|
|
757
|
+
stepName,
|
|
758
|
+
// Keep stepName for queries
|
|
627
759
|
awaitType,
|
|
628
760
|
position,
|
|
629
761
|
config: config2,
|
|
@@ -651,18 +783,80 @@ export function createFlowWiring() {
|
|
|
651
783
|
}
|
|
652
784
|
if (e.type === "await.resolved") {
|
|
653
785
|
const awaitEvent = e;
|
|
654
|
-
const { stepName, triggerData } = awaitEvent;
|
|
786
|
+
const { stepName, triggerData, position } = awaitEvent;
|
|
655
787
|
try {
|
|
656
788
|
if (store.index.updateWithRetry) {
|
|
789
|
+
const awaitKey = `${stepName}:${position}`;
|
|
657
790
|
await store.index.updateWithRetry(indexKey, runId, {
|
|
658
791
|
awaitingSteps: {
|
|
659
|
-
[
|
|
792
|
+
[awaitKey]: {
|
|
660
793
|
status: "resolved",
|
|
661
|
-
|
|
794
|
+
stepName,
|
|
795
|
+
// Keep stepName for queries
|
|
796
|
+
triggerData,
|
|
797
|
+
position
|
|
662
798
|
}
|
|
663
799
|
}
|
|
664
800
|
});
|
|
665
801
|
}
|
|
802
|
+
const queue = useQueueAdapter();
|
|
803
|
+
const { StoreSubjects: StoreSubjects2 } = useStreamTopics();
|
|
804
|
+
const streamName2 = StoreSubjects2.flowRun(runId);
|
|
805
|
+
const inputData = {};
|
|
806
|
+
if (store.stream.read) {
|
|
807
|
+
const events = await store.stream.read(streamName2, { limit: 100 });
|
|
808
|
+
const registry2 = $useFunctionRegistry();
|
|
809
|
+
const flowRegistry2 = (registry2?.flows || {})[flowName];
|
|
810
|
+
const stepMeta2 = flowRegistry2?.steps?.[stepName];
|
|
811
|
+
const subscribes = stepMeta2?.subscribes || [];
|
|
812
|
+
for (const sub of subscribes) {
|
|
813
|
+
const emitEvent = events.find(
|
|
814
|
+
(evt) => evt.type === "emit" && evt.data?.name === sub
|
|
815
|
+
);
|
|
816
|
+
if (emitEvent && emitEvent.data?.payload !== void 0) {
|
|
817
|
+
inputData[sub] = emitEvent.data.payload;
|
|
818
|
+
}
|
|
819
|
+
}
|
|
820
|
+
}
|
|
821
|
+
const { SYSTEM_HANDLERS: SYSTEM_HANDLERS2 } = await import("../../worker/system/index.js");
|
|
822
|
+
const payload = {
|
|
823
|
+
flowId: runId,
|
|
824
|
+
flowName,
|
|
825
|
+
stepName,
|
|
826
|
+
position,
|
|
827
|
+
triggerData,
|
|
828
|
+
input: inputData
|
|
829
|
+
};
|
|
830
|
+
const jobId = `${runId}__${stepName}__await-resolve`;
|
|
831
|
+
const registry = $useFunctionRegistry();
|
|
832
|
+
const flowRegistry = (registry?.flows || {})[flowName];
|
|
833
|
+
const stepMeta = flowRegistry?.steps?.[stepName];
|
|
834
|
+
let stepQueue = stepMeta?.queue;
|
|
835
|
+
if (!stepQueue && flowRegistry?.entry?.step === stepName) {
|
|
836
|
+
stepQueue = flowRegistry.entry.queue;
|
|
837
|
+
}
|
|
838
|
+
if (!stepQueue && registry?.workers) {
|
|
839
|
+
const worker = registry.workers.find((w) => {
|
|
840
|
+
const flowNames = w?.flow?.names || (w?.flow?.name ? [w?.flow?.name] : []);
|
|
841
|
+
const stepMatch = w?.flow?.step === stepName || Array.isArray(w?.flow?.step) && w?.flow?.step.includes(stepName);
|
|
842
|
+
return flowNames.includes(flowName) && stepMatch;
|
|
843
|
+
});
|
|
844
|
+
stepQueue = worker?.queue?.name;
|
|
845
|
+
}
|
|
846
|
+
if (!stepQueue) {
|
|
847
|
+
logger.error("Cannot find queue for step", {
|
|
848
|
+
stepName,
|
|
849
|
+
flowName,
|
|
850
|
+
availableSteps: Object.keys(flowRegistry?.steps || {}),
|
|
851
|
+
entryStep: flowRegistry?.entry?.step
|
|
852
|
+
});
|
|
853
|
+
throw new Error(`Cannot resolve await: queue not found for step ${stepName} in flow ${flowName}`);
|
|
854
|
+
}
|
|
855
|
+
await queue.enqueue(stepQueue, {
|
|
856
|
+
name: SYSTEM_HANDLERS2.AWAIT_RESOLVE,
|
|
857
|
+
data: payload,
|
|
858
|
+
opts: { jobId }
|
|
859
|
+
});
|
|
666
860
|
await checkAndTriggerPendingSteps(flowName, runId, store);
|
|
667
861
|
} catch (err) {
|
|
668
862
|
logger.error("Error handling await resolution", {
|
|
@@ -684,6 +878,63 @@ export function createFlowWiring() {
|
|
|
684
878
|
action
|
|
685
879
|
});
|
|
686
880
|
try {
|
|
881
|
+
const queue = useQueueAdapter();
|
|
882
|
+
const { StoreSubjects: StoreSubjects2 } = useStreamTopics();
|
|
883
|
+
const streamName2 = StoreSubjects2.flowRun(runId);
|
|
884
|
+
const inputData = {};
|
|
885
|
+
if (store.stream.read) {
|
|
886
|
+
const events = await store.stream.read(streamName2, { limit: 100 });
|
|
887
|
+
const registry2 = $useFunctionRegistry();
|
|
888
|
+
const flowRegistry2 = (registry2?.flows || {})[flowName];
|
|
889
|
+
const stepMeta2 = flowRegistry2?.steps?.[stepName];
|
|
890
|
+
const subscribes = stepMeta2?.subscribes || [];
|
|
891
|
+
for (const sub of subscribes) {
|
|
892
|
+
const emitEvent = events.find(
|
|
893
|
+
(evt) => evt.type === "emit" && evt.data?.name === sub
|
|
894
|
+
);
|
|
895
|
+
if (emitEvent && emitEvent.data?.payload !== void 0) {
|
|
896
|
+
inputData[sub] = emitEvent.data.payload;
|
|
897
|
+
}
|
|
898
|
+
}
|
|
899
|
+
}
|
|
900
|
+
const payload = {
|
|
901
|
+
flowId: runId,
|
|
902
|
+
flowName,
|
|
903
|
+
stepName,
|
|
904
|
+
position,
|
|
905
|
+
timeoutAction: action,
|
|
906
|
+
input: inputData
|
|
907
|
+
};
|
|
908
|
+
const jobId = `${runId}__${stepName}__await-timeout`;
|
|
909
|
+
const registry = $useFunctionRegistry();
|
|
910
|
+
const flowRegistry = (registry?.flows || {})[flowName];
|
|
911
|
+
const stepMeta = flowRegistry?.steps?.[stepName];
|
|
912
|
+
let stepQueue = stepMeta?.queue;
|
|
913
|
+
if (!stepQueue && flowRegistry?.entry?.step === stepName) {
|
|
914
|
+
stepQueue = flowRegistry.entry.queue;
|
|
915
|
+
}
|
|
916
|
+
if (!stepQueue && registry?.workers) {
|
|
917
|
+
const worker = registry.workers.find((w) => {
|
|
918
|
+
const flowNames = w?.flow?.names || (w?.flow?.name ? [w?.flow?.name] : []);
|
|
919
|
+
const stepMatch = w?.flow?.step === stepName || Array.isArray(w?.flow?.step) && w?.flow?.step.includes(stepName);
|
|
920
|
+
return flowNames.includes(flowName) && stepMatch;
|
|
921
|
+
});
|
|
922
|
+
stepQueue = worker?.queue?.name;
|
|
923
|
+
}
|
|
924
|
+
if (!stepQueue) {
|
|
925
|
+
logger.error("Cannot find queue for step", {
|
|
926
|
+
stepName,
|
|
927
|
+
flowName,
|
|
928
|
+
availableSteps: Object.keys(flowRegistry?.steps || {}),
|
|
929
|
+
entryStep: flowRegistry?.entry?.step
|
|
930
|
+
});
|
|
931
|
+
throw new Error(`Cannot handle await timeout: queue not found for step ${stepName} in flow ${flowName}`);
|
|
932
|
+
}
|
|
933
|
+
await queue.enqueue(stepQueue, {
|
|
934
|
+
name: SYSTEM_HANDLERS.AWAIT_TIMEOUT,
|
|
935
|
+
data: payload,
|
|
936
|
+
opts: { jobId }
|
|
937
|
+
});
|
|
687
938
|
if (action === "fail") {
|
|
688
939
|
if (store.index.updateWithRetry) {
|
|
689
940
|
await store.index.updateWithRetry(indexKey, runId, {
|
|
@@ -938,56 +1189,9 @@ export function createFlowWiring() {
|
|
|
938
1189
|
const config = useRuntimeConfig();
|
|
939
1190
|
const flowConfig = config.nvent.flow || {};
|
|
940
1191
|
stallDetector = createStallDetector(store, flowConfig.stallDetection);
|
|
941
|
-
if (flowConfig.stallDetection?.enabled) {
|
|
1192
|
+
if (flowConfig.stallDetection?.enabled !== false) {
|
|
942
1193
|
await stallDetector.start();
|
|
943
|
-
|
|
944
|
-
if (scheduleConfig.enabled) {
|
|
945
|
-
try {
|
|
946
|
-
const scheduler = useScheduler();
|
|
947
|
-
logger.info("Scheduling periodic stall detector from flowWiring", {
|
|
948
|
-
checkInterval: `${scheduleConfig.interval / 1e3}s`
|
|
949
|
-
});
|
|
950
|
-
const jobId = await scheduler.schedule({
|
|
951
|
-
id: "stall-detection",
|
|
952
|
-
name: "Flow Stall Detection",
|
|
953
|
-
type: "interval",
|
|
954
|
-
interval: scheduleConfig.interval,
|
|
955
|
-
handler: async () => {
|
|
956
|
-
if (!stallDetector || !wired) {
|
|
957
|
-
logger.debug("Stall detector handler called but wiring stopped");
|
|
958
|
-
return;
|
|
959
|
-
}
|
|
960
|
-
try {
|
|
961
|
-
logger.info("Stall detector running periodic check");
|
|
962
|
-
const analyzedFlows = $useAnalyzedFlows();
|
|
963
|
-
const flowNames = analyzedFlows.map((f) => f.id).filter(Boolean);
|
|
964
|
-
if (flowNames.length > 0) {
|
|
965
|
-
await stallDetector.checkFlowsForStalls(flowNames);
|
|
966
|
-
}
|
|
967
|
-
} catch (error) {
|
|
968
|
-
logger.error("Stall detector periodic check failed", {
|
|
969
|
-
error: error.message,
|
|
970
|
-
stack: error.stack
|
|
971
|
-
});
|
|
972
|
-
}
|
|
973
|
-
},
|
|
974
|
-
metadata: {
|
|
975
|
-
component: "stall-detector",
|
|
976
|
-
stallTimeout: scheduleConfig.stallTimeout,
|
|
977
|
-
checkInterval: scheduleConfig.interval
|
|
978
|
-
}
|
|
979
|
-
});
|
|
980
|
-
stallDetector.setSchedulerJobId(jobId);
|
|
981
|
-
logger.info("Stall detector started and scheduled", { jobId });
|
|
982
|
-
} catch (error) {
|
|
983
|
-
logger.error("Failed to schedule stall detector - periodic checks disabled", {
|
|
984
|
-
error: error.message,
|
|
985
|
-
stack: error.stack
|
|
986
|
-
});
|
|
987
|
-
}
|
|
988
|
-
} else {
|
|
989
|
-
logger.info("Stall detector started (periodic check disabled)");
|
|
990
|
-
}
|
|
1194
|
+
logger.info("Stall detector initialized - using per-flow scheduler jobs");
|
|
991
1195
|
}
|
|
992
1196
|
}
|
|
993
1197
|
async function stop() {
|
|
@@ -380,8 +380,10 @@ export async function startFlowFromTrigger(flowName, triggerName, triggerData) {
|
|
|
380
380
|
(w) => w?.flow?.step === stepName && w?.queue?.name === queueName
|
|
381
381
|
);
|
|
382
382
|
const defaultOpts = entryWorker?.queue?.defaultJobOptions || {};
|
|
383
|
+
const analyzedEntry = flowDef.analyzed?.steps?.[stepName];
|
|
384
|
+
const stepTimeout = analyzedEntry?.stepTimeout;
|
|
383
385
|
const jobId = `${runId}__${stepName}`;
|
|
384
|
-
const opts = { ...defaultOpts, jobId };
|
|
386
|
+
const opts = { ...defaultOpts, jobId, timeout: stepTimeout };
|
|
385
387
|
try {
|
|
386
388
|
await queue.enqueue(queueName, {
|
|
387
389
|
name: stepName,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { defineNitroPlugin, $useWorkerHandlers, $useFunctionRegistry, useQueueAdapter, useHookRegistry } from "#imports";
|
|
2
2
|
import { createJobProcessor } from "../../worker/node/runner.js";
|
|
3
|
+
import { registerSystemHandlersOnQueue } from "../../worker/system/index.js";
|
|
3
4
|
export default defineNitroPlugin(async (nitroApp) => {
|
|
4
5
|
nitroApp.hooks.hook("close", async () => {
|
|
5
6
|
const queueAdapter = useQueueAdapter();
|
|
@@ -11,6 +12,7 @@ export default defineNitroPlugin(async (nitroApp) => {
|
|
|
11
12
|
const handlers = $useWorkerHandlers();
|
|
12
13
|
const registry = $useFunctionRegistry() || { workers: [] };
|
|
13
14
|
const registeredQueues = /* @__PURE__ */ new Set();
|
|
15
|
+
const queuesWithHooks = /* @__PURE__ */ new Set();
|
|
14
16
|
for (const entry of handlers) {
|
|
15
17
|
const { queue, id, handler, module } = entry;
|
|
16
18
|
const w = registry.workers.find((rw) => rw?.id === id || rw?.queue?.name === queue && rw?.absPath === entry.absPath);
|
|
@@ -20,7 +22,7 @@ export default defineNitroPlugin(async (nitroApp) => {
|
|
|
20
22
|
} else {
|
|
21
23
|
jobName = id.includes("/") ? id.split("/").pop() : id;
|
|
22
24
|
}
|
|
23
|
-
if (module
|
|
25
|
+
if (module) {
|
|
24
26
|
const hooks = {};
|
|
25
27
|
if (typeof module.onAwaitRegister === "function") {
|
|
26
28
|
hooks.onAwaitRegister = module.onAwaitRegister;
|
|
@@ -28,7 +30,10 @@ export default defineNitroPlugin(async (nitroApp) => {
|
|
|
28
30
|
if (typeof module.onAwaitResolve === "function") {
|
|
29
31
|
hooks.onAwaitResolve = module.onAwaitResolve;
|
|
30
32
|
}
|
|
31
|
-
if (
|
|
33
|
+
if (typeof module.onAwaitTimeout === "function") {
|
|
34
|
+
hooks.onAwaitTimeout = module.onAwaitTimeout;
|
|
35
|
+
}
|
|
36
|
+
if (Object.keys(hooks).length > 0 && w?.flow) {
|
|
32
37
|
const hookRegistry = useHookRegistry();
|
|
33
38
|
const flowNames = w.flow.names ? Array.isArray(w.flow.names) ? w.flow.names : [w.flow.names] : w.flow.name ? Array.isArray(w.flow.name) ? w.flow.name : [w.flow.name] : [];
|
|
34
39
|
for (const flowName of flowNames) {
|
|
@@ -36,6 +41,20 @@ export default defineNitroPlugin(async (nitroApp) => {
|
|
|
36
41
|
hookRegistry.register(flowName, jobName, hooks);
|
|
37
42
|
}
|
|
38
43
|
}
|
|
44
|
+
queuesWithHooks.add(queue);
|
|
45
|
+
}
|
|
46
|
+
if (w?.flow?.awaitBefore || w?.flow?.awaitAfter) {
|
|
47
|
+
queuesWithHooks.add(queue);
|
|
48
|
+
}
|
|
49
|
+
if (w?.flow?.role === "entry") {
|
|
50
|
+
const flowNames = w.flow.names ? Array.isArray(w.flow.names) ? w.flow.names : [w.flow.names] : w.flow.name ? Array.isArray(w.flow.name) ? w.flow.name : [w.flow.name] : [];
|
|
51
|
+
for (const flowName of flowNames) {
|
|
52
|
+
const flowRegistry = (registry?.flows || {})[flowName];
|
|
53
|
+
if (flowRegistry?.entry?.awaitBefore || flowRegistry?.entry?.awaitAfter) {
|
|
54
|
+
queuesWithHooks.add(queue);
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
39
58
|
}
|
|
40
59
|
}
|
|
41
60
|
if (typeof handler === "function") {
|
|
@@ -52,6 +71,16 @@ export default defineNitroPlugin(async (nitroApp) => {
|
|
|
52
71
|
registeredQueues.add(queue);
|
|
53
72
|
}
|
|
54
73
|
}
|
|
74
|
+
for (const queueName of queuesWithHooks) {
|
|
75
|
+
try {
|
|
76
|
+
const queueWorker = registry.workers.find((w) => w?.queue?.name === queueName);
|
|
77
|
+
const concurrency = queueWorker?.worker?.concurrency;
|
|
78
|
+
registerSystemHandlersOnQueue(queueName, concurrency);
|
|
79
|
+
registeredQueues.add(queueName);
|
|
80
|
+
} catch (err) {
|
|
81
|
+
console.error(`[nvent] Failed to register system handlers on queue ${queueName}:`, err);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
55
84
|
if (queueAdapter.startProcessingQueue) {
|
|
56
85
|
for (const queueName of Array.from(registeredQueues)) {
|
|
57
86
|
queueAdapter.startProcessingQueue(queueName);
|
|
@@ -30,8 +30,8 @@ export default defineEventHandler(async (event) => {
|
|
|
30
30
|
});
|
|
31
31
|
}
|
|
32
32
|
const status = flowEntry.metadata?.status;
|
|
33
|
-
if (status && status !== "running") {
|
|
34
|
-
logger.warn(`Flow is not running`, { flowName, runId, stepName, status });
|
|
33
|
+
if (status && status !== "running" && status !== "awaiting") {
|
|
34
|
+
logger.warn(`Flow is not running or awaiting`, { flowName, runId, stepName, status });
|
|
35
35
|
setResponseStatus(event, 410);
|
|
36
36
|
throw createError({
|
|
37
37
|
statusCode: 410,
|
|
@@ -39,9 +39,32 @@ export default defineEventHandler(async (event) => {
|
|
|
39
39
|
message: `This webhook is no longer valid because the flow is ${status}.`
|
|
40
40
|
});
|
|
41
41
|
}
|
|
42
|
-
const
|
|
43
|
-
|
|
44
|
-
|
|
42
|
+
const awaitingSteps = flowEntry.metadata?.awaitingSteps || {};
|
|
43
|
+
const awaitKeyBefore = `${stepName}:before`;
|
|
44
|
+
const awaitKeyAfter = `${stepName}:after`;
|
|
45
|
+
let awaitState = null;
|
|
46
|
+
let position = "before";
|
|
47
|
+
const awaitStateBefore = awaitingSteps[awaitKeyBefore];
|
|
48
|
+
if (awaitStateBefore && awaitStateBefore.status === "awaiting") {
|
|
49
|
+
awaitState = awaitStateBefore;
|
|
50
|
+
position = "before";
|
|
51
|
+
}
|
|
52
|
+
if (!awaitState) {
|
|
53
|
+
const awaitStateAfter = awaitingSteps[awaitKeyAfter];
|
|
54
|
+
if (awaitStateAfter && awaitStateAfter.status === "awaiting") {
|
|
55
|
+
awaitState = awaitStateAfter;
|
|
56
|
+
position = "after";
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
if (!awaitState) {
|
|
60
|
+
const legacyAwaitState = awaitingSteps[stepName];
|
|
61
|
+
if (legacyAwaitState && legacyAwaitState.status === "awaiting") {
|
|
62
|
+
awaitState = legacyAwaitState;
|
|
63
|
+
position = "after";
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
if (!awaitState) {
|
|
67
|
+
logger.warn(`Step is not awaiting`, { flowName, runId, stepName, awaitingSteps });
|
|
45
68
|
setResponseStatus(event, 410);
|
|
46
69
|
throw createError({
|
|
47
70
|
statusCode: 410,
|
|
@@ -66,7 +89,6 @@ export default defineEventHandler(async (event) => {
|
|
|
66
89
|
message: `This webhook expects ${expectedMethod} requests.`
|
|
67
90
|
});
|
|
68
91
|
}
|
|
69
|
-
const position = awaitState.position || "after";
|
|
70
92
|
let webhookData;
|
|
71
93
|
if (event.method === "GET") {
|
|
72
94
|
webhookData = getRouterParams(event, { decode: true });
|