nvent 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/module.d.mts +4 -185
- package/dist/module.json +3 -3
- package/dist/module.mjs +451 -257
- package/dist/runtime/adapters/base/index.d.ts +6 -0
- package/dist/runtime/adapters/base/index.js +1 -0
- package/dist/runtime/adapters/base/store-validator.d.ts +48 -0
- package/dist/runtime/adapters/base/store-validator.js +147 -0
- package/dist/runtime/adapters/builtin/file-queue.d.ts +67 -0
- package/dist/runtime/adapters/builtin/file-queue.js +499 -0
- package/dist/runtime/adapters/builtin/file-store.d.ts +32 -0
- package/dist/runtime/adapters/builtin/file-store.js +206 -0
- package/dist/runtime/adapters/builtin/file-stream.d.ts +39 -0
- package/dist/runtime/adapters/builtin/file-stream.js +56 -0
- package/dist/runtime/adapters/builtin/index.d.ts +10 -0
- package/dist/runtime/adapters/builtin/index.js +5 -0
- package/dist/runtime/adapters/builtin/memory-queue.d.ts +52 -0
- package/dist/runtime/adapters/builtin/memory-queue.js +243 -0
- package/dist/runtime/adapters/builtin/memory-store.d.ts +68 -0
- package/dist/runtime/adapters/builtin/memory-store.js +333 -0
- package/dist/runtime/adapters/builtin/memory-stream.d.ts +21 -0
- package/dist/runtime/adapters/builtin/memory-stream.js +56 -0
- package/dist/runtime/adapters/factory.d.ts +31 -0
- package/dist/runtime/adapters/factory.js +134 -0
- package/dist/runtime/adapters/index.d.ts +8 -0
- package/dist/runtime/adapters/index.js +3 -0
- package/dist/runtime/adapters/interfaces/index.d.ts +11 -0
- package/dist/runtime/adapters/interfaces/index.js +3 -0
- package/dist/runtime/adapters/interfaces/queue.d.ts +150 -0
- package/dist/runtime/adapters/interfaces/store.d.ts +297 -0
- package/dist/runtime/adapters/interfaces/stream.d.ts +62 -0
- package/dist/runtime/adapters/registry.d.ts +85 -0
- package/dist/runtime/adapters/registry.js +161 -0
- package/dist/runtime/config/index.d.ts +29 -0
- package/dist/runtime/config/index.js +175 -0
- package/dist/runtime/config/types.d.ts +397 -0
- package/dist/runtime/config/types.js +0 -0
- package/dist/runtime/{server-utils/events → events}/eventBus.d.ts +1 -1
- package/dist/runtime/events/types.d.ts +145 -0
- package/dist/runtime/events/types.js +0 -0
- package/dist/runtime/events/utils/scheduleTrigger.d.ts +8 -0
- package/dist/runtime/events/utils/scheduleTrigger.js +69 -0
- package/dist/runtime/events/utils/stallDetector.d.ts +140 -0
- package/dist/runtime/events/utils/stallDetector.js +436 -0
- package/dist/runtime/events/utils/triggerRuntime.d.ts +58 -0
- package/dist/runtime/events/utils/triggerRuntime.js +212 -0
- package/dist/runtime/{server-utils/events → events}/wiring/flowWiring.d.ts +12 -11
- package/dist/runtime/events/wiring/flowWiring.js +1020 -0
- package/dist/runtime/events/wiring/registry.d.ts +19 -0
- package/dist/runtime/events/wiring/registry.js +35 -0
- package/dist/runtime/events/wiring/stateWiring.d.ts +37 -0
- package/dist/runtime/events/wiring/stateWiring.js +92 -0
- package/dist/runtime/events/wiring/streamWiring.d.ts +36 -0
- package/dist/runtime/events/wiring/streamWiring.js +156 -0
- package/dist/runtime/events/wiring/triggerWiring.d.ts +21 -0
- package/dist/runtime/events/wiring/triggerWiring.js +412 -0
- package/dist/runtime/nitro/plugins/00.adapters.d.ts +14 -0
- package/dist/runtime/nitro/plugins/00.adapters.js +73 -0
- package/dist/runtime/nitro/plugins/02.workers.js +63 -0
- package/dist/runtime/nitro/plugins/03.triggers.d.ts +12 -0
- package/dist/runtime/nitro/plugins/03.triggers.js +55 -0
- package/dist/runtime/nitro/routes/webhook.await.d.ts +23 -0
- package/dist/runtime/nitro/routes/webhook.await.js +90 -0
- package/dist/runtime/nitro/routes/webhook.trigger.d.ts +69 -0
- package/dist/runtime/nitro/routes/webhook.trigger.js +64 -0
- package/dist/runtime/nitro/utils/adapters.d.ts +66 -0
- package/dist/runtime/nitro/utils/adapters.js +51 -0
- package/dist/runtime/nitro/utils/awaitPatterns/event.d.ts +15 -0
- package/dist/runtime/nitro/utils/awaitPatterns/event.js +120 -0
- package/dist/runtime/nitro/utils/awaitPatterns/index.d.ts +28 -0
- package/dist/runtime/nitro/utils/awaitPatterns/index.js +55 -0
- package/dist/runtime/nitro/utils/awaitPatterns/schedule.d.ts +16 -0
- package/dist/runtime/nitro/utils/awaitPatterns/schedule.js +78 -0
- package/dist/runtime/nitro/utils/awaitPatterns/time.d.ts +15 -0
- package/dist/runtime/nitro/utils/awaitPatterns/time.js +67 -0
- package/dist/runtime/nitro/utils/awaitPatterns/webhook.d.ts +15 -0
- package/dist/runtime/nitro/utils/awaitPatterns/webhook.js +120 -0
- package/dist/runtime/nitro/utils/defineFunction.d.ts +10 -0
- package/dist/runtime/nitro/utils/defineFunction.js +17 -0
- package/dist/runtime/nitro/utils/defineFunctionConfig.d.ts +310 -0
- package/dist/runtime/nitro/utils/defineFunctionConfig.js +3 -0
- package/dist/runtime/nitro/utils/defineHooks.d.ts +41 -0
- package/dist/runtime/nitro/utils/defineHooks.js +6 -0
- package/dist/runtime/nitro/utils/registerAdapter.d.ts +59 -0
- package/dist/runtime/nitro/utils/registerAdapter.js +13 -0
- package/dist/runtime/nitro/utils/useAwait.d.ts +71 -0
- package/dist/runtime/nitro/utils/useAwait.js +139 -0
- package/dist/runtime/{server-utils → nitro}/utils/useEventManager.d.ts +2 -2
- package/dist/runtime/{server-utils → nitro}/utils/useEventManager.js +1 -1
- package/dist/runtime/nitro/utils/useFlow.d.ts +68 -0
- package/dist/runtime/nitro/utils/useFlow.js +226 -0
- package/dist/runtime/nitro/utils/useHookRegistry.d.ts +34 -0
- package/dist/runtime/nitro/utils/useHookRegistry.js +25 -0
- package/dist/runtime/{server-utils → nitro}/utils/useNventLogger.js +2 -2
- package/dist/runtime/nitro/utils/useRunContext.d.ts +6 -0
- package/dist/runtime/nitro/utils/useRunContext.js +102 -0
- package/dist/runtime/nitro/utils/useStreamTopics.d.ts +83 -0
- package/dist/runtime/nitro/utils/useStreamTopics.js +94 -0
- package/dist/runtime/nitro/utils/useTrigger.d.ts +150 -0
- package/dist/runtime/nitro/utils/useTrigger.js +320 -0
- package/dist/runtime/scheduler/index.d.ts +33 -0
- package/dist/runtime/scheduler/index.js +38 -0
- package/dist/runtime/scheduler/scheduler.d.ts +113 -0
- package/dist/runtime/scheduler/scheduler.js +623 -0
- package/dist/runtime/scheduler/types.d.ts +116 -0
- package/dist/runtime/scheduler/types.js +0 -0
- package/dist/runtime/tsconfig.json +8 -0
- package/dist/runtime/worker/node/runner.d.ts +53 -0
- package/dist/runtime/worker/node/runner.js +327 -0
- package/dist/types.d.mts +2 -2
- package/package.json +16 -46
- package/LICENSE +0 -21
- package/README.md +0 -389
- package/dist/runtime/app/assets/vueflow.css +0 -1
- package/dist/runtime/app/components/ConfirmDialog.d.vue.ts +0 -33
- package/dist/runtime/app/components/ConfirmDialog.vue +0 -121
- package/dist/runtime/app/components/ConfirmDialog.vue.d.ts +0 -33
- package/dist/runtime/app/components/FlowDiagram.d.vue.ts +0 -64
- package/dist/runtime/app/components/FlowDiagram.vue +0 -338
- package/dist/runtime/app/components/FlowDiagram.vue.d.ts +0 -64
- package/dist/runtime/app/components/FlowNodeCard.d.vue.ts +0 -29
- package/dist/runtime/app/components/FlowNodeCard.vue +0 -156
- package/dist/runtime/app/components/FlowNodeCard.vue.d.ts +0 -29
- package/dist/runtime/app/components/FlowRunOverview.d.vue.ts +0 -9
- package/dist/runtime/app/components/FlowRunOverview.vue +0 -291
- package/dist/runtime/app/components/FlowRunOverview.vue.d.ts +0 -9
- package/dist/runtime/app/components/FlowRunStatusBadge.d.vue.ts +0 -14
- package/dist/runtime/app/components/FlowRunStatusBadge.vue +0 -60
- package/dist/runtime/app/components/FlowRunStatusBadge.vue.d.ts +0 -14
- package/dist/runtime/app/components/FlowRunTimeline.d.vue.ts +0 -12
- package/dist/runtime/app/components/FlowRunTimeline.vue +0 -127
- package/dist/runtime/app/components/FlowRunTimeline.vue.d.ts +0 -12
- package/dist/runtime/app/components/FlowScheduleDialog.d.vue.ts +0 -16
- package/dist/runtime/app/components/FlowScheduleDialog.vue +0 -226
- package/dist/runtime/app/components/FlowScheduleDialog.vue.d.ts +0 -16
- package/dist/runtime/app/components/FlowSchedulesList.d.vue.ts +0 -12
- package/dist/runtime/app/components/FlowSchedulesList.vue +0 -99
- package/dist/runtime/app/components/FlowSchedulesList.vue.d.ts +0 -12
- package/dist/runtime/app/components/JobScheduling.d.vue.ts +0 -6
- package/dist/runtime/app/components/JobScheduling.vue +0 -203
- package/dist/runtime/app/components/JobScheduling.vue.d.ts +0 -6
- package/dist/runtime/app/components/ListItem.d.vue.ts +0 -23
- package/dist/runtime/app/components/ListItem.vue +0 -70
- package/dist/runtime/app/components/ListItem.vue.d.ts +0 -23
- package/dist/runtime/app/components/QueueConfigDetails.d.vue.ts +0 -45
- package/dist/runtime/app/components/QueueConfigDetails.vue +0 -412
- package/dist/runtime/app/components/QueueConfigDetails.vue.d.ts +0 -45
- package/dist/runtime/app/components/StatCounter.d.vue.ts +0 -9
- package/dist/runtime/app/components/StatCounter.vue +0 -25
- package/dist/runtime/app/components/StatCounter.vue.d.ts +0 -9
- package/dist/runtime/app/components/TimelineList.d.vue.ts +0 -7
- package/dist/runtime/app/components/TimelineList.vue +0 -210
- package/dist/runtime/app/components/TimelineList.vue.d.ts +0 -7
- package/dist/runtime/app/components/nhealth/component-router.d.vue.ts +0 -46
- package/dist/runtime/app/components/nhealth/component-router.vue +0 -26
- package/dist/runtime/app/components/nhealth/component-router.vue.d.ts +0 -46
- package/dist/runtime/app/components/nhealth/component-shell.d.vue.ts +0 -24
- package/dist/runtime/app/components/nhealth/component-shell.vue +0 -89
- package/dist/runtime/app/components/nhealth/component-shell.vue.d.ts +0 -24
- package/dist/runtime/app/composables/useAnalyzedFlows.d.ts +0 -14
- package/dist/runtime/app/composables/useAnalyzedFlows.js +0 -8
- package/dist/runtime/app/composables/useComponentRouter.d.ts +0 -38
- package/dist/runtime/app/composables/useComponentRouter.js +0 -240
- package/dist/runtime/app/composables/useFlowRunTimeline.d.ts +0 -80
- package/dist/runtime/app/composables/useFlowRunTimeline.js +0 -68
- package/dist/runtime/app/composables/useFlowRuns.d.ts +0 -18
- package/dist/runtime/app/composables/useFlowRuns.js +0 -32
- package/dist/runtime/app/composables/useFlowRunsInfinite.d.ts +0 -24
- package/dist/runtime/app/composables/useFlowRunsInfinite.js +0 -123
- package/dist/runtime/app/composables/useFlowRunsPolling.d.ts +0 -9
- package/dist/runtime/app/composables/useFlowRunsPolling.js +0 -33
- package/dist/runtime/app/composables/useFlowState.d.ts +0 -125
- package/dist/runtime/app/composables/useFlowState.js +0 -211
- package/dist/runtime/app/composables/useFlowWebSocket.d.ts +0 -27
- package/dist/runtime/app/composables/useFlowWebSocket.js +0 -205
- package/dist/runtime/app/composables/useFlowsNavigation.d.ts +0 -10
- package/dist/runtime/app/composables/useFlowsNavigation.js +0 -58
- package/dist/runtime/app/composables/useQueueJobs.d.ts +0 -26
- package/dist/runtime/app/composables/useQueueJobs.js +0 -20
- package/dist/runtime/app/composables/useQueueUpdates.d.ts +0 -26
- package/dist/runtime/app/composables/useQueueUpdates.js +0 -122
- package/dist/runtime/app/composables/useQueues.d.ts +0 -45
- package/dist/runtime/app/composables/useQueues.js +0 -26
- package/dist/runtime/app/composables/useQueuesLive.d.ts +0 -19
- package/dist/runtime/app/composables/useQueuesLive.js +0 -143
- package/dist/runtime/app/pages/flows/index.d.vue.ts +0 -3
- package/dist/runtime/app/pages/flows/index.vue +0 -645
- package/dist/runtime/app/pages/flows/index.vue.d.ts +0 -3
- package/dist/runtime/app/pages/index.d.vue.ts +0 -3
- package/dist/runtime/app/pages/index.vue +0 -34
- package/dist/runtime/app/pages/index.vue.d.ts +0 -3
- package/dist/runtime/app/pages/queues/index.d.vue.ts +0 -3
- package/dist/runtime/app/pages/queues/index.vue +0 -229
- package/dist/runtime/app/pages/queues/index.vue.d.ts +0 -3
- package/dist/runtime/app/pages/queues/job.d.vue.ts +0 -3
- package/dist/runtime/app/pages/queues/job.vue +0 -262
- package/dist/runtime/app/pages/queues/job.vue.d.ts +0 -3
- package/dist/runtime/app/pages/queues/jobs.d.vue.ts +0 -3
- package/dist/runtime/app/pages/queues/jobs.vue +0 -291
- package/dist/runtime/app/pages/queues/jobs.vue.d.ts +0 -3
- package/dist/runtime/app/plugins/vueflow.client.d.ts +0 -2
- package/dist/runtime/app/plugins/vueflow.client.js +0 -11
- package/dist/runtime/constants.d.ts +0 -11
- package/dist/runtime/constants.js +0 -11
- package/dist/runtime/schema.d.ts +0 -37
- package/dist/runtime/schema.js +0 -20
- package/dist/runtime/server/api/_flows/[name]/clear-history.delete.d.ts +0 -10
- package/dist/runtime/server/api/_flows/[name]/clear-history.delete.js +0 -44
- package/dist/runtime/server/api/_flows/[name]/runs.get.d.ts +0 -7
- package/dist/runtime/server/api/_flows/[name]/runs.get.js +0 -53
- package/dist/runtime/server/api/_flows/[name]/schedule.post.js +0 -57
- package/dist/runtime/server/api/_flows/[name]/schedules/[id].delete.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/schedules/[id].delete.js +0 -42
- package/dist/runtime/server/api/_flows/[name]/schedules.get.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/schedules.get.js +0 -48
- package/dist/runtime/server/api/_flows/[name]/start.post.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/start.post.js +0 -9
- package/dist/runtime/server/api/_flows/index.get.d.ts +0 -6
- package/dist/runtime/server/api/_flows/index.get.js +0 -5
- package/dist/runtime/server/api/_flows/ws.d.ts +0 -60
- package/dist/runtime/server/api/_flows/ws.js +0 -188
- package/dist/runtime/server/api/_queues/[name]/job/[id].get.d.ts +0 -2
- package/dist/runtime/server/api/_queues/[name]/job/[id].get.js +0 -9
- package/dist/runtime/server/api/_queues/[name]/job/index.get.d.ts +0 -2
- package/dist/runtime/server/api/_queues/[name]/job/index.get.js +0 -18
- package/dist/runtime/server/api/_queues/index.get.d.ts +0 -2
- package/dist/runtime/server/api/_queues/index.get.js +0 -63
- package/dist/runtime/server/api/_queues/ws.d.ts +0 -48
- package/dist/runtime/server/api/_queues/ws.js +0 -205
- package/dist/runtime/server/plugins/00.event-store.d.ts +0 -13
- package/dist/runtime/server/plugins/00.event-store.js +0 -16
- package/dist/runtime/server/plugins/flow-management.d.ts +0 -13
- package/dist/runtime/server/plugins/flow-management.js +0 -65
- package/dist/runtime/server/plugins/queue-management.d.ts +0 -2
- package/dist/runtime/server/plugins/queue-management.js +0 -27
- package/dist/runtime/server/plugins/state-cleanup.d.ts +0 -11
- package/dist/runtime/server/plugins/state-cleanup.js +0 -93
- package/dist/runtime/server/plugins/worker-management.d.ts +0 -2
- package/dist/runtime/server/plugins/worker-management.js +0 -33
- package/dist/runtime/server/tsconfig.json +0 -3
- package/dist/runtime/server-utils/events/adapters/fileAdapter.d.ts +0 -2
- package/dist/runtime/server-utils/events/adapters/fileAdapter.js +0 -382
- package/dist/runtime/server-utils/events/adapters/memoryAdapter.d.ts +0 -2
- package/dist/runtime/server-utils/events/adapters/memoryAdapter.js +0 -171
- package/dist/runtime/server-utils/events/adapters/redis/redisAdapter.d.ts +0 -2
- package/dist/runtime/server-utils/events/adapters/redis/redisAdapter.js +0 -348
- package/dist/runtime/server-utils/events/adapters/redis/redisPubSubGateway.d.ts +0 -30
- package/dist/runtime/server-utils/events/adapters/redis/redisPubSubGateway.js +0 -82
- package/dist/runtime/server-utils/events/eventStoreFactory.d.ts +0 -19
- package/dist/runtime/server-utils/events/eventStoreFactory.js +0 -44
- package/dist/runtime/server-utils/events/streamNames.d.ts +0 -17
- package/dist/runtime/server-utils/events/streamNames.js +0 -17
- package/dist/runtime/server-utils/events/types.d.ts +0 -63
- package/dist/runtime/server-utils/events/wiring/flowWiring.js +0 -409
- package/dist/runtime/server-utils/events/wiring/registry.d.ts +0 -10
- package/dist/runtime/server-utils/events/wiring/registry.js +0 -24
- package/dist/runtime/server-utils/queue/adapters/bullmq.d.ts +0 -18
- package/dist/runtime/server-utils/queue/adapters/bullmq.js +0 -164
- package/dist/runtime/server-utils/queue/queueFactory.d.ts +0 -3
- package/dist/runtime/server-utils/queue/queueFactory.js +0 -10
- package/dist/runtime/server-utils/queue/types.d.ts +0 -47
- package/dist/runtime/server-utils/state/adapters/redis.d.ts +0 -2
- package/dist/runtime/server-utils/state/adapters/redis.js +0 -42
- package/dist/runtime/server-utils/state/stateFactory.d.ts +0 -3
- package/dist/runtime/server-utils/state/stateFactory.js +0 -17
- package/dist/runtime/server-utils/state/types.d.ts +0 -23
- package/dist/runtime/server-utils/utils/defineQueueConfig.d.ts +0 -154
- package/dist/runtime/server-utils/utils/defineQueueConfig.js +0 -2
- package/dist/runtime/server-utils/utils/defineQueueWorker.d.ts +0 -10
- package/dist/runtime/server-utils/utils/defineQueueWorker.js +0 -17
- package/dist/runtime/server-utils/utils/useEventStore.d.ts +0 -20
- package/dist/runtime/server-utils/utils/useEventStore.js +0 -119
- package/dist/runtime/server-utils/utils/useFlowEngine.d.ts +0 -9
- package/dist/runtime/server-utils/utils/useFlowEngine.js +0 -44
- package/dist/runtime/server-utils/utils/useLogs.d.ts +0 -41
- package/dist/runtime/server-utils/utils/useLogs.js +0 -74
- package/dist/runtime/server-utils/utils/useQueue.d.ts +0 -31
- package/dist/runtime/server-utils/utils/useQueue.js +0 -24
- package/dist/runtime/server-utils/worker/adapter.d.ts +0 -4
- package/dist/runtime/server-utils/worker/adapter.js +0 -66
- package/dist/runtime/server-utils/worker/runner/node.d.ts +0 -27
- package/dist/runtime/server-utils/worker/runner/node.js +0 -196
- package/dist/runtime/types.d.ts +0 -132
- /package/dist/runtime/{server-utils/events/types.js → adapters/interfaces/queue.js} +0 -0
- /package/dist/runtime/{server-utils/queue/types.js → adapters/interfaces/store.js} +0 -0
- /package/dist/runtime/{server-utils/state/types.js → adapters/interfaces/stream.js} +0 -0
- /package/dist/runtime/{server-utils/events → events}/eventBus.js +0 -0
- /package/dist/runtime/{server/plugins/00.ws-lifecycle.d.ts → nitro/plugins/01.ws-lifecycle.d.ts} +0 -0
- /package/dist/runtime/{server/plugins/00.ws-lifecycle.js → nitro/plugins/01.ws-lifecycle.js} +0 -0
- /package/dist/runtime/{server/api/_flows/[name]/schedule.post.d.ts → nitro/plugins/02.workers.d.ts} +0 -0
- /package/dist/runtime/{server-utils → nitro}/utils/useNventLogger.d.ts +0 -0
- /package/dist/runtime/{server-utils → nitro}/utils/wsPeerManager.d.ts +0 -0
- /package/dist/runtime/{server-utils → nitro}/utils/wsPeerManager.js +0 -0
- /package/dist/runtime/{python → worker/python}/get_config.py +0 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Flow Stall Detection System
|
|
3
|
+
*
|
|
4
|
+
* Detects and marks flows that have been in "running" state for too long without activity.
|
|
5
|
+
* Uses a hybrid approach:
|
|
6
|
+
* 1. Lazy detection: Check stall status when flows are queried (zero overhead)
|
|
7
|
+
* 2. Periodic cleanup: Background job that checks all running flows periodically (safety net)
|
|
8
|
+
*
|
|
9
|
+
* A flow is considered "stalled" when:
|
|
10
|
+
* - Status is "running"
|
|
11
|
+
* - No activity (step events) for longer than STALL_TIMEOUT
|
|
12
|
+
* - lastActivityAt timestamp is older than threshold
|
|
13
|
+
*/
|
|
14
|
+
import type { StoreAdapter } from '../../adapters/interfaces/store.js';
|
|
15
|
+
export interface StallDetectorConfig {
|
|
16
|
+
/**
|
|
17
|
+
* Time in milliseconds after which a running flow without activity is considered stalled
|
|
18
|
+
* @default 1800000 (30 minutes)
|
|
19
|
+
*/
|
|
20
|
+
stallTimeout?: number;
|
|
21
|
+
/**
|
|
22
|
+
* Interval in milliseconds for periodic stall checks
|
|
23
|
+
* @default 900000 (15 minutes)
|
|
24
|
+
*/
|
|
25
|
+
checkInterval?: number;
|
|
26
|
+
/**
|
|
27
|
+
* Enable periodic background checks
|
|
28
|
+
* Set to false to use only lazy detection
|
|
29
|
+
* @default true
|
|
30
|
+
*/
|
|
31
|
+
enablePeriodicCheck?: boolean;
|
|
32
|
+
}
|
|
33
|
+
export type FlowStatus = 'running' | 'completed' | 'failed' | 'canceled' | 'stalled';
|
|
34
|
+
export interface FlowActivity {
|
|
35
|
+
runId: string;
|
|
36
|
+
flowName: string;
|
|
37
|
+
status: FlowStatus;
|
|
38
|
+
startedAt: number;
|
|
39
|
+
lastActivityAt: number;
|
|
40
|
+
metadata?: any;
|
|
41
|
+
}
|
|
42
|
+
export declare class FlowStallDetector {
|
|
43
|
+
private store;
|
|
44
|
+
private config;
|
|
45
|
+
private logger;
|
|
46
|
+
private schedulerJobId?;
|
|
47
|
+
private started;
|
|
48
|
+
constructor(store: StoreAdapter, config?: StallDetectorConfig);
|
|
49
|
+
/**
|
|
50
|
+
* Start the periodic stall detector
|
|
51
|
+
* Should be called once per instance after adapters are initialized
|
|
52
|
+
* Runs startup recovery to clean up flows from previous server instances
|
|
53
|
+
*/
|
|
54
|
+
start(): Promise<void>;
|
|
55
|
+
/**
|
|
56
|
+
* Get the configuration for scheduling
|
|
57
|
+
* Returns config needed by flowWiring to register the scheduler job
|
|
58
|
+
*/
|
|
59
|
+
getScheduleConfig(): {
|
|
60
|
+
enabled: boolean;
|
|
61
|
+
interval: number;
|
|
62
|
+
stallTimeout: number;
|
|
63
|
+
};
|
|
64
|
+
/**
|
|
65
|
+
* Set the scheduler job ID (called from flowWiring after scheduling)
|
|
66
|
+
*/
|
|
67
|
+
setSchedulerJobId(jobId: string): void;
|
|
68
|
+
/**
|
|
69
|
+
* Stop the periodic stall detector
|
|
70
|
+
*/
|
|
71
|
+
stop(): Promise<void>;
|
|
72
|
+
/**
|
|
73
|
+
* Get stall timeout for a specific flow
|
|
74
|
+
* Uses flow-specific timeout from analyzed metadata, falls back to global config
|
|
75
|
+
*/
|
|
76
|
+
private getFlowStallTimeout;
|
|
77
|
+
/**
|
|
78
|
+
* Update activity timestamp for a flow
|
|
79
|
+
* Should be called on every step event (started, completed, failed, retry)
|
|
80
|
+
*/
|
|
81
|
+
updateActivity(flowName: string, runId: string): Promise<void>;
|
|
82
|
+
/**
|
|
83
|
+
* Check if a specific flow is stalled (lazy detection)
|
|
84
|
+
* Returns true if the flow should be marked as stalled
|
|
85
|
+
* v0.5: Await-aware - uses flow-specific timeout and skips awaiting flows
|
|
86
|
+
*/
|
|
87
|
+
isStalled(flowName: string, runId: string): Promise<boolean>;
|
|
88
|
+
/**
|
|
89
|
+
* Mark a flow as stalled
|
|
90
|
+
* Emits a flow.stalled event and updates the flow status
|
|
91
|
+
*/
|
|
92
|
+
markAsStalled(flowName: string, runId: string, reason?: string): Promise<void>;
|
|
93
|
+
/**
|
|
94
|
+
* Check all running flows and mark stalled ones
|
|
95
|
+
* This is called by the periodic background job
|
|
96
|
+
*
|
|
97
|
+
* Note: This method requires knowledge of which flows exist.
|
|
98
|
+
* For now, we'll need to pass flow names to check, or iterate known flows from registry.
|
|
99
|
+
*/
|
|
100
|
+
checkFlowsForStalls(flowNames: string[]): Promise<void>;
|
|
101
|
+
/**
|
|
102
|
+
* Run startup recovery to clean up flows left in running state from previous server instance
|
|
103
|
+
* This marks all running flows as stalled since their in-memory state is lost
|
|
104
|
+
* Also validates and cleans up flow stats index
|
|
105
|
+
*/
|
|
106
|
+
private runStartupRecovery;
|
|
107
|
+
/**
|
|
108
|
+
* Validate flow stats index and remove entries for non-existent flows
|
|
109
|
+
* Also corrects running/awaiting counts based on actual scanned data
|
|
110
|
+
*
|
|
111
|
+
* NOTE: We only validate running/awaiting counts because:
|
|
112
|
+
* - They are small snapshot values (usually < 100)
|
|
113
|
+
* - We already scanned all flows during startup recovery
|
|
114
|
+
* - Discrepancies indicate actual bugs (flows stuck in wrong state)
|
|
115
|
+
*
|
|
116
|
+
* We do NOT validate total/success/failure/cancel because:
|
|
117
|
+
* - These are cumulative counters that can be millions in production
|
|
118
|
+
* - Validation would require full table scan (prohibitively expensive)
|
|
119
|
+
* - Minor discrepancies don't affect runtime behavior
|
|
120
|
+
*/
|
|
121
|
+
private validateFlowStats;
|
|
122
|
+
/**
|
|
123
|
+
* Internal method for periodic checks
|
|
124
|
+
* Gets flow names from registry and checks them
|
|
125
|
+
*/
|
|
126
|
+
private checkAllRunningFlows;
|
|
127
|
+
/**
|
|
128
|
+
* Get stall detector statistics
|
|
129
|
+
*/
|
|
130
|
+
getStats(): {
|
|
131
|
+
enabled: boolean;
|
|
132
|
+
periodicCheckEnabled: boolean;
|
|
133
|
+
stallTimeout: number;
|
|
134
|
+
checkInterval: number;
|
|
135
|
+
};
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Create and configure a flow stall detector
|
|
139
|
+
*/
|
|
140
|
+
export declare function createStallDetector(store: StoreAdapter, config?: StallDetectorConfig): FlowStallDetector;
|
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
import { useNventLogger, useStreamTopics, $useAnalyzedFlows, useScheduler } from "#imports";
|
|
2
|
+
const DEFAULT_STALL_TIMEOUT = 30 * 60 * 1e3;
|
|
3
|
+
const DEFAULT_CHECK_INTERVAL = 15 * 60 * 1e3;
|
|
4
|
+
export class FlowStallDetector {
|
|
5
|
+
store;
|
|
6
|
+
config;
|
|
7
|
+
logger = useNventLogger("stall-detector");
|
|
8
|
+
schedulerJobId;
|
|
9
|
+
started = false;
|
|
10
|
+
constructor(store, config = {}) {
|
|
11
|
+
this.store = store;
|
|
12
|
+
this.config = {
|
|
13
|
+
stallTimeout: config.stallTimeout ?? DEFAULT_STALL_TIMEOUT,
|
|
14
|
+
checkInterval: config.checkInterval ?? DEFAULT_CHECK_INTERVAL,
|
|
15
|
+
enablePeriodicCheck: config.enablePeriodicCheck ?? true
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
/**
|
|
19
|
+
* Start the periodic stall detector
|
|
20
|
+
* Should be called once per instance after adapters are initialized
|
|
21
|
+
* Runs startup recovery to clean up flows from previous server instances
|
|
22
|
+
*/
|
|
23
|
+
async start() {
|
|
24
|
+
if (this.started) {
|
|
25
|
+
this.logger.warn("Stall detector already started");
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
this.started = true;
|
|
29
|
+
await this.runStartupRecovery();
|
|
30
|
+
this.logger.info(`Stall detector started - periodicCheck: ${this.config.enablePeriodicCheck}, stallTimeout: ${this.config.stallTimeout / 1e3}s, checkInterval: ${this.config.checkInterval / 1e3}s`);
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Get the configuration for scheduling
|
|
34
|
+
* Returns config needed by flowWiring to register the scheduler job
|
|
35
|
+
*/
|
|
36
|
+
getScheduleConfig() {
|
|
37
|
+
return {
|
|
38
|
+
enabled: this.config.enablePeriodicCheck,
|
|
39
|
+
interval: this.config.checkInterval,
|
|
40
|
+
stallTimeout: this.config.stallTimeout
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Set the scheduler job ID (called from flowWiring after scheduling)
|
|
45
|
+
*/
|
|
46
|
+
setSchedulerJobId(jobId) {
|
|
47
|
+
this.schedulerJobId = jobId;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Stop the periodic stall detector
|
|
51
|
+
*/
|
|
52
|
+
async stop() {
|
|
53
|
+
if (this.schedulerJobId) {
|
|
54
|
+
try {
|
|
55
|
+
const scheduler = useScheduler();
|
|
56
|
+
await scheduler.unschedule(this.schedulerJobId);
|
|
57
|
+
this.schedulerJobId = void 0;
|
|
58
|
+
this.logger.info("Stopped periodic stall detector");
|
|
59
|
+
} catch (error) {
|
|
60
|
+
this.logger.error(`Failed to stop stall detector: ${error.message}`);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
this.started = false;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Get stall timeout for a specific flow
|
|
67
|
+
* Uses flow-specific timeout from analyzed metadata, falls back to global config
|
|
68
|
+
*/
|
|
69
|
+
async getFlowStallTimeout(flowName) {
|
|
70
|
+
try {
|
|
71
|
+
const analyzedFlows = $useAnalyzedFlows();
|
|
72
|
+
const flowMeta = analyzedFlows.find((f) => f.id === flowName);
|
|
73
|
+
if (flowMeta?.stallTimeout) {
|
|
74
|
+
this.logger.debug(`Using flow-specific stall timeout for '${flowName}': ${flowMeta.stallTimeout / 1e3}s`);
|
|
75
|
+
return flowMeta.stallTimeout;
|
|
76
|
+
}
|
|
77
|
+
} catch (error) {
|
|
78
|
+
this.logger.warn(`Failed to get flow-specific stall timeout for '${flowName}': ${error.message}`);
|
|
79
|
+
}
|
|
80
|
+
return this.config.stallTimeout;
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Update activity timestamp for a flow
|
|
84
|
+
* Should be called on every step event (started, completed, failed, retry)
|
|
85
|
+
*/
|
|
86
|
+
async updateActivity(flowName, runId) {
|
|
87
|
+
const { StoreSubjects } = useStreamTopics();
|
|
88
|
+
const indexKey = StoreSubjects.flowRunIndex(flowName);
|
|
89
|
+
try {
|
|
90
|
+
if (!this.store.index.update) {
|
|
91
|
+
this.logger.warn("Store does not support indexUpdate, cannot update activity");
|
|
92
|
+
return;
|
|
93
|
+
}
|
|
94
|
+
await this.store.index.update(indexKey, runId, {
|
|
95
|
+
lastActivityAt: Date.now()
|
|
96
|
+
});
|
|
97
|
+
} catch (error) {
|
|
98
|
+
this.logger.warn(`Failed to update flow activity for '${flowName}' runId '${runId}': ${error.message}`);
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Check if a specific flow is stalled (lazy detection)
|
|
103
|
+
* Returns true if the flow should be marked as stalled
|
|
104
|
+
* v0.5: Await-aware - uses flow-specific timeout and skips awaiting flows
|
|
105
|
+
*/
|
|
106
|
+
async isStalled(flowName, runId) {
|
|
107
|
+
const { StoreSubjects } = useStreamTopics();
|
|
108
|
+
const indexKey = StoreSubjects.flowRunIndex(flowName);
|
|
109
|
+
try {
|
|
110
|
+
if (!this.store.index.get) return false;
|
|
111
|
+
const flowEntry = await this.store.index.get(indexKey, runId);
|
|
112
|
+
if (!flowEntry?.metadata) return false;
|
|
113
|
+
if (flowEntry.metadata.status !== "running") return false;
|
|
114
|
+
const awaitingSteps = flowEntry.metadata.awaitingSteps || {};
|
|
115
|
+
const hasActiveAwaits = Object.keys(awaitingSteps).length > 0;
|
|
116
|
+
if (hasActiveAwaits) {
|
|
117
|
+
this.logger.debug(`Flow '${flowName}' runId '${runId}' has active awaits [${Object.keys(awaitingSteps).join(", ")}], skipping stall check`);
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
const stallTimeout = await this.getFlowStallTimeout(flowName);
|
|
121
|
+
const lastActivity = flowEntry.metadata.lastActivityAt || flowEntry.metadata.startedAt || 0;
|
|
122
|
+
const timeSinceActivity = Date.now() - lastActivity;
|
|
123
|
+
if (timeSinceActivity > stallTimeout) {
|
|
124
|
+
this.logger.info(`Flow detected as stalled (lazy check) - '${flowName}' runId '${runId}': ${Math.round(timeSinceActivity / 1e3)}s since activity (timeout: ${stallTimeout / 1e3}s)`);
|
|
125
|
+
return true;
|
|
126
|
+
}
|
|
127
|
+
return false;
|
|
128
|
+
} catch (error) {
|
|
129
|
+
this.logger.warn(`Failed to check if flow is stalled for '${flowName}' runId '${runId}': ${error.message}`);
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Mark a flow as stalled
|
|
135
|
+
* Emits a flow.stalled event and updates the flow status
|
|
136
|
+
*/
|
|
137
|
+
async markAsStalled(flowName, runId, reason = "No activity timeout") {
|
|
138
|
+
const { StoreSubjects } = useStreamTopics();
|
|
139
|
+
const indexKey = StoreSubjects.flowRunIndex(flowName);
|
|
140
|
+
try {
|
|
141
|
+
if (!this.store.index.get) {
|
|
142
|
+
this.logger.warn("Store does not support indexGet, cannot mark as stalled");
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
const flowEntry = await this.store.index.get(indexKey, runId);
|
|
146
|
+
if (!flowEntry?.metadata) return;
|
|
147
|
+
const previousStatus = flowEntry.metadata.status;
|
|
148
|
+
if (previousStatus !== "running" && previousStatus !== "awaiting") return;
|
|
149
|
+
if (this.store.index.update) {
|
|
150
|
+
await this.store.index.update(indexKey, runId, {
|
|
151
|
+
status: "stalled",
|
|
152
|
+
previousStatus,
|
|
153
|
+
// Track what state it was in before stalling
|
|
154
|
+
stalledAt: Date.now(),
|
|
155
|
+
stallReason: reason
|
|
156
|
+
});
|
|
157
|
+
}
|
|
158
|
+
const streamName = StoreSubjects.flowRun(runId);
|
|
159
|
+
await this.store.stream.append(streamName, {
|
|
160
|
+
type: "flow.stalled",
|
|
161
|
+
runId,
|
|
162
|
+
flowName,
|
|
163
|
+
data: {
|
|
164
|
+
reason,
|
|
165
|
+
previousStatus
|
|
166
|
+
// Include previous status so stats handler knows which counter to decrement
|
|
167
|
+
}
|
|
168
|
+
});
|
|
169
|
+
this.logger.info(`Marked flow as stalled - '${flowName}' runId '${runId}': ${reason}`);
|
|
170
|
+
} catch (error) {
|
|
171
|
+
this.logger.error(`Failed to mark flow as stalled for '${flowName}' runId '${runId}': ${error.message}`);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Check all running flows and mark stalled ones
|
|
176
|
+
* This is called by the periodic background job
|
|
177
|
+
*
|
|
178
|
+
* Note: This method requires knowledge of which flows exist.
|
|
179
|
+
* For now, we'll need to pass flow names to check, or iterate known flows from registry.
|
|
180
|
+
*/
|
|
181
|
+
async checkFlowsForStalls(flowNames) {
|
|
182
|
+
this.logger.info(`Running periodic stall check for ${flowNames.length} flows`);
|
|
183
|
+
try {
|
|
184
|
+
if (!this.store.index.get || !this.store.index.read) {
|
|
185
|
+
this.logger.warn("Store does not support required index operations");
|
|
186
|
+
return;
|
|
187
|
+
}
|
|
188
|
+
const { StoreSubjects } = useStreamTopics();
|
|
189
|
+
let checkedCount = 0;
|
|
190
|
+
let stalledCount = 0;
|
|
191
|
+
for (const flowName of flowNames) {
|
|
192
|
+
const indexKey = StoreSubjects.flowRunIndex(flowName);
|
|
193
|
+
const flowStallTimeout = await this.getFlowStallTimeout(flowName);
|
|
194
|
+
const entries = await this.store.index.read(indexKey, { limit: 1e3 });
|
|
195
|
+
for (const entry of entries) {
|
|
196
|
+
if (!entry.metadata) continue;
|
|
197
|
+
checkedCount++;
|
|
198
|
+
if (entry.metadata.status !== "running" && entry.metadata.status !== "awaiting") continue;
|
|
199
|
+
const awaitingSteps = entry.metadata.awaitingSteps || {};
|
|
200
|
+
const awaitingStepNames = Object.keys(awaitingSteps);
|
|
201
|
+
if (awaitingStepNames.length > 0) {
|
|
202
|
+
let hasOverdueAwaits = false;
|
|
203
|
+
let hasLegacyAwait = false;
|
|
204
|
+
for (const stepName of awaitingStepNames) {
|
|
205
|
+
const awaitState = awaitingSteps[stepName];
|
|
206
|
+
if (awaitState?.status === "awaiting") {
|
|
207
|
+
if (!awaitState.timeoutAt) {
|
|
208
|
+
hasLegacyAwait = true;
|
|
209
|
+
} else if (Date.now() > awaitState.timeoutAt) {
|
|
210
|
+
hasOverdueAwaits = true;
|
|
211
|
+
break;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
if (hasOverdueAwaits) {
|
|
216
|
+
await this.markAsStalled(flowName, entry.id, "Await pattern timed out");
|
|
217
|
+
stalledCount++;
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
if (hasLegacyAwait) {
|
|
221
|
+
this.logger.debug(`Skipping flow with legacy await (no timeout) - '${flowName}' runId '${entry.id}'`);
|
|
222
|
+
}
|
|
223
|
+
continue;
|
|
224
|
+
}
|
|
225
|
+
const lastActivity = entry.metadata.lastActivityAt || entry.metadata.startedAt || 0;
|
|
226
|
+
const timeSinceActivity = Date.now() - lastActivity;
|
|
227
|
+
if (timeSinceActivity > flowStallTimeout) {
|
|
228
|
+
await this.markAsStalled(flowName, entry.id, "Periodic check detected no activity");
|
|
229
|
+
stalledCount++;
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
}
|
|
233
|
+
this.logger.info(`Periodic stall check completed - checked: ${checkedCount}, stalled: ${stalledCount}`);
|
|
234
|
+
} catch (error) {
|
|
235
|
+
this.logger.error("Failed to run periodic stall check", {
|
|
236
|
+
error: error.message
|
|
237
|
+
});
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
/**
|
|
241
|
+
* Run startup recovery to clean up flows left in running state from previous server instance
|
|
242
|
+
* This marks all running flows as stalled since their in-memory state is lost
|
|
243
|
+
* Also validates and cleans up flow stats index
|
|
244
|
+
*/
|
|
245
|
+
async runStartupRecovery() {
|
|
246
|
+
this.logger.info("Running startup recovery to check for orphaned flows and validate stats");
|
|
247
|
+
try {
|
|
248
|
+
if (!this.store.index.get || !this.store.index.read) {
|
|
249
|
+
this.logger.warn("Store does not support required index operations");
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
const analyzedFlows = $useAnalyzedFlows();
|
|
253
|
+
const flowNames = analyzedFlows.map((f) => f.id).filter(Boolean);
|
|
254
|
+
this.logger.info(`Starting flow recovery check for ${flowNames.length} registered flows: [${flowNames.join(", ")}]`);
|
|
255
|
+
if (flowNames.length === 0) {
|
|
256
|
+
this.logger.debug("No flows registered, skipping startup recovery");
|
|
257
|
+
return;
|
|
258
|
+
}
|
|
259
|
+
const { StoreSubjects } = useStreamTopics();
|
|
260
|
+
let recoveredCount = 0;
|
|
261
|
+
const actualCounts = {};
|
|
262
|
+
for (const flowName of flowNames) {
|
|
263
|
+
actualCounts[flowName] = { running: 0, awaiting: 0 };
|
|
264
|
+
const indexKey = StoreSubjects.flowRunIndex(flowName);
|
|
265
|
+
this.logger.debug(`Reading flow run index for '${flowName}': ${indexKey}`);
|
|
266
|
+
const entries = await this.store.index.read(indexKey, { limit: 1e3 });
|
|
267
|
+
const statusCounts = {};
|
|
268
|
+
for (const entry of entries) {
|
|
269
|
+
if (!entry.metadata) {
|
|
270
|
+
this.logger.debug(`Skipping entry without metadata - '${flowName}' runId '${entry.id}'`);
|
|
271
|
+
continue;
|
|
272
|
+
}
|
|
273
|
+
const status = entry.metadata.status || "unknown";
|
|
274
|
+
statusCounts[status] = (statusCounts[status] || 0) + 1;
|
|
275
|
+
if (status === "running") {
|
|
276
|
+
actualCounts[flowName].running++;
|
|
277
|
+
} else if (status === "awaiting") {
|
|
278
|
+
actualCounts[flowName].awaiting++;
|
|
279
|
+
}
|
|
280
|
+
if (entry.metadata.status === "running" || entry.metadata.status === "awaiting") {
|
|
281
|
+
const awaitingSteps = entry.metadata.awaitingSteps || {};
|
|
282
|
+
const awaitingStepNames = Object.keys(awaitingSteps);
|
|
283
|
+
this.logger.info(`Found flow in ${entry.metadata.status} state - '${flowName}' runId '${entry.id}' with ${awaitingStepNames.length} awaiting steps`);
|
|
284
|
+
this.logger.debug(`Flow '${flowName}' runId '${entry.id}' status: ${entry.metadata.status}, awaitingSteps: ${awaitingStepNames.length}`);
|
|
285
|
+
if (awaitingStepNames.length > 0) {
|
|
286
|
+
let hasActiveValidAwaits = false;
|
|
287
|
+
let hasOverdueAwaits = false;
|
|
288
|
+
for (const stepName of awaitingStepNames) {
|
|
289
|
+
const awaitState = awaitingSteps[stepName];
|
|
290
|
+
this.logger.info(`Checking await state for '${flowName}' runId '${entry.id}' step '${stepName}': status=${awaitState?.status}, timeoutAt=${awaitState?.timeoutAt}, resolveAt=${awaitState?.resolveAt}`);
|
|
291
|
+
if (awaitState?.status === "awaiting") {
|
|
292
|
+
const timeoutAt = awaitState.timeoutAt || awaitState.resolveAt;
|
|
293
|
+
if (!timeoutAt) {
|
|
294
|
+
hasActiveValidAwaits = true;
|
|
295
|
+
this.logger.warn(`Found await without timeout (legacy data) - '${flowName}' runId '${entry.id}' step '${stepName}' - treating as valid (timeout tracking was added later)`);
|
|
296
|
+
} else if (Date.now() > timeoutAt) {
|
|
297
|
+
hasOverdueAwaits = true;
|
|
298
|
+
this.logger.warn(`Found overdue await pattern - '${flowName}' runId '${entry.id}' step '${stepName}': timeoutAt=${new Date(timeoutAt).toISOString()}, overdueBy=${Math.round((Date.now() - timeoutAt) / 1e3)}s`);
|
|
299
|
+
} else {
|
|
300
|
+
hasActiveValidAwaits = true;
|
|
301
|
+
this.logger.debug(`Found active valid await - '${flowName}' runId '${entry.id}' step '${stepName}': remaining=${Math.round((timeoutAt - Date.now()) / 1e3)}s`);
|
|
302
|
+
}
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
if (hasOverdueAwaits) {
|
|
306
|
+
this.logger.info(`Marking flow as stalled (overdue awaits) - '${flowName}' runId '${entry.id}'`);
|
|
307
|
+
await this.markAsStalled(flowName, entry.id, "Await pattern resolution failed or expired");
|
|
308
|
+
recoveredCount++;
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
if (hasActiveValidAwaits) {
|
|
312
|
+
if (entry.metadata.status === "running") {
|
|
313
|
+
if (this.store.index.update) {
|
|
314
|
+
await this.store.index.update(indexKey, entry.id, {
|
|
315
|
+
status: "awaiting"
|
|
316
|
+
});
|
|
317
|
+
this.logger.info(`Updated flow status to awaiting (has active awaits) - '${flowName}' runId '${entry.id}' steps: [${awaitingStepNames.join(", ")}]`);
|
|
318
|
+
}
|
|
319
|
+
} else {
|
|
320
|
+
this.logger.debug(`Flow already has awaiting status - '${flowName}' runId '${entry.id}'`);
|
|
321
|
+
}
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
}
|
|
325
|
+
this.logger.info(`Marking flow as stalled (no active awaits) - '${flowName}' runId '${entry.id}'`);
|
|
326
|
+
await this.markAsStalled(flowName, entry.id, "Server restart - flow state lost");
|
|
327
|
+
recoveredCount++;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
const statusSummary = Object.entries(statusCounts).map(([status, count]) => `${status}:${count}`).join(", ");
|
|
331
|
+
this.logger.info(`Flow recovery summary for '${flowName}' - total: ${entries.length}, statuses: {${statusSummary}}`);
|
|
332
|
+
}
|
|
333
|
+
await this.validateFlowStats(flowNames, actualCounts);
|
|
334
|
+
if (recoveredCount > 0) {
|
|
335
|
+
this.logger.info(`Startup recovery completed - marked ${recoveredCount} orphaned flow(s) as stalled`);
|
|
336
|
+
} else {
|
|
337
|
+
this.logger.debug("Startup recovery completed - no orphaned flows found");
|
|
338
|
+
}
|
|
339
|
+
} catch (error) {
|
|
340
|
+
this.logger.error(`Failed to run startup recovery: ${error.message}`);
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
/**
|
|
344
|
+
* Validate flow stats index and remove entries for non-existent flows
|
|
345
|
+
* Also corrects running/awaiting counts based on actual scanned data
|
|
346
|
+
*
|
|
347
|
+
* NOTE: We only validate running/awaiting counts because:
|
|
348
|
+
* - They are small snapshot values (usually < 100)
|
|
349
|
+
* - We already scanned all flows during startup recovery
|
|
350
|
+
* - Discrepancies indicate actual bugs (flows stuck in wrong state)
|
|
351
|
+
*
|
|
352
|
+
* We do NOT validate total/success/failure/cancel because:
|
|
353
|
+
* - These are cumulative counters that can be millions in production
|
|
354
|
+
* - Validation would require full table scan (prohibitively expensive)
|
|
355
|
+
* - Minor discrepancies don't affect runtime behavior
|
|
356
|
+
*/
|
|
357
|
+
async validateFlowStats(validFlowNames, actualCounts) {
|
|
358
|
+
this.logger.debug("Validating flow stats index");
|
|
359
|
+
try {
|
|
360
|
+
if (!this.store.index.read || !this.store.index.delete || !this.store.index.update) {
|
|
361
|
+
this.logger.debug("Store does not support stats validation operations");
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
const { StoreSubjects } = useStreamTopics();
|
|
365
|
+
const statsIndexKey = StoreSubjects.flowIndex();
|
|
366
|
+
const statsEntries = await this.store.index.read(statsIndexKey, { limit: 1e4 });
|
|
367
|
+
let removedCount = 0;
|
|
368
|
+
let correctedCount = 0;
|
|
369
|
+
for (const entry of statsEntries) {
|
|
370
|
+
const flowName = entry.id;
|
|
371
|
+
if (!validFlowNames.includes(flowName)) {
|
|
372
|
+
this.logger.info(`Removing stats for non-existent flow '${flowName}'`);
|
|
373
|
+
await this.store.index.delete(statsIndexKey, flowName);
|
|
374
|
+
removedCount++;
|
|
375
|
+
continue;
|
|
376
|
+
}
|
|
377
|
+
if (!entry.metadata?.stats) {
|
|
378
|
+
this.logger.warn(`Flow stats entry missing stats object for '${flowName}'`);
|
|
379
|
+
continue;
|
|
380
|
+
}
|
|
381
|
+
const stats = entry.metadata.stats;
|
|
382
|
+
const actual = actualCounts[flowName] || { running: 0, awaiting: 0 };
|
|
383
|
+
const runningMismatch = stats.running !== actual.running;
|
|
384
|
+
const awaitingMismatch = stats.awaiting !== actual.awaiting;
|
|
385
|
+
if (runningMismatch || awaitingMismatch) {
|
|
386
|
+
this.logger.warn(`Flow stats mismatch detected for '${flowName}' - stored: running=${stats.running} awaiting=${stats.awaiting}, actual: running=${actual.running} awaiting=${actual.awaiting} - correcting`);
|
|
387
|
+
await this.store.index.update(statsIndexKey, flowName, {
|
|
388
|
+
stats: {
|
|
389
|
+
running: actual.running,
|
|
390
|
+
awaiting: actual.awaiting
|
|
391
|
+
}
|
|
392
|
+
});
|
|
393
|
+
correctedCount++;
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
if (removedCount > 0 || correctedCount > 0) {
|
|
397
|
+
this.logger.info(`Flow stats validation completed - removed ${removedCount} orphaned stats, corrected ${correctedCount} running/awaiting counts`);
|
|
398
|
+
} else {
|
|
399
|
+
this.logger.debug("Flow stats validation completed - all stats accurate");
|
|
400
|
+
}
|
|
401
|
+
} catch (error) {
|
|
402
|
+
this.logger.error(`Failed to validate flow stats: ${error.message}`);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
/**
|
|
406
|
+
* Internal method for periodic checks
|
|
407
|
+
* Gets flow names from registry and checks them
|
|
408
|
+
*/
|
|
409
|
+
async checkAllRunningFlows() {
|
|
410
|
+
try {
|
|
411
|
+
const analyzedFlows = $useAnalyzedFlows();
|
|
412
|
+
const flowNames = analyzedFlows.map((f) => f.id).filter(Boolean);
|
|
413
|
+
if (flowNames.length === 0) {
|
|
414
|
+
this.logger.debug("No flows registered, skipping stall check");
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
await this.checkFlowsForStalls(flowNames);
|
|
418
|
+
} catch (error) {
|
|
419
|
+
this.logger.error(`Failed to run periodic stall check: ${error.message}`);
|
|
420
|
+
}
|
|
421
|
+
}
|
|
422
|
+
/**
|
|
423
|
+
* Get stall detector statistics
|
|
424
|
+
*/
|
|
425
|
+
getStats() {
|
|
426
|
+
return {
|
|
427
|
+
enabled: this.started,
|
|
428
|
+
periodicCheckEnabled: this.config.enablePeriodicCheck,
|
|
429
|
+
stallTimeout: this.config.stallTimeout,
|
|
430
|
+
checkInterval: this.config.checkInterval
|
|
431
|
+
};
|
|
432
|
+
}
|
|
433
|
+
}
|
|
434
|
+
export function createStallDetector(store, config) {
|
|
435
|
+
return new FlowStallDetector(store, config);
|
|
436
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { TriggerEntry, TriggerSubscription } from '../../../registry/types.js';
|
|
2
|
+
import type { StoreAdapter } from '../../adapters/interfaces/store.js';
|
|
3
|
+
/**
|
|
4
|
+
* Centralized trigger runtime management
|
|
5
|
+
* Handles runtime state, payload storage/resolution, and state manipulation
|
|
6
|
+
*
|
|
7
|
+
* This class is the single source of truth for trigger runtime state,
|
|
8
|
+
* used by both useTrigger (public API) and triggerWiring (event orchestration)
|
|
9
|
+
*/
|
|
10
|
+
export declare class TriggerRuntime {
|
|
11
|
+
private state;
|
|
12
|
+
private store;
|
|
13
|
+
private logger;
|
|
14
|
+
constructor(store: StoreAdapter, logger: any);
|
|
15
|
+
get initialized(): boolean;
|
|
16
|
+
setInitialized(value: boolean): void;
|
|
17
|
+
hasTrigger(name: string): boolean;
|
|
18
|
+
getTrigger(name: string): TriggerEntry | undefined;
|
|
19
|
+
getAllTriggers(options?: {
|
|
20
|
+
sortBy?: 'registeredAt' | 'lastActivityAt' | 'name';
|
|
21
|
+
order?: 'asc' | 'desc';
|
|
22
|
+
limit?: number;
|
|
23
|
+
offset?: number;
|
|
24
|
+
}): TriggerEntry[];
|
|
25
|
+
getSubscribedFlows(trigger: string): string[];
|
|
26
|
+
getFlowTriggers(flow: string): string[];
|
|
27
|
+
getSubscription(trigger: string, flow: string): TriggerSubscription | undefined;
|
|
28
|
+
getAllSubscriptions(): TriggerSubscription[];
|
|
29
|
+
getRuntimeStats(): {
|
|
30
|
+
triggerCount: number;
|
|
31
|
+
subscriptionCount: number;
|
|
32
|
+
flowCount: number;
|
|
33
|
+
initialized: boolean;
|
|
34
|
+
};
|
|
35
|
+
addTrigger(name: string, entry: TriggerEntry): void;
|
|
36
|
+
removeTrigger(name: string): void;
|
|
37
|
+
addSubscription(triggerName: string, flowName: string, subscription: TriggerSubscription): void;
|
|
38
|
+
removeSubscription(triggerName: string, flowName: string): void;
|
|
39
|
+
/**
|
|
40
|
+
* Extract a minimal summary from payload for debugging
|
|
41
|
+
* Keeps only small, identifying fields
|
|
42
|
+
*/
|
|
43
|
+
private extractSummary;
|
|
44
|
+
/**
|
|
45
|
+
* Store large payload in KV store and return reference
|
|
46
|
+
* Returns either the original data (if small) or a reference object
|
|
47
|
+
*/
|
|
48
|
+
handleLargePayload(triggerName: string, data: any, threshold: number): Promise<any>;
|
|
49
|
+
/**
|
|
50
|
+
* Resolve payload reference if needed
|
|
51
|
+
* Returns the full payload from KV store, or falls back to summary if expired
|
|
52
|
+
*/
|
|
53
|
+
resolvePayload(data: any): Promise<any>;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* Get or create the singleton TriggerRuntime instance
|
|
57
|
+
*/
|
|
58
|
+
export declare function getTriggerRuntime(store: StoreAdapter, logger: any): TriggerRuntime;
|