nvent 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/module.d.mts +4 -185
- package/dist/module.json +3 -3
- package/dist/module.mjs +451 -257
- package/dist/runtime/adapters/base/index.d.ts +6 -0
- package/dist/runtime/adapters/base/index.js +1 -0
- package/dist/runtime/adapters/base/store-validator.d.ts +48 -0
- package/dist/runtime/adapters/base/store-validator.js +147 -0
- package/dist/runtime/adapters/builtin/file-queue.d.ts +67 -0
- package/dist/runtime/adapters/builtin/file-queue.js +499 -0
- package/dist/runtime/adapters/builtin/file-store.d.ts +32 -0
- package/dist/runtime/adapters/builtin/file-store.js +206 -0
- package/dist/runtime/adapters/builtin/file-stream.d.ts +39 -0
- package/dist/runtime/adapters/builtin/file-stream.js +56 -0
- package/dist/runtime/adapters/builtin/index.d.ts +10 -0
- package/dist/runtime/adapters/builtin/index.js +5 -0
- package/dist/runtime/adapters/builtin/memory-queue.d.ts +52 -0
- package/dist/runtime/adapters/builtin/memory-queue.js +243 -0
- package/dist/runtime/adapters/builtin/memory-store.d.ts +68 -0
- package/dist/runtime/adapters/builtin/memory-store.js +333 -0
- package/dist/runtime/adapters/builtin/memory-stream.d.ts +21 -0
- package/dist/runtime/adapters/builtin/memory-stream.js +56 -0
- package/dist/runtime/adapters/factory.d.ts +31 -0
- package/dist/runtime/adapters/factory.js +134 -0
- package/dist/runtime/adapters/index.d.ts +8 -0
- package/dist/runtime/adapters/index.js +3 -0
- package/dist/runtime/adapters/interfaces/index.d.ts +11 -0
- package/dist/runtime/adapters/interfaces/index.js +3 -0
- package/dist/runtime/adapters/interfaces/queue.d.ts +150 -0
- package/dist/runtime/adapters/interfaces/store.d.ts +297 -0
- package/dist/runtime/adapters/interfaces/stream.d.ts +62 -0
- package/dist/runtime/adapters/registry.d.ts +85 -0
- package/dist/runtime/adapters/registry.js +161 -0
- package/dist/runtime/config/index.d.ts +29 -0
- package/dist/runtime/config/index.js +175 -0
- package/dist/runtime/config/types.d.ts +397 -0
- package/dist/runtime/config/types.js +0 -0
- package/dist/runtime/{server-utils/events → events}/eventBus.d.ts +1 -1
- package/dist/runtime/events/types.d.ts +145 -0
- package/dist/runtime/events/types.js +0 -0
- package/dist/runtime/events/utils/scheduleTrigger.d.ts +8 -0
- package/dist/runtime/events/utils/scheduleTrigger.js +69 -0
- package/dist/runtime/events/utils/stallDetector.d.ts +140 -0
- package/dist/runtime/events/utils/stallDetector.js +436 -0
- package/dist/runtime/events/utils/triggerRuntime.d.ts +58 -0
- package/dist/runtime/events/utils/triggerRuntime.js +212 -0
- package/dist/runtime/{server-utils/events → events}/wiring/flowWiring.d.ts +12 -11
- package/dist/runtime/events/wiring/flowWiring.js +1020 -0
- package/dist/runtime/events/wiring/registry.d.ts +19 -0
- package/dist/runtime/events/wiring/registry.js +35 -0
- package/dist/runtime/events/wiring/stateWiring.d.ts +37 -0
- package/dist/runtime/events/wiring/stateWiring.js +92 -0
- package/dist/runtime/events/wiring/streamWiring.d.ts +36 -0
- package/dist/runtime/events/wiring/streamWiring.js +156 -0
- package/dist/runtime/events/wiring/triggerWiring.d.ts +21 -0
- package/dist/runtime/events/wiring/triggerWiring.js +412 -0
- package/dist/runtime/nitro/plugins/00.adapters.d.ts +14 -0
- package/dist/runtime/nitro/plugins/00.adapters.js +73 -0
- package/dist/runtime/nitro/plugins/02.workers.js +63 -0
- package/dist/runtime/nitro/plugins/03.triggers.d.ts +12 -0
- package/dist/runtime/nitro/plugins/03.triggers.js +55 -0
- package/dist/runtime/nitro/routes/webhook.await.d.ts +23 -0
- package/dist/runtime/nitro/routes/webhook.await.js +90 -0
- package/dist/runtime/nitro/routes/webhook.trigger.d.ts +69 -0
- package/dist/runtime/nitro/routes/webhook.trigger.js +64 -0
- package/dist/runtime/nitro/utils/adapters.d.ts +66 -0
- package/dist/runtime/nitro/utils/adapters.js +51 -0
- package/dist/runtime/nitro/utils/awaitPatterns/event.d.ts +15 -0
- package/dist/runtime/nitro/utils/awaitPatterns/event.js +120 -0
- package/dist/runtime/nitro/utils/awaitPatterns/index.d.ts +28 -0
- package/dist/runtime/nitro/utils/awaitPatterns/index.js +55 -0
- package/dist/runtime/nitro/utils/awaitPatterns/schedule.d.ts +16 -0
- package/dist/runtime/nitro/utils/awaitPatterns/schedule.js +78 -0
- package/dist/runtime/nitro/utils/awaitPatterns/time.d.ts +15 -0
- package/dist/runtime/nitro/utils/awaitPatterns/time.js +67 -0
- package/dist/runtime/nitro/utils/awaitPatterns/webhook.d.ts +15 -0
- package/dist/runtime/nitro/utils/awaitPatterns/webhook.js +120 -0
- package/dist/runtime/nitro/utils/defineFunction.d.ts +10 -0
- package/dist/runtime/nitro/utils/defineFunction.js +17 -0
- package/dist/runtime/nitro/utils/defineFunctionConfig.d.ts +310 -0
- package/dist/runtime/nitro/utils/defineFunctionConfig.js +3 -0
- package/dist/runtime/nitro/utils/defineHooks.d.ts +41 -0
- package/dist/runtime/nitro/utils/defineHooks.js +6 -0
- package/dist/runtime/nitro/utils/registerAdapter.d.ts +59 -0
- package/dist/runtime/nitro/utils/registerAdapter.js +13 -0
- package/dist/runtime/nitro/utils/useAwait.d.ts +71 -0
- package/dist/runtime/nitro/utils/useAwait.js +139 -0
- package/dist/runtime/{server-utils → nitro}/utils/useEventManager.d.ts +2 -2
- package/dist/runtime/{server-utils → nitro}/utils/useEventManager.js +1 -1
- package/dist/runtime/nitro/utils/useFlow.d.ts +68 -0
- package/dist/runtime/nitro/utils/useFlow.js +226 -0
- package/dist/runtime/nitro/utils/useHookRegistry.d.ts +34 -0
- package/dist/runtime/nitro/utils/useHookRegistry.js +25 -0
- package/dist/runtime/{server-utils → nitro}/utils/useNventLogger.js +2 -2
- package/dist/runtime/nitro/utils/useRunContext.d.ts +6 -0
- package/dist/runtime/nitro/utils/useRunContext.js +102 -0
- package/dist/runtime/nitro/utils/useStreamTopics.d.ts +83 -0
- package/dist/runtime/nitro/utils/useStreamTopics.js +94 -0
- package/dist/runtime/nitro/utils/useTrigger.d.ts +150 -0
- package/dist/runtime/nitro/utils/useTrigger.js +320 -0
- package/dist/runtime/scheduler/index.d.ts +33 -0
- package/dist/runtime/scheduler/index.js +38 -0
- package/dist/runtime/scheduler/scheduler.d.ts +113 -0
- package/dist/runtime/scheduler/scheduler.js +623 -0
- package/dist/runtime/scheduler/types.d.ts +116 -0
- package/dist/runtime/scheduler/types.js +0 -0
- package/dist/runtime/tsconfig.json +8 -0
- package/dist/runtime/worker/node/runner.d.ts +53 -0
- package/dist/runtime/worker/node/runner.js +327 -0
- package/dist/types.d.mts +2 -2
- package/package.json +16 -46
- package/LICENSE +0 -21
- package/README.md +0 -389
- package/dist/runtime/app/assets/vueflow.css +0 -1
- package/dist/runtime/app/components/ConfirmDialog.d.vue.ts +0 -33
- package/dist/runtime/app/components/ConfirmDialog.vue +0 -121
- package/dist/runtime/app/components/ConfirmDialog.vue.d.ts +0 -33
- package/dist/runtime/app/components/FlowDiagram.d.vue.ts +0 -64
- package/dist/runtime/app/components/FlowDiagram.vue +0 -338
- package/dist/runtime/app/components/FlowDiagram.vue.d.ts +0 -64
- package/dist/runtime/app/components/FlowNodeCard.d.vue.ts +0 -29
- package/dist/runtime/app/components/FlowNodeCard.vue +0 -156
- package/dist/runtime/app/components/FlowNodeCard.vue.d.ts +0 -29
- package/dist/runtime/app/components/FlowRunOverview.d.vue.ts +0 -9
- package/dist/runtime/app/components/FlowRunOverview.vue +0 -291
- package/dist/runtime/app/components/FlowRunOverview.vue.d.ts +0 -9
- package/dist/runtime/app/components/FlowRunStatusBadge.d.vue.ts +0 -14
- package/dist/runtime/app/components/FlowRunStatusBadge.vue +0 -60
- package/dist/runtime/app/components/FlowRunStatusBadge.vue.d.ts +0 -14
- package/dist/runtime/app/components/FlowRunTimeline.d.vue.ts +0 -12
- package/dist/runtime/app/components/FlowRunTimeline.vue +0 -127
- package/dist/runtime/app/components/FlowRunTimeline.vue.d.ts +0 -12
- package/dist/runtime/app/components/FlowScheduleDialog.d.vue.ts +0 -16
- package/dist/runtime/app/components/FlowScheduleDialog.vue +0 -226
- package/dist/runtime/app/components/FlowScheduleDialog.vue.d.ts +0 -16
- package/dist/runtime/app/components/FlowSchedulesList.d.vue.ts +0 -12
- package/dist/runtime/app/components/FlowSchedulesList.vue +0 -99
- package/dist/runtime/app/components/FlowSchedulesList.vue.d.ts +0 -12
- package/dist/runtime/app/components/JobScheduling.d.vue.ts +0 -6
- package/dist/runtime/app/components/JobScheduling.vue +0 -203
- package/dist/runtime/app/components/JobScheduling.vue.d.ts +0 -6
- package/dist/runtime/app/components/ListItem.d.vue.ts +0 -23
- package/dist/runtime/app/components/ListItem.vue +0 -70
- package/dist/runtime/app/components/ListItem.vue.d.ts +0 -23
- package/dist/runtime/app/components/QueueConfigDetails.d.vue.ts +0 -45
- package/dist/runtime/app/components/QueueConfigDetails.vue +0 -412
- package/dist/runtime/app/components/QueueConfigDetails.vue.d.ts +0 -45
- package/dist/runtime/app/components/StatCounter.d.vue.ts +0 -9
- package/dist/runtime/app/components/StatCounter.vue +0 -25
- package/dist/runtime/app/components/StatCounter.vue.d.ts +0 -9
- package/dist/runtime/app/components/TimelineList.d.vue.ts +0 -7
- package/dist/runtime/app/components/TimelineList.vue +0 -210
- package/dist/runtime/app/components/TimelineList.vue.d.ts +0 -7
- package/dist/runtime/app/components/nhealth/component-router.d.vue.ts +0 -46
- package/dist/runtime/app/components/nhealth/component-router.vue +0 -26
- package/dist/runtime/app/components/nhealth/component-router.vue.d.ts +0 -46
- package/dist/runtime/app/components/nhealth/component-shell.d.vue.ts +0 -24
- package/dist/runtime/app/components/nhealth/component-shell.vue +0 -89
- package/dist/runtime/app/components/nhealth/component-shell.vue.d.ts +0 -24
- package/dist/runtime/app/composables/useAnalyzedFlows.d.ts +0 -14
- package/dist/runtime/app/composables/useAnalyzedFlows.js +0 -8
- package/dist/runtime/app/composables/useComponentRouter.d.ts +0 -38
- package/dist/runtime/app/composables/useComponentRouter.js +0 -240
- package/dist/runtime/app/composables/useFlowRunTimeline.d.ts +0 -80
- package/dist/runtime/app/composables/useFlowRunTimeline.js +0 -68
- package/dist/runtime/app/composables/useFlowRuns.d.ts +0 -18
- package/dist/runtime/app/composables/useFlowRuns.js +0 -32
- package/dist/runtime/app/composables/useFlowRunsInfinite.d.ts +0 -24
- package/dist/runtime/app/composables/useFlowRunsInfinite.js +0 -123
- package/dist/runtime/app/composables/useFlowRunsPolling.d.ts +0 -9
- package/dist/runtime/app/composables/useFlowRunsPolling.js +0 -33
- package/dist/runtime/app/composables/useFlowState.d.ts +0 -125
- package/dist/runtime/app/composables/useFlowState.js +0 -211
- package/dist/runtime/app/composables/useFlowWebSocket.d.ts +0 -27
- package/dist/runtime/app/composables/useFlowWebSocket.js +0 -205
- package/dist/runtime/app/composables/useFlowsNavigation.d.ts +0 -10
- package/dist/runtime/app/composables/useFlowsNavigation.js +0 -58
- package/dist/runtime/app/composables/useQueueJobs.d.ts +0 -26
- package/dist/runtime/app/composables/useQueueJobs.js +0 -20
- package/dist/runtime/app/composables/useQueueUpdates.d.ts +0 -26
- package/dist/runtime/app/composables/useQueueUpdates.js +0 -122
- package/dist/runtime/app/composables/useQueues.d.ts +0 -45
- package/dist/runtime/app/composables/useQueues.js +0 -26
- package/dist/runtime/app/composables/useQueuesLive.d.ts +0 -19
- package/dist/runtime/app/composables/useQueuesLive.js +0 -143
- package/dist/runtime/app/pages/flows/index.d.vue.ts +0 -3
- package/dist/runtime/app/pages/flows/index.vue +0 -645
- package/dist/runtime/app/pages/flows/index.vue.d.ts +0 -3
- package/dist/runtime/app/pages/index.d.vue.ts +0 -3
- package/dist/runtime/app/pages/index.vue +0 -34
- package/dist/runtime/app/pages/index.vue.d.ts +0 -3
- package/dist/runtime/app/pages/queues/index.d.vue.ts +0 -3
- package/dist/runtime/app/pages/queues/index.vue +0 -229
- package/dist/runtime/app/pages/queues/index.vue.d.ts +0 -3
- package/dist/runtime/app/pages/queues/job.d.vue.ts +0 -3
- package/dist/runtime/app/pages/queues/job.vue +0 -262
- package/dist/runtime/app/pages/queues/job.vue.d.ts +0 -3
- package/dist/runtime/app/pages/queues/jobs.d.vue.ts +0 -3
- package/dist/runtime/app/pages/queues/jobs.vue +0 -291
- package/dist/runtime/app/pages/queues/jobs.vue.d.ts +0 -3
- package/dist/runtime/app/plugins/vueflow.client.d.ts +0 -2
- package/dist/runtime/app/plugins/vueflow.client.js +0 -11
- package/dist/runtime/constants.d.ts +0 -11
- package/dist/runtime/constants.js +0 -11
- package/dist/runtime/schema.d.ts +0 -37
- package/dist/runtime/schema.js +0 -20
- package/dist/runtime/server/api/_flows/[name]/clear-history.delete.d.ts +0 -10
- package/dist/runtime/server/api/_flows/[name]/clear-history.delete.js +0 -44
- package/dist/runtime/server/api/_flows/[name]/runs.get.d.ts +0 -7
- package/dist/runtime/server/api/_flows/[name]/runs.get.js +0 -53
- package/dist/runtime/server/api/_flows/[name]/schedule.post.js +0 -57
- package/dist/runtime/server/api/_flows/[name]/schedules/[id].delete.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/schedules/[id].delete.js +0 -42
- package/dist/runtime/server/api/_flows/[name]/schedules.get.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/schedules.get.js +0 -48
- package/dist/runtime/server/api/_flows/[name]/start.post.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/start.post.js +0 -9
- package/dist/runtime/server/api/_flows/index.get.d.ts +0 -6
- package/dist/runtime/server/api/_flows/index.get.js +0 -5
- package/dist/runtime/server/api/_flows/ws.d.ts +0 -60
- package/dist/runtime/server/api/_flows/ws.js +0 -188
- package/dist/runtime/server/api/_queues/[name]/job/[id].get.d.ts +0 -2
- package/dist/runtime/server/api/_queues/[name]/job/[id].get.js +0 -9
- package/dist/runtime/server/api/_queues/[name]/job/index.get.d.ts +0 -2
- package/dist/runtime/server/api/_queues/[name]/job/index.get.js +0 -18
- package/dist/runtime/server/api/_queues/index.get.d.ts +0 -2
- package/dist/runtime/server/api/_queues/index.get.js +0 -63
- package/dist/runtime/server/api/_queues/ws.d.ts +0 -48
- package/dist/runtime/server/api/_queues/ws.js +0 -205
- package/dist/runtime/server/plugins/00.event-store.d.ts +0 -13
- package/dist/runtime/server/plugins/00.event-store.js +0 -16
- package/dist/runtime/server/plugins/flow-management.d.ts +0 -13
- package/dist/runtime/server/plugins/flow-management.js +0 -65
- package/dist/runtime/server/plugins/queue-management.d.ts +0 -2
- package/dist/runtime/server/plugins/queue-management.js +0 -27
- package/dist/runtime/server/plugins/state-cleanup.d.ts +0 -11
- package/dist/runtime/server/plugins/state-cleanup.js +0 -93
- package/dist/runtime/server/plugins/worker-management.d.ts +0 -2
- package/dist/runtime/server/plugins/worker-management.js +0 -33
- package/dist/runtime/server/tsconfig.json +0 -3
- package/dist/runtime/server-utils/events/adapters/fileAdapter.d.ts +0 -2
- package/dist/runtime/server-utils/events/adapters/fileAdapter.js +0 -382
- package/dist/runtime/server-utils/events/adapters/memoryAdapter.d.ts +0 -2
- package/dist/runtime/server-utils/events/adapters/memoryAdapter.js +0 -171
- package/dist/runtime/server-utils/events/adapters/redis/redisAdapter.d.ts +0 -2
- package/dist/runtime/server-utils/events/adapters/redis/redisAdapter.js +0 -348
- package/dist/runtime/server-utils/events/adapters/redis/redisPubSubGateway.d.ts +0 -30
- package/dist/runtime/server-utils/events/adapters/redis/redisPubSubGateway.js +0 -82
- package/dist/runtime/server-utils/events/eventStoreFactory.d.ts +0 -19
- package/dist/runtime/server-utils/events/eventStoreFactory.js +0 -44
- package/dist/runtime/server-utils/events/streamNames.d.ts +0 -17
- package/dist/runtime/server-utils/events/streamNames.js +0 -17
- package/dist/runtime/server-utils/events/types.d.ts +0 -63
- package/dist/runtime/server-utils/events/wiring/flowWiring.js +0 -409
- package/dist/runtime/server-utils/events/wiring/registry.d.ts +0 -10
- package/dist/runtime/server-utils/events/wiring/registry.js +0 -24
- package/dist/runtime/server-utils/queue/adapters/bullmq.d.ts +0 -18
- package/dist/runtime/server-utils/queue/adapters/bullmq.js +0 -164
- package/dist/runtime/server-utils/queue/queueFactory.d.ts +0 -3
- package/dist/runtime/server-utils/queue/queueFactory.js +0 -10
- package/dist/runtime/server-utils/queue/types.d.ts +0 -47
- package/dist/runtime/server-utils/state/adapters/redis.d.ts +0 -2
- package/dist/runtime/server-utils/state/adapters/redis.js +0 -42
- package/dist/runtime/server-utils/state/stateFactory.d.ts +0 -3
- package/dist/runtime/server-utils/state/stateFactory.js +0 -17
- package/dist/runtime/server-utils/state/types.d.ts +0 -23
- package/dist/runtime/server-utils/utils/defineQueueConfig.d.ts +0 -154
- package/dist/runtime/server-utils/utils/defineQueueConfig.js +0 -2
- package/dist/runtime/server-utils/utils/defineQueueWorker.d.ts +0 -10
- package/dist/runtime/server-utils/utils/defineQueueWorker.js +0 -17
- package/dist/runtime/server-utils/utils/useEventStore.d.ts +0 -20
- package/dist/runtime/server-utils/utils/useEventStore.js +0 -119
- package/dist/runtime/server-utils/utils/useFlowEngine.d.ts +0 -9
- package/dist/runtime/server-utils/utils/useFlowEngine.js +0 -44
- package/dist/runtime/server-utils/utils/useLogs.d.ts +0 -41
- package/dist/runtime/server-utils/utils/useLogs.js +0 -74
- package/dist/runtime/server-utils/utils/useQueue.d.ts +0 -31
- package/dist/runtime/server-utils/utils/useQueue.js +0 -24
- package/dist/runtime/server-utils/worker/adapter.d.ts +0 -4
- package/dist/runtime/server-utils/worker/adapter.js +0 -66
- package/dist/runtime/server-utils/worker/runner/node.d.ts +0 -27
- package/dist/runtime/server-utils/worker/runner/node.js +0 -196
- package/dist/runtime/types.d.ts +0 -132
- /package/dist/runtime/{server-utils/events/types.js → adapters/interfaces/queue.js} +0 -0
- /package/dist/runtime/{server-utils/queue/types.js → adapters/interfaces/store.js} +0 -0
- /package/dist/runtime/{server-utils/state/types.js → adapters/interfaces/stream.js} +0 -0
- /package/dist/runtime/{server-utils/events → events}/eventBus.js +0 -0
- /package/dist/runtime/{server/plugins/00.ws-lifecycle.d.ts → nitro/plugins/01.ws-lifecycle.d.ts} +0 -0
- /package/dist/runtime/{server/plugins/00.ws-lifecycle.js → nitro/plugins/01.ws-lifecycle.js} +0 -0
- /package/dist/runtime/{server/api/_flows/[name]/schedule.post.d.ts → nitro/plugins/02.workers.d.ts} +0 -0
- /package/dist/runtime/{server-utils → nitro}/utils/useNventLogger.d.ts +0 -0
- /package/dist/runtime/{server-utils → nitro}/utils/wsPeerManager.d.ts +0 -0
- /package/dist/runtime/{server-utils → nitro}/utils/wsPeerManager.js +0 -0
- /package/dist/runtime/{python → worker/python}/get_config.py +0 -0
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
import { promises as fs } from "node:fs";
|
|
2
|
+
import { join, dirname } from "node:path";
|
|
3
|
+
import { MemoryStoreAdapter } from "./memory-store.js";
|
|
4
|
+
function sanitize(name) {
|
|
5
|
+
return name.replace(/[^\w.-]/g, "_");
|
|
6
|
+
}
|
|
7
|
+
async function ensureDir(path) {
|
|
8
|
+
try {
|
|
9
|
+
await fs.mkdir(path, { recursive: true });
|
|
10
|
+
} catch {
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
export class FileStoreAdapter extends MemoryStoreAdapter {
|
|
14
|
+
options;
|
|
15
|
+
constructor(options) {
|
|
16
|
+
super();
|
|
17
|
+
this.options = options;
|
|
18
|
+
const parentKv = this.kv;
|
|
19
|
+
const parentStream = this.stream;
|
|
20
|
+
const parentIndex = this.index;
|
|
21
|
+
this.kv = {
|
|
22
|
+
get: parentKv.get,
|
|
23
|
+
set: async (key, value, ttl) => {
|
|
24
|
+
await parentKv.set(key, value, ttl);
|
|
25
|
+
const path = this.kvPath(key);
|
|
26
|
+
await ensureDir(dirname(path));
|
|
27
|
+
await fs.writeFile(path, JSON.stringify(value), "utf-8");
|
|
28
|
+
},
|
|
29
|
+
delete: async (key) => {
|
|
30
|
+
await parentKv.delete(key);
|
|
31
|
+
const path = this.kvPath(key);
|
|
32
|
+
try {
|
|
33
|
+
await fs.unlink(path);
|
|
34
|
+
} catch {
|
|
35
|
+
}
|
|
36
|
+
},
|
|
37
|
+
clear: async (pattern) => {
|
|
38
|
+
const count = await parentKv.clear(pattern);
|
|
39
|
+
try {
|
|
40
|
+
const kvDir = join(this.options.dataDir, "kv");
|
|
41
|
+
const files = await fs.readdir(kvDir);
|
|
42
|
+
const regex = new RegExp(`^${pattern.replace(/\*/g, ".*")}$`);
|
|
43
|
+
for (const file of files) {
|
|
44
|
+
if (!file.endsWith(".json")) continue;
|
|
45
|
+
const key = file.replace(".json", "").replace(/_/g, ":");
|
|
46
|
+
if (regex.test(key)) {
|
|
47
|
+
await fs.unlink(join(kvDir, file));
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
} catch {
|
|
51
|
+
}
|
|
52
|
+
return count;
|
|
53
|
+
},
|
|
54
|
+
increment: async (key, by = 1) => {
|
|
55
|
+
const result = await parentKv.increment(key, by);
|
|
56
|
+
const path = this.kvPath(key);
|
|
57
|
+
await ensureDir(dirname(path));
|
|
58
|
+
await fs.writeFile(path, JSON.stringify(result), "utf-8");
|
|
59
|
+
return result;
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
this.stream = {
|
|
63
|
+
append: async (subject, event) => {
|
|
64
|
+
const result = await parentStream.append(subject, event);
|
|
65
|
+
const path = this.streamPath(subject);
|
|
66
|
+
await ensureDir(dirname(path));
|
|
67
|
+
await fs.appendFile(path, JSON.stringify(result) + "\n", "utf-8");
|
|
68
|
+
return result;
|
|
69
|
+
},
|
|
70
|
+
read: parentStream.read,
|
|
71
|
+
subscribe: parentStream.subscribe,
|
|
72
|
+
delete: async (subject) => {
|
|
73
|
+
const deleted = parentStream.delete ? await parentStream.delete(subject) : false;
|
|
74
|
+
if (deleted) {
|
|
75
|
+
const path = this.streamPath(subject);
|
|
76
|
+
try {
|
|
77
|
+
await fs.unlink(path);
|
|
78
|
+
} catch {
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return deleted;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
this.index = {
|
|
85
|
+
add: async (key, id, score, metadata) => {
|
|
86
|
+
await parentIndex.add(key, id, score, metadata);
|
|
87
|
+
const self = this;
|
|
88
|
+
const index = self.sortedIndices?.get(key);
|
|
89
|
+
if (index) {
|
|
90
|
+
const path = this.indexPath(key);
|
|
91
|
+
await ensureDir(dirname(path));
|
|
92
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
get: parentIndex.get,
|
|
96
|
+
read: parentIndex.read,
|
|
97
|
+
update: async (key, id, metadata) => {
|
|
98
|
+
const result = await parentIndex.update(key, id, metadata);
|
|
99
|
+
const self = this;
|
|
100
|
+
const index = self.sortedIndices?.get(key);
|
|
101
|
+
if (index) {
|
|
102
|
+
const path = this.indexPath(key);
|
|
103
|
+
await ensureDir(dirname(path));
|
|
104
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
105
|
+
}
|
|
106
|
+
return result;
|
|
107
|
+
},
|
|
108
|
+
updateWithRetry: async (key, id, metadata, maxRetries) => {
|
|
109
|
+
await parentIndex.updateWithRetry(key, id, metadata, maxRetries);
|
|
110
|
+
const self = this;
|
|
111
|
+
const index = self.sortedIndices?.get(key);
|
|
112
|
+
if (index) {
|
|
113
|
+
const path = this.indexPath(key);
|
|
114
|
+
await ensureDir(dirname(path));
|
|
115
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
increment: async (key, id, field, increment) => {
|
|
119
|
+
const result = await parentIndex.increment(key, id, field, increment);
|
|
120
|
+
const self = this;
|
|
121
|
+
const index = self.sortedIndices?.get(key);
|
|
122
|
+
if (index) {
|
|
123
|
+
const path = this.indexPath(key);
|
|
124
|
+
await ensureDir(dirname(path));
|
|
125
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
126
|
+
}
|
|
127
|
+
return result;
|
|
128
|
+
},
|
|
129
|
+
delete: async (key, id) => {
|
|
130
|
+
const deleted = await parentIndex.delete(key, id);
|
|
131
|
+
if (deleted) {
|
|
132
|
+
const self = this;
|
|
133
|
+
const index = self.sortedIndices?.get(key);
|
|
134
|
+
if (index) {
|
|
135
|
+
const path = this.indexPath(key);
|
|
136
|
+
await ensureDir(dirname(path));
|
|
137
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return deleted;
|
|
141
|
+
}
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
streamPath(subject) {
|
|
145
|
+
return join(this.options.dataDir, "streams", sanitize(subject) + ".ndjson");
|
|
146
|
+
}
|
|
147
|
+
indexPath(key) {
|
|
148
|
+
return join(this.options.dataDir, "indices", sanitize(key) + ".json");
|
|
149
|
+
}
|
|
150
|
+
kvPath(key) {
|
|
151
|
+
return join(this.options.dataDir, "kv", `${sanitize(key)}.json`);
|
|
152
|
+
}
|
|
153
|
+
async init() {
|
|
154
|
+
await ensureDir(this.options.dataDir);
|
|
155
|
+
await ensureDir(join(this.options.dataDir, "streams"));
|
|
156
|
+
await ensureDir(join(this.options.dataDir, "indices"));
|
|
157
|
+
await ensureDir(join(this.options.dataDir, "kv"));
|
|
158
|
+
await this.loadFromDisk();
|
|
159
|
+
}
|
|
160
|
+
async loadFromDisk() {
|
|
161
|
+
const self = this;
|
|
162
|
+
try {
|
|
163
|
+
const streamsDir = join(this.options.dataDir, "streams");
|
|
164
|
+
const files = await fs.readdir(streamsDir);
|
|
165
|
+
for (const file of files) {
|
|
166
|
+
if (!file.endsWith(".ndjson")) continue;
|
|
167
|
+
const subject = file.replace(".ndjson", "").replace(/_/g, ":");
|
|
168
|
+
const content = await fs.readFile(join(streamsDir, file), "utf-8");
|
|
169
|
+
const lines = content.trim().split("\n").filter((l) => l.length > 0);
|
|
170
|
+
const events = lines.map((line) => JSON.parse(line));
|
|
171
|
+
if (!self.eventStreams) self.eventStreams = /* @__PURE__ */ new Map();
|
|
172
|
+
self.eventStreams.set(subject, events);
|
|
173
|
+
}
|
|
174
|
+
} catch {
|
|
175
|
+
}
|
|
176
|
+
try {
|
|
177
|
+
const indicesDir = join(this.options.dataDir, "indices");
|
|
178
|
+
const indexFiles = await fs.readdir(indicesDir);
|
|
179
|
+
for (const file of indexFiles) {
|
|
180
|
+
if (!file.endsWith(".json")) continue;
|
|
181
|
+
const key = file.replace(".json", "").replace(/_/g, ":");
|
|
182
|
+
const content = await fs.readFile(join(indicesDir, file), "utf-8");
|
|
183
|
+
const entries = JSON.parse(content);
|
|
184
|
+
if (!self.sortedIndices) self.sortedIndices = /* @__PURE__ */ new Map();
|
|
185
|
+
self.sortedIndices.set(key, entries);
|
|
186
|
+
}
|
|
187
|
+
} catch {
|
|
188
|
+
}
|
|
189
|
+
try {
|
|
190
|
+
const kvDir = join(this.options.dataDir, "kv");
|
|
191
|
+
const kvFiles = await fs.readdir(kvDir);
|
|
192
|
+
for (const file of kvFiles) {
|
|
193
|
+
if (!file.endsWith(".json")) continue;
|
|
194
|
+
const key = file.replace(".json", "").replace(/_/g, ":");
|
|
195
|
+
const content = await fs.readFile(join(kvDir, file), "utf-8");
|
|
196
|
+
const value = JSON.parse(content);
|
|
197
|
+
if (!self.kvStore) self.kvStore = /* @__PURE__ */ new Map();
|
|
198
|
+
self.kvStore.set(key, value);
|
|
199
|
+
}
|
|
200
|
+
} catch {
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
async close() {
|
|
204
|
+
await super.close();
|
|
205
|
+
}
|
|
206
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* File Stream Adapter
|
|
3
|
+
*
|
|
4
|
+
* DEPRECATED: This adapter is identical to MemoryStreamAdapter
|
|
5
|
+
*
|
|
6
|
+
* For file-based deployments (single instance):
|
|
7
|
+
* - StreamAdapter is NOT needed for cross-instance pub/sub
|
|
8
|
+
* - StoreAdapter.subscribe() handles in-process subscriptions
|
|
9
|
+
* - StoreAdapter handles ALL persistence (events, documents, KV)
|
|
10
|
+
*
|
|
11
|
+
* This adapter exists only for API compatibility.
|
|
12
|
+
* Use MemoryStreamAdapter instead - it's the same implementation.
|
|
13
|
+
*/
|
|
14
|
+
import type { StreamAdapter, StreamEvent, SubscribeOptions, SubscriptionHandle } from '../interfaces/stream.js';
|
|
15
|
+
/**
|
|
16
|
+
* @deprecated Use MemoryStreamAdapter instead. FileStreamAdapter is identical.
|
|
17
|
+
*/
|
|
18
|
+
export interface FileStreamAdapterOptions {
|
|
19
|
+
dataDir?: string;
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* @deprecated Use MemoryStreamAdapter instead
|
|
23
|
+
*
|
|
24
|
+
* Pure in-memory pub/sub (same as MemoryStreamAdapter)
|
|
25
|
+
* - No persistence (StoreAdapter handles that)
|
|
26
|
+
* - Single instance only
|
|
27
|
+
* - Events lost if no subscribers
|
|
28
|
+
*/
|
|
29
|
+
export declare class FileStreamAdapter implements StreamAdapter {
|
|
30
|
+
private subscriptions;
|
|
31
|
+
private subscriptionCounter;
|
|
32
|
+
init(): Promise<void>;
|
|
33
|
+
publish(topic: string, event: StreamEvent): Promise<void>;
|
|
34
|
+
subscribe(topic: string, handler: (event: StreamEvent) => void | Promise<void>, _opts?: SubscribeOptions): Promise<SubscriptionHandle>;
|
|
35
|
+
unsubscribe(handle: SubscriptionHandle): Promise<void>;
|
|
36
|
+
listTopics(): Promise<string[]>;
|
|
37
|
+
getSubscriptionCount(topic: string): Promise<number>;
|
|
38
|
+
shutdown(): Promise<void>;
|
|
39
|
+
}
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
export class FileStreamAdapter {
|
|
2
|
+
subscriptions = /* @__PURE__ */ new Map();
|
|
3
|
+
subscriptionCounter = 0;
|
|
4
|
+
async init() {
|
|
5
|
+
}
|
|
6
|
+
async publish(topic, event) {
|
|
7
|
+
const topicSubscriptions = this.subscriptions.get(topic);
|
|
8
|
+
if (!topicSubscriptions || topicSubscriptions.size === 0) {
|
|
9
|
+
return;
|
|
10
|
+
}
|
|
11
|
+
const eventWithTs = {
|
|
12
|
+
...event,
|
|
13
|
+
timestamp: event.timestamp || Date.now()
|
|
14
|
+
};
|
|
15
|
+
const promises = [];
|
|
16
|
+
Array.from(topicSubscriptions.values()).forEach((handler) => {
|
|
17
|
+
promises.push(handler(eventWithTs));
|
|
18
|
+
});
|
|
19
|
+
await Promise.all(promises);
|
|
20
|
+
}
|
|
21
|
+
async subscribe(topic, handler, _opts) {
|
|
22
|
+
const subscriptionId = `sub-${++this.subscriptionCounter}`;
|
|
23
|
+
if (!this.subscriptions.has(topic)) {
|
|
24
|
+
this.subscriptions.set(topic, /* @__PURE__ */ new Map());
|
|
25
|
+
}
|
|
26
|
+
const topicSubscriptions = this.subscriptions.get(topic);
|
|
27
|
+
topicSubscriptions.set(subscriptionId, handler);
|
|
28
|
+
return {
|
|
29
|
+
id: subscriptionId,
|
|
30
|
+
topic,
|
|
31
|
+
unsubscribe: async () => {
|
|
32
|
+
await this.unsubscribe({ id: subscriptionId, topic, unsubscribe: async () => {
|
|
33
|
+
} });
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
async unsubscribe(handle) {
|
|
38
|
+
const topicSubscriptions = this.subscriptions.get(handle.topic);
|
|
39
|
+
if (topicSubscriptions) {
|
|
40
|
+
topicSubscriptions.delete(handle.id);
|
|
41
|
+
if (topicSubscriptions.size === 0) {
|
|
42
|
+
this.subscriptions.delete(handle.topic);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
async listTopics() {
|
|
47
|
+
return Array.from(this.subscriptions.keys());
|
|
48
|
+
}
|
|
49
|
+
async getSubscriptionCount(topic) {
|
|
50
|
+
const topicSubscriptions = this.subscriptions.get(topic);
|
|
51
|
+
return topicSubscriptions ? topicSubscriptions.size : 0;
|
|
52
|
+
}
|
|
53
|
+
async shutdown() {
|
|
54
|
+
this.subscriptions.clear();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Built-in Adapters
|
|
3
|
+
*
|
|
4
|
+
* In-memory and file-based adapters for development (no external dependencies)
|
|
5
|
+
*/
|
|
6
|
+
export { MemoryQueueAdapter } from './memory-queue.js';
|
|
7
|
+
export { MemoryStreamAdapter } from './memory-stream.js';
|
|
8
|
+
export { MemoryStoreAdapter } from './memory-store.js';
|
|
9
|
+
export { FileQueueAdapter } from './file-queue.js';
|
|
10
|
+
export { FileStoreAdapter } from './file-store.js';
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export { MemoryQueueAdapter } from "./memory-queue.js";
|
|
2
|
+
export { MemoryStreamAdapter } from "./memory-stream.js";
|
|
3
|
+
export { MemoryStoreAdapter } from "./memory-store.js";
|
|
4
|
+
export { FileQueueAdapter } from "./file-queue.js";
|
|
5
|
+
export { FileStoreAdapter } from "./file-store.js";
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory Queue Adapter
|
|
3
|
+
*
|
|
4
|
+
* In-memory queue implementation using fastq for development
|
|
5
|
+
* - Fast and simple
|
|
6
|
+
* - No external dependencies beyond fastq
|
|
7
|
+
* - Data lost on restart
|
|
8
|
+
* - Single instance only
|
|
9
|
+
*
|
|
10
|
+
* Architecture:
|
|
11
|
+
* - QueueAdapter: Manages job storage and queue events
|
|
12
|
+
* - WorkerManager: Executes jobs using fastq
|
|
13
|
+
* - Dispatcher pattern: Routes jobs to handlers by job.name
|
|
14
|
+
*/
|
|
15
|
+
import type { QueueAdapter, JobInput, Job, JobsQuery, ScheduleOptions, JobCounts, QueueEvent, WorkerHandler, WorkerOptions } from '../interfaces/queue.js';
|
|
16
|
+
export interface MemoryQueueAdapterOptions {
|
|
17
|
+
maxQueueSize?: number;
|
|
18
|
+
}
|
|
19
|
+
export declare class MemoryQueueAdapter implements QueueAdapter {
|
|
20
|
+
private jobs;
|
|
21
|
+
private eventListeners;
|
|
22
|
+
private workers;
|
|
23
|
+
private options;
|
|
24
|
+
constructor(options?: MemoryQueueAdapterOptions);
|
|
25
|
+
init(): Promise<void>;
|
|
26
|
+
enqueue(queueName: string, job: JobInput): Promise<string>;
|
|
27
|
+
schedule(queueName: string, job: JobInput, opts?: ScheduleOptions): Promise<string>;
|
|
28
|
+
getJob(_queueName: string, id: string): Promise<Job | null>;
|
|
29
|
+
getJobs(queueName: string, query?: JobsQuery): Promise<Job[]>;
|
|
30
|
+
on(queueName: string, event: QueueEvent, callback: (payload: any) => void): () => void;
|
|
31
|
+
isPaused(queueName: string): Promise<boolean>;
|
|
32
|
+
getJobCounts(queueName: string): Promise<JobCounts>;
|
|
33
|
+
pause(queueName: string): Promise<void>;
|
|
34
|
+
resume(queueName: string): Promise<void>;
|
|
35
|
+
close(): Promise<void>;
|
|
36
|
+
/**
|
|
37
|
+
* Register a worker handler for a queue
|
|
38
|
+
* This is called by the worker registration system
|
|
39
|
+
*/
|
|
40
|
+
registerWorker(queueName: string, jobName: string, handler: WorkerHandler, opts?: WorkerOptions): void;
|
|
41
|
+
/**
|
|
42
|
+
* Start processing waiting jobs for a queue
|
|
43
|
+
* Should be called after all handlers are registered
|
|
44
|
+
*/
|
|
45
|
+
startProcessingQueue(queueName: string): void;
|
|
46
|
+
/**
|
|
47
|
+
* Update job state (called internally and by worker manager)
|
|
48
|
+
*/
|
|
49
|
+
updateJobState(jobId: string, state: Job['state'], extra?: Partial<Job>): void;
|
|
50
|
+
private emitEvent;
|
|
51
|
+
private generateId;
|
|
52
|
+
}
|
|
@@ -0,0 +1,243 @@
|
|
|
1
|
+
import * as fastq from "fastq";
|
|
2
|
+
export class MemoryQueueAdapter {
|
|
3
|
+
jobs = /* @__PURE__ */ new Map();
|
|
4
|
+
eventListeners = /* @__PURE__ */ new Map();
|
|
5
|
+
workers = /* @__PURE__ */ new Map();
|
|
6
|
+
options;
|
|
7
|
+
constructor(options = {}) {
|
|
8
|
+
this.options = {
|
|
9
|
+
maxQueueSize: options.maxQueueSize || 1e3
|
|
10
|
+
};
|
|
11
|
+
}
|
|
12
|
+
async init() {
|
|
13
|
+
}
|
|
14
|
+
async enqueue(queueName, job) {
|
|
15
|
+
const jobId = job.opts?.jobId || this.generateId();
|
|
16
|
+
if (this.jobs.has(jobId)) {
|
|
17
|
+
return jobId;
|
|
18
|
+
}
|
|
19
|
+
if (this.jobs.size >= this.options.maxQueueSize) {
|
|
20
|
+
throw new Error(`Queue ${queueName} is full (max: ${this.options.maxQueueSize})`);
|
|
21
|
+
}
|
|
22
|
+
const internalJob = {
|
|
23
|
+
id: jobId,
|
|
24
|
+
name: job.name,
|
|
25
|
+
data: { ...job.data, __queueName: queueName },
|
|
26
|
+
state: "waiting",
|
|
27
|
+
timestamp: Date.now()
|
|
28
|
+
};
|
|
29
|
+
this.jobs.set(jobId, internalJob);
|
|
30
|
+
this.emitEvent(queueName, "waiting", { jobId, job: internalJob });
|
|
31
|
+
const workerInfo = this.workers.get(queueName);
|
|
32
|
+
if (workerInfo && !workerInfo.paused) {
|
|
33
|
+
workerInfo.queue.push({ jobId, jobName: job.name, data: job.data }).catch((error) => {
|
|
34
|
+
console.error(`[MemoryQueueAdapter] Error processing job ${jobId}:`, error);
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
return jobId;
|
|
38
|
+
}
|
|
39
|
+
async schedule(queueName, job, opts) {
|
|
40
|
+
if (opts?.delay) {
|
|
41
|
+
const jobId = this.generateId();
|
|
42
|
+
const internalJob = {
|
|
43
|
+
id: jobId,
|
|
44
|
+
name: job.name,
|
|
45
|
+
data: { ...job.data, __queueName: queueName },
|
|
46
|
+
state: "delayed",
|
|
47
|
+
timestamp: Date.now()
|
|
48
|
+
};
|
|
49
|
+
this.jobs.set(jobId, internalJob);
|
|
50
|
+
this.emitEvent(queueName, "delayed", { jobId, job: internalJob, delay: opts.delay });
|
|
51
|
+
setTimeout(() => {
|
|
52
|
+
this.enqueue(queueName, job);
|
|
53
|
+
}, opts.delay);
|
|
54
|
+
return jobId;
|
|
55
|
+
}
|
|
56
|
+
if (opts?.cron || opts?.repeat) {
|
|
57
|
+
throw new Error("Cron/repeat scheduling not supported in memory adapter");
|
|
58
|
+
}
|
|
59
|
+
return this.enqueue(queueName, job);
|
|
60
|
+
}
|
|
61
|
+
async getJob(_queueName, id) {
|
|
62
|
+
return this.jobs.get(id) || null;
|
|
63
|
+
}
|
|
64
|
+
async getJobs(queueName, query) {
|
|
65
|
+
let jobs = Array.from(this.jobs.values()).filter((j) => j.data?.__queueName === queueName);
|
|
66
|
+
if (query?.state && query.state.length > 0) {
|
|
67
|
+
jobs = jobs.filter((j) => query.state.includes(j.state));
|
|
68
|
+
}
|
|
69
|
+
if (query?.offset) {
|
|
70
|
+
jobs = jobs.slice(query.offset);
|
|
71
|
+
}
|
|
72
|
+
if (query?.limit) {
|
|
73
|
+
jobs = jobs.slice(0, query.limit);
|
|
74
|
+
}
|
|
75
|
+
return jobs;
|
|
76
|
+
}
|
|
77
|
+
on(queueName, event, callback) {
|
|
78
|
+
const key = `${queueName}:${event}`;
|
|
79
|
+
if (!this.eventListeners.has(key)) {
|
|
80
|
+
this.eventListeners.set(key, []);
|
|
81
|
+
}
|
|
82
|
+
this.eventListeners.get(key).push(callback);
|
|
83
|
+
return () => {
|
|
84
|
+
const listeners = this.eventListeners.get(key);
|
|
85
|
+
if (listeners) {
|
|
86
|
+
const index = listeners.indexOf(callback);
|
|
87
|
+
if (index > -1) {
|
|
88
|
+
listeners.splice(index, 1);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
async isPaused(queueName) {
|
|
94
|
+
const workerInfo = this.workers.get(queueName);
|
|
95
|
+
return workerInfo?.paused || false;
|
|
96
|
+
}
|
|
97
|
+
async getJobCounts(queueName) {
|
|
98
|
+
const jobs = Array.from(this.jobs.values()).filter((j) => j.data?.__queueName === queueName);
|
|
99
|
+
return {
|
|
100
|
+
active: jobs.filter((j) => j.state === "active").length,
|
|
101
|
+
completed: jobs.filter((j) => j.state === "completed").length,
|
|
102
|
+
failed: jobs.filter((j) => j.state === "failed").length,
|
|
103
|
+
delayed: jobs.filter((j) => j.state === "delayed").length,
|
|
104
|
+
waiting: jobs.filter((j) => j.state === "waiting").length,
|
|
105
|
+
paused: jobs.filter((j) => j.state === "paused").length
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
async pause(queueName) {
|
|
109
|
+
const workerInfo = this.workers.get(queueName);
|
|
110
|
+
if (workerInfo) {
|
|
111
|
+
workerInfo.paused = true;
|
|
112
|
+
workerInfo.queue.pause();
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
async resume(queueName) {
|
|
116
|
+
const workerInfo = this.workers.get(queueName);
|
|
117
|
+
if (workerInfo) {
|
|
118
|
+
workerInfo.paused = false;
|
|
119
|
+
workerInfo.queue.resume();
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
async close() {
|
|
123
|
+
const drainPromises = Array.from(this.workers.values()).map((w) => w.queue.drained());
|
|
124
|
+
await Promise.all(drainPromises);
|
|
125
|
+
this.jobs.clear();
|
|
126
|
+
this.eventListeners.clear();
|
|
127
|
+
this.workers.clear();
|
|
128
|
+
}
|
|
129
|
+
// ============================================================
|
|
130
|
+
// Worker Management (used by worker registration system)
|
|
131
|
+
// ============================================================
|
|
132
|
+
/**
|
|
133
|
+
* Register a worker handler for a queue
|
|
134
|
+
* This is called by the worker registration system
|
|
135
|
+
*/
|
|
136
|
+
registerWorker(queueName, jobName, handler, opts) {
|
|
137
|
+
let workerInfo = this.workers.get(queueName);
|
|
138
|
+
if (workerInfo) {
|
|
139
|
+
console.info(`[MemoryQueue] Adding handler for job "${jobName}" to queue "${queueName}"`);
|
|
140
|
+
workerInfo.handlers.set(jobName, handler);
|
|
141
|
+
return;
|
|
142
|
+
}
|
|
143
|
+
console.info(`[MemoryQueue] Creating new worker for queue: ${queueName}`);
|
|
144
|
+
const handlers = /* @__PURE__ */ new Map();
|
|
145
|
+
handlers.set(jobName, handler);
|
|
146
|
+
const dispatcher = async (task) => {
|
|
147
|
+
const handler2 = handlers.get(task.jobName);
|
|
148
|
+
if (!handler2) {
|
|
149
|
+
const error = `No handler for job "${task.jobName}" on queue "${queueName}". Available: ${Array.from(handlers.keys()).join(", ")}`;
|
|
150
|
+
console.error(error);
|
|
151
|
+
throw new Error(error);
|
|
152
|
+
}
|
|
153
|
+
this.updateJobState(task.jobId, "active", { processedOn: Date.now() });
|
|
154
|
+
this.emitEvent(queueName, "active", { jobId: task.jobId });
|
|
155
|
+
try {
|
|
156
|
+
const result = await handler2(task.data, {
|
|
157
|
+
jobId: task.jobId,
|
|
158
|
+
queueName
|
|
159
|
+
// Add more context as needed
|
|
160
|
+
});
|
|
161
|
+
if (result && typeof result === "object" && result.awaiting === true) {
|
|
162
|
+
this.jobs.delete(task.jobId);
|
|
163
|
+
return result;
|
|
164
|
+
}
|
|
165
|
+
this.updateJobState(task.jobId, "completed", {
|
|
166
|
+
returnvalue: result,
|
|
167
|
+
finishedOn: Date.now()
|
|
168
|
+
});
|
|
169
|
+
this.emitEvent(queueName, "completed", { jobId: task.jobId, returnvalue: result });
|
|
170
|
+
return result;
|
|
171
|
+
} catch (err) {
|
|
172
|
+
this.updateJobState(task.jobId, "failed", {
|
|
173
|
+
failedReason: err.message,
|
|
174
|
+
finishedOn: Date.now()
|
|
175
|
+
});
|
|
176
|
+
this.emitEvent(queueName, "failed", {
|
|
177
|
+
jobId: task.jobId,
|
|
178
|
+
failedReason: err.message
|
|
179
|
+
});
|
|
180
|
+
throw err;
|
|
181
|
+
}
|
|
182
|
+
};
|
|
183
|
+
const concurrency = opts?.concurrency || 1;
|
|
184
|
+
const queue = fastq.promise(dispatcher, concurrency);
|
|
185
|
+
const shouldPause = opts?.autorun === false;
|
|
186
|
+
workerInfo = {
|
|
187
|
+
queue,
|
|
188
|
+
handlers,
|
|
189
|
+
paused: shouldPause || false,
|
|
190
|
+
concurrency
|
|
191
|
+
};
|
|
192
|
+
if (shouldPause) {
|
|
193
|
+
queue.pause();
|
|
194
|
+
console.info(`[MemoryQueue] Worker for "${queueName}" created but paused`);
|
|
195
|
+
}
|
|
196
|
+
this.workers.set(queueName, workerInfo);
|
|
197
|
+
}
|
|
198
|
+
/**
|
|
199
|
+
* Start processing waiting jobs for a queue
|
|
200
|
+
* Should be called after all handlers are registered
|
|
201
|
+
*/
|
|
202
|
+
startProcessingQueue(queueName) {
|
|
203
|
+
const workerInfo = this.workers.get(queueName);
|
|
204
|
+
if (!workerInfo || workerInfo.paused) return;
|
|
205
|
+
const waitingJobs = Array.from(this.jobs.values()).filter(
|
|
206
|
+
(j) => j.state === "waiting" && j.data?.__queueName === queueName
|
|
207
|
+
);
|
|
208
|
+
for (const job of waitingJobs) {
|
|
209
|
+
workerInfo.queue.push({ jobId: job.id, jobName: job.name, data: job.data }).catch((error) => {
|
|
210
|
+
console.error(`[MemoryQueueAdapter] Error processing queued job ${job.id}:`, error);
|
|
211
|
+
});
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
// ============================================================
|
|
215
|
+
// Helper Methods
|
|
216
|
+
// ============================================================
|
|
217
|
+
/**
|
|
218
|
+
* Update job state (called internally and by worker manager)
|
|
219
|
+
*/
|
|
220
|
+
updateJobState(jobId, state, extra) {
|
|
221
|
+
const job = this.jobs.get(jobId);
|
|
222
|
+
if (job) {
|
|
223
|
+
job.state = state;
|
|
224
|
+
if (extra) {
|
|
225
|
+
Object.assign(job, extra);
|
|
226
|
+
}
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
emitEvent(queueName, event, payload) {
|
|
230
|
+
const key = `${queueName}:${event}`;
|
|
231
|
+
const listeners = this.eventListeners.get(key) || [];
|
|
232
|
+
for (const callback of listeners) {
|
|
233
|
+
try {
|
|
234
|
+
callback(payload);
|
|
235
|
+
} catch (error) {
|
|
236
|
+
console.error(`[MemoryQueueAdapter] Error in event listener for ${key}:`, error);
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
generateId() {
|
|
241
|
+
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Memory Store Adapter
|
|
3
|
+
*
|
|
4
|
+
* In-memory storage implementation for development
|
|
5
|
+
* Three-tier storage:
|
|
6
|
+
* 1. Event Stream - Append-only event log
|
|
7
|
+
* 2. Sorted Index - Time-ordered metadata storage
|
|
8
|
+
* 3. Key-Value Store - Fast lookups
|
|
9
|
+
*
|
|
10
|
+
* All data is lost on restart (ephemeral)
|
|
11
|
+
*/
|
|
12
|
+
import type { StoreAdapter, EventRecord, EventReadOptions, EventSubscription } from '../interfaces/store.js';
|
|
13
|
+
export declare class MemoryStoreAdapter implements StoreAdapter {
|
|
14
|
+
private eventStreams;
|
|
15
|
+
private eventSubscriptions;
|
|
16
|
+
private subscriptionCounter;
|
|
17
|
+
private kvStore;
|
|
18
|
+
private sortedIndices;
|
|
19
|
+
protected validator: import("../base/index.js").StoreValidator;
|
|
20
|
+
private indexLocks;
|
|
21
|
+
close(): Promise<void>;
|
|
22
|
+
stream: {
|
|
23
|
+
append: (subject: string, event: Omit<EventRecord, "id" | "ts">) => Promise<EventRecord>;
|
|
24
|
+
read: (subject: string, opts?: EventReadOptions) => Promise<EventRecord[]>;
|
|
25
|
+
subscribe: (subject: string, onEvent: (event: EventRecord) => void) => Promise<EventSubscription>;
|
|
26
|
+
delete: (subject: string) => Promise<boolean>;
|
|
27
|
+
};
|
|
28
|
+
private notifySubscribers;
|
|
29
|
+
kv: {
|
|
30
|
+
get: <T = any>(key: string) => Promise<T | null>;
|
|
31
|
+
set: <T = any>(key: string, value: T, _ttl?: number) => Promise<void>;
|
|
32
|
+
delete: (key: string) => Promise<void>;
|
|
33
|
+
clear: (pattern: string) => Promise<number>;
|
|
34
|
+
increment: (key: string, by?: number) => Promise<number>;
|
|
35
|
+
};
|
|
36
|
+
index: {
|
|
37
|
+
add: (key: string, id: string, score: number, metadata?: Record<string, any>) => Promise<void>;
|
|
38
|
+
get: (key: string, id: string) => Promise<{
|
|
39
|
+
id: string;
|
|
40
|
+
score: number;
|
|
41
|
+
metadata?: any;
|
|
42
|
+
} | null>;
|
|
43
|
+
read: (key: string, opts?: {
|
|
44
|
+
offset?: number;
|
|
45
|
+
limit?: number;
|
|
46
|
+
}) => Promise<Array<{
|
|
47
|
+
id: string;
|
|
48
|
+
score: number;
|
|
49
|
+
metadata?: any;
|
|
50
|
+
}>>;
|
|
51
|
+
update: (key: string, id: string, metadata: Record<string, any>) => Promise<boolean>;
|
|
52
|
+
updateWithRetry: (key: string, id: string, metadata: Record<string, any>, maxRetries?: number) => Promise<void>;
|
|
53
|
+
increment: (key: string, id: string, field: string, increment?: number) => Promise<number>;
|
|
54
|
+
delete: (key: string, id: string) => Promise<boolean>;
|
|
55
|
+
};
|
|
56
|
+
/**
|
|
57
|
+
* Acquire a lock for atomic index operations
|
|
58
|
+
* Protected to allow access from FileStoreAdapter
|
|
59
|
+
*/
|
|
60
|
+
protected acquireIndexLock(key: string): Promise<() => void>;
|
|
61
|
+
/**
|
|
62
|
+
* Convert dot notation keys to nested objects
|
|
63
|
+
* e.g., { 'stats.totalFires': 5 } -> { stats: { totalFires: 5 } }
|
|
64
|
+
* null values are preserved for deletion
|
|
65
|
+
*/
|
|
66
|
+
private expandDotNotation;
|
|
67
|
+
private generateId;
|
|
68
|
+
}
|