nvent 0.4.5 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/module.d.mts +1 -1
- package/dist/module.mjs +433 -180
- package/dist/runtime/adapters/base/index.d.ts +6 -0
- package/dist/runtime/adapters/base/index.js +1 -0
- package/dist/runtime/adapters/base/store-validator.d.ts +48 -0
- package/dist/runtime/adapters/base/store-validator.js +147 -0
- package/dist/runtime/adapters/builtin/file-queue.d.ts +15 -1
- package/dist/runtime/adapters/builtin/file-queue.js +70 -6
- package/dist/runtime/adapters/builtin/file-store.d.ts +4 -18
- package/dist/runtime/adapters/builtin/file-store.js +90 -109
- package/dist/runtime/adapters/builtin/memory-queue.js +4 -0
- package/dist/runtime/adapters/builtin/memory-store.d.ts +42 -31
- package/dist/runtime/adapters/builtin/memory-store.js +253 -183
- package/dist/runtime/adapters/factory.d.ts +2 -2
- package/dist/runtime/adapters/factory.js +54 -20
- package/dist/runtime/adapters/interfaces/store.d.ts +177 -113
- package/dist/runtime/config/index.d.ts +2 -2
- package/dist/runtime/config/index.js +14 -6
- package/dist/runtime/config/types.d.ts +32 -2
- package/dist/runtime/events/eventBus.d.ts +1 -1
- package/dist/runtime/events/types.d.ts +31 -2
- package/dist/runtime/events/utils/scheduleTrigger.d.ts +8 -0
- package/dist/runtime/events/utils/scheduleTrigger.js +69 -0
- package/dist/runtime/events/utils/stallDetector.d.ts +44 -3
- package/dist/runtime/events/utils/stallDetector.js +288 -89
- package/dist/runtime/events/utils/triggerRuntime.d.ts +58 -0
- package/dist/runtime/events/utils/triggerRuntime.js +212 -0
- package/dist/runtime/events/wiring/flowWiring.d.ts +11 -5
- package/dist/runtime/events/wiring/flowWiring.js +620 -92
- package/dist/runtime/events/wiring/registry.d.ts +2 -2
- package/dist/runtime/events/wiring/registry.js +8 -6
- package/dist/runtime/events/wiring/streamWiring.d.ts +15 -11
- package/dist/runtime/events/wiring/streamWiring.js +88 -11
- package/dist/runtime/events/wiring/triggerWiring.d.ts +21 -0
- package/dist/runtime/events/wiring/triggerWiring.js +412 -0
- package/dist/runtime/{server → nitro}/plugins/00.adapters.js +8 -4
- package/dist/runtime/{server → nitro}/plugins/02.workers.js +21 -3
- package/dist/runtime/nitro/plugins/03.triggers.d.ts +12 -0
- package/dist/runtime/nitro/plugins/03.triggers.js +55 -0
- package/dist/runtime/nitro/routes/webhook.await.d.ts +23 -0
- package/dist/runtime/nitro/routes/webhook.await.js +90 -0
- package/dist/runtime/nitro/routes/webhook.trigger.d.ts +69 -0
- package/dist/runtime/nitro/routes/webhook.trigger.js +64 -0
- package/dist/runtime/{utils → nitro/utils}/adapters.d.ts +6 -6
- package/dist/runtime/nitro/utils/awaitPatterns/event.d.ts +15 -0
- package/dist/runtime/nitro/utils/awaitPatterns/event.js +120 -0
- package/dist/runtime/nitro/utils/awaitPatterns/index.d.ts +28 -0
- package/dist/runtime/nitro/utils/awaitPatterns/index.js +55 -0
- package/dist/runtime/nitro/utils/awaitPatterns/schedule.d.ts +16 -0
- package/dist/runtime/nitro/utils/awaitPatterns/schedule.js +78 -0
- package/dist/runtime/nitro/utils/awaitPatterns/time.d.ts +15 -0
- package/dist/runtime/nitro/utils/awaitPatterns/time.js +67 -0
- package/dist/runtime/nitro/utils/awaitPatterns/webhook.d.ts +15 -0
- package/dist/runtime/nitro/utils/awaitPatterns/webhook.js +120 -0
- package/dist/runtime/{utils → nitro/utils}/defineFunction.d.ts +2 -2
- package/dist/runtime/{utils → nitro/utils}/defineFunction.js +3 -3
- package/dist/runtime/{utils → nitro/utils}/defineFunctionConfig.d.ts +156 -0
- package/dist/runtime/{utils → nitro/utils}/defineFunctionConfig.js +1 -0
- package/dist/runtime/nitro/utils/defineHooks.d.ts +41 -0
- package/dist/runtime/nitro/utils/defineHooks.js +6 -0
- package/dist/runtime/{utils → nitro/utils}/registerAdapter.d.ts +3 -3
- package/dist/runtime/{utils → nitro/utils}/registerAdapter.js +1 -1
- package/dist/runtime/nitro/utils/useAwait.d.ts +71 -0
- package/dist/runtime/nitro/utils/useAwait.js +139 -0
- package/dist/runtime/{utils → nitro/utils}/useEventManager.d.ts +2 -2
- package/dist/runtime/{utils → nitro/utils}/useEventManager.js +1 -1
- package/dist/runtime/nitro/utils/useFlow.d.ts +68 -0
- package/dist/runtime/nitro/utils/useFlow.js +226 -0
- package/dist/runtime/nitro/utils/useHookRegistry.d.ts +34 -0
- package/dist/runtime/nitro/utils/useHookRegistry.js +25 -0
- package/dist/runtime/nitro/utils/useRunContext.d.ts +6 -0
- package/dist/runtime/nitro/utils/useRunContext.js +102 -0
- package/dist/runtime/nitro/utils/useStreamTopics.d.ts +83 -0
- package/dist/runtime/nitro/utils/useStreamTopics.js +94 -0
- package/dist/runtime/nitro/utils/useTrigger.d.ts +150 -0
- package/dist/runtime/nitro/utils/useTrigger.js +320 -0
- package/dist/runtime/scheduler/index.d.ts +33 -0
- package/dist/runtime/scheduler/index.js +38 -0
- package/dist/runtime/scheduler/scheduler.d.ts +113 -0
- package/dist/runtime/scheduler/scheduler.js +623 -0
- package/dist/runtime/scheduler/types.d.ts +116 -0
- package/dist/runtime/scheduler/types.js +0 -0
- package/dist/runtime/worker/node/runner.d.ts +12 -2
- package/dist/runtime/worker/node/runner.js +141 -37
- package/package.json +6 -6
- package/dist/runtime/server/api/_flows/[name]/clear-history.delete.d.ts +0 -10
- package/dist/runtime/server/api/_flows/[name]/clear-history.delete.js +0 -55
- package/dist/runtime/server/api/_flows/[name]/runs/[runId]/cancel.post.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/runs/[runId]/cancel.post.js +0 -21
- package/dist/runtime/server/api/_flows/[name]/runs.get.d.ts +0 -17
- package/dist/runtime/server/api/_flows/[name]/runs.get.js +0 -64
- package/dist/runtime/server/api/_flows/[name]/schedule.post.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/schedule.post.js +0 -66
- package/dist/runtime/server/api/_flows/[name]/schedules/[id].delete.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/schedules/[id].delete.js +0 -47
- package/dist/runtime/server/api/_flows/[name]/schedules.get.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/schedules.get.js +0 -50
- package/dist/runtime/server/api/_flows/[name]/start.post.d.ts +0 -2
- package/dist/runtime/server/api/_flows/[name]/start.post.js +0 -9
- package/dist/runtime/server/api/_flows/index.get.d.ts +0 -6
- package/dist/runtime/server/api/_flows/index.get.js +0 -5
- package/dist/runtime/server/api/_flows/ws.d.ts +0 -60
- package/dist/runtime/server/api/_flows/ws.js +0 -209
- package/dist/runtime/server/api/_queues/[name]/job/[id].get.d.ts +0 -2
- package/dist/runtime/server/api/_queues/[name]/job/[id].get.js +0 -14
- package/dist/runtime/server/api/_queues/[name]/job/index.get.d.ts +0 -2
- package/dist/runtime/server/api/_queues/[name]/job/index.get.js +0 -27
- package/dist/runtime/server/api/_queues/index.get.d.ts +0 -2
- package/dist/runtime/server/api/_queues/index.get.js +0 -106
- package/dist/runtime/server/api/_queues/ws.d.ts +0 -48
- package/dist/runtime/server/api/_queues/ws.js +0 -215
- package/dist/runtime/utils/useFlowEngine.d.ts +0 -19
- package/dist/runtime/utils/useFlowEngine.js +0 -108
- package/dist/runtime/utils/useStreamTopics.d.ts +0 -72
- package/dist/runtime/utils/useStreamTopics.js +0 -47
- /package/dist/runtime/{server → nitro}/plugins/00.adapters.d.ts +0 -0
- /package/dist/runtime/{server → nitro}/plugins/01.ws-lifecycle.d.ts +0 -0
- /package/dist/runtime/{server → nitro}/plugins/01.ws-lifecycle.js +0 -0
- /package/dist/runtime/{server → nitro}/plugins/02.workers.d.ts +0 -0
- /package/dist/runtime/{utils → nitro/utils}/adapters.js +0 -0
- /package/dist/runtime/{utils → nitro/utils}/useNventLogger.d.ts +0 -0
- /package/dist/runtime/{utils → nitro/utils}/useNventLogger.js +0 -0
- /package/dist/runtime/{utils → nitro/utils}/wsPeerManager.d.ts +0 -0
- /package/dist/runtime/{utils → nitro/utils}/wsPeerManager.js +0 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { StoreValidator, createStoreValidator } from "./store-validator.js";
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Store Adapter Validation
|
|
3
|
+
*
|
|
4
|
+
* Centralized validation logic for index updates to ensure correct usage
|
|
5
|
+
* across all store adapter implementations.
|
|
6
|
+
*/
|
|
7
|
+
export interface StoreValidatorOptions {
|
|
8
|
+
/** Adapter name for error messages */
|
|
9
|
+
adapterName: string;
|
|
10
|
+
/** Enable counter misuse warnings (default: true in development) */
|
|
11
|
+
warnCounters?: boolean;
|
|
12
|
+
/** Throw errors on validation failures (default: true) */
|
|
13
|
+
throwOnError?: boolean;
|
|
14
|
+
}
|
|
15
|
+
export declare class StoreValidator {
|
|
16
|
+
private readonly adapterName;
|
|
17
|
+
private readonly warnCounters;
|
|
18
|
+
private readonly throwOnError;
|
|
19
|
+
constructor(options: StoreValidatorOptions);
|
|
20
|
+
/**
|
|
21
|
+
* Validate index update payload
|
|
22
|
+
* Throws error if dot notation keys are detected (except 'version')
|
|
23
|
+
* Recursively checks all nested objects
|
|
24
|
+
*/
|
|
25
|
+
validateUpdatePayload(payload: Record<string, any>, method: string): void;
|
|
26
|
+
/**
|
|
27
|
+
* Recursively check for dot notation in object keys
|
|
28
|
+
*/
|
|
29
|
+
private checkDotNotationKeys;
|
|
30
|
+
/**
|
|
31
|
+
* Build a nested object suggestion from a dot notation key
|
|
32
|
+
*/
|
|
33
|
+
private buildNestedObjectSuggestion;
|
|
34
|
+
/**
|
|
35
|
+
* Detect potential counter misuse
|
|
36
|
+
* Logs warnings when numeric fields that look like counters are updated
|
|
37
|
+
* via update() instead of increment()
|
|
38
|
+
*/
|
|
39
|
+
private detectCounterMisuse;
|
|
40
|
+
/**
|
|
41
|
+
* Check if a field looks like a counter that should use increment()
|
|
42
|
+
*/
|
|
43
|
+
private isCounterField;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Factory function to create a validator for an adapter
|
|
47
|
+
*/
|
|
48
|
+
export declare function createStoreValidator(adapterName: string, options?: Partial<StoreValidatorOptions>): StoreValidator;
|
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
export class StoreValidator {
|
|
2
|
+
adapterName;
|
|
3
|
+
warnCounters;
|
|
4
|
+
throwOnError;
|
|
5
|
+
constructor(options) {
|
|
6
|
+
this.adapterName = options.adapterName;
|
|
7
|
+
this.warnCounters = options.warnCounters ?? process.env.NODE_ENV === "development";
|
|
8
|
+
this.throwOnError = options.throwOnError ?? true;
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Validate index update payload
|
|
12
|
+
* Throws error if dot notation keys are detected (except 'version')
|
|
13
|
+
* Recursively checks all nested objects
|
|
14
|
+
*/
|
|
15
|
+
validateUpdatePayload(payload, method) {
|
|
16
|
+
this.checkDotNotationKeys(payload, method, []);
|
|
17
|
+
if (this.warnCounters) {
|
|
18
|
+
this.detectCounterMisuse(payload, method);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Recursively check for dot notation in object keys
|
|
23
|
+
*/
|
|
24
|
+
checkDotNotationKeys(obj, method, path) {
|
|
25
|
+
for (const key of Object.keys(obj)) {
|
|
26
|
+
const currentPath = [...path, key];
|
|
27
|
+
if (key.includes(".") && key !== "version") {
|
|
28
|
+
const fullPath = currentPath.join(".");
|
|
29
|
+
const suggestion = this.buildNestedObjectSuggestion(key);
|
|
30
|
+
const message = [
|
|
31
|
+
`[${this.adapterName}] Invalid update payload in ${method}:`,
|
|
32
|
+
`Dot notation key "${key}" detected at path "${fullPath}".`,
|
|
33
|
+
`Keys should not contain dots. Split "${key}" into nested objects: ${suggestion}`
|
|
34
|
+
].join("\n");
|
|
35
|
+
console.error(message);
|
|
36
|
+
if (this.throwOnError) {
|
|
37
|
+
throw new Error(
|
|
38
|
+
`Invalid index update: dot notation keys not supported. Key "${key}" at path "${fullPath}" contains dots.`
|
|
39
|
+
);
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
const value = obj[key];
|
|
43
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
44
|
+
this.checkDotNotationKeys(value, method, currentPath);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* Build a nested object suggestion from a dot notation key
|
|
50
|
+
*/
|
|
51
|
+
buildNestedObjectSuggestion(dotKey) {
|
|
52
|
+
const parts = dotKey.split(".");
|
|
53
|
+
let suggestion = "{ ";
|
|
54
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
55
|
+
suggestion += `${parts[i]}: { `;
|
|
56
|
+
}
|
|
57
|
+
suggestion += `${parts[parts.length - 1]}: value `;
|
|
58
|
+
for (let i = 0; i < parts.length - 1; i++) {
|
|
59
|
+
suggestion += "}";
|
|
60
|
+
}
|
|
61
|
+
suggestion += " }";
|
|
62
|
+
return suggestion;
|
|
63
|
+
}
|
|
64
|
+
/**
|
|
65
|
+
* Detect potential counter misuse
|
|
66
|
+
* Logs warnings when numeric fields that look like counters are updated
|
|
67
|
+
* via update() instead of increment()
|
|
68
|
+
*/
|
|
69
|
+
detectCounterMisuse(payload, method) {
|
|
70
|
+
const checkNested = (obj, path = []) => {
|
|
71
|
+
for (const [key, value] of Object.entries(obj)) {
|
|
72
|
+
const currentPath = [...path, key];
|
|
73
|
+
if (this.isCounterField(key, value, currentPath)) {
|
|
74
|
+
console.warn(
|
|
75
|
+
`[${this.adapterName}] Potential counter misuse in ${method}:`,
|
|
76
|
+
`Field "${currentPath.join(".")}" is numeric.`,
|
|
77
|
+
`Consider using index.increment() for atomic updates instead of ${method}.`
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
if (value && typeof value === "object" && !Array.isArray(value)) {
|
|
81
|
+
checkNested(value, currentPath);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
};
|
|
85
|
+
checkNested(payload);
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Check if a field looks like a counter that should use increment()
|
|
89
|
+
*/
|
|
90
|
+
isCounterField(key, value, path) {
|
|
91
|
+
if (typeof value !== "number") {
|
|
92
|
+
return false;
|
|
93
|
+
}
|
|
94
|
+
if (path.includes("emittedEvents")) {
|
|
95
|
+
return false;
|
|
96
|
+
}
|
|
97
|
+
const metadataFields = [
|
|
98
|
+
"stepCount",
|
|
99
|
+
// Total steps in flow (constant)
|
|
100
|
+
"timeout",
|
|
101
|
+
// Timeout duration
|
|
102
|
+
"delay",
|
|
103
|
+
// Delay duration
|
|
104
|
+
"timestamp",
|
|
105
|
+
// Timestamp values
|
|
106
|
+
"score",
|
|
107
|
+
// Score values
|
|
108
|
+
"priority",
|
|
109
|
+
// Priority values
|
|
110
|
+
"version"
|
|
111
|
+
// Version field
|
|
112
|
+
];
|
|
113
|
+
if (metadataFields.includes(key)) {
|
|
114
|
+
return false;
|
|
115
|
+
}
|
|
116
|
+
if (key.endsWith("At")) {
|
|
117
|
+
return false;
|
|
118
|
+
}
|
|
119
|
+
const counterKeywords = [
|
|
120
|
+
"count",
|
|
121
|
+
"total",
|
|
122
|
+
"running",
|
|
123
|
+
"awaiting",
|
|
124
|
+
"success",
|
|
125
|
+
"failure",
|
|
126
|
+
"cancel",
|
|
127
|
+
"pending",
|
|
128
|
+
"completed",
|
|
129
|
+
"failed",
|
|
130
|
+
"retries",
|
|
131
|
+
"attempts"
|
|
132
|
+
];
|
|
133
|
+
if (counterKeywords.some((keyword) => key.toLowerCase().includes(keyword))) {
|
|
134
|
+
return true;
|
|
135
|
+
}
|
|
136
|
+
if (path.includes("stats")) {
|
|
137
|
+
return true;
|
|
138
|
+
}
|
|
139
|
+
return false;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
export function createStoreValidator(adapterName, options) {
|
|
143
|
+
return new StoreValidator({
|
|
144
|
+
adapterName,
|
|
145
|
+
...options
|
|
146
|
+
});
|
|
147
|
+
}
|
|
@@ -8,7 +8,7 @@
|
|
|
8
8
|
* - Single instance only (no distributed lock)
|
|
9
9
|
*
|
|
10
10
|
* Storage format:
|
|
11
|
-
* - {dataDir}/
|
|
11
|
+
* - {dataDir}/{queueName}/jobs/{jobId}.json - Individual job files
|
|
12
12
|
* - Jobs are loaded on init and kept in memory with fastq
|
|
13
13
|
*/
|
|
14
14
|
import type { QueueAdapter, JobInput, Job, JobsQuery, ScheduleOptions, JobCounts, QueueEvent, WorkerHandler, WorkerOptions } from '../interfaces/queue.js';
|
|
@@ -48,6 +48,20 @@ export declare class FileQueueAdapter implements QueueAdapter {
|
|
|
48
48
|
startProcessingQueue(queueName: string): void;
|
|
49
49
|
private updateJobState;
|
|
50
50
|
private emitEvent;
|
|
51
|
+
/**
|
|
52
|
+
* Clean up completed jobs based on removeOnComplete option
|
|
53
|
+
* - true: delete immediately
|
|
54
|
+
* - false/undefined: keep forever
|
|
55
|
+
* - number: keep last N jobs, delete older ones
|
|
56
|
+
*/
|
|
57
|
+
private cleanupCompletedJobs;
|
|
58
|
+
/**
|
|
59
|
+
* Clean up failed jobs based on removeOnFail option
|
|
60
|
+
* - true: delete immediately
|
|
61
|
+
* - false/undefined: keep forever
|
|
62
|
+
* - number: keep last N jobs, delete older ones
|
|
63
|
+
*/
|
|
64
|
+
private cleanupFailedJobs;
|
|
51
65
|
private generateId;
|
|
52
66
|
private scheduleCronJob;
|
|
53
67
|
}
|
|
@@ -17,12 +17,12 @@ export class FileQueueAdapter {
|
|
|
17
17
|
}
|
|
18
18
|
async init() {
|
|
19
19
|
if (this.initialized) return;
|
|
20
|
-
await fs.mkdir(
|
|
20
|
+
await fs.mkdir(this.options.dataDir, { recursive: true });
|
|
21
21
|
await this.loadJobsFromDisk();
|
|
22
22
|
this.initialized = true;
|
|
23
23
|
}
|
|
24
24
|
async loadJobsFromDisk() {
|
|
25
|
-
const queuesDir =
|
|
25
|
+
const queuesDir = this.options.dataDir;
|
|
26
26
|
try {
|
|
27
27
|
const queueNames = await fs.readdir(queuesDir);
|
|
28
28
|
for (const queueName of queueNames) {
|
|
@@ -43,13 +43,13 @@ export class FileQueueAdapter {
|
|
|
43
43
|
}
|
|
44
44
|
}
|
|
45
45
|
async persistJob(queueName, job) {
|
|
46
|
-
const jobsDir = join(this.options.dataDir,
|
|
46
|
+
const jobsDir = join(this.options.dataDir, queueName, "jobs");
|
|
47
47
|
await fs.mkdir(jobsDir, { recursive: true });
|
|
48
48
|
const jobPath = join(jobsDir, `${job.id}.json`);
|
|
49
49
|
await fs.writeFile(jobPath, JSON.stringify(job, null, 2));
|
|
50
50
|
}
|
|
51
51
|
async deleteJobFile(queueName, jobId) {
|
|
52
|
-
const jobPath = join(this.options.dataDir,
|
|
52
|
+
const jobPath = join(this.options.dataDir, queueName, "jobs", `${jobId}.json`);
|
|
53
53
|
try {
|
|
54
54
|
await fs.unlink(jobPath);
|
|
55
55
|
} catch {
|
|
@@ -78,7 +78,7 @@ export class FileQueueAdapter {
|
|
|
78
78
|
this.emitEvent(queueName, "waiting", { jobId, job: internalJob });
|
|
79
79
|
const workerInfo = this.workers.get(queueName);
|
|
80
80
|
if (workerInfo && !workerInfo.paused) {
|
|
81
|
-
workerInfo.queue.push({ jobId, jobName: job.name, data:
|
|
81
|
+
workerInfo.queue.push({ jobId, jobName: job.name, data: internalJob.data }).catch(() => {
|
|
82
82
|
});
|
|
83
83
|
}
|
|
84
84
|
return jobId;
|
|
@@ -285,12 +285,17 @@ export class FileQueueAdapter {
|
|
|
285
285
|
opts: { attempts: maxAttempts, ...storedJob.opts }
|
|
286
286
|
};
|
|
287
287
|
const result = await handler2(jobLike, {});
|
|
288
|
+
if (result && typeof result === "object" && result.awaiting === true) {
|
|
289
|
+
this.jobs.delete(task.jobId);
|
|
290
|
+
await this.deleteJobFile(queueName, task.jobId);
|
|
291
|
+
return result;
|
|
292
|
+
}
|
|
288
293
|
await this.updateJobState(queueName, task.jobId, "completed", {
|
|
289
294
|
returnvalue: result,
|
|
290
295
|
finishedOn: Date.now()
|
|
291
296
|
});
|
|
292
297
|
this.emitEvent(queueName, "completed", { jobId: task.jobId, returnvalue: result });
|
|
293
|
-
await this.
|
|
298
|
+
await this.cleanupCompletedJobs(queueName, storedJob.opts?.removeOnComplete);
|
|
294
299
|
return result;
|
|
295
300
|
} catch (err) {
|
|
296
301
|
const newAttemptCount = currentAttempts + 1;
|
|
@@ -331,6 +336,7 @@ export class FileQueueAdapter {
|
|
|
331
336
|
attemptsMade: newAttemptCount,
|
|
332
337
|
maxAttempts
|
|
333
338
|
});
|
|
339
|
+
await this.cleanupFailedJobs(queueName, storedJob.opts?.removeOnFail);
|
|
334
340
|
throw err;
|
|
335
341
|
}
|
|
336
342
|
};
|
|
@@ -388,6 +394,64 @@ export class FileQueueAdapter {
|
|
|
388
394
|
}
|
|
389
395
|
}
|
|
390
396
|
}
|
|
397
|
+
/**
|
|
398
|
+
* Clean up completed jobs based on removeOnComplete option
|
|
399
|
+
* - true: delete immediately
|
|
400
|
+
* - false/undefined: keep forever
|
|
401
|
+
* - number: keep last N jobs, delete older ones
|
|
402
|
+
*/
|
|
403
|
+
async cleanupCompletedJobs(queueName, removeOnComplete) {
|
|
404
|
+
if (removeOnComplete === false || removeOnComplete === void 0) {
|
|
405
|
+
return;
|
|
406
|
+
}
|
|
407
|
+
if (removeOnComplete === true) {
|
|
408
|
+
const completedJobs = Array.from(this.jobs.values()).filter(
|
|
409
|
+
(j) => j.state === "completed" && j.data?.__queueName === queueName
|
|
410
|
+
);
|
|
411
|
+
for (const job of completedJobs) {
|
|
412
|
+
this.jobs.delete(job.id);
|
|
413
|
+
await this.deleteJobFile(queueName, job.id);
|
|
414
|
+
}
|
|
415
|
+
return;
|
|
416
|
+
}
|
|
417
|
+
if (typeof removeOnComplete === "number") {
|
|
418
|
+
const completedJobs = Array.from(this.jobs.values()).filter((j) => j.state === "completed" && j.data?.__queueName === queueName).sort((a, b) => (b.finishedOn || 0) - (a.finishedOn || 0));
|
|
419
|
+
const jobsToDelete = completedJobs.slice(removeOnComplete);
|
|
420
|
+
for (const job of jobsToDelete) {
|
|
421
|
+
this.jobs.delete(job.id);
|
|
422
|
+
await this.deleteJobFile(queueName, job.id);
|
|
423
|
+
}
|
|
424
|
+
}
|
|
425
|
+
}
|
|
426
|
+
/**
|
|
427
|
+
* Clean up failed jobs based on removeOnFail option
|
|
428
|
+
* - true: delete immediately
|
|
429
|
+
* - false/undefined: keep forever
|
|
430
|
+
* - number: keep last N jobs, delete older ones
|
|
431
|
+
*/
|
|
432
|
+
async cleanupFailedJobs(queueName, removeOnFail) {
|
|
433
|
+
if (removeOnFail === false || removeOnFail === void 0) {
|
|
434
|
+
return;
|
|
435
|
+
}
|
|
436
|
+
if (removeOnFail === true) {
|
|
437
|
+
const failedJobs = Array.from(this.jobs.values()).filter(
|
|
438
|
+
(j) => j.state === "failed" && j.data?.__queueName === queueName
|
|
439
|
+
);
|
|
440
|
+
for (const job of failedJobs) {
|
|
441
|
+
this.jobs.delete(job.id);
|
|
442
|
+
await this.deleteJobFile(queueName, job.id);
|
|
443
|
+
}
|
|
444
|
+
return;
|
|
445
|
+
}
|
|
446
|
+
if (typeof removeOnFail === "number") {
|
|
447
|
+
const failedJobs = Array.from(this.jobs.values()).filter((j) => j.state === "failed" && j.data?.__queueName === queueName).sort((a, b) => (b.finishedOn || 0) - (a.finishedOn || 0));
|
|
448
|
+
const jobsToDelete = failedJobs.slice(removeOnFail);
|
|
449
|
+
for (const job of jobsToDelete) {
|
|
450
|
+
this.jobs.delete(job.id);
|
|
451
|
+
await this.deleteJobFile(queueName, job.id);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
}
|
|
391
455
|
generateId() {
|
|
392
456
|
return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
|
|
393
457
|
}
|
|
@@ -7,14 +7,13 @@
|
|
|
7
7
|
* - Survives restarts
|
|
8
8
|
* - Single instance only
|
|
9
9
|
*
|
|
10
|
-
* Storage format
|
|
11
|
-
* - {dataDir}/{subject}.ndjson - Event streams (append-only NDJSON)
|
|
10
|
+
* Storage format:
|
|
11
|
+
* - {dataDir}/streams/{subject}.ndjson - Event streams (append-only NDJSON)
|
|
12
12
|
* - {dataDir}/indices/{key}.json - Sorted indices (JSON arrays)
|
|
13
|
-
* - {dataDir}/docs/{collection}/{id}.json - Documents (individual JSON files)
|
|
14
13
|
* - {dataDir}/kv/{key}.json - KV store (individual JSON files)
|
|
15
14
|
*/
|
|
16
|
-
import { MemoryStoreAdapter
|
|
17
|
-
export interface FileStoreAdapterOptions
|
|
15
|
+
import { MemoryStoreAdapter } from './memory-store.js';
|
|
16
|
+
export interface FileStoreAdapterOptions {
|
|
18
17
|
dataDir: string;
|
|
19
18
|
}
|
|
20
19
|
/**
|
|
@@ -23,24 +22,11 @@ export interface FileStoreAdapterOptions extends MemoryStoreAdapterOptions {
|
|
|
23
22
|
*/
|
|
24
23
|
export declare class FileStoreAdapter extends MemoryStoreAdapter {
|
|
25
24
|
private options;
|
|
26
|
-
private parentKvGet;
|
|
27
|
-
private parentKvSet;
|
|
28
|
-
private parentKvDelete;
|
|
29
|
-
private parentKvClear;
|
|
30
|
-
private parentKvIncrement;
|
|
31
25
|
constructor(options: FileStoreAdapterOptions);
|
|
32
26
|
private streamPath;
|
|
33
27
|
private indexPath;
|
|
34
|
-
private docPath;
|
|
35
28
|
private kvPath;
|
|
36
29
|
init(): Promise<void>;
|
|
37
30
|
private loadFromDisk;
|
|
38
|
-
append(subject: string, event: Omit<import('../interfaces/store').EventRecord, 'id' | 'ts'>): Promise<import("..").EventRecord>;
|
|
39
|
-
save(collection: string, id: string, doc: Record<string, any>): Promise<void>;
|
|
40
|
-
delete(collection: string, id: string): Promise<void>;
|
|
41
|
-
indexAdd(key: string, id: string, score: number, metadata?: Record<string, any>): Promise<void>;
|
|
42
|
-
indexUpdate(key: string, id: string, metadata: Record<string, any>): Promise<boolean>;
|
|
43
|
-
indexUpdateWithRetry(key: string, id: string, metadata: Record<string, any>, maxRetries?: number): Promise<void>;
|
|
44
|
-
indexIncrement(key: string, id: string, field: string, increment?: number): Promise<number>;
|
|
45
31
|
close(): Promise<void>;
|
|
46
32
|
}
|
|
@@ -12,32 +12,22 @@ async function ensureDir(path) {
|
|
|
12
12
|
}
|
|
13
13
|
export class FileStoreAdapter extends MemoryStoreAdapter {
|
|
14
14
|
options;
|
|
15
|
-
parentKvGet;
|
|
16
|
-
parentKvSet;
|
|
17
|
-
parentKvDelete;
|
|
18
|
-
parentKvClear;
|
|
19
|
-
parentKvIncrement;
|
|
20
15
|
constructor(options) {
|
|
21
|
-
super(
|
|
16
|
+
super();
|
|
22
17
|
this.options = options;
|
|
23
|
-
const
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
this.parentKvDelete = tempKv.delete;
|
|
27
|
-
this.parentKvClear = tempKv.clear;
|
|
28
|
-
this.parentKvIncrement = tempKv.increment;
|
|
18
|
+
const parentKv = this.kv;
|
|
19
|
+
const parentStream = this.stream;
|
|
20
|
+
const parentIndex = this.index;
|
|
29
21
|
this.kv = {
|
|
30
|
-
get:
|
|
31
|
-
return this.parentKvGet(key);
|
|
32
|
-
},
|
|
22
|
+
get: parentKv.get,
|
|
33
23
|
set: async (key, value, ttl) => {
|
|
34
|
-
await
|
|
24
|
+
await parentKv.set(key, value, ttl);
|
|
35
25
|
const path = this.kvPath(key);
|
|
36
26
|
await ensureDir(dirname(path));
|
|
37
27
|
await fs.writeFile(path, JSON.stringify(value), "utf-8");
|
|
38
28
|
},
|
|
39
29
|
delete: async (key) => {
|
|
40
|
-
await
|
|
30
|
+
await parentKv.delete(key);
|
|
41
31
|
const path = this.kvPath(key);
|
|
42
32
|
try {
|
|
43
33
|
await fs.unlink(path);
|
|
@@ -45,7 +35,7 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
|
|
|
45
35
|
}
|
|
46
36
|
},
|
|
47
37
|
clear: async (pattern) => {
|
|
48
|
-
const count = await
|
|
38
|
+
const count = await parentKv.clear(pattern);
|
|
49
39
|
try {
|
|
50
40
|
const kvDir = join(this.options.dataDir, "kv");
|
|
51
41
|
const files = await fs.readdir(kvDir);
|
|
@@ -62,13 +52,94 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
|
|
|
62
52
|
return count;
|
|
63
53
|
},
|
|
64
54
|
increment: async (key, by = 1) => {
|
|
65
|
-
const result = await
|
|
55
|
+
const result = await parentKv.increment(key, by);
|
|
66
56
|
const path = this.kvPath(key);
|
|
67
57
|
await ensureDir(dirname(path));
|
|
68
58
|
await fs.writeFile(path, JSON.stringify(result), "utf-8");
|
|
69
59
|
return result;
|
|
70
60
|
}
|
|
71
61
|
};
|
|
62
|
+
this.stream = {
|
|
63
|
+
append: async (subject, event) => {
|
|
64
|
+
const result = await parentStream.append(subject, event);
|
|
65
|
+
const path = this.streamPath(subject);
|
|
66
|
+
await ensureDir(dirname(path));
|
|
67
|
+
await fs.appendFile(path, JSON.stringify(result) + "\n", "utf-8");
|
|
68
|
+
return result;
|
|
69
|
+
},
|
|
70
|
+
read: parentStream.read,
|
|
71
|
+
subscribe: parentStream.subscribe,
|
|
72
|
+
delete: async (subject) => {
|
|
73
|
+
const deleted = parentStream.delete ? await parentStream.delete(subject) : false;
|
|
74
|
+
if (deleted) {
|
|
75
|
+
const path = this.streamPath(subject);
|
|
76
|
+
try {
|
|
77
|
+
await fs.unlink(path);
|
|
78
|
+
} catch {
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
return deleted;
|
|
82
|
+
}
|
|
83
|
+
};
|
|
84
|
+
this.index = {
|
|
85
|
+
add: async (key, id, score, metadata) => {
|
|
86
|
+
await parentIndex.add(key, id, score, metadata);
|
|
87
|
+
const self = this;
|
|
88
|
+
const index = self.sortedIndices?.get(key);
|
|
89
|
+
if (index) {
|
|
90
|
+
const path = this.indexPath(key);
|
|
91
|
+
await ensureDir(dirname(path));
|
|
92
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
93
|
+
}
|
|
94
|
+
},
|
|
95
|
+
get: parentIndex.get,
|
|
96
|
+
read: parentIndex.read,
|
|
97
|
+
update: async (key, id, metadata) => {
|
|
98
|
+
const result = await parentIndex.update(key, id, metadata);
|
|
99
|
+
const self = this;
|
|
100
|
+
const index = self.sortedIndices?.get(key);
|
|
101
|
+
if (index) {
|
|
102
|
+
const path = this.indexPath(key);
|
|
103
|
+
await ensureDir(dirname(path));
|
|
104
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
105
|
+
}
|
|
106
|
+
return result;
|
|
107
|
+
},
|
|
108
|
+
updateWithRetry: async (key, id, metadata, maxRetries) => {
|
|
109
|
+
await parentIndex.updateWithRetry(key, id, metadata, maxRetries);
|
|
110
|
+
const self = this;
|
|
111
|
+
const index = self.sortedIndices?.get(key);
|
|
112
|
+
if (index) {
|
|
113
|
+
const path = this.indexPath(key);
|
|
114
|
+
await ensureDir(dirname(path));
|
|
115
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
116
|
+
}
|
|
117
|
+
},
|
|
118
|
+
increment: async (key, id, field, increment) => {
|
|
119
|
+
const result = await parentIndex.increment(key, id, field, increment);
|
|
120
|
+
const self = this;
|
|
121
|
+
const index = self.sortedIndices?.get(key);
|
|
122
|
+
if (index) {
|
|
123
|
+
const path = this.indexPath(key);
|
|
124
|
+
await ensureDir(dirname(path));
|
|
125
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
126
|
+
}
|
|
127
|
+
return result;
|
|
128
|
+
},
|
|
129
|
+
delete: async (key, id) => {
|
|
130
|
+
const deleted = await parentIndex.delete(key, id);
|
|
131
|
+
if (deleted) {
|
|
132
|
+
const self = this;
|
|
133
|
+
const index = self.sortedIndices?.get(key);
|
|
134
|
+
if (index) {
|
|
135
|
+
const path = this.indexPath(key);
|
|
136
|
+
await ensureDir(dirname(path));
|
|
137
|
+
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return deleted;
|
|
141
|
+
}
|
|
142
|
+
};
|
|
72
143
|
}
|
|
73
144
|
streamPath(subject) {
|
|
74
145
|
return join(this.options.dataDir, "streams", sanitize(subject) + ".ndjson");
|
|
@@ -76,9 +147,6 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
|
|
|
76
147
|
indexPath(key) {
|
|
77
148
|
return join(this.options.dataDir, "indices", sanitize(key) + ".json");
|
|
78
149
|
}
|
|
79
|
-
docPath(collection, id) {
|
|
80
|
-
return join(this.options.dataDir, "docs", collection, `${id}.json`);
|
|
81
|
-
}
|
|
82
150
|
kvPath(key) {
|
|
83
151
|
return join(this.options.dataDir, "kv", `${sanitize(key)}.json`);
|
|
84
152
|
}
|
|
@@ -86,7 +154,6 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
|
|
|
86
154
|
await ensureDir(this.options.dataDir);
|
|
87
155
|
await ensureDir(join(this.options.dataDir, "streams"));
|
|
88
156
|
await ensureDir(join(this.options.dataDir, "indices"));
|
|
89
|
-
await ensureDir(join(this.options.dataDir, "docs"));
|
|
90
157
|
await ensureDir(join(this.options.dataDir, "kv"));
|
|
91
158
|
await this.loadFromDisk();
|
|
92
159
|
}
|
|
@@ -119,27 +186,6 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
|
|
|
119
186
|
}
|
|
120
187
|
} catch {
|
|
121
188
|
}
|
|
122
|
-
try {
|
|
123
|
-
const docsDir = join(this.options.dataDir, "docs");
|
|
124
|
-
const collections = await fs.readdir(docsDir);
|
|
125
|
-
for (const collection of collections) {
|
|
126
|
-
const collectionDir = join(docsDir, collection);
|
|
127
|
-
const docFiles = await fs.readdir(collectionDir);
|
|
128
|
-
if (!self.documents) self.documents = /* @__PURE__ */ new Map();
|
|
129
|
-
if (!self.documents.has(collection)) {
|
|
130
|
-
self.documents.set(collection, /* @__PURE__ */ new Map());
|
|
131
|
-
}
|
|
132
|
-
const collectionMap = self.documents.get(collection);
|
|
133
|
-
for (const file of docFiles) {
|
|
134
|
-
if (!file.endsWith(".json")) continue;
|
|
135
|
-
const id = file.replace(".json", "");
|
|
136
|
-
const content = await fs.readFile(join(collectionDir, file), "utf-8");
|
|
137
|
-
const doc = JSON.parse(content);
|
|
138
|
-
collectionMap.set(id, doc);
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
} catch {
|
|
142
|
-
}
|
|
143
189
|
try {
|
|
144
190
|
const kvDir = join(this.options.dataDir, "kv");
|
|
145
191
|
const kvFiles = await fs.readdir(kvDir);
|
|
@@ -154,71 +200,6 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
|
|
|
154
200
|
} catch {
|
|
155
201
|
}
|
|
156
202
|
}
|
|
157
|
-
// Override methods to add persistence
|
|
158
|
-
async append(subject, event) {
|
|
159
|
-
const result = await super.append(subject, event);
|
|
160
|
-
const path = this.streamPath(subject);
|
|
161
|
-
await ensureDir(dirname(path));
|
|
162
|
-
await fs.appendFile(path, JSON.stringify(result) + "\n", "utf-8");
|
|
163
|
-
return result;
|
|
164
|
-
}
|
|
165
|
-
async save(collection, id, doc) {
|
|
166
|
-
await super.save(collection, id, doc);
|
|
167
|
-
const path = this.docPath(collection, id);
|
|
168
|
-
await ensureDir(dirname(path));
|
|
169
|
-
await fs.writeFile(path, JSON.stringify(doc, null, 2), "utf-8");
|
|
170
|
-
}
|
|
171
|
-
async delete(collection, id) {
|
|
172
|
-
await super.delete(collection, id);
|
|
173
|
-
const path = this.docPath(collection, id);
|
|
174
|
-
try {
|
|
175
|
-
await fs.unlink(path);
|
|
176
|
-
} catch {
|
|
177
|
-
}
|
|
178
|
-
}
|
|
179
|
-
// Override index operations
|
|
180
|
-
async indexAdd(key, id, score, metadata) {
|
|
181
|
-
await super.indexAdd(key, id, score, metadata);
|
|
182
|
-
const self = this;
|
|
183
|
-
const index = self.sortedIndices?.get(key);
|
|
184
|
-
if (index) {
|
|
185
|
-
const path = this.indexPath(key);
|
|
186
|
-
await ensureDir(dirname(path));
|
|
187
|
-
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
188
|
-
}
|
|
189
|
-
}
|
|
190
|
-
async indexUpdate(key, id, metadata) {
|
|
191
|
-
const result = await super.indexUpdate(key, id, metadata);
|
|
192
|
-
const self = this;
|
|
193
|
-
const index = self.sortedIndices?.get(key);
|
|
194
|
-
if (index) {
|
|
195
|
-
const path = this.indexPath(key);
|
|
196
|
-
await ensureDir(dirname(path));
|
|
197
|
-
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
198
|
-
}
|
|
199
|
-
return result;
|
|
200
|
-
}
|
|
201
|
-
async indexUpdateWithRetry(key, id, metadata, maxRetries) {
|
|
202
|
-
await super.indexUpdateWithRetry(key, id, metadata, maxRetries);
|
|
203
|
-
const self = this;
|
|
204
|
-
const index = self.sortedIndices?.get(key);
|
|
205
|
-
if (index) {
|
|
206
|
-
const path = this.indexPath(key);
|
|
207
|
-
await ensureDir(dirname(path));
|
|
208
|
-
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
209
|
-
}
|
|
210
|
-
}
|
|
211
|
-
async indexIncrement(key, id, field, increment) {
|
|
212
|
-
const result = await super.indexIncrement(key, id, field, increment);
|
|
213
|
-
const self = this;
|
|
214
|
-
const index = self.sortedIndices?.get(key);
|
|
215
|
-
if (index) {
|
|
216
|
-
const path = this.indexPath(key);
|
|
217
|
-
await ensureDir(dirname(path));
|
|
218
|
-
await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
|
|
219
|
-
}
|
|
220
|
-
return result;
|
|
221
|
-
}
|
|
222
203
|
async close() {
|
|
223
204
|
await super.close();
|
|
224
205
|
}
|
|
@@ -158,6 +158,10 @@ export class MemoryQueueAdapter {
|
|
|
158
158
|
queueName
|
|
159
159
|
// Add more context as needed
|
|
160
160
|
});
|
|
161
|
+
if (result && typeof result === "object" && result.awaiting === true) {
|
|
162
|
+
this.jobs.delete(task.jobId);
|
|
163
|
+
return result;
|
|
164
|
+
}
|
|
161
165
|
this.updateJobState(task.jobId, "completed", {
|
|
162
166
|
returnvalue: result,
|
|
163
167
|
finishedOn: Date.now()
|