nvent 0.4.5 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/dist/module.d.mts +1 -1
  2. package/dist/module.mjs +433 -175
  3. package/dist/runtime/adapters/base/index.d.ts +6 -0
  4. package/dist/runtime/adapters/base/index.js +1 -0
  5. package/dist/runtime/adapters/base/store-validator.d.ts +48 -0
  6. package/dist/runtime/adapters/base/store-validator.js +147 -0
  7. package/dist/runtime/adapters/builtin/file-queue.d.ts +15 -1
  8. package/dist/runtime/adapters/builtin/file-queue.js +70 -6
  9. package/dist/runtime/adapters/builtin/file-store.d.ts +4 -18
  10. package/dist/runtime/adapters/builtin/file-store.js +90 -109
  11. package/dist/runtime/adapters/builtin/memory-queue.js +4 -0
  12. package/dist/runtime/adapters/builtin/memory-store.d.ts +42 -31
  13. package/dist/runtime/adapters/builtin/memory-store.js +253 -183
  14. package/dist/runtime/adapters/factory.d.ts +2 -2
  15. package/dist/runtime/adapters/factory.js +54 -20
  16. package/dist/runtime/adapters/interfaces/store.d.ts +177 -113
  17. package/dist/runtime/config/index.d.ts +2 -2
  18. package/dist/runtime/config/index.js +14 -6
  19. package/dist/runtime/config/types.d.ts +32 -2
  20. package/dist/runtime/events/eventBus.d.ts +1 -1
  21. package/dist/runtime/events/types.d.ts +31 -2
  22. package/dist/runtime/events/utils/scheduleTrigger.d.ts +8 -0
  23. package/dist/runtime/events/utils/scheduleTrigger.js +69 -0
  24. package/dist/runtime/events/utils/stallDetector.d.ts +44 -3
  25. package/dist/runtime/events/utils/stallDetector.js +288 -89
  26. package/dist/runtime/events/utils/triggerRuntime.d.ts +58 -0
  27. package/dist/runtime/events/utils/triggerRuntime.js +212 -0
  28. package/dist/runtime/events/wiring/flowWiring.d.ts +11 -5
  29. package/dist/runtime/events/wiring/flowWiring.js +620 -92
  30. package/dist/runtime/events/wiring/registry.d.ts +2 -2
  31. package/dist/runtime/events/wiring/registry.js +8 -6
  32. package/dist/runtime/events/wiring/streamWiring.d.ts +15 -11
  33. package/dist/runtime/events/wiring/streamWiring.js +88 -11
  34. package/dist/runtime/events/wiring/triggerWiring.d.ts +21 -0
  35. package/dist/runtime/events/wiring/triggerWiring.js +412 -0
  36. package/dist/runtime/{server → nitro}/plugins/00.adapters.js +8 -4
  37. package/dist/runtime/{server → nitro}/plugins/02.workers.js +21 -3
  38. package/dist/runtime/nitro/plugins/03.triggers.d.ts +12 -0
  39. package/dist/runtime/nitro/plugins/03.triggers.js +55 -0
  40. package/dist/runtime/nitro/routes/webhook.await.d.ts +23 -0
  41. package/dist/runtime/nitro/routes/webhook.await.js +90 -0
  42. package/dist/runtime/nitro/routes/webhook.trigger.d.ts +69 -0
  43. package/dist/runtime/nitro/routes/webhook.trigger.js +64 -0
  44. package/dist/runtime/{utils → nitro/utils}/adapters.d.ts +6 -6
  45. package/dist/runtime/nitro/utils/awaitPatterns/event.d.ts +15 -0
  46. package/dist/runtime/nitro/utils/awaitPatterns/event.js +120 -0
  47. package/dist/runtime/nitro/utils/awaitPatterns/index.d.ts +28 -0
  48. package/dist/runtime/nitro/utils/awaitPatterns/index.js +55 -0
  49. package/dist/runtime/nitro/utils/awaitPatterns/schedule.d.ts +16 -0
  50. package/dist/runtime/nitro/utils/awaitPatterns/schedule.js +78 -0
  51. package/dist/runtime/nitro/utils/awaitPatterns/time.d.ts +15 -0
  52. package/dist/runtime/nitro/utils/awaitPatterns/time.js +67 -0
  53. package/dist/runtime/nitro/utils/awaitPatterns/webhook.d.ts +15 -0
  54. package/dist/runtime/nitro/utils/awaitPatterns/webhook.js +120 -0
  55. package/dist/runtime/{utils → nitro/utils}/defineFunction.d.ts +2 -2
  56. package/dist/runtime/{utils → nitro/utils}/defineFunction.js +3 -3
  57. package/dist/runtime/{utils → nitro/utils}/defineFunctionConfig.d.ts +156 -0
  58. package/dist/runtime/{utils → nitro/utils}/defineFunctionConfig.js +1 -0
  59. package/dist/runtime/nitro/utils/defineHooks.d.ts +41 -0
  60. package/dist/runtime/nitro/utils/defineHooks.js +6 -0
  61. package/dist/runtime/{utils → nitro/utils}/registerAdapter.d.ts +3 -3
  62. package/dist/runtime/{utils → nitro/utils}/registerAdapter.js +1 -1
  63. package/dist/runtime/nitro/utils/useAwait.d.ts +71 -0
  64. package/dist/runtime/nitro/utils/useAwait.js +139 -0
  65. package/dist/runtime/{utils → nitro/utils}/useEventManager.d.ts +2 -2
  66. package/dist/runtime/{utils → nitro/utils}/useEventManager.js +1 -1
  67. package/dist/runtime/nitro/utils/useFlow.d.ts +68 -0
  68. package/dist/runtime/nitro/utils/useFlow.js +226 -0
  69. package/dist/runtime/nitro/utils/useHookRegistry.d.ts +34 -0
  70. package/dist/runtime/nitro/utils/useHookRegistry.js +25 -0
  71. package/dist/runtime/nitro/utils/useRunContext.d.ts +6 -0
  72. package/dist/runtime/nitro/utils/useRunContext.js +102 -0
  73. package/dist/runtime/nitro/utils/useStreamTopics.d.ts +83 -0
  74. package/dist/runtime/nitro/utils/useStreamTopics.js +94 -0
  75. package/dist/runtime/nitro/utils/useTrigger.d.ts +150 -0
  76. package/dist/runtime/nitro/utils/useTrigger.js +320 -0
  77. package/dist/runtime/scheduler/index.d.ts +33 -0
  78. package/dist/runtime/scheduler/index.js +38 -0
  79. package/dist/runtime/scheduler/scheduler.d.ts +113 -0
  80. package/dist/runtime/scheduler/scheduler.js +623 -0
  81. package/dist/runtime/scheduler/types.d.ts +116 -0
  82. package/dist/runtime/scheduler/types.js +0 -0
  83. package/dist/runtime/worker/node/runner.d.ts +12 -2
  84. package/dist/runtime/worker/node/runner.js +141 -37
  85. package/package.json +6 -6
  86. package/dist/runtime/server/api/_flows/[name]/clear-history.delete.d.ts +0 -10
  87. package/dist/runtime/server/api/_flows/[name]/clear-history.delete.js +0 -55
  88. package/dist/runtime/server/api/_flows/[name]/runs/[runId]/cancel.post.d.ts +0 -2
  89. package/dist/runtime/server/api/_flows/[name]/runs/[runId]/cancel.post.js +0 -21
  90. package/dist/runtime/server/api/_flows/[name]/runs.get.d.ts +0 -17
  91. package/dist/runtime/server/api/_flows/[name]/runs.get.js +0 -64
  92. package/dist/runtime/server/api/_flows/[name]/schedule.post.d.ts +0 -2
  93. package/dist/runtime/server/api/_flows/[name]/schedule.post.js +0 -66
  94. package/dist/runtime/server/api/_flows/[name]/schedules/[id].delete.d.ts +0 -2
  95. package/dist/runtime/server/api/_flows/[name]/schedules/[id].delete.js +0 -47
  96. package/dist/runtime/server/api/_flows/[name]/schedules.get.d.ts +0 -2
  97. package/dist/runtime/server/api/_flows/[name]/schedules.get.js +0 -50
  98. package/dist/runtime/server/api/_flows/[name]/start.post.d.ts +0 -2
  99. package/dist/runtime/server/api/_flows/[name]/start.post.js +0 -9
  100. package/dist/runtime/server/api/_flows/index.get.d.ts +0 -6
  101. package/dist/runtime/server/api/_flows/index.get.js +0 -5
  102. package/dist/runtime/server/api/_flows/ws.d.ts +0 -60
  103. package/dist/runtime/server/api/_flows/ws.js +0 -209
  104. package/dist/runtime/server/api/_queues/[name]/job/[id].get.d.ts +0 -2
  105. package/dist/runtime/server/api/_queues/[name]/job/[id].get.js +0 -14
  106. package/dist/runtime/server/api/_queues/[name]/job/index.get.d.ts +0 -2
  107. package/dist/runtime/server/api/_queues/[name]/job/index.get.js +0 -27
  108. package/dist/runtime/server/api/_queues/index.get.d.ts +0 -2
  109. package/dist/runtime/server/api/_queues/index.get.js +0 -106
  110. package/dist/runtime/server/api/_queues/ws.d.ts +0 -48
  111. package/dist/runtime/server/api/_queues/ws.js +0 -215
  112. package/dist/runtime/utils/useFlowEngine.d.ts +0 -19
  113. package/dist/runtime/utils/useFlowEngine.js +0 -108
  114. package/dist/runtime/utils/useStreamTopics.d.ts +0 -72
  115. package/dist/runtime/utils/useStreamTopics.js +0 -47
  116. /package/dist/runtime/{server → nitro}/plugins/00.adapters.d.ts +0 -0
  117. /package/dist/runtime/{server → nitro}/plugins/01.ws-lifecycle.d.ts +0 -0
  118. /package/dist/runtime/{server → nitro}/plugins/01.ws-lifecycle.js +0 -0
  119. /package/dist/runtime/{server → nitro}/plugins/02.workers.d.ts +0 -0
  120. /package/dist/runtime/{utils → nitro/utils}/adapters.js +0 -0
  121. /package/dist/runtime/{utils → nitro/utils}/useNventLogger.d.ts +0 -0
  122. /package/dist/runtime/{utils → nitro/utils}/useNventLogger.js +0 -0
  123. /package/dist/runtime/{utils → nitro/utils}/wsPeerManager.d.ts +0 -0
  124. /package/dist/runtime/{utils → nitro/utils}/wsPeerManager.js +0 -0
@@ -0,0 +1,6 @@
1
+ /**
2
+ * Adapter base utilities
3
+ * Exported for use by external adapter packages
4
+ */
5
+ export { StoreValidator, createStoreValidator } from './store-validator.js';
6
+ export type { StoreValidatorOptions } from './store-validator.js';
@@ -0,0 +1 @@
1
+ export { StoreValidator, createStoreValidator } from "./store-validator.js";
@@ -0,0 +1,48 @@
1
+ /**
2
+ * Store Adapter Validation
3
+ *
4
+ * Centralized validation logic for index updates to ensure correct usage
5
+ * across all store adapter implementations.
6
+ */
7
+ export interface StoreValidatorOptions {
8
+ /** Adapter name for error messages */
9
+ adapterName: string;
10
+ /** Enable counter misuse warnings (default: true in development) */
11
+ warnCounters?: boolean;
12
+ /** Throw errors on validation failures (default: true) */
13
+ throwOnError?: boolean;
14
+ }
15
+ export declare class StoreValidator {
16
+ private readonly adapterName;
17
+ private readonly warnCounters;
18
+ private readonly throwOnError;
19
+ constructor(options: StoreValidatorOptions);
20
+ /**
21
+ * Validate index update payload
22
+ * Throws error if dot notation keys are detected (except 'version')
23
+ * Recursively checks all nested objects
24
+ */
25
+ validateUpdatePayload(payload: Record<string, any>, method: string): void;
26
+ /**
27
+ * Recursively check for dot notation in object keys
28
+ */
29
+ private checkDotNotationKeys;
30
+ /**
31
+ * Build a nested object suggestion from a dot notation key
32
+ */
33
+ private buildNestedObjectSuggestion;
34
+ /**
35
+ * Detect potential counter misuse
36
+ * Logs warnings when numeric fields that look like counters are updated
37
+ * via update() instead of increment()
38
+ */
39
+ private detectCounterMisuse;
40
+ /**
41
+ * Check if a field looks like a counter that should use increment()
42
+ */
43
+ private isCounterField;
44
+ }
45
+ /**
46
+ * Factory function to create a validator for an adapter
47
+ */
48
+ export declare function createStoreValidator(adapterName: string, options?: Partial<StoreValidatorOptions>): StoreValidator;
@@ -0,0 +1,147 @@
1
+ export class StoreValidator {
2
+ adapterName;
3
+ warnCounters;
4
+ throwOnError;
5
+ constructor(options) {
6
+ this.adapterName = options.adapterName;
7
+ this.warnCounters = options.warnCounters ?? process.env.NODE_ENV === "development";
8
+ this.throwOnError = options.throwOnError ?? true;
9
+ }
10
+ /**
11
+ * Validate index update payload
12
+ * Throws error if dot notation keys are detected (except 'version')
13
+ * Recursively checks all nested objects
14
+ */
15
+ validateUpdatePayload(payload, method) {
16
+ this.checkDotNotationKeys(payload, method, []);
17
+ if (this.warnCounters) {
18
+ this.detectCounterMisuse(payload, method);
19
+ }
20
+ }
21
+ /**
22
+ * Recursively check for dot notation in object keys
23
+ */
24
+ checkDotNotationKeys(obj, method, path) {
25
+ for (const key of Object.keys(obj)) {
26
+ const currentPath = [...path, key];
27
+ if (key.includes(".") && key !== "version") {
28
+ const fullPath = currentPath.join(".");
29
+ const suggestion = this.buildNestedObjectSuggestion(key);
30
+ const message = [
31
+ `[${this.adapterName}] Invalid update payload in ${method}:`,
32
+ `Dot notation key "${key}" detected at path "${fullPath}".`,
33
+ `Keys should not contain dots. Split "${key}" into nested objects: ${suggestion}`
34
+ ].join("\n");
35
+ console.error(message);
36
+ if (this.throwOnError) {
37
+ throw new Error(
38
+ `Invalid index update: dot notation keys not supported. Key "${key}" at path "${fullPath}" contains dots.`
39
+ );
40
+ }
41
+ }
42
+ const value = obj[key];
43
+ if (value && typeof value === "object" && !Array.isArray(value)) {
44
+ this.checkDotNotationKeys(value, method, currentPath);
45
+ }
46
+ }
47
+ }
48
+ /**
49
+ * Build a nested object suggestion from a dot notation key
50
+ */
51
+ buildNestedObjectSuggestion(dotKey) {
52
+ const parts = dotKey.split(".");
53
+ let suggestion = "{ ";
54
+ for (let i = 0; i < parts.length - 1; i++) {
55
+ suggestion += `${parts[i]}: { `;
56
+ }
57
+ suggestion += `${parts[parts.length - 1]}: value `;
58
+ for (let i = 0; i < parts.length - 1; i++) {
59
+ suggestion += "}";
60
+ }
61
+ suggestion += " }";
62
+ return suggestion;
63
+ }
64
+ /**
65
+ * Detect potential counter misuse
66
+ * Logs warnings when numeric fields that look like counters are updated
67
+ * via update() instead of increment()
68
+ */
69
+ detectCounterMisuse(payload, method) {
70
+ const checkNested = (obj, path = []) => {
71
+ for (const [key, value] of Object.entries(obj)) {
72
+ const currentPath = [...path, key];
73
+ if (this.isCounterField(key, value, currentPath)) {
74
+ console.warn(
75
+ `[${this.adapterName}] Potential counter misuse in ${method}:`,
76
+ `Field "${currentPath.join(".")}" is numeric.`,
77
+ `Consider using index.increment() for atomic updates instead of ${method}.`
78
+ );
79
+ }
80
+ if (value && typeof value === "object" && !Array.isArray(value)) {
81
+ checkNested(value, currentPath);
82
+ }
83
+ }
84
+ };
85
+ checkNested(payload);
86
+ }
87
+ /**
88
+ * Check if a field looks like a counter that should use increment()
89
+ */
90
+ isCounterField(key, value, path) {
91
+ if (typeof value !== "number") {
92
+ return false;
93
+ }
94
+ if (path.includes("emittedEvents")) {
95
+ return false;
96
+ }
97
+ const metadataFields = [
98
+ "stepCount",
99
+ // Total steps in flow (constant)
100
+ "timeout",
101
+ // Timeout duration
102
+ "delay",
103
+ // Delay duration
104
+ "timestamp",
105
+ // Timestamp values
106
+ "score",
107
+ // Score values
108
+ "priority",
109
+ // Priority values
110
+ "version"
111
+ // Version field
112
+ ];
113
+ if (metadataFields.includes(key)) {
114
+ return false;
115
+ }
116
+ if (key.endsWith("At")) {
117
+ return false;
118
+ }
119
+ const counterKeywords = [
120
+ "count",
121
+ "total",
122
+ "running",
123
+ "awaiting",
124
+ "success",
125
+ "failure",
126
+ "cancel",
127
+ "pending",
128
+ "completed",
129
+ "failed",
130
+ "retries",
131
+ "attempts"
132
+ ];
133
+ if (counterKeywords.some((keyword) => key.toLowerCase().includes(keyword))) {
134
+ return true;
135
+ }
136
+ if (path.includes("stats")) {
137
+ return true;
138
+ }
139
+ return false;
140
+ }
141
+ }
142
+ export function createStoreValidator(adapterName, options) {
143
+ return new StoreValidator({
144
+ adapterName,
145
+ ...options
146
+ });
147
+ }
@@ -8,7 +8,7 @@
8
8
  * - Single instance only (no distributed lock)
9
9
  *
10
10
  * Storage format:
11
- * - {dataDir}/queues/{queueName}/jobs/{jobId}.json - Individual job files
11
+ * - {dataDir}/{queueName}/jobs/{jobId}.json - Individual job files
12
12
  * - Jobs are loaded on init and kept in memory with fastq
13
13
  */
14
14
  import type { QueueAdapter, JobInput, Job, JobsQuery, ScheduleOptions, JobCounts, QueueEvent, WorkerHandler, WorkerOptions } from '../interfaces/queue.js';
@@ -48,6 +48,20 @@ export declare class FileQueueAdapter implements QueueAdapter {
48
48
  startProcessingQueue(queueName: string): void;
49
49
  private updateJobState;
50
50
  private emitEvent;
51
+ /**
52
+ * Clean up completed jobs based on removeOnComplete option
53
+ * - true: delete immediately
54
+ * - false/undefined: keep forever
55
+ * - number: keep last N jobs, delete older ones
56
+ */
57
+ private cleanupCompletedJobs;
58
+ /**
59
+ * Clean up failed jobs based on removeOnFail option
60
+ * - true: delete immediately
61
+ * - false/undefined: keep forever
62
+ * - number: keep last N jobs, delete older ones
63
+ */
64
+ private cleanupFailedJobs;
51
65
  private generateId;
52
66
  private scheduleCronJob;
53
67
  }
@@ -17,12 +17,12 @@ export class FileQueueAdapter {
17
17
  }
18
18
  async init() {
19
19
  if (this.initialized) return;
20
- await fs.mkdir(join(this.options.dataDir, "queues"), { recursive: true });
20
+ await fs.mkdir(this.options.dataDir, { recursive: true });
21
21
  await this.loadJobsFromDisk();
22
22
  this.initialized = true;
23
23
  }
24
24
  async loadJobsFromDisk() {
25
- const queuesDir = join(this.options.dataDir, "queues");
25
+ const queuesDir = this.options.dataDir;
26
26
  try {
27
27
  const queueNames = await fs.readdir(queuesDir);
28
28
  for (const queueName of queueNames) {
@@ -43,13 +43,13 @@ export class FileQueueAdapter {
43
43
  }
44
44
  }
45
45
  async persistJob(queueName, job) {
46
- const jobsDir = join(this.options.dataDir, "queues", queueName, "jobs");
46
+ const jobsDir = join(this.options.dataDir, queueName, "jobs");
47
47
  await fs.mkdir(jobsDir, { recursive: true });
48
48
  const jobPath = join(jobsDir, `${job.id}.json`);
49
49
  await fs.writeFile(jobPath, JSON.stringify(job, null, 2));
50
50
  }
51
51
  async deleteJobFile(queueName, jobId) {
52
- const jobPath = join(this.options.dataDir, "queues", queueName, "jobs", `${jobId}.json`);
52
+ const jobPath = join(this.options.dataDir, queueName, "jobs", `${jobId}.json`);
53
53
  try {
54
54
  await fs.unlink(jobPath);
55
55
  } catch {
@@ -78,7 +78,7 @@ export class FileQueueAdapter {
78
78
  this.emitEvent(queueName, "waiting", { jobId, job: internalJob });
79
79
  const workerInfo = this.workers.get(queueName);
80
80
  if (workerInfo && !workerInfo.paused) {
81
- workerInfo.queue.push({ jobId, jobName: job.name, data: job.data }).catch(() => {
81
+ workerInfo.queue.push({ jobId, jobName: job.name, data: internalJob.data }).catch(() => {
82
82
  });
83
83
  }
84
84
  return jobId;
@@ -285,12 +285,17 @@ export class FileQueueAdapter {
285
285
  opts: { attempts: maxAttempts, ...storedJob.opts }
286
286
  };
287
287
  const result = await handler2(jobLike, {});
288
+ if (result && typeof result === "object" && result.awaiting === true) {
289
+ this.jobs.delete(task.jobId);
290
+ await this.deleteJobFile(queueName, task.jobId);
291
+ return result;
292
+ }
288
293
  await this.updateJobState(queueName, task.jobId, "completed", {
289
294
  returnvalue: result,
290
295
  finishedOn: Date.now()
291
296
  });
292
297
  this.emitEvent(queueName, "completed", { jobId: task.jobId, returnvalue: result });
293
- await this.deleteJobFile(queueName, task.jobId);
298
+ await this.cleanupCompletedJobs(queueName, storedJob.opts?.removeOnComplete);
294
299
  return result;
295
300
  } catch (err) {
296
301
  const newAttemptCount = currentAttempts + 1;
@@ -331,6 +336,7 @@ export class FileQueueAdapter {
331
336
  attemptsMade: newAttemptCount,
332
337
  maxAttempts
333
338
  });
339
+ await this.cleanupFailedJobs(queueName, storedJob.opts?.removeOnFail);
334
340
  throw err;
335
341
  }
336
342
  };
@@ -388,6 +394,64 @@ export class FileQueueAdapter {
388
394
  }
389
395
  }
390
396
  }
397
+ /**
398
+ * Clean up completed jobs based on removeOnComplete option
399
+ * - true: delete immediately
400
+ * - false/undefined: keep forever
401
+ * - number: keep last N jobs, delete older ones
402
+ */
403
+ async cleanupCompletedJobs(queueName, removeOnComplete) {
404
+ if (removeOnComplete === false || removeOnComplete === void 0) {
405
+ return;
406
+ }
407
+ if (removeOnComplete === true) {
408
+ const completedJobs = Array.from(this.jobs.values()).filter(
409
+ (j) => j.state === "completed" && j.data?.__queueName === queueName
410
+ );
411
+ for (const job of completedJobs) {
412
+ this.jobs.delete(job.id);
413
+ await this.deleteJobFile(queueName, job.id);
414
+ }
415
+ return;
416
+ }
417
+ if (typeof removeOnComplete === "number") {
418
+ const completedJobs = Array.from(this.jobs.values()).filter((j) => j.state === "completed" && j.data?.__queueName === queueName).sort((a, b) => (b.finishedOn || 0) - (a.finishedOn || 0));
419
+ const jobsToDelete = completedJobs.slice(removeOnComplete);
420
+ for (const job of jobsToDelete) {
421
+ this.jobs.delete(job.id);
422
+ await this.deleteJobFile(queueName, job.id);
423
+ }
424
+ }
425
+ }
426
+ /**
427
+ * Clean up failed jobs based on removeOnFail option
428
+ * - true: delete immediately
429
+ * - false/undefined: keep forever
430
+ * - number: keep last N jobs, delete older ones
431
+ */
432
+ async cleanupFailedJobs(queueName, removeOnFail) {
433
+ if (removeOnFail === false || removeOnFail === void 0) {
434
+ return;
435
+ }
436
+ if (removeOnFail === true) {
437
+ const failedJobs = Array.from(this.jobs.values()).filter(
438
+ (j) => j.state === "failed" && j.data?.__queueName === queueName
439
+ );
440
+ for (const job of failedJobs) {
441
+ this.jobs.delete(job.id);
442
+ await this.deleteJobFile(queueName, job.id);
443
+ }
444
+ return;
445
+ }
446
+ if (typeof removeOnFail === "number") {
447
+ const failedJobs = Array.from(this.jobs.values()).filter((j) => j.state === "failed" && j.data?.__queueName === queueName).sort((a, b) => (b.finishedOn || 0) - (a.finishedOn || 0));
448
+ const jobsToDelete = failedJobs.slice(removeOnFail);
449
+ for (const job of jobsToDelete) {
450
+ this.jobs.delete(job.id);
451
+ await this.deleteJobFile(queueName, job.id);
452
+ }
453
+ }
454
+ }
391
455
  generateId() {
392
456
  return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
393
457
  }
@@ -7,14 +7,13 @@
7
7
  * - Survives restarts
8
8
  * - Single instance only
9
9
  *
10
- * Storage format (matches existing file adapter):
11
- * - {dataDir}/{subject}.ndjson - Event streams (append-only NDJSON)
10
+ * Storage format:
11
+ * - {dataDir}/streams/{subject}.ndjson - Event streams (append-only NDJSON)
12
12
  * - {dataDir}/indices/{key}.json - Sorted indices (JSON arrays)
13
- * - {dataDir}/docs/{collection}/{id}.json - Documents (individual JSON files)
14
13
  * - {dataDir}/kv/{key}.json - KV store (individual JSON files)
15
14
  */
16
- import { MemoryStoreAdapter, type MemoryStoreAdapterOptions } from './memory-store.js';
17
- export interface FileStoreAdapterOptions extends MemoryStoreAdapterOptions {
15
+ import { MemoryStoreAdapter } from './memory-store.js';
16
+ export interface FileStoreAdapterOptions {
18
17
  dataDir: string;
19
18
  }
20
19
  /**
@@ -23,24 +22,11 @@ export interface FileStoreAdapterOptions extends MemoryStoreAdapterOptions {
23
22
  */
24
23
  export declare class FileStoreAdapter extends MemoryStoreAdapter {
25
24
  private options;
26
- private parentKvGet;
27
- private parentKvSet;
28
- private parentKvDelete;
29
- private parentKvClear;
30
- private parentKvIncrement;
31
25
  constructor(options: FileStoreAdapterOptions);
32
26
  private streamPath;
33
27
  private indexPath;
34
- private docPath;
35
28
  private kvPath;
36
29
  init(): Promise<void>;
37
30
  private loadFromDisk;
38
- append(subject: string, event: Omit<import('../interfaces/store').EventRecord, 'id' | 'ts'>): Promise<import("..").EventRecord>;
39
- save(collection: string, id: string, doc: Record<string, any>): Promise<void>;
40
- delete(collection: string, id: string): Promise<void>;
41
- indexAdd(key: string, id: string, score: number, metadata?: Record<string, any>): Promise<void>;
42
- indexUpdate(key: string, id: string, metadata: Record<string, any>): Promise<boolean>;
43
- indexUpdateWithRetry(key: string, id: string, metadata: Record<string, any>, maxRetries?: number): Promise<void>;
44
- indexIncrement(key: string, id: string, field: string, increment?: number): Promise<number>;
45
31
  close(): Promise<void>;
46
32
  }
@@ -12,32 +12,22 @@ async function ensureDir(path) {
12
12
  }
13
13
  export class FileStoreAdapter extends MemoryStoreAdapter {
14
14
  options;
15
- parentKvGet;
16
- parentKvSet;
17
- parentKvDelete;
18
- parentKvClear;
19
- parentKvIncrement;
20
15
  constructor(options) {
21
- super(options);
16
+ super();
22
17
  this.options = options;
23
- const tempKv = this.kv;
24
- this.parentKvGet = tempKv.get;
25
- this.parentKvSet = tempKv.set;
26
- this.parentKvDelete = tempKv.delete;
27
- this.parentKvClear = tempKv.clear;
28
- this.parentKvIncrement = tempKv.increment;
18
+ const parentKv = this.kv;
19
+ const parentStream = this.stream;
20
+ const parentIndex = this.index;
29
21
  this.kv = {
30
- get: async (key) => {
31
- return this.parentKvGet(key);
32
- },
22
+ get: parentKv.get,
33
23
  set: async (key, value, ttl) => {
34
- await this.parentKvSet(key, value, ttl);
24
+ await parentKv.set(key, value, ttl);
35
25
  const path = this.kvPath(key);
36
26
  await ensureDir(dirname(path));
37
27
  await fs.writeFile(path, JSON.stringify(value), "utf-8");
38
28
  },
39
29
  delete: async (key) => {
40
- await this.parentKvDelete(key);
30
+ await parentKv.delete(key);
41
31
  const path = this.kvPath(key);
42
32
  try {
43
33
  await fs.unlink(path);
@@ -45,7 +35,7 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
45
35
  }
46
36
  },
47
37
  clear: async (pattern) => {
48
- const count = await this.parentKvClear(pattern);
38
+ const count = await parentKv.clear(pattern);
49
39
  try {
50
40
  const kvDir = join(this.options.dataDir, "kv");
51
41
  const files = await fs.readdir(kvDir);
@@ -62,13 +52,94 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
62
52
  return count;
63
53
  },
64
54
  increment: async (key, by = 1) => {
65
- const result = await this.parentKvIncrement(key, by);
55
+ const result = await parentKv.increment(key, by);
66
56
  const path = this.kvPath(key);
67
57
  await ensureDir(dirname(path));
68
58
  await fs.writeFile(path, JSON.stringify(result), "utf-8");
69
59
  return result;
70
60
  }
71
61
  };
62
+ this.stream = {
63
+ append: async (subject, event) => {
64
+ const result = await parentStream.append(subject, event);
65
+ const path = this.streamPath(subject);
66
+ await ensureDir(dirname(path));
67
+ await fs.appendFile(path, JSON.stringify(result) + "\n", "utf-8");
68
+ return result;
69
+ },
70
+ read: parentStream.read,
71
+ subscribe: parentStream.subscribe,
72
+ delete: async (subject) => {
73
+ const deleted = parentStream.delete ? await parentStream.delete(subject) : false;
74
+ if (deleted) {
75
+ const path = this.streamPath(subject);
76
+ try {
77
+ await fs.unlink(path);
78
+ } catch {
79
+ }
80
+ }
81
+ return deleted;
82
+ }
83
+ };
84
+ this.index = {
85
+ add: async (key, id, score, metadata) => {
86
+ await parentIndex.add(key, id, score, metadata);
87
+ const self = this;
88
+ const index = self.sortedIndices?.get(key);
89
+ if (index) {
90
+ const path = this.indexPath(key);
91
+ await ensureDir(dirname(path));
92
+ await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
93
+ }
94
+ },
95
+ get: parentIndex.get,
96
+ read: parentIndex.read,
97
+ update: async (key, id, metadata) => {
98
+ const result = await parentIndex.update(key, id, metadata);
99
+ const self = this;
100
+ const index = self.sortedIndices?.get(key);
101
+ if (index) {
102
+ const path = this.indexPath(key);
103
+ await ensureDir(dirname(path));
104
+ await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
105
+ }
106
+ return result;
107
+ },
108
+ updateWithRetry: async (key, id, metadata, maxRetries) => {
109
+ await parentIndex.updateWithRetry(key, id, metadata, maxRetries);
110
+ const self = this;
111
+ const index = self.sortedIndices?.get(key);
112
+ if (index) {
113
+ const path = this.indexPath(key);
114
+ await ensureDir(dirname(path));
115
+ await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
116
+ }
117
+ },
118
+ increment: async (key, id, field, increment) => {
119
+ const result = await parentIndex.increment(key, id, field, increment);
120
+ const self = this;
121
+ const index = self.sortedIndices?.get(key);
122
+ if (index) {
123
+ const path = this.indexPath(key);
124
+ await ensureDir(dirname(path));
125
+ await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
126
+ }
127
+ return result;
128
+ },
129
+ delete: async (key, id) => {
130
+ const deleted = await parentIndex.delete(key, id);
131
+ if (deleted) {
132
+ const self = this;
133
+ const index = self.sortedIndices?.get(key);
134
+ if (index) {
135
+ const path = this.indexPath(key);
136
+ await ensureDir(dirname(path));
137
+ await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
138
+ }
139
+ }
140
+ return deleted;
141
+ }
142
+ };
72
143
  }
73
144
  streamPath(subject) {
74
145
  return join(this.options.dataDir, "streams", sanitize(subject) + ".ndjson");
@@ -76,9 +147,6 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
76
147
  indexPath(key) {
77
148
  return join(this.options.dataDir, "indices", sanitize(key) + ".json");
78
149
  }
79
- docPath(collection, id) {
80
- return join(this.options.dataDir, "docs", collection, `${id}.json`);
81
- }
82
150
  kvPath(key) {
83
151
  return join(this.options.dataDir, "kv", `${sanitize(key)}.json`);
84
152
  }
@@ -86,7 +154,6 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
86
154
  await ensureDir(this.options.dataDir);
87
155
  await ensureDir(join(this.options.dataDir, "streams"));
88
156
  await ensureDir(join(this.options.dataDir, "indices"));
89
- await ensureDir(join(this.options.dataDir, "docs"));
90
157
  await ensureDir(join(this.options.dataDir, "kv"));
91
158
  await this.loadFromDisk();
92
159
  }
@@ -119,27 +186,6 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
119
186
  }
120
187
  } catch {
121
188
  }
122
- try {
123
- const docsDir = join(this.options.dataDir, "docs");
124
- const collections = await fs.readdir(docsDir);
125
- for (const collection of collections) {
126
- const collectionDir = join(docsDir, collection);
127
- const docFiles = await fs.readdir(collectionDir);
128
- if (!self.documents) self.documents = /* @__PURE__ */ new Map();
129
- if (!self.documents.has(collection)) {
130
- self.documents.set(collection, /* @__PURE__ */ new Map());
131
- }
132
- const collectionMap = self.documents.get(collection);
133
- for (const file of docFiles) {
134
- if (!file.endsWith(".json")) continue;
135
- const id = file.replace(".json", "");
136
- const content = await fs.readFile(join(collectionDir, file), "utf-8");
137
- const doc = JSON.parse(content);
138
- collectionMap.set(id, doc);
139
- }
140
- }
141
- } catch {
142
- }
143
189
  try {
144
190
  const kvDir = join(this.options.dataDir, "kv");
145
191
  const kvFiles = await fs.readdir(kvDir);
@@ -154,71 +200,6 @@ export class FileStoreAdapter extends MemoryStoreAdapter {
154
200
  } catch {
155
201
  }
156
202
  }
157
- // Override methods to add persistence
158
- async append(subject, event) {
159
- const result = await super.append(subject, event);
160
- const path = this.streamPath(subject);
161
- await ensureDir(dirname(path));
162
- await fs.appendFile(path, JSON.stringify(result) + "\n", "utf-8");
163
- return result;
164
- }
165
- async save(collection, id, doc) {
166
- await super.save(collection, id, doc);
167
- const path = this.docPath(collection, id);
168
- await ensureDir(dirname(path));
169
- await fs.writeFile(path, JSON.stringify(doc, null, 2), "utf-8");
170
- }
171
- async delete(collection, id) {
172
- await super.delete(collection, id);
173
- const path = this.docPath(collection, id);
174
- try {
175
- await fs.unlink(path);
176
- } catch {
177
- }
178
- }
179
- // Override index operations
180
- async indexAdd(key, id, score, metadata) {
181
- await super.indexAdd(key, id, score, metadata);
182
- const self = this;
183
- const index = self.sortedIndices?.get(key);
184
- if (index) {
185
- const path = this.indexPath(key);
186
- await ensureDir(dirname(path));
187
- await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
188
- }
189
- }
190
- async indexUpdate(key, id, metadata) {
191
- const result = await super.indexUpdate(key, id, metadata);
192
- const self = this;
193
- const index = self.sortedIndices?.get(key);
194
- if (index) {
195
- const path = this.indexPath(key);
196
- await ensureDir(dirname(path));
197
- await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
198
- }
199
- return result;
200
- }
201
- async indexUpdateWithRetry(key, id, metadata, maxRetries) {
202
- await super.indexUpdateWithRetry(key, id, metadata, maxRetries);
203
- const self = this;
204
- const index = self.sortedIndices?.get(key);
205
- if (index) {
206
- const path = this.indexPath(key);
207
- await ensureDir(dirname(path));
208
- await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
209
- }
210
- }
211
- async indexIncrement(key, id, field, increment) {
212
- const result = await super.indexIncrement(key, id, field, increment);
213
- const self = this;
214
- const index = self.sortedIndices?.get(key);
215
- if (index) {
216
- const path = this.indexPath(key);
217
- await ensureDir(dirname(path));
218
- await fs.writeFile(path, JSON.stringify(index, null, 2), "utf-8");
219
- }
220
- return result;
221
- }
222
203
  async close() {
223
204
  await super.close();
224
205
  }
@@ -158,6 +158,10 @@ export class MemoryQueueAdapter {
158
158
  queueName
159
159
  // Add more context as needed
160
160
  });
161
+ if (result && typeof result === "object" && result.awaiting === true) {
162
+ this.jobs.delete(task.jobId);
163
+ return result;
164
+ }
161
165
  this.updateJobState(task.jobId, "completed", {
162
166
  returnvalue: result,
163
167
  finishedOn: Date.now()