nvent 0.4.4 → 0.5.0

package/dist/runtime/adapters/builtin/file-queue.js

@@ -0,0 +1,499 @@
+import * as fastq from "fastq";
+import { promises as fs } from "node:fs";
+import { join } from "node:path";
+import cronParser from "cron-parser";
+export class FileQueueAdapter {
+  jobs = /* @__PURE__ */ new Map();
+  eventListeners = /* @__PURE__ */ new Map();
+  workers = /* @__PURE__ */ new Map();
+  scheduledJobs = /* @__PURE__ */ new Map();
+  options;
+  initialized = false;
+  constructor(options) {
+    this.options = {
+      dataDir: options.dataDir,
+      maxQueueSize: options.maxQueueSize || 1e4
+    };
+  }
+  async init() {
+    if (this.initialized) return;
+    await fs.mkdir(this.options.dataDir, { recursive: true });
+    await this.loadJobsFromDisk();
+    this.initialized = true;
+  }
+  async loadJobsFromDisk() {
+    const queuesDir = this.options.dataDir;
+    try {
+      const queueNames = await fs.readdir(queuesDir);
+      for (const queueName of queueNames) {
+        const jobsDir = join(queuesDir, queueName, "jobs");
+        try {
+          const jobFiles = await fs.readdir(jobsDir);
+          for (const jobFile of jobFiles) {
+            if (!jobFile.endsWith(".json")) continue;
+            const jobPath = join(jobsDir, jobFile);
+            const jobData = await fs.readFile(jobPath, "utf-8");
+            const job = JSON.parse(jobData);
+            this.jobs.set(job.id, job);
+          }
+        } catch {
+        }
+      }
+    } catch {
+    }
+  }
+  async persistJob(queueName, job) {
+    const jobsDir = join(this.options.dataDir, queueName, "jobs");
+    await fs.mkdir(jobsDir, { recursive: true });
+    const jobPath = join(jobsDir, `${job.id}.json`);
+    await fs.writeFile(jobPath, JSON.stringify(job, null, 2));
+  }
+  async deleteJobFile(queueName, jobId) {
+    const jobPath = join(this.options.dataDir, queueName, "jobs", `${jobId}.json`);
+    try {
+      await fs.unlink(jobPath);
+    } catch {
+    }
+  }
+  async enqueue(queueName, job) {
+    const jobId = job.opts?.jobId || this.generateId();
+    if (this.jobs.has(jobId)) {
+      return jobId;
+    }
+    if (this.jobs.size >= this.options.maxQueueSize) {
+      throw new Error(`Queue ${queueName} is full (max: ${this.options.maxQueueSize})`);
+    }
+    const internalJob = {
+      id: jobId,
+      name: job.name,
+      data: { ...job.data, __queueName: queueName },
+      state: "waiting",
+      timestamp: Date.now(),
+      attemptsMade: 0,
+      opts: job.opts
+      // Store job options for retry logic
+    };
+    this.jobs.set(jobId, internalJob);
+    await this.persistJob(queueName, internalJob);
+    this.emitEvent(queueName, "waiting", { jobId, job: internalJob });
+    const workerInfo = this.workers.get(queueName);
+    if (workerInfo && !workerInfo.paused) {
+      workerInfo.queue.push({ jobId, jobName: job.name, data: internalJob.data }).catch(() => {
+      });
+    }
+    return jobId;
+  }
+  async schedule(queueName, job, opts) {
+    if (opts?.delay) {
+      const jobId = this.generateId();
+      const internalJob = {
+        id: jobId,
+        name: job.name,
+        data: { ...job.data, __queueName: queueName },
+        state: "delayed",
+        timestamp: Date.now()
+      };
+      this.jobs.set(jobId, internalJob);
+      await this.persistJob(queueName, internalJob);
+      this.emitEvent(queueName, "delayed", { jobId, job: internalJob, delay: opts.delay });
+      setTimeout(() => {
+        this.enqueue(queueName, job);
+      }, opts.delay);
+      return jobId;
+    }
+    if (opts?.cron || opts?.repeat) {
+      const scheduleId = this.generateId();
+      const cronPattern = opts.cron || opts.repeat?.pattern;
+      if (!cronPattern) {
+        throw new Error("Cron pattern is required for scheduled jobs");
+      }
+      const scheduledJob = {
+        queueName,
+        job,
+        opts,
+        repeatCount: 0
+      };
+      this.scheduledJobs.set(scheduleId, scheduledJob);
+      this.scheduleCronJob(scheduleId, scheduledJob);
+      console.info(`[FileQueue] Scheduled job "${job.name}" with cron pattern "${cronPattern}" (id: ${scheduleId})`);
+      return scheduleId;
+    }
+    return this.enqueue(queueName, job);
+  }
+  async getJob(_queueName, id) {
+    return this.jobs.get(id) || null;
+  }
+  async getJobs(queueName, query) {
+    let jobs = Array.from(this.jobs.values()).filter((j) => j.data?.__queueName === queueName);
+    if (query?.state && query.state.length > 0) {
+      jobs = jobs.filter((j) => query.state.includes(j.state));
+    }
+    if (query?.offset) {
+      jobs = jobs.slice(query.offset);
+    }
+    if (query?.limit) {
+      jobs = jobs.slice(0, query.limit);
+    }
+    return jobs;
+  }
+  on(queueName, event, callback) {
+    const key = `${queueName}:${event}`;
+    if (!this.eventListeners.has(key)) {
+      this.eventListeners.set(key, []);
+    }
+    this.eventListeners.get(key).push(callback);
+    return () => {
+      const listeners = this.eventListeners.get(key);
+      if (listeners) {
+        const index = listeners.indexOf(callback);
+        if (index > -1) {
+          listeners.splice(index, 1);
+        }
+      }
+    };
+  }
+  async isPaused(queueName) {
+    const workerInfo = this.workers.get(queueName);
+    return workerInfo?.paused || false;
+  }
+  async getJobCounts(queueName) {
+    const jobs = Array.from(this.jobs.values()).filter((j) => j.data?.__queueName === queueName);
+    return {
+      active: jobs.filter((j) => j.state === "active").length,
+      completed: jobs.filter((j) => j.state === "completed").length,
+      failed: jobs.filter((j) => j.state === "failed").length,
+      delayed: jobs.filter((j) => j.state === "delayed").length,
+      waiting: jobs.filter((j) => j.state === "waiting").length,
+      paused: jobs.filter((j) => j.state === "paused").length
+    };
+  }
+  async getScheduledJobs(queueName) {
+    const scheduled = [];
+    for (const [scheduleId, scheduledJob] of this.scheduledJobs.entries()) {
+      if (scheduledJob.queueName === queueName) {
+        const cronPattern = scheduledJob.opts.cron || scheduledJob.opts.repeat?.pattern;
+        let nextRun;
+        if (cronPattern) {
+          try {
+            const interval = cronParser.parseExpression(cronPattern);
+            nextRun = interval.next().toDate();
+          } catch {
+          }
+        }
+        scheduled.push({
+          id: scheduleId,
+          jobName: scheduledJob.job.name,
+          queueName: scheduledJob.queueName,
+          cron: scheduledJob.opts.cron,
+          pattern: scheduledJob.opts.repeat?.pattern,
+          nextRun,
+          repeatCount: scheduledJob.repeatCount,
+          limit: scheduledJob.opts.repeat?.limit
+        });
+      }
+    }
+    return scheduled;
+  }
+  async removeScheduledJob(scheduleId) {
+    const scheduledJob = this.scheduledJobs.get(scheduleId);
+    if (!scheduledJob) {
+      return false;
+    }
+    if (scheduledJob.timerId) {
+      clearTimeout(scheduledJob.timerId);
+    }
+    this.scheduledJobs.delete(scheduleId);
+    console.info(`[FileQueue] Removed scheduled job: ${scheduleId}`);
+    return true;
+  }
+  async pause(queueName) {
+    const workerInfo = this.workers.get(queueName);
+    if (workerInfo) {
+      workerInfo.paused = true;
+      workerInfo.queue.pause();
+    }
+  }
+  async resume(queueName) {
+    const workerInfo = this.workers.get(queueName);
+    if (workerInfo) {
+      workerInfo.paused = false;
+      workerInfo.queue.resume();
+    }
+  }
+  async close() {
+    for (const [scheduleId, scheduledJob] of this.scheduledJobs.entries()) {
+      if (scheduledJob.timerId) {
+        clearTimeout(scheduledJob.timerId);
+      }
+      console.info(`[FileQueue] Stopped scheduled job: ${scheduleId}`);
+    }
+    this.scheduledJobs.clear();
+    const drainPromises = Array.from(this.workers.values()).map((w) => w.queue.drained());
+    await Promise.all(drainPromises);
+    const persistPromises = [];
+    for (const [_jobId, job] of Array.from(this.jobs.entries())) {
+      const queueName = job.data?.__queueName || "default";
+      persistPromises.push(this.persistJob(queueName, job));
+    }
+    await Promise.all(persistPromises);
+    this.jobs.clear();
+    this.eventListeners.clear();
+    this.workers.clear();
+  }
+  // ============================================================
+  // Worker Management
+  // ============================================================
+  registerWorker(queueName, jobName, handler, opts) {
+    const requestedConcurrency = opts?.concurrency || 1;
+    let workerInfo = this.workers.get(queueName);
+    if (workerInfo) {
+      console.info(`[FileQueue] Adding handler for job "${jobName}" to queue "${queueName}"`);
+      workerInfo.handlers.set(jobName, handler);
+      if (requestedConcurrency > workerInfo.concurrency) {
+        workerInfo.concurrency = requestedConcurrency;
+        workerInfo.queue.concurrency = requestedConcurrency;
+        console.info(`[FileQueue] Updated queue "${queueName}" concurrency to ${requestedConcurrency}`);
+      }
+      return;
+    }
+    console.info(`[FileQueue] Creating new worker for queue: ${queueName}`);
+    const handlers = /* @__PURE__ */ new Map();
+    handlers.set(jobName, handler);
+    const dispatcher = async (task) => {
+      const handler2 = handlers.get(task.jobName);
+      if (!handler2) {
+        const error = `No handler for job "${task.jobName}" on queue "${queueName}". Available: ${Array.from(handlers.keys()).join(", ")}`;
+        console.error(error);
+        throw new Error(error);
+      }
+      const storedJob = this.jobs.get(task.jobId);
+      if (!storedJob) {
+        throw new Error(`Job ${task.jobId} not found`);
+      }
+      const currentAttempts = storedJob.attemptsMade || 0;
+      const maxAttempts = storedJob.opts?.attempts || 1;
+      await this.updateJobState(queueName, task.jobId, "active", {
+        processedOn: Date.now()
+      });
+      this.emitEvent(queueName, "active", { jobId: task.jobId });
+      try {
+        const jobLike = {
+          id: task.jobId,
+          name: task.jobName,
+          data: task.data,
+          attemptsMade: currentAttempts,
+          opts: { attempts: maxAttempts, ...storedJob.opts }
+        };
+        const result = await handler2(jobLike, {});
+        if (result && typeof result === "object" && result.awaiting === true) {
+          this.jobs.delete(task.jobId);
+          await this.deleteJobFile(queueName, task.jobId);
+          return result;
+        }
+        await this.updateJobState(queueName, task.jobId, "completed", {
+          returnvalue: result,
+          finishedOn: Date.now()
+        });
+        this.emitEvent(queueName, "completed", { jobId: task.jobId, returnvalue: result });
+        await this.cleanupCompletedJobs(queueName, storedJob.opts?.removeOnComplete);
+        return result;
+      } catch (err) {
+        const newAttemptCount = currentAttempts + 1;
+        if (newAttemptCount < maxAttempts) {
+          let retryDelay = 0;
+          if (storedJob.opts?.backoff) {
+            const { type, delay } = storedJob.opts.backoff;
+            if (type === "exponential") {
+              retryDelay = delay * Math.pow(2, newAttemptCount - 1);
+            } else {
+              retryDelay = delay;
+            }
+          }
+          await this.updateJobState(queueName, task.jobId, "waiting", {
+            attemptsMade: newAttemptCount,
+            failedReason: `Attempt ${newAttemptCount}/${maxAttempts} failed: ${err.message}`
+          });
+          console.info(
+            `[FileQueue] Retrying job ${task.jobId} (attempt ${newAttemptCount + 1}/${maxAttempts}) after ${retryDelay}ms delay`
+          );
+          const currentWorkerInfo = this.workers.get(queueName);
+          if (currentWorkerInfo) {
+            setTimeout(() => {
+              currentWorkerInfo.queue.push(task).catch(() => {
+              });
+            }, retryDelay);
+          }
+          throw err;
+        }
+        await this.updateJobState(queueName, task.jobId, "failed", {
+          failedReason: `Failed after ${newAttemptCount} attempts: ${err.message}`,
+          finishedOn: Date.now(),
+          attemptsMade: newAttemptCount
+        });
+        this.emitEvent(queueName, "failed", {
+          jobId: task.jobId,
+          failedReason: err.message,
+          attemptsMade: newAttemptCount,
+          maxAttempts
+        });
+        await this.cleanupFailedJobs(queueName, storedJob.opts?.removeOnFail);
+        throw err;
+      }
+    };
+    const concurrency = opts?.concurrency || 1;
+    const queue = fastq.promise(dispatcher, concurrency);
+    const shouldPause = opts?.autorun === false;
+    workerInfo = {
+      queue,
+      handlers,
+      paused: shouldPause || false,
+      concurrency
+    };
+    if (shouldPause) {
+      queue.pause();
+      console.info(`[FileQueue] Worker for "${queueName}" created but paused`);
+    }
+    this.workers.set(queueName, workerInfo);
+  }
+  /**
+   * Start processing waiting jobs for a queue
+   * Should be called after all handlers are registered
+   */
+  startProcessingQueue(queueName) {
+    const workerInfo = this.workers.get(queueName);
+    if (!workerInfo || workerInfo.paused) return;
+    const waitingJobs = Array.from(this.jobs.values()).filter(
+      (j) => j.state === "waiting" && j.data?.__queueName === queueName
+    );
+    for (const job of waitingJobs) {
+      workerInfo.queue.push({ jobId: job.id, jobName: job.name, data: job.data }).catch(() => {
+      });
+    }
+  }
+  // ============================================================
+  // Helper Methods
+  // ============================================================
+  async updateJobState(queueName, jobId, state, extra) {
+    const job = this.jobs.get(jobId);
+    if (job) {
+      job.state = state;
+      if (extra) {
+        Object.assign(job, extra);
+      }
+      await this.persistJob(queueName, job);
+    }
+  }
+  emitEvent(queueName, event, payload) {
+    const key = `${queueName}:${event}`;
+    const listeners = this.eventListeners.get(key) || [];
+    for (const callback of listeners) {
+      try {
+        callback(payload);
+      } catch (error) {
+        console.error(`[FileQueueAdapter] Error in event listener for ${key}:`, error);
+      }
+    }
+  }
+  /**
+   * Clean up completed jobs based on removeOnComplete option
+   * - true: delete immediately
+   * - false/undefined: keep forever
+   * - number: keep last N jobs, delete older ones
+   */
+  async cleanupCompletedJobs(queueName, removeOnComplete) {
+    if (removeOnComplete === false || removeOnComplete === void 0) {
+      return;
+    }
+    if (removeOnComplete === true) {
+      const completedJobs = Array.from(this.jobs.values()).filter(
+        (j) => j.state === "completed" && j.data?.__queueName === queueName
+      );
+      for (const job of completedJobs) {
+        this.jobs.delete(job.id);
+        await this.deleteJobFile(queueName, job.id);
+      }
+      return;
+    }
+    if (typeof removeOnComplete === "number") {
+      const completedJobs = Array.from(this.jobs.values()).filter((j) => j.state === "completed" && j.data?.__queueName === queueName).sort((a, b) => (b.finishedOn || 0) - (a.finishedOn || 0));
+      const jobsToDelete = completedJobs.slice(removeOnComplete);
+      for (const job of jobsToDelete) {
+        this.jobs.delete(job.id);
+        await this.deleteJobFile(queueName, job.id);
+      }
+    }
+  }
+  /**
+   * Clean up failed jobs based on removeOnFail option
+   * - true: delete immediately
+   * - false/undefined: keep forever
+   * - number: keep last N jobs, delete older ones
+   */
+  async cleanupFailedJobs(queueName, removeOnFail) {
+    if (removeOnFail === false || removeOnFail === void 0) {
+      return;
+    }
+    if (removeOnFail === true) {
+      const failedJobs = Array.from(this.jobs.values()).filter(
+        (j) => j.state === "failed" && j.data?.__queueName === queueName
+      );
+      for (const job of failedJobs) {
+        this.jobs.delete(job.id);
+        await this.deleteJobFile(queueName, job.id);
+      }
+      return;
+    }
+    if (typeof removeOnFail === "number") {
+      const failedJobs = Array.from(this.jobs.values()).filter((j) => j.state === "failed" && j.data?.__queueName === queueName).sort((a, b) => (b.finishedOn || 0) - (a.finishedOn || 0));
+      const jobsToDelete = failedJobs.slice(removeOnFail);
+      for (const job of jobsToDelete) {
+        this.jobs.delete(job.id);
+        await this.deleteJobFile(queueName, job.id);
+      }
+    }
+  }
+  generateId() {
+    return `${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
+  }
+  scheduleCronJob(scheduleId, scheduledJob) {
+    const cronPattern = scheduledJob.opts.cron || scheduledJob.opts.repeat?.pattern;
+    if (!cronPattern) {
+      console.error(`[FileQueue] No cron pattern found for schedule ${scheduleId}`);
+      return;
+    }
+    try {
+      const interval = cronParser.parseExpression(cronPattern);
+      const nextRun = interval.next().toDate();
+      const now = /* @__PURE__ */ new Date();
+      const delay = nextRun.getTime() - now.getTime();
+      console.info(
+        `[FileQueue] Next run for schedule ${scheduleId}: ${nextRun.toISOString()} (in ${Math.round(delay / 1e3)}s)`
+      );
+      scheduledJob.timerId = setTimeout(async () => {
+        const limit = scheduledJob.opts.repeat?.limit;
+        if (limit !== void 0 && scheduledJob.repeatCount >= limit) {
+          console.info(`[FileQueue] Schedule ${scheduleId} reached repeat limit (${limit})`);
+          this.scheduledJobs.delete(scheduleId);
+          return;
+        }
+        try {
+          await this.enqueue(scheduledJob.queueName, scheduledJob.job);
+          scheduledJob.repeatCount++;
+          const shouldContinue = limit === void 0 || scheduledJob.repeatCount < limit;
+          if (shouldContinue) {
+            this.scheduleCronJob(scheduleId, scheduledJob);
+          } else {
+            console.info(`[FileQueue] Schedule ${scheduleId} completed all ${limit} runs`);
+            this.scheduledJobs.delete(scheduleId);
+          }
+        } catch (error) {
+          console.error(`[FileQueue] Error executing scheduled job ${scheduleId}:`, error);
+          this.scheduleCronJob(scheduleId, scheduledJob);
+        }
+      }, delay);
+    } catch (error) {
+      console.error(`[FileQueue] Error parsing cron expression "${cronPattern}":`, error);
+      throw new Error(`Invalid cron pattern: ${cronPattern}`);
+    }
+  }
+}

package/dist/runtime/adapters/builtin/file-store.d.ts

@@ -0,0 +1,32 @@
+/**
+ * File Store Adapter
+ *
+ * File-based storage for development/small deployments
+ * - Extends MemoryStoreAdapter with persistence
+ * - Fast in-memory access backed by file system
+ * - Survives restarts
+ * - Single instance only
+ *
+ * Storage format:
+ * - {dataDir}/streams/{subject}.ndjson - Event streams (append-only NDJSON)
+ * - {dataDir}/indices/{key}.json - Sorted indices (JSON arrays)
+ * - {dataDir}/kv/{key}.json - KV store (individual JSON files)
+ */
+import { MemoryStoreAdapter } from './memory-store.js';
+export interface FileStoreAdapterOptions {
+    dataDir: string;
+}
+/**
+ * File-backed store adapter
+ * Extends memory store and persists on every write using same format as existing file adapter
+ */
+export declare class FileStoreAdapter extends MemoryStoreAdapter {
+    private options;
+    constructor(options: FileStoreAdapterOptions);
+    private streamPath;
+    private indexPath;
+    private kvPath;
+    init(): Promise<void>;
+    private loadFromDisk;
+    close(): Promise<void>;
+}