@nicnocquee/dataqueue 1.16.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) Nico Prananta 2024
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/cli.cjs

@@ -0,0 +1,38 @@
+#!/usr/bin/env node
+
+const { spawnSync } = require('child_process');
+const path = require('path');
+
+function printUsage() {
+  console.log('Usage: dataqueue-cli migrate');
+  process.exit(1);
+}
+
+const [, , command] = process.argv;
+
+if (command === 'migrate') {
+  const migrationsDir = path.join(__dirname, 'migrations');
+  const dbUrl = process.env.PG_DATAQUEUE_DATABASE;
+  console.log(dbUrl);
+  if (!dbUrl) {
+    console.error(
+      'Error: PG_DATAQUEUE_DATABASE environment variable must be set to your Postgres connection string.',
+    );
+    process.exit(1);
+  }
+  const result = spawnSync(
+    'npx',
+    [
+      'node-pg-migrate',
+      'up',
+      '-d',
+      'PG_DATAQUEUE_DATABASE',
+      '-m',
+      migrationsDir,
+    ],
+    { stdio: 'inherit' },
+  );
+  process.exit(result.status);
+} else {
+  printUsage();
+}

package/dist/index.cjs

@@ -0,0 +1,663 @@
+'use strict';
+
+var async_hooks = require('async_hooks');
+var pg = require('pg');
+
+// src/types.ts
+var JobEventType = /* @__PURE__ */ ((JobEventType2) => {
+  JobEventType2["Added"] = "added";
+  JobEventType2["Processing"] = "processing";
+  JobEventType2["Completed"] = "completed";
+  JobEventType2["Failed"] = "failed";
+  JobEventType2["Cancelled"] = "cancelled";
+  JobEventType2["Retried"] = "retried";
+  return JobEventType2;
+})(JobEventType || {});
+var FailureReason = /* @__PURE__ */ ((FailureReason3) => {
+  FailureReason3["Timeout"] = "timeout";
+  FailureReason3["HandlerError"] = "handler_error";
+  FailureReason3["NoHandler"] = "no_handler";
+  return FailureReason3;
+})(FailureReason || {});
+var logStorage = new async_hooks.AsyncLocalStorage();
+var setLogContext = (verbose) => {
+  logStorage.enterWith({ verbose });
+};
+var getLogContext = () => {
+  return logStorage.getStore();
+};
+var log = (message) => {
+  const context = getLogContext();
+  if (context?.verbose) {
+    console.log(message);
+  }
+};
+
+// src/queue.ts
+var recordJobEvent = async (pool, jobId, eventType, metadata) => {
+  const client = await pool.connect();
+  try {
+    await client.query(
+      `INSERT INTO job_events (job_id, event_type, metadata) VALUES ($1, $2, $3)`,
+      [jobId, eventType, metadata ? JSON.stringify(metadata) : null]
+    );
+  } catch (error) {
+    log(`Error recording job event for job ${jobId}: ${error}`);
+  } finally {
+    client.release();
+  }
+};
+var addJob = async (pool, {
+  job_type,
+  payload,
+  max_attempts = 3,
+  priority = 0,
+  run_at = null,
+  timeoutMs = void 0
+}) => {
+  const client = await pool.connect();
+  try {
+    let result;
+    if (run_at) {
+      result = await client.query(
+        `INSERT INTO job_queue 
+          (job_type, payload, max_attempts, priority, run_at, timeout_ms) 
+         VALUES ($1, $2, $3, $4, $5, $6) 
+         RETURNING id`,
+        [job_type, payload, max_attempts, priority, run_at, timeoutMs ?? null]
+      );
+      log(
+        `Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)}, run_at ${run_at.toISOString()}, priority ${priority}, max_attempts ${max_attempts} job_type ${job_type}`
+      );
+    } else {
+      result = await client.query(
+        `INSERT INTO job_queue 
+          (job_type, payload, max_attempts, priority, timeout_ms) 
+         VALUES ($1, $2, $3, $4, $5) 
+         RETURNING id`,
+        [job_type, payload, max_attempts, priority, timeoutMs ?? null]
+      );
+      log(
+        `Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)}, priority ${priority}, max_attempts ${max_attempts} job_type ${job_type}`
+      );
+    }
+    await recordJobEvent(pool, result.rows[0].id, "added" /* Added */, {
+      job_type,
+      payload
+    });
+    return result.rows[0].id;
+  } catch (error) {
+    log(`Error adding job: ${error}`);
+    throw error;
+  } finally {
+    client.release();
+  }
+};
+var getJob = async (pool, id) => {
+  const client = await pool.connect();
+  try {
+    const result = await client.query("SELECT * FROM job_queue WHERE id = $1", [
+      id
+    ]);
+    if (result.rows.length === 0) {
+      log(`Job ${id} not found`);
+      return null;
+    }
+    log(`Found job ${id}`);
+    return {
+      ...result.rows[0],
+      payload: result.rows[0].payload,
+      timeout_ms: result.rows[0].timeout_ms,
+      failure_reason: result.rows[0].failure_reason
+    };
+  } catch (error) {
+    log(`Error getting job ${id}: ${error}`);
+    throw error;
+  } finally {
+    client.release();
+  }
+};
+var getJobsByStatus = async (pool, status, limit = 100, offset = 0) => {
+  const client = await pool.connect();
+  try {
+    const result = await client.query(
+      "SELECT * FROM job_queue WHERE status = $1 ORDER BY created_at DESC LIMIT $2 OFFSET $3",
+      [status, limit, offset]
+    );
+    log(`Found ${result.rows.length} jobs by status ${status}`);
+    return result.rows.map((row) => ({
+      ...row,
+      payload: row.payload,
+      timeout_ms: row.timeout_ms,
+      failure_reason: row.failure_reason
+    }));
+  } catch (error) {
+    log(`Error getting jobs by status ${status}: ${error}`);
+    throw error;
+  } finally {
+    client.release();
+  }
+};
+var getNextBatch = async (pool, workerId, batchSize = 10, jobType) => {
+  const client = await pool.connect();
+  try {
+    await client.query("BEGIN");
+    let jobTypeFilter = "";
+    let params = [workerId, batchSize];
+    if (jobType) {
+      if (Array.isArray(jobType)) {
+        jobTypeFilter = ` AND job_type = ANY($3)`;
+        params.push(jobType);
+      } else {
+        jobTypeFilter = ` AND job_type = $3`;
+        params.push(jobType);
+      }
+    }
+    const result = await client.query(
+      `
+      UPDATE job_queue
+      SET status = 'processing', 
+          locked_at = NOW(), 
+          locked_by = $1,
+          attempts = attempts + 1,
+          updated_at = NOW(),
+          pending_reason = NULL,
+          started_at = COALESCE(started_at, NOW()),
+          last_retried_at = CASE WHEN attempts > 0 THEN NOW() ELSE last_retried_at END
+      WHERE id IN (
+        SELECT id FROM job_queue
+        WHERE (status = 'pending' OR (status = 'failed' AND next_attempt_at <= NOW()))
+        AND (attempts < max_attempts)
+        AND run_at <= NOW()
+        ${jobTypeFilter}
+        ORDER BY priority DESC, created_at ASC
+        LIMIT $2
+        FOR UPDATE SKIP LOCKED
+      )
+      RETURNING *
+    `,
+      params
+    );
+    log(`Found ${result.rows.length} jobs to process`);
+    await client.query("COMMIT");
+    for (const row of result.rows) {
+      await recordJobEvent(pool, row.id, "processing" /* Processing */);
+    }
+    return result.rows.map((row) => ({
+      ...row,
+      payload: row.payload,
+      timeout_ms: row.timeout_ms
+    }));
+  } catch (error) {
+    log(`Error getting next batch: ${error}`);
+    await client.query("ROLLBACK");
+    throw error;
+  } finally {
+    client.release();
+  }
+};
+var completeJob = async (pool, jobId) => {
+  const client = await pool.connect();
+  try {
+    await client.query(
+      `
+      UPDATE job_queue
+      SET status = 'completed', updated_at = NOW(), completed_at = NOW()
+      WHERE id = $1
+    `,
+      [jobId]
+    );
+    await recordJobEvent(pool, jobId, "completed" /* Completed */);
+  } catch (error) {
+    log(`Error completing job ${jobId}: ${error}`);
+    throw error;
+  } finally {
+    log(`Completed job ${jobId}`);
+    client.release();
+  }
+};
+var failJob = async (pool, jobId, error, failureReason) => {
+  const client = await pool.connect();
+  try {
+    await client.query(
+      `
+      UPDATE job_queue
+      SET status = 'failed', 
+          updated_at = NOW(),
+          next_attempt_at = CASE 
+            WHEN attempts < max_attempts THEN NOW() + (POWER(2, attempts) * INTERVAL '1 minute')
+            ELSE NULL
+          END,
+          error_history = COALESCE(error_history, '[]'::jsonb) || $2::jsonb,
+          failure_reason = $3,
+          last_failed_at = NOW()
+      WHERE id = $1
+    `,
+      [
+        jobId,
+        JSON.stringify([
+          {
+            message: error.message || String(error),
+            timestamp: (/* @__PURE__ */ new Date()).toISOString()
+          }
+        ]),
+        failureReason ?? null
+      ]
+    );
+    await recordJobEvent(pool, jobId, "failed" /* Failed */, {
+      message: error.message || String(error),
+      failureReason
+    });
+  } catch (error2) {
+    log(`Error failing job ${jobId}: ${error2}`);
+    throw error2;
+  } finally {
+    log(`Failed job ${jobId}`);
+    client.release();
+  }
+};
+var retryJob = async (pool, jobId) => {
+  const client = await pool.connect();
+  try {
+    await client.query(
+      `
+      UPDATE job_queue
+      SET status = 'pending', 
+          updated_at = NOW(),
+          locked_at = NULL,
+          locked_by = NULL,
+          next_attempt_at = NOW(),
+          last_retried_at = NOW()
+      WHERE id = $1
+    `,
+      [jobId]
+    );
+    await recordJobEvent(pool, jobId, "retried" /* Retried */);
+  } catch (error) {
+    log(`Error retrying job ${jobId}: ${error}`);
+    throw error;
+  } finally {
+    log(`Retried job ${jobId}`);
+    client.release();
+  }
+};
+var cleanupOldJobs = async (pool, daysToKeep = 30) => {
+  const client = await pool.connect();
+  try {
+    const result = await client.query(`
+      DELETE FROM job_queue
+      WHERE status = 'completed'
+      AND updated_at < NOW() - INTERVAL '${daysToKeep} days'
+      RETURNING id
+    `);
+    log(`Deleted ${result.rowCount} old jobs`);
+    return result.rowCount || 0;
+  } catch (error) {
+    log(`Error cleaning up old jobs: ${error}`);
+    throw error;
+  } finally {
+    client.release();
+  }
+};
+var cancelJob = async (pool, jobId) => {
+  const client = await pool.connect();
+  try {
+    await client.query(
+      `
+      UPDATE job_queue
+      SET status = 'cancelled', updated_at = NOW(), last_cancelled_at = NOW()
+      WHERE id = $1 AND status = 'pending'
+    `,
+      [jobId]
+    );
+    await recordJobEvent(pool, jobId, "cancelled" /* Cancelled */);
+  } catch (error) {
+    log(`Error cancelling job ${jobId}: ${error}`);
+    throw error;
+  } finally {
+    log(`Cancelled job ${jobId}`);
+    client.release();
+  }
+};
+var cancelAllUpcomingJobs = async (pool, filters) => {
+  const client = await pool.connect();
+  try {
+    let query = `
+      UPDATE job_queue
+      SET status = 'cancelled', updated_at = NOW()
+      WHERE status = 'pending'`;
+    const params = [];
+    let paramIdx = 1;
+    if (filters) {
+      if (filters.job_type) {
+        query += ` AND job_type = $${paramIdx++}`;
+        params.push(filters.job_type);
+      }
+      if (filters.priority !== void 0) {
+        query += ` AND priority = $${paramIdx++}`;
+        params.push(filters.priority);
+      }
+      if (filters.run_at) {
+        query += ` AND run_at = $${paramIdx++}`;
+        params.push(filters.run_at);
+      }
+    }
+    query += "\nRETURNING id";
+    const result = await client.query(query, params);
+    log(`Cancelled ${result.rowCount} jobs`);
+    return result.rowCount || 0;
+  } catch (error) {
+    log(`Error cancelling upcoming jobs: ${error}`);
+    throw error;
+  } finally {
+    client.release();
+  }
+};
+var getAllJobs = async (pool, limit = 100, offset = 0) => {
+  const client = await pool.connect();
+  try {
+    const result = await client.query(
+      "SELECT * FROM job_queue ORDER BY created_at DESC LIMIT $1 OFFSET $2",
+      [limit, offset]
+    );
+    log(`Found ${result.rows.length} jobs (all)`);
+    return result.rows.map((row) => ({
+      ...row,
+      payload: row.payload,
+      timeout_ms: row.timeout_ms
+    }));
+  } catch (error) {
+    log(`Error getting all jobs: ${error}`);
+    throw error;
+  } finally {
+    client.release();
+  }
+};
+var setPendingReasonForUnpickedJobs = async (pool, reason, jobType) => {
+  const client = await pool.connect();
+  try {
+    let jobTypeFilter = "";
+    let params = [reason];
+    if (jobType) {
+      if (Array.isArray(jobType)) {
+        jobTypeFilter = ` AND job_type = ANY($2)`;
+        params.push(jobType);
+      } else {
+        jobTypeFilter = ` AND job_type = $2`;
+        params.push(jobType);
+      }
+    }
+    await client.query(
+      `UPDATE job_queue SET pending_reason = $1 WHERE status = 'pending'${jobTypeFilter}`,
+      params
+    );
+  } finally {
+    client.release();
+  }
+};
+var reclaimStuckJobs = async (pool, maxProcessingTimeMinutes = 10) => {
+  const client = await pool.connect();
+  try {
+    const result = await client.query(
+      `
+      UPDATE job_queue
+      SET status = 'pending', locked_at = NULL, locked_by = NULL, updated_at = NOW()
+      WHERE status = 'processing'
+        AND locked_at < NOW() - INTERVAL '${maxProcessingTimeMinutes} minutes'
+      RETURNING id
+      `
+    );
+    log(`Reclaimed ${result.rowCount} stuck jobs`);
+    return result.rowCount || 0;
+  } catch (error) {
+    log(`Error reclaiming stuck jobs: ${error}`);
+    throw error;
+  } finally {
+    client.release();
+  }
+};
+var getJobEvents = async (pool, jobId) => {
+  const client = await pool.connect();
+  try {
+    const res = await client.query(
+      "SELECT * FROM job_events WHERE job_id = $1 ORDER BY created_at ASC",
+      [jobId]
+    );
+    return res.rows;
+  } finally {
+    client.release();
+  }
+};
+
+// src/processor.ts
+async function processJobWithHandlers(pool, job, jobHandlers) {
+  const handler = jobHandlers[job.job_type];
+  if (!handler) {
+    await setPendingReasonForUnpickedJobs(
+      pool,
+      `No handler registered for job type: ${job.job_type}`,
+      job.job_type
+    );
+    await failJob(
+      pool,
+      job.id,
+      new Error(`No handler registered for job type: ${job.job_type}`),
+      "no_handler" /* NoHandler */
+    );
+    return;
+  }
+  const timeoutMs = job.timeout_ms ?? void 0;
+  let timeoutId;
+  const controller = new AbortController();
+  try {
+    const jobPromise = handler(job.payload, controller.signal);
+    if (timeoutMs && timeoutMs > 0) {
+      await Promise.race([
+        jobPromise,
+        new Promise((_, reject) => {
+          timeoutId = setTimeout(() => {
+            controller.abort();
+            const timeoutError = new Error(
+              `Job timed out after ${timeoutMs} ms`
+            );
+            timeoutError.failureReason = "timeout" /* Timeout */;
+            reject(timeoutError);
+          }, timeoutMs);
+        })
+      ]);
+    } else {
+      await jobPromise;
+    }
+    if (timeoutId) clearTimeout(timeoutId);
+    await completeJob(pool, job.id);
+  } catch (error) {
+    if (timeoutId) clearTimeout(timeoutId);
+    console.error(`Error processing job ${job.id}:`, error);
+    let failureReason = "handler_error" /* HandlerError */;
+    if (error && typeof error === "object" && "failureReason" in error && error.failureReason === "timeout" /* Timeout */) {
+      failureReason = "timeout" /* Timeout */;
+    }
+    await failJob(
+      pool,
+      job.id,
+      error instanceof Error ? error : new Error(String(error)),
+      failureReason
+    );
+  }
+}
+async function processBatchWithHandlers(pool, workerId, batchSize, jobType, jobHandlers, concurrency) {
+  const jobs = await getNextBatch(
+    pool,
+    workerId,
+    batchSize,
+    jobType
+  );
+  if (!concurrency || concurrency >= jobs.length) {
+    await Promise.all(
+      jobs.map((job) => processJobWithHandlers(pool, job, jobHandlers))
+    );
+    return jobs.length;
+  }
+  let idx = 0;
+  let running = 0;
+  let finished = 0;
+  return new Promise((resolve, reject) => {
+    const next = () => {
+      if (finished === jobs.length) return resolve(jobs.length);
+      while (running < concurrency && idx < jobs.length) {
+        const job = jobs[idx++];
+        running++;
+        processJobWithHandlers(pool, job, jobHandlers).then(() => {
+          running--;
+          finished++;
+          next();
+        }).catch((err) => {
+          running--;
+          finished++;
+          next();
+        });
+      }
+    };
+    next();
+  });
+}
+var createProcessor = (pool, handlers, options = {}) => {
+  const {
+    workerId = `worker-${Math.random().toString(36).substring(2, 9)}`,
+    batchSize = 10,
+    pollInterval = 5e3,
+    onError = (error) => console.error("Job processor error:", error),
+    jobType,
+    concurrency = 3
+  } = options;
+  let running = false;
+  let intervalId = null;
+  setLogContext(options.verbose ?? false);
+  const processJobs = async () => {
+    if (!running) return 0;
+    log(
+      `Processing jobs with workerId: ${workerId}${jobType ? ` and jobType: ${Array.isArray(jobType) ? jobType.join(",") : jobType}` : ""}`
+    );
+    try {
+      const processed = await processBatchWithHandlers(
+        pool,
+        workerId,
+        batchSize,
+        jobType,
+        handlers,
+        concurrency
+      );
+      return processed;
+    } catch (error) {
+      onError(error instanceof Error ? error : new Error(String(error)));
+    }
+    return 0;
+  };
+  return {
+    /**
+     * Start the job processor in the background.
+     * - This will run periodically (every pollInterval milliseconds or 5 seconds if not provided) and process jobs as they become available.
+     * - You have to call the stop method to stop the processor.
+     */
+    startInBackground: () => {
+      if (running) return;
+      log(`Starting job processor with workerId: ${workerId}`);
+      running = true;
+      const processBatches = async () => {
+        if (!running) return;
+        const processed = await processJobs();
+        if (processed === batchSize && running) {
+          setImmediate(processBatches);
+        }
+      };
+      processBatches();
+      intervalId = setInterval(processJobs, pollInterval);
+    },
+    /**
+     * Stop the job processor that runs in the background
+     */
+    stop: () => {
+      log(`Stopping job processor with workerId: ${workerId}`);
+      running = false;
+      if (intervalId) {
+        clearInterval(intervalId);
+        intervalId = null;
+      }
+    },
+    /**
+     * Start the job processor synchronously.
+     * - This will process all jobs immediately and then stop.
+     * - The pollInterval is ignored.
+     */
+    start: async () => {
+      log(`Starting job processor with workerId: ${workerId}`);
+      running = true;
+      const processed = await processJobs();
+      running = false;
+      return processed;
+    },
+    isRunning: () => running
+  };
+};
+var createPool = (config) => {
+  return new pg.Pool(config);
+};
+
+// src/index.ts
+var initJobQueue = async (config) => {
+  const { databaseConfig } = config;
+  const pool = createPool(databaseConfig);
+  setLogContext(config.verbose ?? false);
+  return {
+    // Job queue operations
+    addJob: withLogContext(
+      (job) => addJob(pool, job),
+      config.verbose ?? false
+    ),
+    getJob: withLogContext(
+      (id) => getJob(pool, id),
+      config.verbose ?? false
+    ),
+    getJobsByStatus: withLogContext(
+      (status, limit, offset) => getJobsByStatus(pool, status, limit, offset),
+      config.verbose ?? false
+    ),
+    getAllJobs: withLogContext(
+      (limit, offset) => getAllJobs(pool, limit, offset),
+      config.verbose ?? false
+    ),
+    retryJob: (jobId) => retryJob(pool, jobId),
+    cleanupOldJobs: (daysToKeep) => cleanupOldJobs(pool, daysToKeep),
+    cancelJob: withLogContext(
+      (jobId) => cancelJob(pool, jobId),
+      config.verbose ?? false
+    ),
+    cancelAllUpcomingJobs: withLogContext(
+      (filters) => cancelAllUpcomingJobs(pool, filters),
+      config.verbose ?? false
+    ),
+    reclaimStuckJobs: withLogContext(
+      (maxProcessingTimeMinutes) => reclaimStuckJobs(pool, maxProcessingTimeMinutes),
+      config.verbose ?? false
+    ),
+    // Job processing
+    createProcessor: (handlers, options) => createProcessor(pool, handlers, options),
+    // Advanced access (for custom operations)
+    getPool: () => pool,
+    // Job events
+    getJobEvents: withLogContext(
+      (jobId) => getJobEvents(pool, jobId),
+      config.verbose ?? false
+    )
+  };
+};
+var withLogContext = (fn, verbose) => (...args) => {
+  setLogContext(verbose);
+  return fn(...args);
+};
+
+exports.FailureReason = FailureReason;
+exports.JobEventType = JobEventType;
+exports.initJobQueue = initJobQueue;
+//# sourceMappingURL=index.cjs.map
+//# sourceMappingURL=index.cjs.map