bullmq-dash 0.2.5 → 0.2.7

package/dist/index.js

@@ -276,7 +276,8 @@ var profileSchema = z.object({
   }).strict().optional(),
   pollInterval: z.coerce.number().int().positive().optional(),
   prefix: z.string().optional(),
-  queues: z.array(z.string()).optional()
+  queues: z.array(z.string()).optional(),
+  retentionMs: z.coerce.number().int().positive().optional()
 }).strict();
 var profilesFileSchema = z.object({
   defaultProfile: z.string().optional(),
@@ -408,6 +409,7 @@ function parseRedisUrl(input) {
 }
 
 // src/config.ts
+var DEFAULT_RETENTION_MS = 7 * 24 * 60 * 60 * 1000;
 var configSchema = z2.object({
   redis: z2.object({
     host: z2.string().default("localhost"),
@@ -419,7 +421,8 @@ var configSchema = z2.object({
   }),
   pollInterval: z2.coerce.number().int().positive().default(3000),
   prefix: z2.string().default("bull"),
-  queueNames: z2.array(z2.string()).optional()
+  queueNames: z2.array(z2.string()).optional(),
+  retentionMs: z2.coerce.number().int().positive().default(DEFAULT_RETENTION_MS)
 });
 var packageVersion = null;
 var HELP_TEXT = `
@@ -631,28 +634,33 @@ Examples:
 var RESOURCE_COMMANDS = new Set(["queues", "jobs", "schedulers"]);
 var ACTIONS = new Set(["list", "get", "retry", "delete"]);
 function extractSubcommand(argv) {
-  const positionals = [];
-  const flagArgv = [];
-  let donePositionals = false;
-  for (let i = 0;i < argv.length; i++) {
-    if (donePositionals) {
-      flagArgv.push(argv[i]);
-      continue;
-    }
-    const arg = argv[i];
-    if (arg.startsWith("-")) {
-      donePositionals = true;
-      flagArgv.push(arg);
-      continue;
-    }
-    positionals.push(arg);
+  const firstFlagIndex = argv.findIndex((arg) => arg.startsWith("-"));
+  if (firstFlagIndex === -1) {
+    return { positionals: argv, flagArgv: [] };
   }
-  return { positionals, flagArgv };
+  return {
+    positionals: argv.slice(0, firstFlagIndex),
+    flagArgv: argv.slice(firstFlagIndex)
+  };
 }
 function showSubcommandHelp(text) {
   console.log(text);
   process.exit(0);
 }
+function assertArgCount(positionals, count, usage) {
+  if (positionals.length > count) {
+    writeError(`Unexpected arguments: ${positionals.slice(count).join(" ")}`, "CONFIG_ERROR", `Usage: ${usage}`);
+    process.exit(2);
+  }
+}
+function getRequiredArg(positionals, index, name, usage) {
+  const arg = positionals[index];
+  if (!arg) {
+    writeError(`Missing required argument: <${name}>`, "CONFIG_ERROR", `Usage: ${usage}`);
+    process.exit(2);
+  }
+  return arg;
+}
 function parseSubcommand(positionals, help, jobState, pageSize, since, name, dryRun, yes = false) {
   if (positionals.length === 0)
     return;
@@ -684,24 +692,15 @@ function parseSubcommand(positionals, help, jobState, pageSize, since, name, dry
       if (action === "list") {
         if (help)
           showSubcommandHelp(QUEUES_LIST_HELP);
-        if (positionals.length > 2) {
-          writeError(`Unexpected arguments: ${positionals.slice(2).join(" ")}`, "CONFIG_ERROR", "Usage: queues list [options]");
-          process.exit(2);
-        }
+        assertArgCount(positionals, 2, "queues list [options]");
         return { kind: "queues-list" };
       }
       if (action === "delete") {
         if (help)
           showSubcommandHelp(QUEUES_DELETE_HELP);
-        const queue = positionals[2];
-        if (!queue) {
-          writeError("Missing required argument: <queue>", "CONFIG_ERROR", "Usage: queues delete <queue> [--dry-run] [--yes]");
-          process.exit(2);
-        }
-        if (positionals.length > 3) {
-          writeError(`Unexpected arguments: ${positionals.slice(3).join(" ")}`, "CONFIG_ERROR", "Usage: queues delete <queue> [--dry-run] [--yes]");
-          process.exit(2);
-        }
+        const usage = "queues delete <queue> [--dry-run] [--yes]";
+        const queue = getRequiredArg(positionals, 2, "queue", usage);
+        assertArgCount(positionals, 3, usage);
         return { kind: "queues-delete", queue, dryRun, yes };
       }
       writeError(`Invalid action '${action}' for queues`, "CONFIG_ERROR", "Available actions: list, delete. Use --help for usage.");
@@ -711,44 +710,26 @@ function parseSubcommand(positionals, help, jobState, pageSize, since, name, dry
       if (action === "list") {
         if (help)
           showSubcommandHelp(JOBS_LIST_HELP);
-        const queue = positionals[2];
-        if (!queue) {
-          writeError("Missing required argument: <queue>", "CONFIG_ERROR", "Usage: jobs list <queue> [--job-state <state>] [--page-size <n>]");
-          process.exit(2);
-        }
-        if (positionals.length > 3) {
-          writeError(`Unexpected arguments: ${positionals.slice(3).join(" ")}`, "CONFIG_ERROR", "Usage: jobs list <queue> [--job-state <state>] [--page-size <n>]");
-          process.exit(2);
-        }
+        const usage = "jobs list <queue> [--job-state <state>] [--page-size <n>]";
+        const queue = getRequiredArg(positionals, 2, "queue", usage);
+        assertArgCount(positionals, 3, usage);
         return { kind: "jobs-list", queue, jobState, pageSize };
       }
       if (action === "get") {
         if (help)
           showSubcommandHelp(JOBS_GET_HELP);
-        const queue = positionals[2];
-        const jobId = positionals[3];
-        if (!queue || !jobId) {
-          writeError("Missing required arguments: <queue> <job-id>", "CONFIG_ERROR", "Usage: jobs get <queue> <job-id>");
-          process.exit(2);
-        }
-        if (positionals.length > 4) {
-          writeError(`Unexpected arguments: ${positionals.slice(4).join(" ")}`, "CONFIG_ERROR", "Usage: jobs get <queue> <job-id>");
-          process.exit(2);
-        }
+        const usage = "jobs get <queue> <job-id>";
+        const queue = getRequiredArg(positionals, 2, "queue", usage);
+        const jobId = getRequiredArg(positionals, 3, "job-id", usage);
+        assertArgCount(positionals, 4, usage);
         return { kind: "jobs-get", queue, jobId };
       }
       if (action === "retry") {
         if (help)
           showSubcommandHelp(JOBS_RETRY_HELP);
-        const queue = positionals[2];
-        if (!queue) {
-          writeError("Missing required argument: <queue>", "CONFIG_ERROR", "Usage: jobs retry <queue> --job-state failed [--since <duration>] [--name <pattern>] [--dry-run]");
-          process.exit(2);
-        }
-        if (positionals.length > 3) {
-          writeError(`Unexpected arguments: ${positionals.slice(3).join(" ")}`, "CONFIG_ERROR", "Usage: jobs retry <queue> --job-state failed [options]");
-          process.exit(2);
-        }
+        const usage = "jobs retry <queue> --job-state failed [--since <duration>] [--name <pattern>] [--dry-run]";
+        const queue = getRequiredArg(positionals, 2, "queue", usage);
+        assertArgCount(positionals, 3, usage);
         if (!jobState) {
           writeError("--job-state is required for 'jobs retry'", "CONFIG_ERROR", "Use --job-state failed. Other states are not currently supported.");
           process.exit(2);
@@ -766,30 +747,18 @@ function parseSubcommand(positionals, help, jobState, pageSize, since, name, dry
       if (action === "list") {
         if (help)
           showSubcommandHelp(SCHEDULERS_LIST_HELP);
-        const queue = positionals[2];
-        if (!queue) {
-          writeError("Missing required argument: <queue>", "CONFIG_ERROR", "Usage: schedulers list <queue> [--page-size <n>]");
-          process.exit(2);
-        }
-        if (positionals.length > 3) {
-          writeError(`Unexpected arguments: ${positionals.slice(3).join(" ")}`, "CONFIG_ERROR", "Usage: schedulers list <queue> [--page-size <n>]");
-          process.exit(2);
-        }
+        const usage = "schedulers list <queue> [--page-size <n>]";
+        const queue = getRequiredArg(positionals, 2, "queue", usage);
+        assertArgCount(positionals, 3, usage);
         return { kind: "schedulers-list", queue, pageSize };
       }
       if (action === "get") {
         if (help)
           showSubcommandHelp(SCHEDULERS_GET_HELP);
-        const queue = positionals[2];
-        const schedulerId = positionals[3];
-        if (!queue || !schedulerId) {
-          writeError("Missing required arguments: <queue> <scheduler-id>", "CONFIG_ERROR", "Usage: schedulers get <queue> <scheduler-id>");
-          process.exit(2);
-        }
-        if (positionals.length > 4) {
-          writeError(`Unexpected arguments: ${positionals.slice(4).join(" ")}`, "CONFIG_ERROR", "Usage: schedulers get <queue> <scheduler-id>");
-          process.exit(2);
-        }
+        const usage = "schedulers get <queue> <scheduler-id>";
+        const queue = getRequiredArg(positionals, 2, "queue", usage);
+        const schedulerId = getRequiredArg(positionals, 3, "scheduler-id", usage);
+        assertArgCount(positionals, 4, usage);
         return { kind: "schedulers-get", queue, schedulerId };
       }
       writeError(`Invalid action '${action}' for schedulers`, "CONFIG_ERROR", "Available actions: list, get. Use --help for usage.");
@@ -985,7 +954,8 @@ function loadConfig(cliArgs, profile) {
     },
     pollInterval: cliArgs.pollInterval ?? p?.pollInterval,
     prefix: cliArgs.prefix ?? p?.prefix,
-    queueNames: cliArgs.queues ?? p?.queues
+    queueNames: cliArgs.queues ?? p?.queues,
+    retentionMs: p?.retentionMs
   };
   const result = configSchema.safeParse(raw);
   if (!result.success) {
@@ -1182,7 +1152,14 @@ async function deleteQueue(queueName, dryRun = false) {
 // src/data/jobs.ts
 var PAGE_SIZE = 25;
 var DEFAULT_MAX_RESULTS = 1000;
-var SYNC_JOB_TYPES = ["active", "waiting", "completed", "failed", "delayed", "prioritized"];
+var SYNC_JOB_TYPES = [
+  "active",
+  "waiting",
+  "completed",
+  "failed",
+  "delayed",
+  "prioritized"
+];
 var SYNC_PAGE_SIZE = 5000;
 var VALID_JOB_STATUSES = ["wait", "active", "completed", "failed", "delayed"];
 async function getAllJobs(queueName, status, maxResults = DEFAULT_MAX_RESULTS, includeData = false) {
@@ -1754,13 +1731,24 @@ CREATE TABLE IF NOT EXISTS jobs (
   state TEXT NOT NULL,
   timestamp INTEGER,
   data_preview TEXT,
+  removed_at INTEGER,
   PRIMARY KEY (queue, id)
 );
 
 CREATE INDEX IF NOT EXISTS idx_jobs_queue_state ON jobs(queue, state);
 CREATE INDEX IF NOT EXISTS idx_jobs_name ON jobs(name);
 CREATE INDEX IF NOT EXISTS idx_jobs_timestamp ON jobs(timestamp);
+CREATE INDEX IF NOT EXISTS idx_jobs_active
+  ON jobs(queue, state, timestamp) WHERE removed_at IS NULL;
 `;
+function migrateRemovedAtColumn(database) {
+  const cols = database.prepare("PRAGMA table_info(jobs)").all();
+  if (cols.length === 0)
+    return;
+  if (cols.some((c) => c.name === "removed_at"))
+    return;
+  database.exec("ALTER TABLE jobs ADD COLUMN removed_at INTEGER");
+}
 var FTS_SCHEMA = `
 CREATE VIRTUAL TABLE IF NOT EXISTS jobs_fts USING fts5(
   id,
@@ -1802,6 +1790,7 @@ function createSqliteDb(dbPath) {
   db = new Database(path);
   db.exec("PRAGMA journal_mode=WAL");
   db.exec("PRAGMA synchronous=NORMAL");
+  migrateRemovedAtColumn(db);
   db.exec(SCHEMA);
   db.exec(FTS_SCHEMA);
   db.exec(SYNC_STATE_SCHEMA);
@@ -1897,6 +1886,15 @@ function insertStagingBatch(queue, jobs) {
   });
   insert(jobs);
 }
+function findResurrectedIdsByStagingDiff(queue) {
+  const database = getSqliteDb();
+  const rows = database.prepare(`
+    SELECT s.id FROM sync_staging s
+    JOIN jobs j ON s.queue = j.queue AND s.id = j.id
+    WHERE s.queue = ? AND j.removed_at IS NOT NULL
+  `).all(queue);
+  return rows.map((r) => r.id);
+}
 function findNewIdsByStagingDiff(queue) {
   const database = getSqliteDb();
   return database.prepare(`
@@ -1910,7 +1908,7 @@ function findChangedIdsByStagingDiff(queue) {
   return database.prepare(`
     SELECT s.id, s.state FROM sync_staging s
     JOIN jobs j ON s.queue = j.queue AND s.id = j.id
-    WHERE s.queue = ? AND s.state != j.state
+    WHERE s.queue = ? AND j.removed_at IS NULL AND s.state != j.state
   `).all(queue);
 }
 function findStaleIdsByStagingDiff(queue) {
@@ -1918,11 +1916,11 @@ function findStaleIdsByStagingDiff(queue) {
   const rows = database.prepare(`
     SELECT j.id FROM jobs j
     LEFT JOIN sync_staging s ON j.queue = s.queue AND j.id = s.id
-    WHERE j.queue = ? AND s.id IS NULL
+    WHERE j.queue = ? AND j.removed_at IS NULL AND s.id IS NULL
   `).all(queue);
   return rows.map((r) => r.id);
 }
-function deleteJobsByIds(queue, ids) {
+function softDeleteJobsByIds(queue, ids, now) {
   if (ids.length === 0)
     return 0;
   const database = getSqliteDb();
@@ -1931,12 +1929,25 @@ function deleteJobsByIds(queue, ids) {
     for (let i = 0;i < ids.length; i += BATCH_SIZE) {
       const batch = ids.slice(i, i + BATCH_SIZE);
       const placeholders = batch.map(() => "?").join(",");
-      database.prepare(`DELETE FROM jobs WHERE queue = ? AND id IN (${placeholders})`).run(queue, ...batch);
+      database.prepare(`UPDATE jobs SET removed_at = ? WHERE queue = ? AND id IN (${placeholders})`).run(now, queue, ...batch);
     }
   });
   run();
   return ids.length;
 }
+function compactRemovedJobs(now, retentionMs) {
+  const database = getSqliteDb();
+  const cutoff = now - retentionMs;
+  const filter = "removed_at IS NOT NULL AND removed_at < ?";
+  const run = database.transaction(() => {
+    const { n } = database.prepare(`SELECT COUNT(*) as n FROM jobs WHERE ${filter}`).get(cutoff);
+    if (n > 0) {
+      database.prepare(`DELETE FROM jobs WHERE ${filter}`).run(cutoff);
+    }
+    return n;
+  });
+  return run();
+}
 function dropSyncStaging() {
   const database = getSqliteDb();
   database.exec("DROP TABLE IF EXISTS sync_staging");
@@ -1993,7 +2004,7 @@ async function syncQueue(queueName) {
     return {
       inserted: 0,
       stateUpdated: 0,
-      deleted: 0,
+      softDeleted: 0,
       total: 0,
       error: message
     };
@@ -2014,6 +2025,10 @@ async function syncQueue(queueName) {
       total += batch.length;
     }
     assertSameConnection();
+    const resurrected = findResurrectedIdsByStagingDiff(queueName);
+    if (resurrected.length > 0) {
+      throw new Error(`Resurrected job IDs detected in queue "${queueName}": ` + `${resurrected.join(", ")}. ` + `Soft-deleted IDs are not allowed to reappear in Redis.`);
+    }
     const newStubs = findNewIdsByStagingDiff(queueName);
     if (newStubs.length > 0) {
       upsertJobStubs(queueName, newStubs);
@@ -2023,15 +2038,16 @@ async function syncQueue(queueName) {
       upsertJobStubs(queueName, changed);
     }
     const staleIds = findStaleIdsByStagingDiff(queueName).filter((id) => !wasPolledSince(queueName, id, syncStart));
-    const deleted = deleteJobsByIds(queueName, staleIds);
+    const now = Date.now();
+    const softDeleted = softDeleteJobsByIds(queueName, staleIds, now);
     upsertSyncState(queueName, {
       jobCount: total,
-      syncedAt: Date.now()
+      syncedAt: now
     });
     return {
       inserted: newStubs.length,
       stateUpdated: changed.length,
-      deleted,
+      softDeleted,
       total
     };
   } catch (error) {
@@ -2040,7 +2056,7 @@ async function syncQueue(queueName) {
     return {
       inserted: 0,
       stateUpdated: 0,
-      deleted: 0,
+      softDeleted: 0,
       total: 0,
       error: message
     };
@@ -2065,12 +2081,13 @@ async function fullSync() {
     return {
       queues: 0,
       totalInserted: 0,
-      totalDeleted: 0,
+      totalSoftDeleted: 0,
+      totalCompacted: 0,
       errors: [{ queue: "", error: message }]
     };
   }
   let totalInserted = 0;
-  let totalDeleted = 0;
+  let totalSoftDeleted = 0;
   const errors = [];
   for (const q of queues) {
     const result = await syncQueue(q);
@@ -2078,10 +2095,17 @@ async function fullSync() {
       errors.push({ queue: q, error: result.error });
     } else {
       totalInserted += result.inserted;
-      totalDeleted += result.deleted;
+      totalSoftDeleted += result.softDeleted;
     }
   }
-  return { queues: queues.length, totalInserted, totalDeleted, errors };
+  const totalCompacted = compactRemovedJobs(Date.now(), getConfig().retentionMs);
+  return {
+    queues: queues.length,
+    totalInserted,
+    totalSoftDeleted,
+    totalCompacted,
+    errors
+  };
 }
 
 // src/polling.ts
@@ -3824,12 +3848,6 @@ function formatTs(ts) {
     return "-";
   return new Date(ts).toISOString().replace("T", " ").slice(0, 19);
 }
-function padRight(str, len) {
-  return str.length >= len ? str : str + " ".repeat(len - str.length);
-}
-function padLeft(str, len) {
-  return str.length >= len ? str : " ".repeat(len - str.length) + str;
-}
 function table(headers, rows, align) {
   const cols = headers.length;
   const widths = headers.map((h) => h.length);
@@ -3838,7 +3856,10 @@ function table(headers, rows, align) {
       widths[i] = Math.max(widths[i], (row[i] ?? "").length);
     }
   }
-  const pad = (val, i) => (align?.[i] ?? "l") === "r" ? padLeft(val, widths[i]) : padRight(val, widths[i]);
+  const pad = (val, i) => {
+    const width = widths[i];
+    return (align?.[i] ?? "l") === "r" ? val.padStart(width) : val.padEnd(width);
+  };
   const headerLine = headers.map(pad).join("  ");
   const separator = widths.map((w) => "-".repeat(w)).join("  ");
   const bodyLines = rows.map((row) => row.map((val, i) => pad(val, i)).join("  "));
@@ -4096,6 +4117,12 @@ function formatJobsRetry(r) {
 
 // src/json-reporter.ts
 import readline2 from "readline";
+function createResponse(data) {
+  return {
+    timestamp: new Date().toISOString(),
+    ...data
+  };
+}
 async function fetchQueuesOverview() {
   const queueNames = await discoverQueueNames();
   const queues = await Promise.all(queueNames.map((name) => getQueueStats(name)));
@@ -4107,25 +4134,23 @@ async function fetchQueuesOverview() {
     delayed: acc.delayed + q.counts.delayed,
     total: acc.total + q.total
   }), { wait: 0, active: 0, completed: 0, failed: 0, delayed: 0, total: 0 });
-  return {
-    timestamp: new Date().toISOString(),
+  return createResponse({
     queues,
     metrics: {
       queueCount: queues.length,
       jobCounts
     }
-  };
+  });
 }
 async function fetchQueuesDelete(queueName, dryRun) {
   const result = await deleteQueue(queueName, dryRun);
-  return {
-    timestamp: new Date().toISOString(),
+  return createResponse({
     queue: queueName,
     deleted: !dryRun,
     dryRun,
     jobCounts: result.counts,
     totalJobs: result.counts.wait + result.counts.active + result.counts.completed + result.counts.failed + result.counts.delayed
-  };
+  });
 }
 async function fetchJobsList(queueName, jobState, maxResults) {
   const { jobs, total } = await getAllJobs(queueName, jobState, maxResults, true);
@@ -4139,13 +4164,12 @@ async function fetchJobsList(queueName, jobState, maxResults) {
     })));
     markPolledWrites(queueName, jobs.map((j) => j.id));
   } catch {}
-  return {
-    timestamp: new Date().toISOString(),
+  return createResponse({
     queue: queueName,
     jobState: jobState ?? "all",
     jobs,
     total
-  };
+  });
 }
 function computeRetryExitCode(result) {
   if (!result.dryRun && result.errors.length > 0)
@@ -4159,8 +4183,7 @@ async function fetchJobsRetry(queueName, jobState, since, name, pageSize, dryRun
     filter.since = since;
   if (name !== undefined)
     filter.name = name;
-  return {
-    timestamp: new Date().toISOString(),
+  return createResponse({
     command: "jobs-retry",
     dryRun,
     queue: queueName,
@@ -4171,7 +4194,7 @@ async function fetchJobsRetry(queueName, jobState, since, name, pageSize, dryRun
     sampleJobIds: result.sampleJobIds,
     totalFailed: result.totalFailed,
     truncated: result.truncated
-  };
+  });
 }
 async function fetchJobDetail(queueName, jobId) {
   const job = await getJobDetail(queueName, jobId);
@@ -4182,20 +4205,18 @@ async function fetchJobDetail(queueName, jobId) {
     } catch {}
     process.exit(1);
   }
-  return {
-    timestamp: new Date().toISOString(),
+  return createResponse({
     queue: queueName,
     job
-  };
+  });
 }
 async function fetchSchedulersList(queueName, maxResults) {
   const { schedulers, total } = await getAllJobSchedulers(queueName, maxResults);
-  return {
-    timestamp: new Date().toISOString(),
+  return createResponse({
     queue: queueName,
     schedulers,
     total
-  };
+  });
 }
 async function fetchSchedulerDetail(queueName, schedulerKey) {
   const scheduler = await getJobSchedulerDetail(queueName, schedulerKey);
@@ -4206,11 +4227,10 @@ async function fetchSchedulerDetail(queueName, schedulerKey) {
     } catch {}
     process.exit(1);
   }
-  return {
-    timestamp: new Date().toISOString(),
+  return createResponse({
     queue: queueName,
     scheduler
-  };
+  });
 }
 function validateJobState(jobState) {
   if (!jobState)
@@ -4382,5 +4402,5 @@ async function main() {
 }
 main();
 
-//# debugId=E5F5BDE7473A4E0164756E2164756E21
+//# debugId=ABA0BC9822B1617064756E2164756E21
 //# sourceMappingURL=index.js.map