@nicnocquee/dataqueue 1.33.0 → 1.35.0-beta.20260224075710

package/src/backends/redis.ts

@@ -14,6 +14,7 @@ import {
   EditCronScheduleOptions,
   WaitpointRecord,
   CreateTokenOptions,
+  AddJobOptions,
 } from '../types.js';
 import {
   QueueBackend,
@@ -61,6 +62,7 @@ function parseTimeoutString(timeout: string): number {
 }
 import {
   ADD_JOB_SCRIPT,
+  ADD_JOBS_SCRIPT,
   GET_NEXT_BATCH_SCRIPT,
   COMPLETE_JOB_SCRIPT,
   FAIL_JOB_SCRIPT,
@@ -162,9 +164,28 @@ function deserializeJob<PayloadMap, T extends JobType<PayloadMap>>(
     waitUntil: dateOrNull(h.waitUntil),
     waitTokenId: nullish(h.waitTokenId) as string | null | undefined,
     stepData: parseStepData(h.stepData),
+    retryDelay: numOrNull(h.retryDelay),
+    retryBackoff:
+      h.retryBackoff === 'true'
+        ? true
+        : h.retryBackoff === 'false'
+          ? false
+          : null,
+    retryDelayMax: numOrNull(h.retryDelayMax),
+    output: parseJsonField(h.output),
   };
 }
 
+/** Parse a JSON field from a Redis hash, returning null for missing/null values. */
+function parseJsonField(raw: string | undefined): unknown {
+  if (!raw || raw === 'null') return null;
+  try {
+    return JSON.parse(raw);
+  } catch {
+    return null;
+  }
+}
+
 /** Parse step data from a Redis hash field. */
 function parseStepData(
   raw: string | undefined,
@@ -181,8 +202,30 @@ export class RedisBackend implements QueueBackend {
   private client: RedisType;
   private prefix: string;
 
-  constructor(redisConfig: RedisJobQueueConfig['redisConfig']) {
-    // Dynamically require ioredis to avoid hard dep
+  /**
+   * Create a RedisBackend.
+   *
+   * @param configOrClient - Either `redisConfig` from the config file (the
+   *   library creates a new ioredis client) or an existing ioredis client
+   *   instance (bring your own).
+   * @param keyPrefix - Key prefix, only used when `configOrClient` is an
+   *   external client. Ignored when `redisConfig` is passed (uses
+   *   `redisConfig.keyPrefix` instead). Default: `'dq:'`.
+   */
+  constructor(
+    configOrClient: RedisJobQueueConfig['redisConfig'] | RedisType,
+    keyPrefix?: string,
+  ) {
+    if (configOrClient && typeof (configOrClient as any).eval === 'function') {
+      this.client = configOrClient as RedisType;
+      this.prefix = keyPrefix ?? 'dq:';
+      return;
+    }
+
+    const redisConfig = configOrClient as NonNullable<
+      RedisJobQueueConfig['redisConfig']
+    >;
+
     let IORedis: any;
     try {
       const _require = createRequire(import.meta.url);
@@ -260,17 +303,29 @@ export class RedisBackend implements QueueBackend {
 
   // ── Job CRUD ──────────────────────────────────────────────────────────
 
-  async addJob<PayloadMap, T extends JobType<PayloadMap>>({
-    jobType,
-    payload,
-    maxAttempts = 3,
-    priority = 0,
-    runAt = null,
-    timeoutMs = undefined,
-    forceKillOnTimeout = false,
-    tags = undefined,
-    idempotencyKey = undefined,
-  }: JobOptions<PayloadMap, T>): Promise<number> {
+  async addJob<PayloadMap, T extends JobType<PayloadMap>>(
+    {
+      jobType,
+      payload,
+      maxAttempts = 3,
+      priority = 0,
+      runAt = null,
+      timeoutMs = undefined,
+      forceKillOnTimeout = false,
+      tags = undefined,
+      idempotencyKey = undefined,
+      retryDelay = undefined,
+      retryBackoff = undefined,
+      retryDelayMax = undefined,
+    }: JobOptions<PayloadMap, T>,
+    options?: AddJobOptions,
+  ): Promise<number> {
+    if (options?.db) {
+      throw new Error(
+        'The db option is not supported with the Redis backend. ' +
+          'Transactional job creation is only available with PostgreSQL.',
+      );
+    }
     const now = this.nowMs();
     const runAtMs = runAt ? runAt.getTime() : 0;
 
@@ -288,6 +343,9 @@ export class RedisBackend implements QueueBackend {
       tags ? JSON.stringify(tags) : 'null',
       idempotencyKey ?? 'null',
       now,
+      retryDelay !== undefined ? retryDelay.toString() : 'null',
+      retryBackoff !== undefined ? retryBackoff.toString() : 'null',
+      retryDelayMax !== undefined ? retryDelayMax.toString() : 'null',
     )) as number;
 
     const jobId = Number(result);
@@ -303,6 +361,83 @@ export class RedisBackend implements QueueBackend {
     return jobId;
   }
 
+  /**
+   * Insert multiple jobs atomically via a single Lua script.
+   * Returns IDs in the same order as the input array.
+   */
+  async addJobs<PayloadMap, T extends JobType<PayloadMap>>(
+    jobs: JobOptions<PayloadMap, T>[],
+    options?: AddJobOptions,
+  ): Promise<number[]> {
+    if (jobs.length === 0) return [];
+
+    if (options?.db) {
+      throw new Error(
+        'The db option is not supported with the Redis backend. ' +
+          'Transactional job creation is only available with PostgreSQL.',
+      );
+    }
+
+    const now = this.nowMs();
+
+    const jobsPayload = jobs.map((job) => ({
+      jobType: job.jobType,
+      payload: JSON.stringify(job.payload),
+      maxAttempts: job.maxAttempts ?? 3,
+      priority: job.priority ?? 0,
+      runAtMs: job.runAt ? job.runAt.getTime() : 0,
+      timeoutMs:
+        job.timeoutMs !== undefined ? job.timeoutMs.toString() : 'null',
+      forceKillOnTimeout: job.forceKillOnTimeout ? 'true' : 'false',
+      tags: job.tags ? JSON.stringify(job.tags) : 'null',
+      idempotencyKey: job.idempotencyKey ?? 'null',
+      retryDelay:
+        job.retryDelay !== undefined ? job.retryDelay.toString() : 'null',
+      retryBackoff:
+        job.retryBackoff !== undefined ? job.retryBackoff.toString() : 'null',
+      retryDelayMax:
+        job.retryDelayMax !== undefined ? job.retryDelayMax.toString() : 'null',
+    }));
+
+    const result = (await this.client.eval(
+      ADD_JOBS_SCRIPT,
+      1,
+      this.prefix,
+      JSON.stringify(jobsPayload),
+      now,
+    )) as number[];
+
+    const ids = result.map(Number);
+    log(`Batch-inserted ${jobs.length} jobs, IDs: [${ids.join(', ')}]`);
+
+    // Record events for newly inserted jobs (skip idempotency duplicates)
+    const existingIdempotencyIds = new Set<number>();
+    for (let i = 0; i < jobs.length; i++) {
+      if (jobs[i].idempotencyKey) {
+        // If the returned ID existed before this batch, it was a duplicate.
+        // We detect this by checking if the same ID appears for a different
+        // idempotency-keyed job (unlikely) or by checking if the ID was less
+        // than what we'd expect. The simplest approach: record events for all,
+        // since the Lua script returns the existing ID for duplicates but
+        // doesn't tell us if it was newly created. We can compare: if
+        // multiple jobs have the same idempotency key in the batch and got
+        // the same ID, only record once.
+        if (existingIdempotencyIds.has(ids[i])) {
+          continue;
+        }
+        existingIdempotencyIds.add(ids[i]);
+      }
+      await this.recordJobEvent(ids[i], JobEventType.Added, {
+        jobType: jobs[i].jobType,
+        payload: jobs[i].payload,
+        tags: jobs[i].tags,
+        idempotencyKey: jobs[i].idempotencyKey,
+      });
+    }
+
+    return ids;
+  }
+
   async getJob<PayloadMap, T extends JobType<PayloadMap>>(
     id: number,
   ): Promise<JobRecord<PayloadMap, T> | null> {
@@ -469,9 +604,18 @@ export class RedisBackend implements QueueBackend {
     return jobs;
   }
 
-  async completeJob(jobId: number): Promise<void> {
+  async completeJob(jobId: number, output?: unknown): Promise<void> {
     const now = this.nowMs();
-    await this.client.eval(COMPLETE_JOB_SCRIPT, 1, this.prefix, jobId, now);
+    const outputArg =
+      output !== undefined ? JSON.stringify(output) : '__NONE__';
+    await this.client.eval(
+      COMPLETE_JOB_SCRIPT,
+      1,
+      this.prefix,
+      jobId,
+      now,
+      outputArg,
+    );
     await this.recordJobEvent(jobId, JobEventType.Completed);
     log(`Completed job ${jobId}`);
   }
@@ -535,6 +679,24 @@ export class RedisBackend implements QueueBackend {
     }
   }
 
+  // ── Output ────────────────────────────────────────────────────────────
+
+  async updateOutput(jobId: number, output: unknown): Promise<void> {
+    try {
+      const now = this.nowMs();
+      await this.client.hset(
+        `${this.prefix}job:${jobId}`,
+        'output',
+        JSON.stringify(output),
+        'updatedAt',
+        now.toString(),
+      );
+      log(`Updated output for job ${jobId}`);
+    } catch (error) {
+      log(`Error updating output for job ${jobId}: ${error}`);
+    }
+  }
+
   // ── Job management ────────────────────────────────────────────────────
 
   async retryJob(jobId: number): Promise<void> {
@@ -657,6 +819,31 @@ export class RedisBackend implements QueueBackend {
       }
       metadata.tags = updates.tags;
     }
+    if (updates.retryDelay !== undefined) {
+      fields.push(
+        'retryDelay',
+        updates.retryDelay !== null ? updates.retryDelay.toString() : 'null',
+      );
+      metadata.retryDelay = updates.retryDelay;
+    }
+    if (updates.retryBackoff !== undefined) {
+      fields.push(
+        'retryBackoff',
+        updates.retryBackoff !== null
+          ? updates.retryBackoff.toString()
+          : 'null',
+      );
+      metadata.retryBackoff = updates.retryBackoff;
+    }
+    if (updates.retryDelayMax !== undefined) {
+      fields.push(
+        'retryDelayMax',
+        updates.retryDelayMax !== null
+          ? updates.retryDelayMax.toString()
+          : 'null',
+      );
+      metadata.retryDelayMax = updates.retryDelayMax;
+    }
 
     if (fields.length === 0) {
       log(`No fields to update for job ${jobId}`);
@@ -1236,6 +1423,18 @@ export class RedisBackend implements QueueBackend {
       now.toString(),
       'updatedAt',
       now.toString(),
+      'retryDelay',
+      input.retryDelay !== null && input.retryDelay !== undefined
+        ? input.retryDelay.toString()
+        : 'null',
+      'retryBackoff',
+      input.retryBackoff !== null && input.retryBackoff !== undefined
+        ? input.retryBackoff.toString()
+        : 'null',
+      'retryDelayMax',
+      input.retryDelayMax !== null && input.retryDelayMax !== undefined
+        ? input.retryDelayMax.toString()
+        : 'null',
     ];
 
     await (this.client as any).hmset(key, ...fields);
@@ -1417,6 +1616,28 @@ export class RedisBackend implements QueueBackend {
     if (updates.allowOverlap !== undefined) {
       fields.push('allowOverlap', updates.allowOverlap ? 'true' : 'false');
     }
+    if (updates.retryDelay !== undefined) {
+      fields.push(
+        'retryDelay',
+        updates.retryDelay !== null ? updates.retryDelay.toString() : 'null',
+      );
+    }
+    if (updates.retryBackoff !== undefined) {
+      fields.push(
+        'retryBackoff',
+        updates.retryBackoff !== null
+          ? updates.retryBackoff.toString()
+          : 'null',
+      );
+    }
+    if (updates.retryDelayMax !== undefined) {
+      fields.push(
+        'retryDelayMax',
+        updates.retryDelayMax !== null
+          ? updates.retryDelayMax.toString()
+          : 'null',
+      );
+    }
     if (nextRunAt !== undefined) {
       const val = nextRunAt !== null ? nextRunAt.getTime().toString() : 'null';
       fields.push('nextRunAt', val);
@@ -1557,6 +1778,14 @@ export class RedisBackend implements QueueBackend {
       nextRunAt: dateOrNull(h.nextRunAt),
       createdAt: new Date(Number(h.createdAt)),
       updatedAt: new Date(Number(h.updatedAt)),
+      retryDelay: numOrNull(h.retryDelay),
+      retryBackoff:
+        h.retryBackoff === 'true'
+          ? true
+          : h.retryBackoff === 'false'
+            ? false
+            : null,
+      retryDelayMax: numOrNull(h.retryDelayMax),
     };
   }
 

package/src/cli.test.ts

@@ -23,6 +23,10 @@ function makeDeps() {
     spawnSyncImpl: vi.fn(() => makeSpawnSyncReturns(0)),
     migrationsDir: '/migrations',
     runInitImpl: vi.fn(),
+    runInstallSkillsImpl: vi.fn(),
+    runInstallRulesImpl: vi.fn(async () => {}),
+    runInstallMcpImpl: vi.fn(async () => {}),
+    startMcpServerImpl: vi.fn(async () => ({}) as any),
   } satisfies CliDeps;
 }
 
@@ -138,4 +142,65 @@ describe('runCli', () => {
     runCli(['node', 'cli.js', 'migrate'], deps);
     expect(deps.exit).toHaveBeenCalledWith(1);
   });
+
+  it('routes install-skills command to runInstallSkillsImpl', () => {
+    // Act
+    runCli(['node', 'cli.js', 'install-skills'], deps);
+
+    // Assert
+    expect(deps.runInstallSkillsImpl).toHaveBeenCalledWith(
+      expect.objectContaining({
+        log: deps.log,
+        error: deps.error,
+        exit: deps.exit,
+      }),
+    );
+  });
+
+  it('routes install-rules command to runInstallRulesImpl', () => {
+    // Act
+    runCli(['node', 'cli.js', 'install-rules'], deps);
+
+    // Assert
+    expect(deps.runInstallRulesImpl).toHaveBeenCalledWith(
+      expect.objectContaining({
+        log: deps.log,
+        error: deps.error,
+        exit: deps.exit,
+      }),
+    );
+  });
+
+  it('routes install-mcp command to runInstallMcpImpl', () => {
+    // Act
+    runCli(['node', 'cli.js', 'install-mcp'], deps);
+
+    // Assert
+    expect(deps.runInstallMcpImpl).toHaveBeenCalledWith(
+      expect.objectContaining({
+        log: deps.log,
+        error: deps.error,
+        exit: deps.exit,
+      }),
+    );
+  });
+
+  it('routes mcp command to startMcpServerImpl', () => {
+    // Act
+    runCli(['node', 'cli.js', 'mcp'], deps);
+
+    // Assert
+    expect(deps.startMcpServerImpl).toHaveBeenCalled();
+  });
+
+  it('shows new commands in usage output', () => {
+    // Act
+    runCli(['node', 'cli.js'], deps);
+
+    // Assert
+    expect(deps.log).toHaveBeenCalledWith('  dataqueue-cli install-skills');
+    expect(deps.log).toHaveBeenCalledWith('  dataqueue-cli install-rules');
+    expect(deps.log).toHaveBeenCalledWith('  dataqueue-cli install-mcp');
+    expect(deps.log).toHaveBeenCalledWith('  dataqueue-cli mcp');
+  });
 });

package/src/cli.ts

@@ -3,6 +3,13 @@ import { spawnSync, SpawnSyncReturns } from 'child_process';
 import path from 'path';
 import { fileURLToPath } from 'url';
 import { InitDeps, runInit } from './init-command.js';
+import {
+  runInstallSkills,
+  InstallSkillsDeps,
+} from './install-skills-command.js';
+import { runInstallRules, InstallRulesDeps } from './install-rules-command.js';
+import { runInstallMcp, InstallMcpDeps } from './install-mcp-command.js';
+import { startMcpServer } from './mcp-server.js';
 
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
@@ -15,6 +22,13 @@ export interface CliDeps {
   migrationsDir?: string;
   initDeps?: InitDeps;
   runInitImpl?: (deps?: InitDeps) => void;
+  installSkillsDeps?: InstallSkillsDeps;
+  runInstallSkillsImpl?: (deps?: InstallSkillsDeps) => void;
+  installRulesDeps?: InstallRulesDeps;
+  runInstallRulesImpl?: (deps?: InstallRulesDeps) => Promise<void>;
+  installMcpDeps?: InstallMcpDeps;
+  runInstallMcpImpl?: (deps?: InstallMcpDeps) => Promise<void>;
+  startMcpServerImpl?: typeof startMcpServer;
 }
 
 export function runCli(
@@ -27,19 +41,27 @@ export function runCli(
     migrationsDir = path.join(__dirname, '../migrations'),
     initDeps,
     runInitImpl = runInit,
+    installSkillsDeps,
+    runInstallSkillsImpl = runInstallSkills,
+    installRulesDeps,
+    runInstallRulesImpl = runInstallRules,
+    installMcpDeps,
+    runInstallMcpImpl = runInstallMcp,
+    startMcpServerImpl = startMcpServer,
   }: CliDeps = {},
 ): void {
   const [, , command, ...restArgs] = argv;
 
-  /**
-   * Prints CLI usage and exits with non-zero code.
-   */
   function printUsage() {
     log('Usage:');
     log(
       '  dataqueue-cli migrate [--envPath <path>] [-s <schema> | --schema <schema>]',
     );
     log('  dataqueue-cli init');
+    log('  dataqueue-cli install-skills');
+    log('  dataqueue-cli install-rules');
+    log('  dataqueue-cli install-mcp');
+    log('  dataqueue-cli mcp');
     log('');
     log('Options for migrate:');
     log(
@@ -49,24 +71,13 @@ export function runCli(
       '  -s, --schema <schema>  Set the schema to use (passed to node-pg-migrate)',
     );
     log('');
-    log('Notes:');
+    log('AI tooling commands:');
+    log('  install-skills     Install DataQueue skill files for AI assistants');
+    log('  install-rules      Install DataQueue agent rules for AI clients');
     log(
-      '  - The PG_DATAQUEUE_DATABASE environment variable must be set to your Postgres connection string.',
-    );
-    log(
-      '  - For managed Postgres (e.g., DigitalOcean) with SSL, set PGSSLMODE=require and PGSSLROOTCERT to your CA .crt file.',
-    );
-    log(
-      '    Example: PGSSLMODE=require NODE_EXTRA_CA_CERTS=/absolute/path/to/ca.crt PG_DATAQUEUE_DATABASE=... npx dataqueue-cli migrate',
-    );
-    log('');
-    log('Notes for init:');
-    log(
-      '  - Supports both Next.js App Router and Pages Router (prefers App Router if both exist).',
-    );
-    log(
-      '  - Scaffolds endpoint, cron.sh, queue placeholder, and package.json entries.',
+      '  install-mcp        Configure the DataQueue MCP server for AI clients',
     );
+    log('  mcp                Start the DataQueue MCP server (stdio)');
     exit(1);
   }
 
@@ -115,6 +126,32 @@ export function runCli(
       exit,
       ...initDeps,
     });
+  } else if (command === 'install-skills') {
+    runInstallSkillsImpl({
+      log,
+      error,
+      exit,
+      ...installSkillsDeps,
+    });
+  } else if (command === 'install-rules') {
+    runInstallRulesImpl({
+      log,
+      error,
+      exit,
+      ...installRulesDeps,
+    });
+  } else if (command === 'install-mcp') {
+    runInstallMcpImpl({
+      log,
+      error,
+      exit,
+      ...installMcpDeps,
+    });
+  } else if (command === 'mcp') {
+    startMcpServerImpl().catch((err) => {
+      error('Failed to start MCP server:', err);
+      exit(1);
+    });
   } else {
     printUsage();
   }

package/src/db-util.ts

@@ -27,7 +27,7 @@ function loadPemOrFile(value?: string): string | undefined {
  *   }
  */
 export const createPool = (
-  config: PostgresJobQueueConfig['databaseConfig'],
+  config: NonNullable<PostgresJobQueueConfig['databaseConfig']>,
 ): Pool => {
   let searchPath: string | undefined;
   let ssl: any = undefined;