@nicnocquee/dataqueue 1.19.2 → 1.20.0

package/src/queue.test.ts

@@ -5,8 +5,6 @@ import { createTestDbAndPool, destroyTestDb } from './test-util.js';
 import { JobEvent, JobEventType } from './types.js';
 import { objectKeysToCamelCase } from './utils.js';
 
-// Example integration test setup
-
 describe('queue integration', () => {
   let pool: Pool;
   let dbName: string;
@@ -501,3 +499,252 @@ describe('job lifecycle timestamp columns', () => {
     expect(job.lastRetriedAt).not.toBeNull();
   });
 });
+
+describe('tags feature', () => {
+  let pool: Pool;
+  let dbName: string;
+
+  beforeEach(async () => {
+    const setup = await createTestDbAndPool();
+    pool = setup.pool;
+    dbName = setup.dbName;
+  });
+
+  afterEach(async () => {
+    await pool.end();
+    await destroyTestDb(dbName);
+  });
+
+  it('should add a job with tags and retrieve it by tags (all mode)', async () => {
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'tagged@example.com' },
+      tags: ['welcome', 'user:1'],
+    });
+    const jobs = await queue.getJobsByTags(pool, ['welcome'], 'all');
+    expect(jobs.map((j) => j.id)).toContain(jobId);
+    expect(jobs[0].tags).toContain('welcome');
+    expect(jobs[0].tags).toContain('user:1');
+  });
+
+  it('should only return jobs that match all specified tags (all mode)', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'a@example.com' },
+        tags: ['foo', 'bar'],
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'b@example.com' },
+        tags: ['foo'],
+      },
+    );
+
+    const jobId3 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'c@example.com' },
+        tags: ['foo', 'bar', 'baz'],
+      },
+    );
+    const jobs = await queue.getJobsByTags(pool, ['foo', 'bar'], 'all');
+    expect(jobs.map((j) => j.id)).toContain(jobId1);
+    expect(jobs.map((j) => j.id)).not.toContain(jobId2);
+    expect(jobs.map((j) => j.id)).toContain(jobId3);
+  });
+
+  it('should return jobs with exactly the same tags (exact mode)', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'a@example.com' },
+        tags: ['foo', 'bar'],
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'b@example.com' },
+        tags: ['foo', 'bar', 'baz'],
+      },
+    );
+    const jobs = await queue.getJobsByTags(pool, ['foo', 'bar'], 'exact');
+    expect(jobs.map((j) => j.id)).toContain(jobId1);
+    expect(jobs.map((j) => j.id)).not.toContain(jobId2);
+  });
+
+  it('should return jobs that have any of the given tags (any mode)', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'a@example.com' },
+        tags: ['foo', 'bar'],
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'b@example.com' },
+        tags: ['baz'],
+      },
+    );
+    const jobs = await queue.getJobsByTags(pool, ['bar', 'baz'], 'any');
+    expect(jobs.map((j) => j.id)).toContain(jobId1);
+    expect(jobs.map((j) => j.id)).toContain(jobId2);
+  });
+
+  it('should return jobs that have none of the given tags (none mode)', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'a@example.com' },
+        tags: ['foo'],
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'b@example.com' },
+        tags: ['bar'],
+      },
+    );
+    const jobId3 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'c@example.com' },
+        tags: ['baz'],
+      },
+    );
+    const jobs = await queue.getJobsByTags(pool, ['foo', 'bar'], 'none');
+    expect(jobs.map((j) => j.id)).toContain(jobId3);
+    expect(jobs.map((j) => j.id)).not.toContain(jobId1);
+    expect(jobs.map((j) => j.id)).not.toContain(jobId2);
+  });
+
+  it('should handle jobs with no tags', async () => {
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'notag@example.com' },
+    });
+    const jobs = await queue.getJobsByTags(pool, ['anytag'], 'all');
+    expect(jobs.map((j) => j.id)).not.toContain(jobId);
+  });
+
+  it('should cancel jobs by tags (all mode)', async () => {
+    const jobId = await queue.addJob<{ email: { to: string } }, 'email'>(pool, {
+      jobType: 'email',
+      payload: { to: 'cancelme@example.com' },
+      tags: ['cancel', 'urgent'],
+    });
+    const cancelled = await queue.cancelAllUpcomingJobs(pool, {
+      tags: { values: ['cancel', 'urgent'], mode: 'all' },
+    });
+    expect(cancelled).toBeGreaterThanOrEqual(1);
+    const job = await queue.getJob(pool, jobId);
+    expect(job?.status).toBe('cancelled');
+  });
+
+  it('should cancel jobs by tags (exact mode)', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'cancel1@example.com' },
+        tags: ['cancel', 'urgent'],
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'cancel2@example.com' },
+        tags: ['cancel', 'urgent', 'other'],
+      },
+    );
+    const cancelled = await queue.cancelAllUpcomingJobs(pool, {
+      tags: { values: ['cancel', 'urgent'], mode: 'exact' },
+    });
+    expect(cancelled).toBe(1);
+    const job1 = await queue.getJob(pool, jobId1);
+    const job2 = await queue.getJob(pool, jobId2);
+    expect(job1?.status).toBe('cancelled');
+    expect(job2?.status).toBe('pending');
+  });
+
+  it('should cancel jobs by tags (any mode)', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'cancel1@example.com' },
+        tags: ['cancel', 'urgent'],
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'cancel2@example.com' },
+        tags: ['other'],
+      },
+    );
+    const cancelled = await queue.cancelAllUpcomingJobs(pool, {
+      tags: { values: ['cancel', 'other'], mode: 'any' },
+    });
+    expect(cancelled).toBe(2);
+    const job1 = await queue.getJob(pool, jobId1);
+    const job2 = await queue.getJob(pool, jobId2);
+    expect(job1?.status).toBe('cancelled');
+    expect(job2?.status).toBe('cancelled');
+  });
+
+  it('should cancel jobs by tags (none mode)', async () => {
+    const jobId1 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'cancel1@example.com' },
+        tags: ['foo'],
+      },
+    );
+    const jobId2 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'cancel2@example.com' },
+        tags: ['bar'],
+      },
+    );
+    const jobId3 = await queue.addJob<{ email: { to: string } }, 'email'>(
+      pool,
+      {
+        jobType: 'email',
+        payload: { to: 'keep@example.com' },
+        tags: ['baz'],
+      },
+    );
+    const cancelled = await queue.cancelAllUpcomingJobs(pool, {
+      tags: { values: ['foo', 'bar'], mode: 'none' },
+    });
+    expect(cancelled).toBe(1);
+    const job1 = await queue.getJob(pool, jobId1);
+    const job2 = await queue.getJob(pool, jobId2);
+    const job3 = await queue.getJob(pool, jobId3);
+    expect(job1?.status).toBe('pending');
+    expect(job2?.status).toBe('pending');
+    expect(job3?.status).toBe('cancelled');
+  });
+});

package/src/queue.ts

@@ -5,6 +5,7 @@ import {
   FailureReason,
   JobEvent,
   JobEventType,
+  TagQueryMode,
 } from './types.js';
 import { log } from './log-context.js';
 
@@ -43,6 +44,7 @@ export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
     priority = 0,
     runAt = null,
     timeoutMs = undefined,
+    tags = undefined,
   }: JobOptions<PayloadMap, T>,
 ): Promise<number> => {
   const client = await pool.connect();
@@ -51,29 +53,45 @@ export const addJob = async <PayloadMap, T extends keyof PayloadMap & string>(
     if (runAt) {
       result = await client.query(
         `INSERT INTO job_queue 
-          (job_type, payload, max_attempts, priority, run_at, timeout_ms) 
-         VALUES ($1, $2, $3, $4, $5, $6) 
+          (job_type, payload, max_attempts, priority, run_at, timeout_ms, tags) 
+         VALUES ($1, $2, $3, $4, $5, $6, $7) 
          RETURNING id`,
-        [jobType, payload, maxAttempts, priority, runAt, timeoutMs ?? null],
+        [
+          jobType,
+          payload,
+          maxAttempts,
+          priority,
+          runAt,
+          timeoutMs ?? null,
+          tags ?? null,
+        ],
       );
       log(
-        `Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)}, runAt ${runAt.toISOString()}, priority ${priority}, maxAttempts ${maxAttempts} jobType ${jobType}`,
+        `Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)}, runAt ${runAt.toISOString()}, priority ${priority}, maxAttempts ${maxAttempts} jobType ${jobType}, tags ${JSON.stringify(tags)}`,
       );
     } else {
       result = await client.query(
         `INSERT INTO job_queue 
-          (job_type, payload, max_attempts, priority, timeout_ms) 
-         VALUES ($1, $2, $3, $4, $5) 
+          (job_type, payload, max_attempts, priority, timeout_ms, tags) 
+         VALUES ($1, $2, $3, $4, $5, $6) 
          RETURNING id`,
-        [jobType, payload, maxAttempts, priority, timeoutMs ?? null],
+        [
+          jobType,
+          payload,
+          maxAttempts,
+          priority,
+          timeoutMs ?? null,
+          tags ?? null,
+        ],
       );
       log(
-        `Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)}, priority ${priority}, maxAttempts ${maxAttempts} jobType ${jobType}`,
+        `Added job ${result.rows[0].id}: payload ${JSON.stringify(payload)}, priority ${priority}, maxAttempts ${maxAttempts} jobType ${jobType}, tags ${JSON.stringify(tags)}`,
       );
     }
     await recordJobEvent(pool, result.rows[0].id, JobEventType.Added, {
       jobType,
       payload,
+      tags,
     });
     return result.rows[0].id;
   } catch (error) {
@@ -400,7 +418,12 @@ export const cancelJob = async (pool: Pool, jobId: number): Promise<void> => {
  */
 export const cancelAllUpcomingJobs = async (
   pool: Pool,
-  filters?: { jobType?: string; priority?: number; runAt?: Date },
+  filters?: {
+    jobType?: string;
+    priority?: number;
+    runAt?: Date;
+    tags?: { values: string[]; mode?: TagQueryMode };
+  },
 ): Promise<number> => {
   const client = await pool.connect();
   try {
@@ -423,6 +446,35 @@ export const cancelAllUpcomingJobs = async (
         query += ` AND run_at = $${paramIdx++}`;
         params.push(filters.runAt);
       }
+      if (
+        filters.tags &&
+        filters.tags.values &&
+        filters.tags.values.length > 0
+      ) {
+        const mode = filters.tags.mode || 'all';
+        const tagValues = filters.tags.values;
+        switch (mode) {
+          case 'exact':
+            query += ` AND tags = $${paramIdx++}`;
+            params.push(tagValues);
+            break;
+          case 'all':
+            query += ` AND tags @> $${paramIdx++}`;
+            params.push(tagValues);
+            break;
+          case 'any':
+            query += ` AND tags && $${paramIdx++}`;
+            params.push(tagValues);
+            break;
+          case 'none':
+            query += ` AND NOT (tags && $${paramIdx++})`;
+            params.push(tagValues);
+            break;
+          default:
+            query += ` AND tags @> $${paramIdx++}`;
+            params.push(tagValues);
+        }
+      }
     }
     query += '\nRETURNING id';
     const result = await client.query(query, params);
@@ -548,3 +600,64 @@ export const getJobEvents = async (
     client.release();
   }
 };
+
+/**
+ * Get jobs by tags (matches all specified tags)
+ */
+export const getJobsByTags = async <
+  PayloadMap,
+  T extends keyof PayloadMap & string,
+>(
+  pool: Pool,
+  tags: string[],
+  mode: TagQueryMode = 'all',
+  limit = 100,
+  offset = 0,
+): Promise<JobRecord<PayloadMap, T>[]> => {
+  const client = await pool.connect();
+  try {
+    let query = `SELECT id, job_type AS "jobType", payload, status, max_attempts AS "maxAttempts", attempts, priority, run_at AS "runAt", timeout_ms AS "timeoutMs", created_at AS "createdAt", updated_at AS "updatedAt", started_at AS "startedAt", completed_at AS "completedAt", last_failed_at AS "lastFailedAt", locked_at AS "lockedAt", locked_by AS "lockedBy", error_history AS "errorHistory", failure_reason AS "failureReason", next_attempt_at AS "nextAttemptAt", last_failed_at AS "lastFailedAt", last_retried_at AS "lastRetriedAt", last_cancelled_at AS "lastCancelledAt", pending_reason AS "pendingReason", tags
+       FROM job_queue`;
+    let params: any[] = [];
+    switch (mode) {
+      case 'exact':
+        query += ' WHERE tags = $1';
+        params = [tags];
+        break;
+      case 'all':
+        query += ' WHERE tags @> $1';
+        params = [tags];
+        break;
+      case 'any':
+        query += ' WHERE tags && $1';
+        params = [tags];
+        break;
+      case 'none':
+        query += ' WHERE NOT (tags && $1)';
+        params = [tags];
+        break;
+      default:
+        query += ' WHERE tags @> $1';
+        params = [tags];
+    }
+    query += ' ORDER BY created_at DESC LIMIT $2 OFFSET $3';
+    params.push(limit, offset);
+    const result = await client.query(query, params);
+    log(
+      `Found ${result.rows.length} jobs by tags ${JSON.stringify(tags)} (mode: ${mode})`,
+    );
+    return result.rows.map((job) => ({
+      ...job,
+      payload: job.payload,
+      timeoutMs: job.timeoutMs,
+      failureReason: job.failureReason,
+    }));
+  } catch (error) {
+    log(
+      `Error getting jobs by tags ${JSON.stringify(tags)} (mode: ${mode}): ${error}`,
+    );
+    throw error;
+  } finally {
+    client.release();
+  }
+};

package/src/test-util.ts

@@ -1,8 +1,12 @@
 import { Pool } from 'pg';
 import { randomUUID } from 'crypto';
-import { join } from 'path';
+import { fileURLToPath } from 'url';
+import { join, dirname } from 'path';
 import { runner } from 'node-pg-migrate';
 
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
 export async function createTestDbAndPool() {
   const baseDatabaseUrl =
     process.env.PG_TEST_URL ||

package/src/types.ts

@@ -13,6 +13,10 @@ export interface JobOptions<PayloadMap, T extends JobType<PayloadMap>> {
    * Timeout for this job in milliseconds. If not set, uses the processor default or unlimited.
    */
   timeoutMs?: number;
+  /**
+   * Tags for this job. Used for grouping, searching, or batch operations.
+   */
+  tags?: string[];
 }
 
 export enum JobEventType {
@@ -89,6 +93,10 @@ export interface JobRecord<PayloadMap, T extends JobType<PayloadMap>> {
    * The time the job was last cancelled.
    */
   lastCancelledAt: Date | null;
+  /**
+   * Tags for this job. Used for grouping, searching, or batch operations.
+   */
+  tags?: string[];
 }
 
 export type JobHandler<PayloadMap, T extends keyof PayloadMap> = (
@@ -169,6 +177,8 @@ export interface JobQueueConfig {
   verbose?: boolean;
 }
 
+export type TagQueryMode = 'exact' | 'all' | 'any' | 'none';
+
 export interface JobQueue<PayloadMap> {
   /**
    * Add a job to the job queue.
@@ -193,6 +203,21 @@ export interface JobQueue<PayloadMap> {
     limit?: number,
     offset?: number,
   ) => Promise<JobRecord<PayloadMap, T>[]>;
+  /**
+   * Get jobs by tag(s).
+   * - Modes:
+   *   - 'exact': Jobs with exactly the same tags (no more, no less)
+   *   - 'all': Jobs that have all the given tags (can have more)
+   *   - 'any': Jobs that have at least one of the given tags
+   *   - 'none': Jobs that have none of the given tags
+   * - Default mode is 'all'.
+   */
+  getJobsByTags: <T extends JobType<PayloadMap>>(
+    tags: string[],
+    mode?: TagQueryMode,
+    limit?: number,
+    offset?: number,
+  ) => Promise<JobRecord<PayloadMap, T>[]>;
   /**
    * Get all jobs.
    */
@@ -229,11 +254,13 @@ export interface JobQueue<PayloadMap> {
    *   - jobType: The job type to cancel.
    *   - priority: The priority of the job to cancel.
    *   - runAt: The time the job is scheduled to run at.
+   *   - tags: An object with 'values' (string[]) and 'mode' (TagQueryMode) for tag-based cancellation.
    */
   cancelAllUpcomingJobs: (filters?: {
     jobType?: string;
     priority?: number;
     runAt?: Date;
+    tags?: { values: string[]; mode?: TagQueryMode };
   }) => Promise<number>;
   /**
    * Create a job processor. Handlers must be provided per-processor.