@nicnocquee/dataqueue 1.38.0 → 1.39.0-beta.20260322125514

package/src/backends/redis-scripts.ts

@@ -32,7 +32,8 @@ const SCORE_RANGE = '1000000000000000'; // 1e15
  * KEYS: [prefix]
  * ARGV: [jobType, payloadJson, maxAttempts, priority, runAtMs, timeoutMs,
  *        forceKillOnTimeout, tagsJson, idempotencyKey, nowMs,
- *        retryDelay, retryBackoff, retryDelayMax, deadLetterJobType, groupId, groupTier]
+ *        retryDelay, retryBackoff, retryDelayMax, deadLetterJobType, groupId, groupTier,
+ *        dependsOnJobIdsJson, dependsOnTagsJson]
  * Returns: job ID (number)
  */
 export const ADD_JOB_SCRIPT = `
@@ -53,6 +54,8 @@ local retryDelayMax = ARGV[13]    -- "null" or seconds string
 local deadLetterJobType = ARGV[14] -- "null" or jobType string
 local groupId = ARGV[15]          -- "null" or group ID
 local groupTier = ARGV[16]        -- "null" or group tier
+local dependsOnJobIdsJson = ARGV[17] -- "null" or JSON array of job ids
+local dependsOnTagsJson = ARGV[18]   -- "null" or JSON array of tags
 
 -- Idempotency check
 if idempotencyKey ~= "null" then
@@ -104,9 +107,18 @@ redis.call('HMSET', jobKey,
   'deadLetteredAt', 'null',
   'deadLetterJobId', 'null',
   'groupId', groupId,
-  'groupTier', groupTier
+  'groupTier', groupTier,
+  'dependsOnJobIds', dependsOnJobIdsJson,
+  'dependsOnTags', dependsOnTagsJson
 )
 
+if dependsOnJobIdsJson ~= "null" then
+  local depIds = cjson.decode(dependsOnJobIdsJson)
+  for _, parentId in ipairs(depIds) do
+    redis.call('SADD', prefix .. 'dep:' .. tostring(parentId), tostring(id))
+  end
+end
+
 -- Status index
 redis.call('SADD', prefix .. 'status:pending', id)
 
@@ -155,6 +167,9 @@ return id
  *     runAtMs, timeoutMs, forceKillOnTimeout, tags (JSON or "null"),
  *     idempotencyKey, retryDelay, retryBackoff, retryDelayMax, deadLetterJobType
  * Returns: array of job IDs (one per input job, in order)
+ *
+ * Note: JSON null decodes to cjson.null, which is truthy in Lua; optional fields must
+ * be normalized with explicit nil/cjson.null checks before tostring/cjson.decode.
  */
 export const ADD_JOBS_SCRIPT = `
 local prefix = KEYS[1]
@@ -180,6 +195,8 @@ for i, job in ipairs(jobs) do
   local deadLetterJobType = tostring(job.deadLetterJobType)
   local groupId = tostring(job.groupId)
   local groupTier = tostring(job.groupTier)
+  local dependsOnJobIdsJson = (job.dependsOnJobIds ~= nil and job.dependsOnJobIds ~= cjson.null) and tostring(job.dependsOnJobIds) or "null"
+  local dependsOnTagsJson = (job.dependsOnTags ~= nil and job.dependsOnTags ~= cjson.null) and tostring(job.dependsOnTags) or "null"
 
   -- Idempotency check
   local skip = false
@@ -234,9 +251,18 @@ for i, job in ipairs(jobs) do
       'deadLetteredAt', 'null',
       'deadLetterJobId', 'null',
       'groupId', groupId,
-      'groupTier', groupTier
+      'groupTier', groupTier,
+      'dependsOnJobIds', dependsOnJobIdsJson,
+      'dependsOnTags', dependsOnTagsJson
     )
 
+    if dependsOnJobIdsJson ~= "null" then
+      local depIds = cjson.decode(dependsOnJobIdsJson)
+      for _, parentId in ipairs(depIds) do
+        redis.call('SADD', prefix .. 'dep:' .. tostring(parentId), tostring(id))
+      end
+    end
+
     -- Status index
     redis.call('SADD', prefix .. 'status:pending', id)
 
@@ -409,6 +435,48 @@ for i = 1, #candidates, 2 do
           end
         end
 
+        if canClaim then
+          local depIdsJson = redis.call('HGET', jk, 'dependsOnJobIds')
+          local depTagsJson = redis.call('HGET', jk, 'dependsOnTags')
+          local depsOk = true
+          if depIdsJson and depIdsJson ~= 'null' then
+            local dids = cjson.decode(depIdsJson)
+            for _, pid in ipairs(dids) do
+              local pst = redis.call('HGET', prefix .. 'job:' .. pid, 'status')
+              if pst ~= 'completed' then depsOk = false break end
+            end
+          end
+          if depsOk and depTagsJson and depTagsJson ~= 'null' then
+            local req = cjson.decode(depTagsJson)
+            if #req > 0 then
+              for _, stname in ipairs({'pending','processing','waiting'}) do
+                local members = redis.call('SMEMBERS', prefix .. 'status:' .. stname)
+                for _, oid in ipairs(members) do
+                  if oid ~= jobId then
+                    local otags = redis.call('HGET', prefix .. 'job:' .. oid, 'tags')
+                    if otags and otags ~= 'null' then
+                      local oarr = cjson.decode(otags)
+                      local tagset = {}
+                      for _, t in ipairs(oarr) do tagset[t] = true end
+                      local all = true
+                      for _, rt in ipairs(req) do
+                        if not tagset[rt] then all = false break end
+                      end
+                      if all then depsOk = false break end
+                    end
+                  end
+                end
+                if not depsOk then break end
+              end
+            end
+          end
+          if not depsOk then
+            table.insert(putBack, score)
+            table.insert(putBack, jobId)
+            canClaim = false
+          end
+        end
+
         if canClaim then
           -- Claim this job
           local attempts = tonumber(redis.call('HGET', jk, 'attempts'))
@@ -495,6 +563,14 @@ if groupId and groupId ~= 'null' then
   end
 end
 
+local depIdsJson = redis.call('HGET', jk, 'dependsOnJobIds')
+if depIdsJson and depIdsJson ~= 'null' then
+  local dids = cjson.decode(depIdsJson)
+  for _, pid in ipairs(dids) do
+    redis.call('SREM', prefix .. 'dep:' .. tostring(pid), jobId)
+  end
+end
+
 return 1
 `;
 
@@ -650,7 +726,11 @@ if nextAttemptAt == 'null' and deadLetterJobType and deadLetterJobType ~= 'null'
     'retryDelayMax', 'null',
     'deadLetterJobType', 'null',
     'deadLetteredAt', 'null',
-    'deadLetterJobId', 'null'
+    'deadLetterJobId', 'null',
+    'dependsOnJobIds', 'null',
+    'dependsOnTags', 'null',
+    'groupId', 'null',
+    'groupTier', 'null'
   )
 
   redis.call('SADD', prefix .. 'status:pending', deadLetterJobId)
@@ -665,6 +745,14 @@ if nextAttemptAt == 'null' and deadLetterJobType and deadLetterJobType ~= 'null'
   )
 end
 
+local depIdsJsonFail = redis.call('HGET', jk, 'dependsOnJobIds')
+if depIdsJsonFail and depIdsJsonFail ~= 'null' then
+  local dids = cjson.decode(depIdsJsonFail)
+  for _, pid in ipairs(dids) do
+    redis.call('SREM', prefix .. 'dep:' .. tostring(pid), jobId)
+  end
+end
+
 return deadLetterJobId
 `;
 
@@ -743,6 +831,14 @@ redis.call('ZREM', prefix .. 'queue', jobId)
 redis.call('ZREM', prefix .. 'delayed', jobId)
 redis.call('ZREM', prefix .. 'waiting', jobId)
 
+local depIdsJsonCan = redis.call('HGET', jk, 'dependsOnJobIds')
+if depIdsJsonCan and depIdsJsonCan ~= 'null' then
+  local dids = cjson.decode(depIdsJsonCan)
+  for _, pid in ipairs(dids) do
+    redis.call('SREM', prefix .. 'dep:' .. tostring(pid), jobId)
+  end
+end
+
 return 1
 `;
 

package/src/backends/redis.ts

@@ -23,6 +23,11 @@ import {
   CronScheduleInput,
 } from '../backend.js';
 import { log } from '../log-context.js';
+import {
+  normalizeDependsOn,
+  resolveDependsOnJobIdsForBatch,
+  tagsAreSuperset,
+} from '../job-dependencies.js';
 
 const MAX_TIMEOUT_MS = 365 * 24 * 60 * 60 * 1000;
 
@@ -181,9 +186,31 @@ function deserializeJob<PayloadMap, T extends JobType<PayloadMap>>(
     groupId: nullish(h.groupId) as string | null | undefined,
     groupTier: nullish(h.groupTier) as string | null | undefined,
     output: parseJsonField(h.output),
+    dependsOnJobIds: parseOptionalIntArray(h.dependsOnJobIds),
+    dependsOnTags: parseOptionalStringArray(h.dependsOnTags),
   };
 }
 
+function parseOptionalIntArray(raw: string | undefined): number[] | null {
+  if (!raw || raw === 'null') return null;
+  try {
+    const arr = JSON.parse(raw) as number[];
+    return Array.isArray(arr) && arr.length > 0 ? arr : null;
+  } catch {
+    return null;
+  }
+}
+
+function parseOptionalStringArray(raw: string | undefined): string[] | null {
+  if (!raw || raw === 'null') return null;
+  try {
+    const arr = JSON.parse(raw) as string[];
+    return Array.isArray(arr) && arr.length > 0 ? arr : null;
+  } catch {
+    return null;
+  }
+}
+
 /** Parse a JSON field from a Redis hash, returning null for missing/null values. */
 function parseJsonField(raw: string | undefined): unknown {
   if (!raw || raw === 'null') return null;
@@ -271,6 +298,76 @@ export class RedisBackend implements QueueBackend {
     return Date.now();
   }
 
+  /**
+   * Cancel pending/waiting jobs that depend on seed jobs (job id or tag), transitively.
+   *
+   * @param initialSeeds - Job ids that failed or were cancelled.
+   * @param rootJobId - Root id for event metadata.
+   */
+  private async propagateDependencyCancellationsRedis(
+    initialSeeds: number[],
+    rootJobId: number,
+  ): Promise<void> {
+    const cancelled = new Set<number>();
+    let frontier = [...new Set(initialSeeds.filter((id) => id > 0))];
+
+    while (frontier.length > 0) {
+      const pendingRaw = await this.client.sunion(
+        `${this.prefix}status:pending`,
+        `${this.prefix}status:waiting`,
+      );
+      const toCancel: number[] = [];
+
+      for (const pidStr of pendingRaw) {
+        const pid = Number(pidStr);
+        if (cancelled.has(pid)) continue;
+        const job = await this.getJob<any, any>(pid);
+        if (!job || (job.status !== 'pending' && job.status !== 'waiting')) {
+          continue;
+        }
+
+        for (const seedId of frontier) {
+          if (pid === seedId) continue;
+          const seedJob = await this.getJob<any, any>(seedId);
+          if (!seedJob) continue;
+
+          const byJobId = job.dependsOnJobIds?.includes(seedId) ?? false;
+          const byTag =
+            job.dependsOnTags &&
+            job.dependsOnTags.length > 0 &&
+            tagsAreSuperset(seedJob.tags, job.dependsOnTags);
+
+          if (byJobId || byTag) {
+            toCancel.push(pid);
+            break;
+          }
+        }
+      }
+
+      if (toCancel.length === 0) break;
+
+      const now = this.nowMs();
+      for (const jid of toCancel) {
+        const ok = await this.client.eval(
+          CANCEL_JOB_SCRIPT,
+          1,
+          this.prefix,
+          jid,
+          now,
+        );
+        if (Number(ok) === 1) {
+          cancelled.add(jid);
+          await this.recordJobEvent(jid, JobEventType.Cancelled, {
+            rootJobId,
+            dependencyCascade: true,
+          });
+        }
+      }
+
+      frontier = toCancel;
+    }
+  }
+
   // ── Events ──────────────────────────────────────────────────────────
 
   async recordJobEvent(
@@ -327,6 +424,7 @@ export class RedisBackend implements QueueBackend {
       retryDelayMax = undefined,
       deadLetterJobType = undefined,
       group = undefined,
+      dependsOn,
     }: JobOptions<PayloadMap, T>,
     options?: AddJobOptions,
   ): Promise<number> {
@@ -336,6 +434,20 @@ export class RedisBackend implements QueueBackend {
           'Transactional job creation is only available with PostgreSQL.',
       );
     }
+    const { jobIds: depJobIdsRaw, tags: depTags } =
+      normalizeDependsOn(dependsOn);
+    if (depJobIdsRaw?.some((id) => id < 0)) {
+      throw new Error(
+        'dependsOn.jobIds: batch-relative (negative) ids are only supported in addJobs()',
+      );
+    }
+    const dependsOnJobIdsJson =
+      depJobIdsRaw && depJobIdsRaw.length > 0
+        ? JSON.stringify(depJobIdsRaw)
+        : 'null';
+    const dependsOnTagsJson =
+      depTags && depTags.length > 0 ? JSON.stringify(depTags) : 'null';
+
     const now = this.nowMs();
     const runAtMs = runAt ? runAt.getTime() : 0;
 
@@ -359,6 +471,8 @@ export class RedisBackend implements QueueBackend {
       deadLetterJobType ?? 'null',
       group?.id ?? 'null',
       group?.tier ?? 'null',
+      dependsOnJobIdsJson,
+      dependsOnTagsJson,
     )) as number;
 
     const jobId = Number(result);
@@ -370,6 +484,10 @@ export class RedisBackend implements QueueBackend {
       payload,
       tags,
       idempotencyKey,
+      dependsOn:
+        dependsOnJobIdsJson !== 'null' || dependsOnTagsJson !== 'null'
+          ? dependsOn
+          : undefined,
     });
     return jobId;
   }
@@ -391,29 +509,67 @@ export class RedisBackend implements QueueBackend {
       );
     }
 
+    const needsSequential = jobs.some((j) => {
+      const n = normalizeDependsOn(j.dependsOn);
+      return Boolean(n.jobIds?.length || n.tags?.length);
+    });
+
+    if (needsSequential) {
+      const ids: number[] = [];
+      for (let i = 0; i < jobs.length; i++) {
+        let job = jobs[i]!;
+        const nd = normalizeDependsOn(job.dependsOn);
+        if (nd.jobIds?.some((id) => id < 0)) {
+          const resolvedJobIds = resolveDependsOnJobIdsForBatch(
+            nd.jobIds!,
+            ids,
+          );
+          job = {
+            ...job,
+            dependsOn: {
+              jobIds: resolvedJobIds,
+              tags: job.dependsOn?.tags,
+            },
+          };
+        }
+        ids.push(await this.addJob(job));
+      }
+      log(
+        `Batch-inserted ${jobs.length} jobs (sequential), IDs: [${ids.join(', ')}]`,
+      );
+      return ids;
+    }
+
     const now = this.nowMs();
 
-    const jobsPayload = jobs.map((job) => ({
-      jobType: job.jobType,
-      payload: JSON.stringify(job.payload),
-      maxAttempts: job.maxAttempts ?? 3,
-      priority: job.priority ?? 0,
-      runAtMs: job.runAt ? job.runAt.getTime() : 0,
-      timeoutMs:
-        job.timeoutMs !== undefined ? job.timeoutMs.toString() : 'null',
-      forceKillOnTimeout: job.forceKillOnTimeout ? 'true' : 'false',
-      tags: job.tags ? JSON.stringify(job.tags) : 'null',
-      idempotencyKey: job.idempotencyKey ?? 'null',
-      retryDelay:
-        job.retryDelay !== undefined ? job.retryDelay.toString() : 'null',
-      retryBackoff:
-        job.retryBackoff !== undefined ? job.retryBackoff.toString() : 'null',
-      retryDelayMax:
-        job.retryDelayMax !== undefined ? job.retryDelayMax.toString() : 'null',
-      deadLetterJobType: job.deadLetterJobType ?? 'null',
-      groupId: job.group?.id ?? 'null',
-      groupTier: job.group?.tier ?? 'null',
-    }));
+    const jobsPayload = jobs.map((job) => {
+      const nd = normalizeDependsOn(job.dependsOn);
+      return {
+        jobType: job.jobType,
+        payload: JSON.stringify(job.payload),
+        maxAttempts: job.maxAttempts ?? 3,
+        priority: job.priority ?? 0,
+        runAtMs: job.runAt ? job.runAt.getTime() : 0,
+        timeoutMs:
+          job.timeoutMs !== undefined ? job.timeoutMs.toString() : 'null',
+        forceKillOnTimeout: job.forceKillOnTimeout ? 'true' : 'false',
+        tags: job.tags ? JSON.stringify(job.tags) : 'null',
+        idempotencyKey: job.idempotencyKey ?? 'null',
+        retryDelay:
+          job.retryDelay !== undefined ? job.retryDelay.toString() : 'null',
+        retryBackoff:
+          job.retryBackoff !== undefined ? job.retryBackoff.toString() : 'null',
+        retryDelayMax:
+          job.retryDelayMax !== undefined
+            ? job.retryDelayMax.toString()
+            : 'null',
+        deadLetterJobType: job.deadLetterJobType ?? 'null',
+        groupId: job.group?.id ?? 'null',
+        groupTier: job.group?.tier ?? 'null',
+        dependsOnJobIds: nd.jobIds?.length ? JSON.stringify(nd.jobIds) : null,
+        dependsOnTags: nd.tags?.length ? JSON.stringify(nd.tags) : null,
+      };
+    });
 
     const result = (await this.client.eval(
       ADD_JOBS_SCRIPT,
@@ -665,6 +821,7 @@ export class RedisBackend implements QueueBackend {
       failureReason,
       deadLetterJobId,
     });
+    await this.propagateDependencyCancellationsRedis([jobId], jobId);
     if (deadLetterJobId) {
       const sourceJob = await this.client.hget(
         `${this.prefix}job:${jobId}`,
@@ -743,9 +900,22 @@ export class RedisBackend implements QueueBackend {
 
   async cancelJob(jobId: number): Promise<void> {
     const now = this.nowMs();
-    await this.client.eval(CANCEL_JOB_SCRIPT, 1, this.prefix, jobId, now);
-    await this.recordJobEvent(jobId, JobEventType.Cancelled);
-    log(`Cancelled job ${jobId}`);
+    const ok = await this.client.eval(
+      CANCEL_JOB_SCRIPT,
+      1,
+      this.prefix,
+      jobId,
+      now,
+    );
+    if (Number(ok) === 1) {
+      await this.recordJobEvent(jobId, JobEventType.Cancelled);
+      await this.propagateDependencyCancellationsRedis([jobId], jobId);
+      log(`Cancelled job ${jobId}`);
+    } else {
+      log(
+        `Job ${jobId} could not be cancelled (not in pending/waiting state or does not exist)`,
+      );
+    }
   }
 
   async cancelAllUpcomingJobs(filters?: JobFilters): Promise<number> {

package/src/index.ts

@@ -442,6 +442,14 @@ const withLogContext =
   };
 
 export * from './types.js';
+export {
+  batchDepRef,
+  normalizeDependsOn,
+  resolveDependsOnJobIdsForBatch,
+  tagsAreSuperset,
+  validatePrerequisiteJobIdsExist,
+  assertNoDependencyCycle,
+} from './job-dependencies.js';
 export { QueueBackend, CronScheduleInput } from './backend.js';
 export { PostgresBackend } from './backends/postgres.js';
 export {

package/src/job-dependencies.test.ts

@@ -0,0 +1,129 @@
+import { describe, it, expect, vi } from 'vitest';
+import {
+  assertNoDependencyCycle,
+  batchDepRef,
+  normalizeDependsOn,
+  resolveDependsOnJobIdsForBatch,
+  tagsAreSuperset,
+  validatePrerequisiteJobIdsExist,
+} from './job-dependencies.js';
+import type { DatabaseClient } from './types.js';
+
+describe('batchDepRef', () => {
+  it('returns negative index encoding', () => {
+    expect(batchDepRef(0)).toBe(-1);
+    expect(batchDepRef(2)).toBe(-3);
+  });
+
+  it('throws on invalid index', () => {
+    expect(() => batchDepRef(-1)).toThrow();
+    expect(() => batchDepRef(1.5)).toThrow();
+  });
+});
+
+describe('normalizeDependsOn', () => {
+  it('returns undefined for empty input', () => {
+    expect(normalizeDependsOn(undefined)).toEqual({
+      jobIds: undefined,
+      tags: undefined,
+    });
+  });
+
+  it('deduplicates and drops empty', () => {
+    expect(
+      normalizeDependsOn({ jobIds: [1, 1, 2], tags: ['a', 'a', 'b'] }),
+    ).toEqual({
+      jobIds: [1, 2],
+      tags: ['a', 'b'],
+    });
+  });
+});
+
+describe('resolveDependsOnJobIdsForBatch', () => {
+  it('resolves negative placeholders', () => {
+    expect(resolveDependsOnJobIdsForBatch([-1, -2], [10, 20])).toEqual([
+      10, 20,
+    ]);
+  });
+
+  it('passes through positive ids', () => {
+    expect(resolveDependsOnJobIdsForBatch([5, -1], [99])).toEqual([5, 99]);
+  });
+
+  it('throws when index out of range', () => {
+    expect(() => resolveDependsOnJobIdsForBatch([-3], [1, 2])).toThrow();
+  });
+});
+
+describe('tagsAreSuperset', () => {
+  it('returns false for empty required', () => {
+    expect(tagsAreSuperset(['a'], [])).toBe(false);
+  });
+
+  it('checks inclusion', () => {
+    expect(tagsAreSuperset(['a', 'b'], ['a'])).toBe(true);
+    expect(tagsAreSuperset(['a'], ['a', 'b'])).toBe(false);
+    expect(tagsAreSuperset(null, ['a'])).toBe(false);
+  });
+});
+
+describe('validatePrerequisiteJobIdsExist', () => {
+  it('no-ops for empty', async () => {
+    const client: DatabaseClient = {
+      query: vi.fn(),
+    };
+    await validatePrerequisiteJobIdsExist(client, []);
+    expect(client.query).not.toHaveBeenCalled();
+  });
+
+  it('throws when count mismatches', async () => {
+    const client: DatabaseClient = {
+      query: vi.fn().mockResolvedValue({ rows: [{ c: 1 }], rowCount: 1 }),
+    };
+    await expect(
+      validatePrerequisiteJobIdsExist(client, [1, 2]),
+    ).rejects.toThrow(/do not exist/);
+  });
+
+  it('resolves when all exist', async () => {
+    const client: DatabaseClient = {
+      query: vi.fn().mockResolvedValue({ rows: [{ c: 2 }], rowCount: 1 }),
+    };
+    await validatePrerequisiteJobIdsExist(client, [1, 2]);
+    expect(client.query).toHaveBeenCalledTimes(1);
+  });
+});
+
+describe('assertNoDependencyCycle', () => {
+  it('throws on self-dependency', async () => {
+    const client: DatabaseClient = { query: vi.fn() };
+    await expect(assertNoDependencyCycle(client, 1, [1])).rejects.toThrow(
+      /cannot depend on itself/,
+    );
+  });
+
+  it('throws when cycle detected', async () => {
+    const client: DatabaseClient = {
+      query: vi
+        .fn()
+        .mockResolvedValue({ rows: [{ '?column?': 1 }], rowCount: 1 }),
+    };
+    await expect(assertNoDependencyCycle(client, 5, [1, 2])).rejects.toThrow(
+      /cycle/,
+    );
+  });
+
+  it('no-ops when no deps', async () => {
+    const client: DatabaseClient = { query: vi.fn() };
+    await assertNoDependencyCycle(client, 1, []);
+    expect(client.query).not.toHaveBeenCalled();
+  });
+
+  it('no-ops when query returns no cycle', async () => {
+    const client: DatabaseClient = {
+      query: vi.fn().mockResolvedValue({ rows: [], rowCount: 0 }),
+    };
+    await assertNoDependencyCycle(client, 5, [1]);
+    expect(client.query).toHaveBeenCalledTimes(1);
+  });
+});