bullmq 5.44.1 → 5.52.0

package/dist/esm/commands/moveToWaitingChildren-8.lua

@@ -36,13 +36,12 @@ local timestamp = ARGV[3]
 local jobId = ARGV[4]
 
 --- Includes
---- @include "includes/moveParentFromWaitingChildrenToFailed"
---- @include "includes/moveParentToWaitIfNeeded"
---- @include "includes/removeDeduplicationKeyIfNeeded"
+--- @include "includes/moveChildFromDependenciesIfNeeded"
+--- @include "includes/removeDeduplicationKeyIfNeededOnFinalization"
 --- @include "includes/removeJobsOnFail"
 --- @include "includes/removeLock"
 
-local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId,
+local function moveToWaitingChildren(activeKey, waitingChildrenKey, jobId,
     timestamp)
   local score = tonumber(timestamp)
 
@@ -62,7 +61,7 @@ if rcall("EXISTS", jobKey) == 1 then
     -- TODO: refactor this logic in an include later
     local jobAttributes = rcall("HMGET", jobKey, "parent", "deid", "opts")
 
-    removeDeduplicationKeyIfNeeded(ARGV[5], jobAttributes[2])
+    removeDeduplicationKeyIfNeededOnFinalization(ARGV[5], jobAttributes[2], jobId)
   
     local failedReason = "children are failed"
     rcall("ZADD", failedKey, timestamp, jobId)
@@ -74,33 +73,7 @@ if rcall("EXISTS", jobKey) == 1 then
     local rawOpts = jobAttributes[3]
     local opts = cjson.decode(rawOpts)
 
-    if rawParentData ~= false then
-      if opts['fpof'] then
-        local parentData = cjson.decode(rawParentData)
-        local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
-        local parentUnsuccesssful = parentKey .. ":unsuccessful"
-        rcall("ZADD", parentUnsuccesssful, timestamp, jobKey)                        
-        moveParentFromWaitingChildrenToFailed(
-            parentData['queueKey'],
-            parentData['queueKey'] .. ':' .. parentData['id'],
-            parentData['id'],
-            jobKey,
-            timestamp
-        )
-      elseif opts['idof'] or opts['rdof'] then
-        local parentData = cjson.decode(rawParentData)
-        local parentKey = parentData['queueKey'] .. ':' .. parentData['id']
-        local dependenciesSet = parentKey .. ":dependencies"
-        if rcall("SREM", dependenciesSet, jobKey) == 1 then
-          moveParentToWaitIfNeeded(parentData['queueKey'], dependenciesSet,
-                                    parentKey, parentData['id'], timestamp)
-          if opts['idof'] then
-            local failedSet = parentKey .. ":failed"
-            rcall("HSET", failedSet, jobKey, failedReason)
-          end
-        end
-      end
-    end
+    moveChildFromDependenciesIfNeeded(rawParentData, jobKey, failedReason, timestamp)
 
     removeJobsOnFail(ARGV[5], failedKey, jobId, opts, timestamp)
 

package/dist/esm/commands/removeJob-3.lua

@@ -1,14 +1,15 @@
 --[[
-    Remove a job from all the queues it may be in as well as all its data.
+    Remove a job from all the statuses it may be in as well as all its data.
     In order to be able to remove a job, it cannot be active.
 
     Input:
-      KEYS[1] queue prefix
+      KEYS[1] jobKey
       KEYS[2] meta key
       KEYS[3] repeat key
 
       ARGV[1] jobId
       ARGV[2] remove children
+      ARGV[3] queue prefix
 
     Events:
       'removed'
@@ -17,80 +18,15 @@
 local rcall = redis.call
 
 -- Includes
---- @include "includes/destructureJobKey"
---- @include "includes/getOrSetMaxEvents"
 --- @include "includes/isJobSchedulerJob"
 --- @include "includes/isLocked"
---- @include "includes/removeDeduplicationKey"
---- @include "includes/removeJobFromAnyState"
---- @include "includes/removeJobKeys"
---- @include "includes/removeParentDependencyKey"
+--- @include "includes/removeJobWithChildren"
 
-local function removeJob(prefix, jobId, parentKey, removeChildren)
-    local jobKey = prefix .. jobId;
-
-    removeParentDependencyKey(jobKey, false, parentKey, nil)
-
-    if removeChildren == "1" then
-        -- Check if this job has children
-        -- If so, we are going to try to remove the children recursively in deep first way because
-        -- if some job is locked we must exit with and error.
-        -- local countProcessed = rcall("HLEN", jobKey .. ":processed")
-        local processed = rcall("HGETALL", jobKey .. ":processed")
-
-        if (#processed > 0) then
-            for i = 1, #processed, 2 do
-                local childJobId = getJobIdFromKey(processed[i])
-                local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
-                removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
-            end
-        end
-
-        local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
-        if (#dependencies > 0) then
-            for i, childJobKey in ipairs(dependencies) do
-                -- We need to get the jobId for this job.
-                local childJobId = getJobIdFromKey(childJobKey)
-                local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
-                removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
-            end
-        end
-
-        local failed = rcall("HGETALL", jobKey .. ":failed")
-
-        if (#failed > 0) then
-            for i = 1, #failed, 2 do
-                local childJobId = getJobIdFromKey(failed[i])
-                local childJobPrefix = getJobKeyPrefix(failed[i], childJobId)
-                removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
-            end
-        end
-
-        local unsuccessful = rcall("ZRANGE", jobKey .. ":unsuccessful", 0, -1)
-
-        if (#unsuccessful > 0) then
-            for i = 1, #unsuccessful, 1 do
-                local childJobId = getJobIdFromKey(unsuccessful[i])
-                local childJobPrefix = getJobKeyPrefix(unsuccessful[i], childJobId)
-                removeJob(childJobPrefix, childJobId, jobKey, removeChildren)
-            end
-        end
-    end
-
-    local prev = removeJobFromAnyState(prefix, jobId)
-
-    removeDeduplicationKey(prefix, jobKey)
-    if removeJobKeys(jobKey) > 0 then
-        local maxEvents = getOrSetMaxEvents(KEYS[2])
-        rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed", "jobId", jobId, "prev",
-            prev)
-    end
-end
-
-local prefix = KEYS[1]
 local jobId = ARGV[1]
 local shouldRemoveChildren = ARGV[2]
-local jobKey = prefix .. jobId
+local prefix = ARGV[3]
+local jobKey = KEYS[1]
+local meta = KEYS[2]
 local repeatKey = KEYS[3]
 
 if isJobSchedulerJob(jobId, jobKey, repeatKey) then
@@ -98,7 +34,13 @@ if isJobSchedulerJob(jobId, jobKey, repeatKey) then
 end
 
 if not isLocked(prefix, jobId, shouldRemoveChildren) then
-    removeJob(prefix, jobId, nil, shouldRemoveChildren)
+    local options = {
+        removeChildren = shouldRemoveChildren == "1",
+        ignoreProcessed = false,
+        ignoreLocked = false
+    }
+
+    removeJobWithChildren(prefix, meta, jobId, nil, options)
     return 1
 end
 return 0

package/dist/esm/commands/removeUnprocessedChildren-2.lua

@@ -0,0 +1,31 @@
+--[[
+    Remove a job from all the statuses it may be in as well as all its data.
+    In order to be able to remove a job, it cannot be active.
+
+    Input:
+      KEYS[1] jobKey
+      KEYS[2] meta key
+      
+      ARGV[1] prefix
+      ARGV[2] jobId
+
+    Events:
+      'removed' for every children removed
+]]
+
+-- Includes
+--- @include "includes/removeJobWithChildren"
+
+local prefix = ARGV[1]
+local jobId = ARGV[2]
+
+local jobKey = KEYS[1]
+local metaKey = KEYS[2]
+
+local options = {
+  removeChildren = "1",
+  ignoreProcessed = true,
+  ignoreLocked = true
+}
+
+removeJobChildren(prefix, metaKey, jobKey, options) 

package/dist/esm/commands/updateJobScheduler-12.lua

@@ -25,7 +25,8 @@
 
     Output:
       next delayed job id  - OK
-]] local rcall = redis.call
+]]
+local rcall = redis.call
 local repeatKey = KEYS[1]
 local delayedKey = KEYS[2]
 local waitKey = KEYS[3]
@@ -48,37 +49,42 @@ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
 
 -- Validate that scheduler exists.
 local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
-if prevMillis ~= false then
+if prevMillis then
     local currentDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
 
-    if producerId == currentDelayedJobId and rcall("EXISTS", nextDelayedJobKey) ~= 1 then
-        local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data")
-
-        rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
-        rcall("HINCRBY", schedulerKey, "ic", 1)
-
+    if producerId == currentDelayedJobId then
         local eventsKey = KEYS[9]
         local maxEvents = getOrSetMaxEvents(metaKey)
 
-        rcall("INCR", KEYS[8])
+        if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
+            local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data")
 
-        -- TODO: remove this workaround in next breaking change,
-        -- all job-schedulers must save job data
-        local templateData = schedulerAttributes[2] or ARGV[3]
+            rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
+            rcall("HINCRBY", schedulerKey, "ic", 1)
 
-        if templateData and templateData ~= '{}' then
-            rcall("HSET", schedulerKey, "data", templateData)
-        end
+            rcall("INCR", KEYS[8])
 
-        addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[4], waitKey, pausedKey, 
-            KEYS[12], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey, 
-            schedulerAttributes[1], maxEvents, ARGV[5], templateData or '{}', jobSchedulerId)
+            -- TODO: remove this workaround in next breaking change,
+            -- all job-schedulers must save job data
+            local templateData = schedulerAttributes[2] or ARGV[3]
 
-        -- TODO: remove this workaround in next breaking change
-        if KEYS[11] ~= "" then
-            rcall("HSET", KEYS[11], "nrjid", nextDelayedJobId)
-        end
+            if templateData and templateData ~= '{}' then
+                rcall("HSET", schedulerKey, "data", templateData)
+            end
 
-        return nextDelayedJobId .. "" -- convert to string
+            addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[4], waitKey, pausedKey, 
+                KEYS[12], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey, 
+                schedulerAttributes[1], maxEvents, ARGV[5], templateData or '{}', jobSchedulerId)
+
+            -- TODO: remove this workaround in next breaking change
+            if KEYS[11] ~= "" then
+                rcall("HSET", KEYS[11], "nrjid", nextDelayedJobId)
+            end
+
+            return nextDelayedJobId .. "" -- convert to string
+        else
+            rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
+                "duplicated", "jobId", nextDelayedJobId)
+        end
     end
 end

package/dist/esm/interfaces/job-json.d.ts

@@ -1,11 +1,11 @@
-import { RedisJobOptions } from '../types';
+import { JobProgress, RedisJobOptions } from '../types';
 import { ParentKeys } from './parent';
 export interface JobJson {
     id: string;
     name: string;
     data: string;
     opts: RedisJobOptions;
-    progress: number | object;
+    progress: JobProgress;
     attemptsMade: number;
     attemptsStarted: number;
     finishedOn?: number;
@@ -21,6 +21,7 @@ export interface JobJson {
     debounceId?: string;
     deduplicationId?: string;
     processedBy?: string;
+    stalledCounter: number;
 }
 export interface JobJsonRaw {
     id: string;
@@ -43,6 +44,8 @@ export interface JobJsonRaw {
     rjk?: string;
     nrjid?: string;
     atm?: string;
+    defa?: string;
+    stc?: string;
     ats?: string;
     pb?: string;
 }

package/dist/esm/interfaces/minimal-job.d.ts

@@ -1,4 +1,4 @@
-import { JobsOptions, JobJsonSandbox } from '../types';
+import { JobsOptions, JobJsonSandbox, JobProgress } from '../types';
 import { JobJson } from './job-json';
 import { ParentKeys } from './parent';
 export type BulkJobOptions = Omit<JobsOptions, 'repeat'>;
@@ -15,15 +15,33 @@ export interface MoveToWaitingChildrenOpts {
         queue: string;
     };
 }
+export interface DependencyOpts {
+    /**
+     * Cursor value to be passed for pagination
+     */
+    cursor?: number;
+    /**
+     * Max quantity of jobs to be retrieved
+     */
+    count?: number;
+}
 export interface DependenciesOpts {
-    processed?: {
-        cursor?: number;
-        count?: number;
-    };
-    unprocessed?: {
-        cursor?: number;
-        count?: number;
-    };
+    /**
+     * Options for failed child pagination
+     */
+    failed?: DependencyOpts;
+    /**
+     * Options for ignored child pagination
+     */
+    ignored?: DependencyOpts;
+    /**
+     * Options for processed child pagination
+     */
+    processed?: DependencyOpts;
+    /**
+     * Options for unprocessed child pagination
+     */
+    unprocessed?: DependencyOpts;
 }
 /**
  * MinimalJob
@@ -46,7 +64,7 @@ export interface MinimalJob<DataType = any, ReturnType = any, NameType extends s
      * The progress a job has performed so far.
      * @defaultValue 0
      */
-    progress: number | object;
+    progress: JobProgress;
     /**
      * The value returned by the processor when processing this job.
      * @defaultValue null
@@ -116,7 +134,7 @@ export interface MinimalJob<DataType = any, ReturnType = any, NameType extends s
      *
      * @param progress - number or object to be saved as progress.
      */
-    updateProgress(progress: number | object): Promise<void>;
+    updateProgress(progress: JobProgress): Promise<void>;
     /**
      * Logs one row of log data.
      *

package/dist/esm/interfaces/queue-options.d.ts

@@ -16,7 +16,7 @@ export interface QueueBaseOptions {
     connection: ConnectionOptions;
     /**
      * Denotes commands should retry indefinitely.
-     * @deprecated
+     * @deprecated not in use anymore.
      */
     blockingConnection?: boolean;
     /**
@@ -84,7 +84,7 @@ export interface RepeatBaseOptions extends QueueBaseOptions {
 /**
  * Options for QueueEvents
  */
-export interface QueueEventsOptions extends QueueBaseOptions {
+export interface QueueEventsOptions extends Omit<QueueBaseOptions, 'telemetry'> {
     /**
      * Condition to start listening to events at instance creation.
      */
@@ -100,3 +100,7 @@ export interface QueueEventsOptions extends QueueBaseOptions {
      */
     blockingTimeout?: number;
 }
+/**
+ * Options for QueueEventsProducer
+ */
+export type QueueEventsProducerOptions = Omit<QueueBaseOptions, 'telemetry'>;

package/dist/esm/interfaces/repeat-options.d.ts

@@ -45,7 +45,7 @@ export interface RepeatOptions extends Omit<ParserOptions, 'iterator'> {
     prevMillis?: number;
     /**
      * Internal property to store de job id
-     * @deprecated
+     * @deprecated not in use anymore
      */
     jobId?: string;
 }

package/dist/esm/interfaces/sandboxed-job.d.ts

@@ -1,4 +1,4 @@
-import { JobJsonSandbox, JobsOptions } from '../types';
+import { JobJsonSandbox, JobProgress, JobsOptions } from '../types';
 /**
  * @see {@link https://docs.bullmq.io/guide/workers/sandboxed-processors}
  */
@@ -8,6 +8,6 @@ export interface SandboxedJob<T = any, R = any> extends Omit<JobJsonSandbox, 'da
     moveToDelayed: (timestamp: number, token?: string) => Promise<void>;
     log: (row: any) => void;
     updateData: (data: any) => Promise<void>;
-    updateProgress: (value: object | number) => Promise<void>;
+    updateProgress: (value: JobProgress) => Promise<void>;
     returnValue: R;
 }

package/dist/esm/interfaces/sandboxed-options.d.ts

@@ -8,7 +8,7 @@ export interface SandboxedOptions {
      * Note: This option can only be used when specifying
      * a file for the processor argument.
      *
-     * @default false
+     * @defaultValue false
      */
     useWorkerThreads?: boolean;
     /**

package/dist/esm/interfaces/telemetry.d.ts

@@ -33,8 +33,8 @@ export interface ContextManager<Context = any> {
     /**
      * Creates a new context and sets it as active for the fn passed as last argument
      *
-     * @param context -
-     * @param fn -
+     * @param context - the context to set as active
+     * @param fn - the function to execute with the context
      */
     with<A extends (...args: any[]) => any>(context: Context, fn: A): ReturnType<A>;
     /**
@@ -46,15 +46,15 @@ export interface ContextManager<Context = any> {
      * is the mechanism used to propagate the context across a distributed
      * application.
      *
-     * @param context -
+     * @param context - the current context
      */
     getMetadata(context: Context): string;
     /**
      * Creates a new context from a serialized version effectively
      * linking the new context to the parent context.
      *
-     * @param activeContext -
-     * @param metadata -
+     * @param activeContext - the current active context
+     * @param metadata - the serialized version of the context
      */
     fromMetadata(activeContext: Context, metadata: string): Context;
 }
@@ -68,9 +68,10 @@ export interface Tracer<Context = any> {
      * context. If the context is not provided, the current active context should be
      * used.
      *
-     * @param name -
-     * @param options -
-     * @param context -
+     * @param name - span name
+     * @param options - span options
+     * @param context - optional context
+     * @returns - the created span
      */
     startSpan(name: string, options?: SpanOptions, context?: Context): Span;
 }
@@ -85,33 +86,35 @@ export interface Span<Context = any> {
      * setSpanOnContext sets the span on the context. This is useful when you want
      * to propagate the span across the application.
      *
-     * @param ctx
+     * @param ctx - context to set the span on
+     * @returns - the context with the span set on it
      */
     setSpanOnContext(ctx: Context): Context;
     /**
      * setAttribute sets an attribute on the span.
      *
-     * @param ctx
+     * @param key - attribute key
+     * @param value - attribute value
      */
     setAttribute(key: string, value: AttributeValue): void;
     /**
      * setAttributes sets multiple attributes on the span.
      *
-     * @param attributes
+     * @param attributes - attributes to set
      */
     setAttributes(attributes: Attributes): void;
     /**
      * addEvent adds an event to the span.
      *
-     * @param name
-     * @param attributes
+     * @param name - event name
+     * @param attributes - event attributes
      */
     addEvent(name: string, attributes?: Attributes): void;
     /**
      * recordException records an exception on the span.
      *
-     * @param exception
-     * @param time
+     * @param exception - exception to record
+     * @param time - time to record the exception
      */
     recordException(exception: Exception, time?: Time): void;
     /**

package/dist/esm/interfaces/worker-options.d.ts

@@ -20,14 +20,14 @@ export interface WorkerOptions extends QueueBaseOptions, SandboxedOptions {
     /**
      * Condition to start processor at instance creation.
      *
-     * @default true
+     * @defaultValue true
      */
     autorun?: boolean;
     /**
      * Amount of jobs that a single worker is allowed to work on
      * in parallel.
      *
-     * @default 1
+     * @defaultValue 1
      * @see {@link https://docs.bullmq.io/guide/workers/concurrency}
      */
     concurrency?: number;
@@ -46,13 +46,13 @@ export interface WorkerOptions extends QueueBaseOptions, SandboxedOptions {
      * to the `wait` state. If this is exceeded, the job is moved
      * to `failed`.
      *
-     * @default 1
+     * @defaultValue 1
      */
     maxStalledCount?: number;
     /**
      * Number of milliseconds between stallness checks.
      *
-     * @default 30000
+     * @defaultValue 30000
      */
     stalledInterval?: number;
     /**
@@ -72,7 +72,7 @@ export interface WorkerOptions extends QueueBaseOptions, SandboxedOptions {
      *  perform stalled checkd and move jobs back to wait for jobs being processed
      *  by this worker.
      *
-     *  @default false
+     *  @defaultValue false
      */
     skipStalledCheck?: boolean;
     /**
@@ -80,13 +80,13 @@ export interface WorkerOptions extends QueueBaseOptions, SandboxedOptions {
      *  after lockDuration and moved back to the wait queue (if the stalled check is
      *  not disabled)
      *
-     *  @default false
+     *  @defaultValue false
      */
     skipLockRenewal?: boolean;
     /**
      * Number of seconds to long poll for jobs when the queue is empty.
      *
-     * @default 5
+     * @defaultValue 5
      */
     drainDelay?: number;
     /**
@@ -95,7 +95,7 @@ export interface WorkerOptions extends QueueBaseOptions, SandboxedOptions {
      * be picked up by the stalled checker and move back to wait so that another worker
      * can process it again.
      *
-     * @default 30000
+     * @defaultValue 30000
      */
     lockDuration?: number;
     /**
@@ -108,7 +108,7 @@ export interface WorkerOptions extends QueueBaseOptions, SandboxedOptions {
     /**
      * This is an internal option that should not be modified.
      *
-     * @default 15000
+     * @defaultValue 15000
      */
     runRetryDelay?: number;
     /**
@@ -120,7 +120,7 @@ export interface WorkerOptions extends QueueBaseOptions, SandboxedOptions {
      * Note: This option can only be used when specifying
      * a file for the processor argument.
      *
-     * @default false
+     * @defaultValue false
      */
     useWorkerThreads?: boolean;
     /**