bullmq 4.0.0-beta.1 → 4.0.0

package/src/commands/moveToActive-8.lua

@@ -1,79 +0,0 @@
---[[
-  Move next job to be processed to active, lock it and fetch its data. The job
-  may be delayed, in that case we need to move it to the delayed set instead.
-
-  Input:
-      KEYS[1] wait key
-      KEYS[2] active key
-      KEYS[3] priority key
-      KEYS[4] stream events key
-      KEYS[5] stalled key
-
-      -- Rate limiting
-      KEYS[6] rate limiter key
-      KEYS[7] delayed key
-
-      -- Delay events
-      KEYS[8] delay stream key
-
-      -- Arguments
-      ARGV[1] key prefix
-      ARGV[2] timestamp
-      ARGV[3] optional jobid
-
-      ARGV[4] optional jobs per time unit (rate limiter)
-      ARGV[5] optional time unit (rate limiter)
-]]
-
-local jobId
-local rcall = redis.call
-
-if(ARGV[3] ~= "") then
-  jobId = ARGV[3]
-
-  -- clean stalled key
-  rcall("SREM", KEYS[5], jobId)
-else
-  -- move from wait to active
-  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
-end
-
-if jobId then
-  -- Check if we need to perform rate limiting.
-  local maxJobs = tonumber(ARGV[4])
-
-  if(maxJobs) then
-    local rateLimiterKey = KEYS[6];
-    local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
-    -- check if rate limit hit
-    if jobCounter > maxJobs then
-      local exceedingJobs = jobCounter - maxJobs
-      local delay = tonumber(rcall("PTTL", rateLimiterKey)) + ((exceedingJobs - 1) * ARGV[5]) / maxJobs;
-      local timestamp = delay + tonumber(ARGV[2])
-      
-      -- put job into delayed queue
-      rcall("ZADD", KEYS[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId);
-      rcall("XADD", KEYS[4], "*", "event", "delayed", "jobId", jobId, "delay", timestamp);
-      rcall("XADD", KEYS[8], "*", "nextTimestamp", timestamp);
-      -- remove from active queue
-      rcall("LREM", KEYS[2], 1, jobId)
-      return
-    else
-      if jobCounter == 1 then
-        rcall("PEXPIRE", rateLimiterKey, ARGV[5])
-      end
-    end
-  end
-
-  local jobKey = ARGV[1] .. jobId
-
-  rcall("ZREM", KEYS[3], jobId) -- remove from priority
-
-  rcall("XADD", KEYS[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
-
-  rcall("HSET", jobKey, "processedOn", ARGV[2])
-
-  return {rcall("HGETALL", jobKey), jobId} -- get job data
-else
-  rcall("XADD", KEYS[4], "*", "event", "drained");
-end

package/src/commands/moveToDelayed-5.lua

@@ -1,39 +0,0 @@
---[[
-  Moves job from active to delayed set.
-
-  Input: 
-    KEYS[1] active key
-    KEYS[2] delayed key
-    KEYS[3] job key
-    KEYS[4] events stream
-    KEYS[5] delayed stream
-
-    ARGV[1] delayedTimestamp
-    ARGV[2] the id of the job
-    ARGV[3] queue token
-
-  Output:
-    0 - OK
-   -1 - Missing job.
-
-  Events:
-    - delayed key.
-]]
-local rcall = redis.call
-
-if rcall("EXISTS", KEYS[3]) == 1 then
-
-  local jobId = ARGV[2]
-  local score = tonumber(ARGV[1])
-  local delayedTimestamp = (score / 0x1000)
-  
-  rcall("ZADD", KEYS[2], score, jobId)
-  rcall("LREM", KEYS[1], 0, jobId)
-
-  rcall("XADD", KEYS[4], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
-  rcall("XADD", KEYS[5], "*", "nextTimestamp", delayedTimestamp);
-
-  return 0
-else
-  return -1
-end

package/src/commands/moveToFinished-7.lua

@@ -1,92 +0,0 @@
---[[
-  Move job from active to a finished status (completed o failed)
-  A job can only be moved to completed if it was active.
-
-     Input:
-      KEYS[1] active key
-      KEYS[2] completed/failed key
-      KEYS[3] jobId key
-
-      KEYS[4] wait key
-      KEYS[5] priority key
-      KEYS[6] event stream key
-      KEYS[7] meta key
-
-      ARGV[1]  jobId
-      ARGV[2]  timestamp
-      ARGV[3]  msg property
-      ARGV[4]  return value / failed reason
-      ARGV[5]  target (completed/failed)
-      ARGV[6]  shouldRemove
-      ARGV[7]  event data (? maybe just send jobid).
-      ARGV[8]  fetch next?
-      ARGV[9]  keys prefix
-
-     Output:
-      0 OK
-      -1 Missing key.
-
-     Events:
-      'completed/failed'
-]] local rcall = redis.call
-
-if rcall("EXISTS", KEYS[3]) == 1 then -- // Make sure job exists
-
-    -- Remove from active list (if not active we shall return error)
-    local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[1])
-
-    -- What if we just ignore this? I think it is good to know regardless.
-    --[[   if(numRemovedElements < 1) then
-    return -2
-  end
- ]]
-    -- Remove job?
-    local removeJobs = tonumber(ARGV[6])
-    if removeJobs ~= 1 then
-        -- Add to complete/failed set
-        rcall("ZADD", KEYS[2], ARGV[2], ARGV[1])
-        rcall("HMSET", KEYS[3], ARGV[3], ARGV[4], "finishedOn", ARGV[2]) -- "returnvalue" / "failedReason" and "finishedOn"
-
-        -- Remove old jobs?
-        if removeJobs and removeJobs > 1 then
-            local start = removeJobs - 1
-            local jobIds = rcall("ZREVRANGE", KEYS[2], start, -1)
-            for i, jobId in ipairs(jobIds) do
-                local jobKey = ARGV[9] .. jobId
-                local jobLogKey = jobKey .. ':logs'
-                rcall("DEL", jobKey, jobLogKey)
-            end
-            rcall("ZREMRANGEBYRANK", KEYS[2], 0, -removeJobs)
-        end
-    else
-        local jobLogKey = KEYS[3] .. ':logs'
-        rcall("DEL", KEYS[3], jobLogKey)
-    end
-
-    rcall("XADD", KEYS[6], "*", "event", ARGV[5], "jobId", ARGV[1], ARGV[3],
-          ARGV[4])
-
-    -- Try to get next job to avoid an extra roundtrip if the queue is not closing, 
-    -- and not rate limited.
-    if (ARGV[8] == "1") then
-        -- move from wait to active 
-        local jobId = rcall("RPOPLPUSH", KEYS[4], KEYS[1])
-        if jobId then
-            local jobKey = ARGV[9] .. jobId
-
-            rcall("ZREM", KEYS[5], jobId) -- remove from priority
-            rcall("XADD", KEYS[6], "*", "event", "active", "jobId", jobId,
-                  "prev", "waiting")
-            rcall("HSET", jobKey, "processedOn", ARGV[2])
-
-            return {rcall("HGETALL", jobKey), jobId} -- get job data
-        end
-    end
-
-    local maxEvents = rcall("HGET", KEYS[7], "opts.maxLenEvents")
-    rcall("XTRIM", KEYS[6], "MAXLEN", "~", maxEvents)
-
-    return 0
-else
-    return -1
-end

package/src/commands/pause-4.lua

@@ -1,27 +0,0 @@
---[[
-  Pauses or resumes a queue globably.
-
-   Input:
-      KEYS[1] 'wait' or 'paused''
-      KEYS[2] 'paused' or 'wait'
-      KEYS[3] 'meta'
-      KEYS[4] events stream key
-
-      ARGV[1] 'paused' or 'resumed'
-
-    Event:
-      publish paused or resumed event.
-]]
-local rcall = redis.call
-
-if rcall("EXISTS", KEYS[1]) == 1 then
-  rcall("RENAME", KEYS[1], KEYS[2])
-end
-
-if ARGV[1] == "paused" then
-  rcall("HSET", KEYS[3], "paused", 1)
-else
-  rcall("HDEL", KEYS[3], "paused")
-end
-
-rcall("XADD", KEYS[4], "*", "event", ARGV[1]);

package/src/commands/promote-4.lua

@@ -1,49 +0,0 @@
---[[
-  Promotes a job that is currently "delayed" to the "waiting" state
-
-     Input:
-      KEYS[1] 'delayed'
-      KEYS[2] 'wait'
-      KEYS[3] 'priority'
-      KEYS[4] 'event stream'
-
-      ARGV[1]  queue.toKey('')
-      ARGV[2]  jobId
-
-     Events:
-      'waiting'
-]]
-local rcall = redis.call;
-local jobId = ARGV[2]
-
-if redis.call("ZREM", KEYS[1], jobId) == 1 then
-  local priority = tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
-
-  local target = KEYS[2];
-
-  if priority == 0 then
-    -- LIFO or FIFO
-    rcall("LPUSH", target, jobId)
-  else
-    -- Priority add
-    rcall("ZADD", KEYS[3], priority, jobId)
-    local count = rcall("ZCOUNT", KEYS[3], 0, priority)
-
-    local len = rcall("LLEN", target)
-    local id = rcall("LINDEX", target, len - (count - 1))
-    if id then
-      rcall("LINSERT", target, "BEFORE", id, jobId)
-    else
-      rcall("RPUSH", target, jobId)
-    end
-  end
-
-  -- Emit waiting event (wait..ing@token)
-  rcall("XADD", KEYS[4], "*", "event", "waiting", "jobId", jobId, "prev", "delayed");
-
-  rcall("HSET", ARGV[1] .. jobId, "delay", 0)
-
-  return 0
-else
-  return -1
-end

package/src/commands/releaseLock-1.lua

@@ -1,19 +0,0 @@
---[[
-  Release lock
-
-     Input:
-        KEYS[1] 'lock',
-      
-        ARGV[1]  token
-        ARGV[2]  lock duration in milliseconds
-      
-      Output:
-        "OK" if lock extented succesfully.
-]]
-local rcall = redis.call
-
-if rcall("GET", KEYS[1]) == ARGV[1] then
-  return rcall("DEL", KEYS[1])
-else
-  return 0
-end

package/src/commands/removeJob-10.lua

@@ -1,37 +0,0 @@
---[[
-    Remove a job from all the queues it may be in as well as all its data.
-    In order to be able to remove a job, it cannot be active.
-
-     Input:
-      KEYS[1] 'active',
-      KEYS[2] 'wait',
-      KEYS[3] 'delayed',
-      KEYS[4] 'paused',
-      KEYS[5] 'completed',
-      KEYS[6] 'failed',
-      KEYS[7] 'priority',
-      KEYS[8] jobId
-      KEYS[9] job logs
-      KEYS[10] events stream
-
-      ARGV[1]  jobId
-
-     Events:
-      'removed'
-]]
-
-local rcall = redis.call
-local jobId = ARGV[1]
-rcall("LREM", KEYS[1], 0, jobId)
-rcall("LREM", KEYS[2], 0, jobId)
-rcall("ZREM", KEYS[3], jobId)
-rcall("LREM", KEYS[4], 0, jobId)
-rcall("ZREM", KEYS[5], jobId)
-rcall("ZREM", KEYS[6], jobId)
-rcall("ZREM", KEYS[7], jobId)
-rcall("DEL", KEYS[8])
-rcall("DEL", KEYS[9])
-
-rcall("XADD", KEYS[10], "*", "event", "removed", "jobId", jobId, "prev", "TBD");
-
-return 1

package/src/commands/reprocessJob-4.lua

@@ -1,36 +0,0 @@
---[[
-  Attempts to reprocess a job
-
-  Input:
-    KEYS[1] job key
-    KEYS[2] events stream
-    KEYS[3] job state
-    KEYS[4] wait key
-
-    ARGV[1] job.id,
-    ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
-
-  Output:
-    1 means the operation was a success
-    0 means the job does not exist
-    -2 means the job was not found in the expected set.
-
-  Events:
-    emits 'added' if succesfully moved job to wait.
-]]
-local rcall = redis.call;
-if (rcall("EXISTS", KEYS[1]) == 1) then
-  local jobId = ARGV[1]
-  if (rcall("ZREM", KEYS[3], jobId) == 1) then
-    rcall(ARGV[2], KEYS[4], jobId)
-    rcall(ARGV[2], KEYS[4] .. ":added", jobId)
-
-    -- Emit waiting event
-    rcall("XADD", KEYS[2], "*", "event", "waiting", "jobId", jobId);
-    return 1
-  else
-    return -2
-  end
-else
-  return 0
-end

package/src/commands/retryJob-4.lua

@@ -1,34 +0,0 @@
---[[
-  Retries a failed job by moving it back to the wait queue.
-
-    Input:
-      KEYS[1] 'active',
-      KEYS[2] 'wait'
-      KEYS[3] jobId
-      KEYS[4] events stream
-
-      ARGV[1]  pushCmd
-      ARGV[2]  jobId
-      ARGV[3]  token
-
-    Events:
-      'prefix:added'
-
-    Output:
-     0  - OK
-     -1 - Missing key
-]]
-local rcall = redis.call
-
-if rcall("EXISTS", KEYS[3]) == 1 then
-
-  rcall("LREM", KEYS[1], 0, ARGV[2])
-  rcall(ARGV[1], KEYS[2], ARGV[2])
-
-  -- Emit waiting event
-  rcall("XADD", KEYS[4], "*", "event", "waiting", "jobId", ARGV[2], "prev", "failed");
-  
-  return 0
-else
-  return -1
-end

package/src/commands/takeLock-1.lua

@@ -1,17 +0,0 @@
---[[
-  Takes a lock
-
-     Input:
-        KEYS[1] 'lock',
-      
-        ARGV[1]  token
-        ARGV[2]  lock duration in milliseconds
-      
-      Output:
-        "OK" if lock extented succesfully.
-]]
-if redis.call("SET", KEYS[1], ARGV[1], "NX", "PX", ARGV[2]) then
-  return 1
-else
-  return 0
-end

package/src/commands/updateDelaySet-7.lua

@@ -1,71 +0,0 @@
---[[
-  Updates the delay set, by picking a delayed job that should
-  be processed now.
-
-     Input:
-      KEYS[1] 'delayed'
-      KEYS[2] 'wait'
-      KEYS[3] 'priority'
-      KEYS[4] 'paused'
-      KEYS[5] 'meta'
-
-      KEYS[6] event's stream
-      KEYS[7] delayed stream
-
-      ARGV[1] queue.toKey('')
-      ARGV[2] delayed timestamp
-
-     Events:
-      'waiting'
-]] local rcall = redis.call
-
--- Try to get as much as 1000 jobs at once
-local jobs = rcall("ZRANGEBYSCORE", KEYS[1], 0, tonumber(ARGV[2]) * 0x1000,
-                   "LIMIT", 0, 1000)
-
-if (#jobs > 0) then
-    rcall("ZREM", KEYS[1], unpack(jobs))
-
-    -- check if we need to use push in paused instead of waiting
-    local target
-    if rcall("HEXISTS", KEYS[5], "paused") ~= 1 then
-        target = KEYS[2]
-    else
-        target = KEYS[4]
-    end
-
-    for _, jobId in ipairs(jobs) do
-        local priority =
-            tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
-
-        if priority == 0 then
-            -- LIFO or FIFO
-            rcall("LPUSH", target, jobId)
-        else
-            -- Priority add
-            rcall("ZADD", KEYS[3], priority, jobId)
-            local count = rcall("ZCOUNT", KEYS[3], 0, priority)
-
-            local len = rcall("LLEN", target)
-            local id = rcall("LINDEX", target, len - (count - 1))
-            if id then
-                rcall("LINSERT", target, "BEFORE", id, jobId)
-            else
-                rcall("RPUSH", target, jobId)
-            end
-        end
-
-        -- Emit waiting event
-        rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", jobId, "prev",
-              "delayed")
-        rcall("HSET", ARGV[1] .. jobId, "delay", 0)
-    end
-end
-
-local nextTimestamp = rcall("ZRANGE", KEYS[1], 0, 0, "WITHSCORES")[2]
-local id
-if (nextTimestamp ~= nil) then
-    id = rcall("XADD", KEYS[7], "*", "nextTimestamp", nextTimestamp / 0x1000)
-end
-
-return {nextTimestamp, id}

package/src/commands/updateProgress-2.lua

@@ -1,15 +0,0 @@
---[[
-  Update job progress
-
-     Input:
-        KEYS[1] Job id key
-        KEYS[2] event stream key
-      
-        ARGV[1] id
-        ARGV[2] progress
-        
-      Event:
-        progress(jobId, progress)
-]]
-redis.call("HSET", KEYS[1], "progress", ARGV[2])
-redis.call("XADD", KEYS[2], "*", "event", "progress", "jobId", ARGV[1], "data", ARGV[2]);

package/src/enums/index.ts

@@ -1 +0,0 @@
-export * from './retry-errors.enum';

package/src/enums/retry-errors.enum.ts

@@ -1,5 +0,0 @@
-export enum RetryErrors {
-  JobNotFailed = -2,
-  JobIsActive = -1,
-  JobNotExist = 0,
-}

package/src/interfaces/advanced-options.ts

@@ -1,28 +0,0 @@
-export interface AdvancedOptions {
-  // How often check for stalled jobs (use 0 for never checking).
-  stalledInterval?: number;
-
-  // Max amount of times a stalled job will be re-processed.
-  maxStalledCount?: number;
-
-  // Poll interval for delayed jobs and added jobs.
-  guardInterval?: number;
-
-  // delay before processing next job in case of internal error.
-  retryProcessDelay?: number;
-
-  // A set of custom backoff strategies keyed by name.
-  backoffStrategies?: {};
-
-  // A timeout for when the queue is in drained state (empty waiting for jobs).
-  drainDelay?: number;
-}
-
-export const AdvancedOptionsDefaults: AdvancedOptions = {
-  stalledInterval: 30000,
-  maxStalledCount: 1,
-  guardInterval: 5000,
-  retryProcessDelay: 5000,
-  backoffStrategies: {},
-  drainDelay: 5,
-};

package/src/interfaces/backoff-options.ts

@@ -1,4 +0,0 @@
-export interface BackoffOptions {
-  type: string;
-  delay?: number;
-}

package/src/interfaces/index.ts

@@ -1,9 +0,0 @@
-export * from './advanced-options';
-export * from './backoff-options';
-export * from './jobs-options';
-export * from './queue-options';
-export * from './queue-scheduler-options';
-export * from './rate-limiter-options';
-export * from './redis-options';
-export * from './repeat-options';
-export * from './worker-options';

package/src/interfaces/jobs-options.ts

@@ -1,53 +0,0 @@
-import { RepeatOptions } from './repeat-options';
-import { BackoffOptions } from './backoff-options';
-
-export interface JobsOptions {
-  //  default Date.now()
-  timestamp?: number;
-
-  // Ranges from 1 (highest priority) to MAX_INT  (lowest priority). Note that
-  // using priorities has a slight impact on performance,
-  // so do not use it if not required.
-  priority?: number;
-
-  // An amount of miliseconds to wait until this job can be processed.
-  // Note that for accurate delays, worker and producers
-  // should have their clocks synchronized.
-  delay?: number;
-
-  // The total number of attempts to try the job until it completes.
-  attempts?: number;
-
-  // Repeat job according to a cron specification.
-  repeat?: RepeatOptions;
-
-  // Backoff setting for automatic retries if the job fails
-  backoff?: number | BackoffOptions;
-
-  // if true, adds the job to the right of the queue instead of the left (default false)
-  lifo?: boolean;
-
-  // The number of milliseconds after which the job should be
-  // fail with a timeout error [optional]
-  timeout?: number;
-
-  // Override the job ID - by default, the job ID is a unique
-  // integer, but you can use this setting to override it.
-  // If you use this option, it is up to you to ensure the
-  // jobId is unique. If you attempt to add a job with an id that
-  // already exists, it will not be added.
-  jobId?: string;
-
-  // If true, removes the job when it successfully completes
-  // A number specify the max amount of jobs to keep.
-  // Default behavior is to keep the job in the completed set.
-  removeOnComplete?: boolean | number;
-
-  // If true, removes the job when it fails after all attempts.
-  // A number specify the max amount of jobs to keep.
-  // Default behavior is to keep the job in the failed set.
-  removeOnFail?: boolean | number;
-
-  // Limits the amount of stack trace lines that will be recorded in the stacktrace.
-  stackTraceLimit?: number;
-}

package/src/interfaces/queue-options.ts

@@ -1,30 +0,0 @@
-import { JobsOptions } from '../interfaces';
-
-import IORedis from 'ioredis';
-import { ConnectionOptions } from './redis-options';
-
-export enum ClientType {
-  blocking = 'blocking',
-  normal = 'normal',
-}
-
-export interface QueueBaseOptions {
-  connection?: ConnectionOptions;
-  client?: IORedis.Redis;
-  prefix?: string; // prefix for all queue keys.
-}
-
-export interface QueueOptions extends QueueBaseOptions {
-  defaultJobOptions?: JobsOptions;
-
-  streams?: {
-    events: {
-      maxLen: number; // Max aproximated length for streams
-    };
-  };
-}
-
-export interface QueueEventsOptions extends QueueBaseOptions {
-  lastEventId?: string;
-  blockingTimeout?: number;
-}

package/src/interfaces/queue-scheduler-options.ts

@@ -1,6 +0,0 @@
-import { QueueBaseOptions } from '../interfaces';
-
-export interface QueueSchedulerOptions extends QueueBaseOptions {
-  maxStalledCount?: number;
-  stalledInterval?: number;
-}

package/src/interfaces/rate-limiter-options.ts

@@ -1,7 +0,0 @@
-export interface RateLimiterOptions {
-  // Max number of jobs processed
-  max: number;
-
-  // per duration in milliseconds
-  duration: number;
-}