bullmq 1.74.3 → 1.76.0

package/dist/esm/commands/includes/collectMetrics.lua

@@ -0,0 +1,42 @@
+--[[
+  Functions to collect metrics based on a current and previous count of jobs.
+  Granualarity is fixed at 1 minute.
+]] local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
+                                 timestamp)
+    -- Increment current count
+    local count = rcall("HINCRBY", metaKey, "count", 1) - 1
+
+    -- Compute how many data points we need to add to the list, N.
+    local prevTS = rcall("HGET", metaKey, "prevTS")
+
+    if not prevTS then
+        -- If prevTS is nil, set it to the current timestamp
+        rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
+        return
+    end
+
+    local N = math.floor((timestamp - prevTS) / 60000)
+
+    if N > 0 then
+        local delta = count - rcall("HGET", metaKey, "prevCount")
+        -- If N > 1, add N-1 zeros to the list
+        if N > 1 then
+            local points = {}
+            points[1] = delta
+            for i = 2, N do
+                points[i] = 0
+            end
+            rcall("LPUSH", dataPointsList, unpack(points))
+        else
+            -- LPUSH delta to the list
+            rcall("LPUSH", dataPointsList, delta)
+        end
+
+        -- LTRIM to keep list to its max size
+        rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
+
+        -- update prev count with current count
+        rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
+    end
+
+end

package/dist/esm/commands/moveToActive-8.lua

@@ -55,16 +55,26 @@ if jobId then
   if(maxJobs) then
     local rateLimiterKey = KEYS[6];
 
+    local groupKey
     if(ARGV[8]) then
-      local groupKey = string.match(jobId, "[^:]+$")
-      if groupKey ~= nil then
+      groupKey = string.match(jobId, "[^:]+$")
+      if groupKey ~= jobId then
         rateLimiterKey = rateLimiterKey .. ":" .. groupKey
       end
     end
 
-    local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
+    local jobCounter
+    
+    if groupKey ~= nil then
+      if rateLimiterKey ~= KEYS[6] then
+        jobCounter = tonumber(rcall("INCR", rateLimiterKey))
+      end
+    else
+      jobCounter = tonumber(rcall("INCR", rateLimiterKey))
+    end
+
     -- check if rate limit hit
-    if jobCounter > maxJobs then
+    if jobCounter ~= nil and jobCounter > maxJobs then
       local exceedingJobs = jobCounter - maxJobs
       local expireTime = tonumber(rcall("PTTL", rateLimiterKey))
       local delay = expireTime + ((exceedingJobs - 1) * ARGV[7]) / maxJobs;

package/dist/esm/commands/{moveToFinished-8.lua → moveToFinished-9.lua}

@@ -13,6 +13,7 @@
       KEYS[6] event stream key
       KEYS[7] meta key
       KEYS[8] stalled key
+      KEYS[9] metrics key
 
       ARGV[1]  jobId
       ARGV[2]  timestamp
@@ -30,6 +31,7 @@
       ARGV[14] parentKey
       ARGV[15] max attempts
       ARGV[16] attemptsMade
+      ARGV[17] maxMetricsSize
 
     Output:
       0 OK
@@ -40,15 +42,15 @@
 
     Events:
       'completed/failed'
-]]
-local rcall = redis.call
+]] local rcall = redis.call
 
--- Includes
+--- Includes
 --- @include "includes/destructureJobKey"
 --- @include "includes/moveJobFromWaitToActive"
 --- @include "includes/removeJob"
 --- @include "includes/trimEvents"
 --- @include "includes/updateParentDepsIfNeeded"
+--- @include "includes/collectMetrics"
 
 local jobIdKey = KEYS[3]
 if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
@@ -72,9 +74,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
     -- Remove from active list (if not active we shall return error)
     local numRemovedElements = rcall("LREM", KEYS[1], -1, jobId)
 
-    if (numRemovedElements < 1) then
-      return -3
-    end
+    if (numRemovedElements < 1) then return -3 end
 
     -- Trim events before emiting them to avoid trimming events emitted in this script
     trimEvents(KEYS[7], KEYS[6])
@@ -117,14 +117,18 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
         if maxAge ~= nil then
             local start = timestamp - maxAge * 1000
             local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
-            for i, jobId in ipairs(jobIds) do removeJob(jobId, false, prefix) end
+            for i, jobId in ipairs(jobIds) do
+                removeJob(jobId, false, prefix)
+            end
             rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
         end
 
         if maxCount ~= nil and maxCount > 0 then
             local start = maxCount
             local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
-            for i, jobId in ipairs(jobIds) do removeJob(jobId, false, prefix) end
+            for i, jobId in ipairs(jobIds) do
+                removeJob(jobId, false, prefix)
+            end
             rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
         end
     else
@@ -141,6 +145,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
         end
     end
 
+    -- Collect metrics
+    if ARGV[17] ~= "" then
+        collectMetrics(KEYS[9], KEYS[9]..':data', ARGV[17], timestamp)
+    end
+
     -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
     -- and not rate limited.
     if (ARGV[8] == "1") then

package/dist/esm/commands/obliterate-2.lua

@@ -75,10 +75,19 @@ if(maxCount <= 0) then
 end
 
 if(maxCount > 0) then
-  rcall("DEL", baseKey .. 'priority', baseKey .. 'events',
-    baseKey .. 'delay', baseKey .. 'stalled-check',
-    baseKey .. 'stalled', baseKey .. 'id',
-    baseKey .. 'meta', baseKey .. 'repeat')
+  rcall("DEL", 
+    baseKey .. 'priority',
+    baseKey .. 'events',
+    baseKey .. 'delay', 
+    baseKey .. 'stalled-check',
+    baseKey .. 'stalled',
+    baseKey .. 'id',
+    baseKey .. 'meta',
+    baseKey .. 'repeat',
+    baseKey .. 'metrics:completed',
+    baseKey .. 'metrics:completed:data',
+    baseKey .. 'metrics:failed',
+    baseKey .. 'metrics:failed:data')
   return 0
 else
   return 1

package/dist/esm/commands/retryJobs-4.lua

@@ -4,7 +4,7 @@
   Input:
     KEYS[1] base key
     KEYS[2] events stream
-    KEYS[3] failed state key
+    KEYS[3] state key (failed, completed)
     KEYS[4] wait state key
 
     ARGV[1]  count
@@ -28,7 +28,7 @@ local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount)
 if (#jobs > 0) then
   for i, key in ipairs(jobs) do
     local jobKey = baseKey .. key
-    rcall("HDEL", jobKey, "finishedOn", "processedOn", "failedReason")
+    rcall("HDEL", jobKey, "finishedOn", "processedOn", "failedReason", "returnvalue")
 
     -- Emit waiting event
     rcall("XADD", KEYS[2], "*", "event", "waiting", "jobId", key);

package/dist/esm/enums/index.d.ts

@@ -1 +1,2 @@
 export * from './error-code.enum';
+export * from './metrics-time';

package/dist/esm/enums/index.js

@@ -1,2 +1,3 @@
 export * from './error-code.enum';
+export * from './metrics-time';
 //# sourceMappingURL=index.js.map

package/dist/esm/enums/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/enums/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/enums/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,gBAAgB,CAAC"}

package/dist/esm/enums/metrics-time.d.ts

@@ -0,0 +1,10 @@
+export declare enum MetricsTime {
+    ONE_MINUTE = 1,
+    FIVE_MINUTES = 5,
+    FIFTEEN_MINUTES = 15,
+    THIRTY_MINUTES = 30,
+    ONE_HOUR = 60,
+    ONE_WEEK = 10080,
+    TWO_WEEKS = 20160,
+    ONE_MONTH = 80640
+}

package/dist/esm/enums/metrics-time.js

@@ -0,0 +1,12 @@
+export var MetricsTime;
+(function (MetricsTime) {
+    MetricsTime[MetricsTime["ONE_MINUTE"] = 1] = "ONE_MINUTE";
+    MetricsTime[MetricsTime["FIVE_MINUTES"] = 5] = "FIVE_MINUTES";
+    MetricsTime[MetricsTime["FIFTEEN_MINUTES"] = 15] = "FIFTEEN_MINUTES";
+    MetricsTime[MetricsTime["THIRTY_MINUTES"] = 30] = "THIRTY_MINUTES";
+    MetricsTime[MetricsTime["ONE_HOUR"] = 60] = "ONE_HOUR";
+    MetricsTime[MetricsTime["ONE_WEEK"] = 10080] = "ONE_WEEK";
+    MetricsTime[MetricsTime["TWO_WEEKS"] = 20160] = "TWO_WEEKS";
+    MetricsTime[MetricsTime["ONE_MONTH"] = 80640] = "ONE_MONTH";
+})(MetricsTime || (MetricsTime = {}));
+//# sourceMappingURL=metrics-time.js.map

package/dist/esm/enums/metrics-time.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics-time.js","sourceRoot":"","sources":["../../../src/enums/metrics-time.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,WASX;AATD,WAAY,WAAW;IACrB,yDAAc,CAAA;IACd,6DAAgB,CAAA;IAChB,oEAAoB,CAAA;IACpB,kEAAmB,CAAA;IACnB,sDAAa,CAAA;IACb,yDAAsB,CAAA;IACtB,2DAA2B,CAAA;IAC3B,2DAA+B,CAAA;AACjC,CAAC,EATW,WAAW,KAAX,WAAW,QAStB"}

package/dist/esm/interfaces/index.d.ts

@@ -7,6 +7,8 @@ export * from './flow-job';
 export * from './job-json';
 export * from './jobs-options';
 export * from './keep-jobs';
+export * from './metrics-options';
+export * from './metrics';
 export * from './parent-command';
 export * from './parent-message';
 export * from './parent';

package/dist/esm/interfaces/index.js

@@ -7,6 +7,8 @@ export * from './flow-job';
 export * from './job-json';
 export * from './jobs-options';
 export * from './keep-jobs';
+export * from './metrics-options';
+export * from './metrics';
 export * from './parent-command';
 export * from './parent-message';
 export * from './parent';

package/dist/esm/interfaces/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,YAAY,CAAC;AAC3B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,kBAAkB,CAAC;AACjC,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC"}

package/dist/esm/interfaces/metrics-options.d.ts

@@ -0,0 +1,12 @@
+/**
+ *
+ *
+ */
+export interface MetricsOptions {
+    /**
+     * Enable gathering metrics for finished jobs.
+     * Output refers to all finished jobs, completed or
+     * failed.
+     */
+    maxDataPoints?: number;
+}

package/dist/esm/interfaces/metrics-options.js

@@ -0,0 +1 @@
+//# sourceMappingURL=metrics-options.js.map

package/dist/esm/interfaces/metrics-options.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics-options.js","sourceRoot":"","sources":["../../../src/interfaces/metrics-options.ts"],"names":[],"mappings":""}

package/dist/esm/interfaces/metrics.d.ts

@@ -0,0 +1,9 @@
+export interface Metrics {
+    meta: {
+        count: number;
+        prevTS: number;
+        prevCount: number;
+    };
+    data: number[];
+    count: number;
+}

package/dist/esm/interfaces/metrics.js

@@ -0,0 +1 @@
+//# sourceMappingURL=metrics.js.map

package/dist/esm/interfaces/metrics.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../../src/interfaces/metrics.ts"],"names":[],"mappings":""}

package/dist/esm/interfaces/worker-options.d.ts

@@ -2,6 +2,7 @@ import { Job } from '../classes/job';
 import { AdvancedOptions } from './advanced-options';
 import { QueueBaseOptions } from './queue-options';
 import { RateLimiterOptions } from './rate-limiter-options';
+import { MetricsOptions } from './metrics-options';
 /**
  * An async function that receives `Job`s and handles them.
  */
@@ -19,9 +20,15 @@ export interface WorkerOptions extends QueueBaseOptions {
      */
     concurrency?: number;
     /**
+     * Enable rate limiter
      * @see {@link https://docs.bullmq.io/guide/rate-limiting}
      */
     limiter?: RateLimiterOptions;
+    /**
+     * Enable collect metrics.
+     * @see {@link https://docs.bullmq.io/guide/metrics}
+     */
+    metrics?: MetricsOptions;
     skipDelayCheck?: boolean;
     drainDelay?: number;
     lockDuration?: number;

package/dist/{cjs/types/finished-target.d.ts → esm/types/finished-status.d.ts}

@@ -1,2 +1,2 @@
-export declare type FinishedTarget = 'completed' | 'failed';
+export declare type FinishedStatus = 'completed' | 'failed';
 export declare type FinishedPropValAttribute = 'returnvalue' | 'failedReason';

package/dist/esm/types/finished-status.js

@@ -0,0 +1 @@
+//# sourceMappingURL=finished-status.js.map

package/dist/esm/types/finished-status.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"finished-status.js","sourceRoot":"","sources":["../../../src/types/finished-status.ts"],"names":[],"mappings":""}

package/dist/esm/types/index.d.ts

@@ -1,3 +1,3 @@
-export * from './finished-target';
+export * from './finished-status';
 export * from './job-json-sandbox';
 export * from './job-type';

package/dist/esm/types/index.js

@@ -1,4 +1,4 @@
-export * from './finished-target';
+export * from './finished-status';
 export * from './job-json-sandbox';
 export * from './job-type';
 //# sourceMappingURL=index.js.map

package/dist/esm/types/job-type.d.ts

@@ -1,3 +1,3 @@
-import { FinishedTarget } from './finished-target';
-export declare type JobState = FinishedTarget | 'active' | 'delayed' | 'waiting' | 'waiting-children';
+import { FinishedStatus } from './finished-status';
+export declare type JobState = FinishedStatus | 'active' | 'delayed' | 'waiting' | 'waiting-children';
 export declare type JobType = JobState | 'paused' | 'repeat' | 'wait';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "bullmq",
-  "version": "1.74.3",
+  "version": "1.76.0",
   "description": "Queue for messages and jobs based on Redis",
   "main": "dist/cjs/index.js",
   "module": "dist/esm/index.js",

package/dist/cjs/types/finished-target.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"finished-target.js","sourceRoot":"","sources":["../../../src/types/finished-target.ts"],"names":[],"mappings":""}

package/dist/esm/types/finished-target.js

@@ -1 +0,0 @@
-//# sourceMappingURL=finished-target.js.map

package/dist/esm/types/finished-target.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"finished-target.js","sourceRoot":"","sources":["../../../src/types/finished-target.ts"],"names":[],"mappings":""}