bullmq 1.54.5 → 1.56.0

package/dist/commands/index.js

@@ -1,74 +1,10 @@
-/**
- * Load redis lua scripts.
- * The name of the script must have the following format:
- *
- * cmdName-numKeys.lua
- *
- * cmdName must be in camel case format.
- *
- * For example:
- * moveToFinish-3.lua
- *
- */
-'use strict';
+"use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.loadIncludes = exports.load = void 0;
-const lodash_1 = require("lodash");
-const path = require("path");
-const fs = require("fs");
-const util = require("util");
-const readdir = util.promisify(fs.readdir);
-const readFile = util.promisify(fs.readFile);
-const exists = util.promisify(fs.exists);
-exports.load = async function (client, pathname, extraIncludes) {
-    const scripts = await loadScripts(pathname, extraIncludes);
-    scripts.forEach((command) => {
-        // Only define the command if not already defined
-        if (!client[command.name]) {
-            client.defineCommand(command.name, command.options);
-        }
-    });
-};
-async function loadScripts(dir, extraIncludes) {
-    const files = await readdir(dir);
-    const includes = await exports.loadIncludes(dir, extraIncludes);
-    const luaFiles = files.filter((file) => path.extname(file) === '.lua');
-    if (luaFiles.length === 0) {
-        /**
-         * To prevent unclarified runtime error "updateDelayset is not a function
-         * @see https://github.com/OptimalBits/bull/issues/920
-         */
-        throw new Error('No .lua files found!');
-    }
-    const commands = [];
-    for (let i = 0; i < luaFiles.length; i++) {
-        const file = luaFiles[i];
-        if (path.extname(file) === '.lua') {
-            const longName = path.basename(file, '.lua');
-            const name = longName.split('-')[0];
-            const numberOfKeys = parseInt(longName.split('-')[1]);
-            const lua = (await readFile(path.join(dir, file))).toString();
-            const compiled = lodash_1.template(lua);
-            commands.push({
-                name,
-                options: { numberOfKeys, lua: compiled(includes) },
-            });
-        }
-    }
-    return commands;
-}
-exports.loadIncludes = async function (dir, extraIncludes) {
-    const includes = extraIncludes || {};
-    const includesDir = path.join(dir, 'includes');
-    if (await exists(includesDir)) {
-        const includesFiles = await readdir(includesDir);
-        for (let i = 0; i < includesFiles.length; i++) {
-            const file = includesFiles[i];
-            const lua = await readFile(path.join(includesDir, file));
-            const name = path.basename(file, '.lua');
-            includes[name] = lua.toString();
-        }
-    }
-    return includes;
-};
+exports.scriptLoader = exports.ScriptLoader = exports.ScriptLoaderError = void 0;
+const script_loader_1 = require("./script-loader");
+Object.defineProperty(exports, "ScriptLoader", { enumerable: true, get: function () { return script_loader_1.ScriptLoader; } });
+var script_loader_2 = require("./script-loader");
+Object.defineProperty(exports, "ScriptLoaderError", { enumerable: true, get: function () { return script_loader_2.ScriptLoaderError; } });
+const scriptLoader = new script_loader_1.ScriptLoader();
+exports.scriptLoader = scriptLoader;
 //# sourceMappingURL=index.js.map

package/dist/commands/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/commands/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,YAAY,CAAC;;;AACb,mCAAkC;AAGlC,6BAA6B;AAC7B,yBAAyB;AACzB,6BAA6B;AAE7B,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,OAAO,CAAC,CAAC;AAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC;AAC7C,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,MAAM,CAAC,CAAC;AAU5B,QAAA,IAAI,GAAG,KAAK,WACvB,MAAmB,EACnB,QAAgB,EAChB,aAAsC;IAEtC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,EAAE,aAAa,CAAC,CAAC;IAE3D,OAAO,CAAC,OAAO,CAAC,CAAC,OAAgB,EAAE,EAAE;QACnC,iDAAiD;QACjD,IAAI,CAAE,MAAc,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE;YAClC,MAAM,CAAC,aAAa,CAAC,OAAO,CAAC,IAAI,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC;SACrD;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,KAAK,UAAU,WAAW,CACxB,GAAW,EACX,aAAsC;IAEtC,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC;IAEjC,MAAM,QAAQ,GAAG,MAAM,oBAAY,CAAC,GAAG,EAAE,aAAa,CAAC,CAAC;IAExD,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAC3B,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,MAAM,CAChD,CAAC;IAEF,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;QACzB;;;WAGG;QACH,MAAM,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;KACzC;IAED,MAAM,QAAQ,GAAG,EAAE,CAAC;IAEpB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACxC,MAAM,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzB,IAAI,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,KAAK,MAAM,EAAE;YACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC7C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACpC,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAEtD,MAAM,GAAG,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC;YAC9D,MAAM,QAAQ,GAAG,iBAAQ,CAAC,GAAG,CAAC,CAAC;YAE/B,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI;gBACJ,OAAO,EAAE,EAAE,YAAY,EAAE,GAAG,EAAE,QAAQ,CAAC,QAAQ,CAAC,EAAE;aACnD,CAAC,CAAC;SACJ;KACF;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC;AAEY,QAAA,YAAY,GAAG,KAAK,WAC/B,GAAW,EACX,aAAsC;IAEtC,MAAM,QAAQ,GAAG,aAAa,IAAI,EAAE,CAAC;IACrC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,CAAC;IAE/C,IAAI,MAAM,MAAM,CAAC,WAAW,CAAC,EAAE;QAC7B,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,CAAC;QAEjD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC7C,MAAM,IAAI,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAC9B,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC,CAAC;YACzD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YACzC,QAAQ,CAAC,IAAI,CAAC,GAAG,GAAG,CAAC,QAAQ,EAAE,CAAC;SACjC;KACF;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/commands/index.ts"],"names":[],"mappings":";;;AAAA,mDAA+C;AAKtC,6FALA,4BAAY,OAKA;AAJrB,iDAA6E;AAA3C,kHAAA,iBAAiB,OAAA;AAEnD,MAAM,YAAY,GAAG,IAAI,4BAAY,EAAE,CAAC;AAEjB,oCAAY"}

package/dist/commands/moveStalledJobsToWait-8.lua

@@ -75,8 +75,8 @@ if (#stalling > 0) then
                 if (stalledCount > MAX_STALLED_JOB_COUNT) then
                     local failedReason = "job stalled more than allowable limit" 
                     rcall("ZADD", KEYS[4], ARGV[3], jobId)
-                    rcall("HSET", jobKey, "failedReason",
-                          failedReason)
+                    rcall("HMSET", jobKey, "failedReason",
+                          failedReason, "finishedOn", ARGV[3])
                     rcall("XADD", KEYS[8], "*", "event", "failed", "jobId",
                           jobId, 'prev', 'active', 'failedReason',
                           failedReason)

package/dist/commands/moveToFinished-8.lua

@@ -42,8 +42,8 @@
 local rcall = redis.call
 
 -- Includes
-<%= updateParentDepsIfNeeded %>
-<%= destructureJobKey %>
+--- @include "includes/updateParentDepsIfNeeded"
+--- @include "includes/destructureJobKey"
 
 local jobIdKey = KEYS[3]
 if rcall("EXISTS",jobIdKey) == 1 then -- // Make sure job exists
@@ -70,8 +70,16 @@ if rcall("EXISTS",jobIdKey) == 1 then -- // Make sure job exists
     if(numRemovedElements < 1) then
       return -3
     end
-    
-    -- If job has a parent we need to 
+
+
+    -- Trim events before emiting them to avoid trimming events emitted in this script
+    local maxEvents = rcall("HGET", KEYS[7], "opts.maxLenEvents")
+    if (maxEvents == false) then
+       maxEvents = 10000
+    end
+    rcall("XTRIM", KEYS[6], "MAXLEN", "~", maxEvents)
+
+    -- If job has a parent we need to
     -- 1) remove this job id from parents dependencies
     -- 2) move the job Id to parent "processed" set
     -- 3) push the results into parent "results" list
@@ -83,15 +91,15 @@ if rcall("EXISTS",jobIdKey) == 1 then -- // Make sure job exists
       parentId = getJobIdFromKey(ARGV[14])
       parentQueueKey = getJobKeyPrefix(ARGV[14], ":" .. parentId)
     end
-    if parentId ~= "" and ARGV[5] == "completed" then 
+    if parentId ~= "" and ARGV[5] == "completed" then
         local parentKey =  parentQueueKey .. ":" .. parentId
         local dependenciesSet = parentKey .. ":dependencies"
         local result = rcall("SREM", dependenciesSet, jobIdKey)
-        if result == 1 then 
+        if result == 1 then
           updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet, parentId, jobIdKey, ARGV[4])
         end
     end
-    
+
     -- Remove job?
     local removeJobs = tonumber(ARGV[6])
     if removeJobs ~= 1 then
@@ -140,15 +148,11 @@ if rcall("EXISTS",jobIdKey) == 1 then -- // Make sure job exists
             rcall("HSET", jobKey, "processedOn", ARGV[2])
 
             return {rcall("HGETALL", jobKey), jobId} -- get job data
+        else
+          rcall("XADD", KEYS[6], "*", "event", "drained");
         end
     end
 
-    local maxEvents = rcall("HGET", KEYS[7], "opts.maxLenEvents")
-    if (maxEvents == false) then
-      maxEvents = 10000
-    end
-    rcall("XTRIM", KEYS[6], "MAXLEN", "~", maxEvents)
-
     return 0
 else
     return -1

package/dist/commands/removeJob-1.lua

@@ -13,7 +13,7 @@
 local rcall = redis.call
 
 -- Includes
-<%= destructureJobKey %>
+--- @include "includes/destructureJobKey"
 
 -- recursively check if there are no locks on the
 -- jobs to be removed.
@@ -45,13 +45,13 @@ local function removeJob( prefix, jobId)
     local jobKey = prefix .. jobId;
 
     -- Check if this job has a parent. If so we will just remove it from
-    -- the parent child list, but if it is the last child we should move the parent to "wait/paused" 
+    -- the parent child list, but if it is the last child we should move the parent to "wait/paused"
     -- which requires code from "moveToFinished"
     local parentKey = rcall("HGET", jobKey, "parentKey")
     if( (type(parentKey) == "string") and parentKey ~= "" and (rcall("EXISTS", parentKey) == 1)) then
         local parentDependenciesKey = parentKey .. ":dependencies"
         local result = rcall("SREM", parentDependenciesKey, jobKey)
-        if rcall("SCARD", parentDependenciesKey) == 0 then 
+        if rcall("SCARD", parentDependenciesKey) == 0 then
             local parentId = getJobIdFromKey(parentKey)
             local parentPrefix = getJobKeyPrefix(parentKey, parentId)
 

package/dist/commands/script-loader.d.ts

@@ -0,0 +1,120 @@
+import { RedisClient } from '../interfaces';
+export interface Command {
+    name: string;
+    options: {
+        numberOfKeys: number;
+        lua: string;
+    };
+}
+/**
+ * Script metadata
+ */
+export interface ScriptMetadata {
+    /**
+     * Name of the script
+     */
+    name: string;
+    numberOfKeys?: number;
+    /**
+     * The path to the script. For includes, this is the normalized path,
+     * whereas it may not be normalized for the top-level parent
+     */
+    path: string;
+    /**
+     * The raw script content
+     */
+    content: string;
+    /**
+     * A hash of the normalized path for easy replacement in the parent
+     */
+    token: string;
+    /**
+     * Metadata on the scripts that this script includes
+     */
+    includes: ScriptMetadata[];
+}
+export declare class ScriptLoaderError extends Error {
+    readonly path: string;
+    /**
+     * The include stack
+     */
+    readonly includes: string[];
+    readonly line: number;
+    readonly position: number;
+    constructor(message: string, path: string, stack?: string[], line?: number, position?: number);
+}
+/**
+ * Lua script loader with include support
+ */
+export declare class ScriptLoader {
+    /**
+     * Map an alias to a path
+     */
+    private pathMapper;
+    private clientScripts;
+    /**
+     * Cache commands by dir
+     */
+    private commandCache;
+    private rootPath;
+    constructor();
+    /**
+     * Add a script path mapping. Allows includes of the form "<includes>/utils.lua" where `includes` is a user
+     * defined path
+     * @param name - the name of the mapping. Note: do not include angle brackets
+     * @param mappedPath - if a relative path is passed, it's relative to the *caller* of this function.
+     * Mapped paths are also accepted, e.g. "~/server/scripts/lua" or "<base>/includes"
+     */
+    addPathMapping(name: string, mappedPath: string): void;
+    /**
+     * Resolve the script path considering path mappings
+     * @param scriptName - the name of the script
+     * @param stack - the include stack, for nicer errors
+     */
+    resolvePath(scriptName: string, stack?: string[]): string;
+    /**
+     * Recursively collect all scripts included in a file
+     * @param file - the parent file
+     * @param cache - a cache for file metadata to increase efficiency. Since a file can be included
+     * multiple times, we make sure to load it only once.
+     * @param stack - internal stack to prevent circular references
+     */
+    private resolveDependencies;
+    /**
+     * Parse a (top-level) lua script
+     * @param filename - the full path to the script
+     * @param content - the content of the script
+     * @param cache - cache
+     */
+    parseScript(filename: string, content: string, cache?: Map<string, ScriptMetadata>): Promise<ScriptMetadata>;
+    /**
+     * Construct the final version of a file by interpolating its includes in dependency order.
+     * @param file - the file whose content we want to construct
+     * @param processed - a cache to keep track of which includes have already been processed
+     */
+    interpolate(file: ScriptMetadata, processed?: Set<string>): string;
+    loadCommand(filename: string, cache?: Map<string, ScriptMetadata>): Promise<Command>;
+    /**
+     * Load redis lua scripts.
+     * The name of the script must have the following format:
+     *
+     * cmdName-numKeys.lua
+     *
+     * cmdName must be in camel case format.
+     *
+     * For example:
+     * moveToFinish-3.lua
+     *
+     */
+    loadScripts(dir?: string, cache?: Map<string, ScriptMetadata>): Promise<Command[]>;
+    /**
+     * Attach all lua scripts in a given directory to a client instance
+     * @param client - redis client to attach script to
+     * @param pathname - the path to the directory containing the scripts
+     */
+    load(client: RedisClient, pathname: string): Promise<void>;
+    /**
+     * Clears the command cache
+     */
+    clearCache(): void;
+}