bullmq 4.0.0-beta.1 → 4.0.0

package/dist/esm/commands/retryJob-9.lua

@@ -0,0 +1,69 @@
+--[[
+  Retries a failed job by moving it back to the wait queue.
+
+    Input:
+      KEYS[1] 'active',
+      KEYS[2] 'wait'
+      KEYS[3] 'paused'
+      KEYS[4] job key
+      KEYS[5] 'meta'
+      KEYS[6] events stream
+      KEYS[7] delayed key
+      KEYS[8] prioritized key
+      KEYS[9] 'pc' priority counter
+
+      ARGV[1]  key prefix
+      ARGV[2]  timestamp
+      ARGV[3]  pushCmd
+      ARGV[4]  jobId
+      ARGV[5]  token
+
+    Events:
+      'waiting'
+
+    Output:
+     0  - OK
+     -1 - Missing key
+     -2 - Missing lock
+]]
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/addJobWithPriority"
+--- @include "includes/getTargetQueueList"
+--- @include "includes/promoteDelayedJobs"
+
+local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
+-- Check if there are delayed jobs that we can move to wait.
+-- test example: when there are delayed jobs between retries
+promoteDelayedJobs(KEYS[7], KEYS[2], target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], paused, KEYS[9])
+
+if rcall("EXISTS", KEYS[4]) == 1 then
+
+  if ARGV[5] ~= "0" then
+    local lockKey = KEYS[4] .. ':lock'
+    if rcall("GET", lockKey) == ARGV[5] then
+      rcall("DEL", lockKey)
+    else
+      return -2
+    end
+  end
+
+  rcall("LREM", KEYS[1], 0, ARGV[4])
+
+  local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0
+
+  -- Standard or priority add
+  if priority == 0 then
+    rcall(ARGV[3], target, ARGV[4])
+  else
+    addJobWithPriority(KEYS[2], KEYS[8], priority, paused, ARGV[4], KEYS[9])
+  end
+
+  -- Emit waiting event
+  rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", ARGV[4], "prev", "failed")
+
+  return 0
+else
+  return -1
+end

package/dist/esm/commands/retryJobs-6.lua

@@ -0,0 +1,53 @@
+--[[
+  Attempts to retry all failed jobs
+
+  Input:
+    KEYS[1] base key
+    KEYS[2] events stream
+    KEYS[3] state key (failed, completed)
+    KEYS[4] 'wait'
+    KEYS[5] 'paused'
+    KEYS[6] 'meta'
+
+    ARGV[1] count
+    ARGV[2] timestamp
+    ARGV[3] prev state
+
+  Output:
+    1  means the operation is not completed
+    0  means the operation is completed
+]]
+local maxCount = tonumber(ARGV[1])
+local timestamp = tonumber(ARGV[2])
+
+local rcall = redis.call;
+
+-- Includes
+--- @include "includes/batches"
+--- @include "includes/getTargetQueueList"
+
+local target = getTargetQueueList(KEYS[6], KEYS[4], KEYS[5])
+
+local jobs = rcall('ZRANGEBYSCORE', KEYS[3], 0, timestamp, 'LIMIT', 0, maxCount)
+if (#jobs > 0) then
+  for i, key in ipairs(jobs) do
+    local jobKey = KEYS[1] .. key
+    rcall("HDEL", jobKey, "finishedOn", "processedOn", "failedReason", "returnvalue")
+
+    -- Emit waiting event
+    rcall("XADD", KEYS[2], "*", "event", "waiting", "jobId", key, "prev", ARGV[3]);
+  end
+
+  for from, to in batches(#jobs, 7000) do
+    rcall("ZREM", KEYS[3], unpack(jobs, from, to))
+    rcall("LPUSH", target, unpack(jobs, from, to))
+  end
+end
+
+maxCount = maxCount - #jobs
+
+if(maxCount <= 0) then
+  return 1
+end
+
+return 0

package/dist/esm/commands/saveStacktrace-1.lua

@@ -0,0 +1,22 @@
+--[[
+  Save stacktrace and failedReason.
+
+  Input:
+    KEYS[1] job key
+
+    ARGV[1]  stacktrace
+    ARGV[2]  failedReason
+
+  Output:
+     0 - OK
+    -1 - Missing key
+]]
+local rcall = redis.call
+
+if rcall("EXISTS", KEYS[1]) == 1 then
+  rcall("HMSET", KEYS[1], "stacktrace", ARGV[1], "failedReason", ARGV[2])
+
+  return 0
+else
+  return -1
+end

package/dist/esm/commands/script-loader.d.ts

@@ -0,0 +1,119 @@
+import { RedisClient } from '../interfaces';
+export interface Command {
+    name: string;
+    options: {
+        numberOfKeys: number;
+        lua: string;
+    };
+}
+/**
+ * Script metadata
+ */
+export interface ScriptMetadata {
+    /**
+     * Name of the script
+     */
+    name: string;
+    numberOfKeys?: number;
+    /**
+     * The path to the script. For includes, this is the normalized path,
+     * whereas it may not be normalized for the top-level parent
+     */
+    path: string;
+    /**
+     * The raw script content
+     */
+    content: string;
+    /**
+     * A hash of the normalized path for easy replacement in the parent
+     */
+    token: string;
+    /**
+     * Metadata on the scripts that this script includes
+     */
+    includes: ScriptMetadata[];
+}
+export declare class ScriptLoaderError extends Error {
+    /**
+     * The include stack
+     */
+    readonly includes: string[];
+    readonly line: number;
+    readonly position: number;
+    constructor(message: string, path: string, stack?: string[], line?: number, position?: number);
+}
+/**
+ * Lua script loader with include support
+ */
+export declare class ScriptLoader {
+    /**
+     * Map an alias to a path
+     */
+    private pathMapper;
+    private clientScripts;
+    /**
+     * Cache commands by dir
+     */
+    private commandCache;
+    private rootPath;
+    constructor();
+    /**
+     * Add a script path mapping. Allows includes of the form "<includes>/utils.lua" where `includes` is a user
+     * defined path
+     * @param name - the name of the mapping. Note: do not include angle brackets
+     * @param mappedPath - if a relative path is passed, it's relative to the *caller* of this function.
+     * Mapped paths are also accepted, e.g. "~/server/scripts/lua" or "<base>/includes"
+     */
+    addPathMapping(name: string, mappedPath: string): void;
+    /**
+     * Resolve the script path considering path mappings
+     * @param scriptName - the name of the script
+     * @param stack - the include stack, for nicer errors
+     */
+    resolvePath(scriptName: string, stack?: string[]): string;
+    /**
+     * Recursively collect all scripts included in a file
+     * @param file - the parent file
+     * @param cache - a cache for file metadata to increase efficiency. Since a file can be included
+     * multiple times, we make sure to load it only once.
+     * @param stack - internal stack to prevent circular references
+     */
+    private resolveDependencies;
+    /**
+     * Parse a (top-level) lua script
+     * @param filename - the full path to the script
+     * @param content - the content of the script
+     * @param cache - cache
+     */
+    parseScript(filename: string, content: string, cache?: Map<string, ScriptMetadata>): Promise<ScriptMetadata>;
+    /**
+     * Construct the final version of a file by interpolating its includes in dependency order.
+     * @param file - the file whose content we want to construct
+     * @param processed - a cache to keep track of which includes have already been processed
+     */
+    interpolate(file: ScriptMetadata, processed?: Set<string>): string;
+    loadCommand(filename: string, cache?: Map<string, ScriptMetadata>): Promise<Command>;
+    /**
+     * Load redis lua scripts.
+     * The name of the script must have the following format:
+     *
+     * cmdName-numKeys.lua
+     *
+     * cmdName must be in camel case format.
+     *
+     * For example:
+     * moveToFinish-3.lua
+     *
+     */
+    loadScripts(dir?: string, cache?: Map<string, ScriptMetadata>): Promise<Command[]>;
+    /**
+     * Attach all lua scripts in a given directory to a client instance
+     * @param client - redis client to attach script to
+     * @param pathname - the path to the directory containing the scripts
+     */
+    load(client: RedisClient, pathname: string, cache?: Map<string, ScriptMetadata>): Promise<void>;
+    /**
+     * Clears the command cache
+     */
+    clearCache(): void;
+}

package/dist/esm/commands/script-loader.js

@@ -0,0 +1,403 @@
+import { createHash } from 'crypto';
+import { glob, hasMagic } from 'glob';
+import * as path from 'path';
+import * as fs from 'fs';
+import { promisify } from 'util';
+const readFile = promisify(fs.readFile);
+const readdir = promisify(fs.readdir);
+const GlobOptions = { dot: true, silent: false };
+const IncludeRegex = /^[-]{2,3}[ \t]*@include[ \t]+(["'])(.+?)\1[; \t\n]*$/m;
+const EmptyLineRegex = /^\s*[\r\n]/gm;
+export class ScriptLoaderError extends Error {
+    constructor(message, path, stack = [], line, position = 0) {
+        super(message);
+        // Ensure the name of this error is the same as the class name
+        this.name = this.constructor.name;
+        Error.captureStackTrace(this, this.constructor);
+        this.includes = stack;
+        this.line = line !== null && line !== void 0 ? line : 0;
+        this.position = position;
+    }
+}
+const isPossiblyMappedPath = (path) => path && ['~', '<'].includes(path[0]);
+const hasFilenamePattern = (path) => hasMagic(path, GlobOptions);
+/**
+ * Lua script loader with include support
+ */
+export class ScriptLoader {
+    constructor() {
+        /**
+         * Map an alias to a path
+         */
+        this.pathMapper = new Map();
+        this.clientScripts = new WeakMap();
+        /**
+         * Cache commands by dir
+         */
+        this.commandCache = new Map();
+        this.rootPath = getPkgJsonDir();
+        this.pathMapper.set('~', this.rootPath);
+        this.pathMapper.set('rootDir', this.rootPath);
+        this.pathMapper.set('base', __dirname);
+    }
+    /**
+     * Add a script path mapping. Allows includes of the form "<includes>/utils.lua" where `includes` is a user
+     * defined path
+     * @param name - the name of the mapping. Note: do not include angle brackets
+     * @param mappedPath - if a relative path is passed, it's relative to the *caller* of this function.
+     * Mapped paths are also accepted, e.g. "~/server/scripts/lua" or "<base>/includes"
+     */
+    addPathMapping(name, mappedPath) {
+        let resolved;
+        if (isPossiblyMappedPath(mappedPath)) {
+            resolved = this.resolvePath(mappedPath);
+        }
+        else {
+            const caller = getCallerFile();
+            const callerPath = path.dirname(caller);
+            resolved = path.normalize(path.resolve(callerPath, mappedPath));
+        }
+        const last = resolved.length - 1;
+        if (resolved[last] === path.sep) {
+            resolved = resolved.substr(0, last);
+        }
+        this.pathMapper.set(name, resolved);
+    }
+    /**
+     * Resolve the script path considering path mappings
+     * @param scriptName - the name of the script
+     * @param stack - the include stack, for nicer errors
+     */
+    resolvePath(scriptName, stack = []) {
+        const first = scriptName[0];
+        if (first === '~') {
+            scriptName = path.join(this.rootPath, scriptName.substr(2));
+        }
+        else if (first === '<') {
+            const p = scriptName.indexOf('>');
+            if (p > 0) {
+                const name = scriptName.substring(1, p);
+                const mappedPath = this.pathMapper.get(name);
+                if (!mappedPath) {
+                    throw new ScriptLoaderError(`No path mapping found for "${name}"`, scriptName, stack);
+                }
+                scriptName = path.join(mappedPath, scriptName.substring(p + 1));
+            }
+        }
+        return path.normalize(scriptName);
+    }
+    /**
+     * Recursively collect all scripts included in a file
+     * @param file - the parent file
+     * @param cache - a cache for file metadata to increase efficiency. Since a file can be included
+     * multiple times, we make sure to load it only once.
+     * @param stack - internal stack to prevent circular references
+     */
+    async resolveDependencies(file, cache, isInclude = false, stack = []) {
+        cache = cache !== null && cache !== void 0 ? cache : new Map();
+        if (stack.includes(file.path)) {
+            throw new ScriptLoaderError(`circular reference: "${file.path}"`, file.path, stack);
+        }
+        stack.push(file.path);
+        function findPos(content, match) {
+            const pos = content.indexOf(match);
+            const arr = content.slice(0, pos).split('\n');
+            return {
+                line: arr.length,
+                column: arr[arr.length - 1].length + match.indexOf('@include') + 1,
+            };
+        }
+        function raiseError(msg, match) {
+            const pos = findPos(file.content, match);
+            throw new ScriptLoaderError(msg, file.path, stack, pos.line, pos.column);
+        }
+        let res;
+        let content = file.content;
+        while ((res = IncludeRegex.exec(content)) !== null) {
+            const [match, , reference] = res;
+            const includeFilename = isPossiblyMappedPath(reference)
+                ? // mapped paths imply absolute reference
+                    this.resolvePath(ensureExt(reference), stack)
+                : // include path is relative to the file being processed
+                    path.resolve(path.dirname(file.path), ensureExt(reference));
+            let includePaths;
+            if (hasFilenamePattern(includeFilename)) {
+                const filesMatched = await getFilenamesByPattern(includeFilename);
+                includePaths = filesMatched.map((x) => path.resolve(x));
+            }
+            else {
+                includePaths = [includeFilename];
+            }
+            includePaths = includePaths.filter((file) => path.extname(file) === '.lua');
+            if (includePaths.length === 0) {
+                raiseError(`include not found: "${reference}"`, match);
+            }
+            const tokens = [];
+            for (let i = 0; i < includePaths.length; i++) {
+                const includePath = includePaths[i];
+                const hasInclude = file.includes.find((x) => x.path === includePath);
+                if (hasInclude) {
+                    /**
+                     * We have something like
+                     * --- \@include "a"
+                     * ...
+                     * --- \@include "a"
+                     */
+                    raiseError(`file "${reference}" already included in "${file.path}"`, match);
+                }
+                let includeMetadata = cache.get(includePath);
+                let token;
+                if (!includeMetadata) {
+                    const { name, numberOfKeys } = splitFilename(includePath);
+                    let childContent = '';
+                    try {
+                        const buf = await readFile(includePath, { flag: 'r' });
+                        childContent = buf.toString();
+                    }
+                    catch (err) {
+                        if (err.code === 'ENOENT') {
+                            raiseError(`include not found: "${reference}"`, match);
+                        }
+                        else {
+                            throw err;
+                        }
+                    }
+                    // this represents a normalized version of the path to make replacement easy
+                    token = getPathHash(includePath);
+                    includeMetadata = {
+                        name,
+                        numberOfKeys,
+                        path: includePath,
+                        content: childContent,
+                        token,
+                        includes: [],
+                    };
+                    cache.set(includePath, includeMetadata);
+                }
+                else {
+                    token = includeMetadata.token;
+                }
+                tokens.push(token);
+                file.includes.push(includeMetadata);
+                await this.resolveDependencies(includeMetadata, cache, true, stack);
+            }
+            // Replace @includes with normalized path hashes
+            const substitution = tokens.join('\n');
+            content = content.replace(match, substitution);
+        }
+        file.content = content;
+        if (isInclude) {
+            cache.set(file.path, file);
+        }
+        else {
+            cache.set(file.name, file);
+        }
+        stack.pop();
+    }
+    /**
+     * Parse a (top-level) lua script
+     * @param filename - the full path to the script
+     * @param content - the content of the script
+     * @param cache - cache
+     */
+    async parseScript(filename, content, cache) {
+        const { name, numberOfKeys } = splitFilename(filename);
+        const meta = cache === null || cache === void 0 ? void 0 : cache.get(name);
+        if ((meta === null || meta === void 0 ? void 0 : meta.content) === content) {
+            return meta;
+        }
+        const fileInfo = {
+            path: filename,
+            token: getPathHash(filename),
+            content,
+            name,
+            numberOfKeys,
+            includes: [],
+        };
+        await this.resolveDependencies(fileInfo, cache);
+        return fileInfo;
+    }
+    /**
+     * Construct the final version of a file by interpolating its includes in dependency order.
+     * @param file - the file whose content we want to construct
+     * @param processed - a cache to keep track of which includes have already been processed
+     */
+    interpolate(file, processed) {
+        processed = processed || new Set();
+        let content = file.content;
+        file.includes.forEach((child) => {
+            const emitted = processed.has(child.path);
+            const fragment = this.interpolate(child, processed);
+            const replacement = emitted ? '' : fragment;
+            if (!replacement) {
+                content = replaceAll(content, child.token, '');
+            }
+            else {
+                // replace the first instance with the dependency
+                content = content.replace(child.token, replacement);
+                // remove the rest
+                content = replaceAll(content, child.token, '');
+            }
+            processed.add(child.path);
+        });
+        return content;
+    }
+    async loadCommand(filename, cache) {
+        filename = path.resolve(filename);
+        const { name: scriptName } = splitFilename(filename);
+        let script = cache === null || cache === void 0 ? void 0 : cache.get(scriptName);
+        if (!script) {
+            const content = (await readFile(filename)).toString();
+            script = await this.parseScript(filename, content, cache);
+        }
+        const lua = removeEmptyLines(this.interpolate(script));
+        const { name, numberOfKeys } = script;
+        return {
+            name,
+            options: { numberOfKeys: numberOfKeys, lua },
+        };
+    }
+    /**
+     * Load redis lua scripts.
+     * The name of the script must have the following format:
+     *
+     * cmdName-numKeys.lua
+     *
+     * cmdName must be in camel case format.
+     *
+     * For example:
+     * moveToFinish-3.lua
+     *
+     */
+    async loadScripts(dir, cache) {
+        dir = path.normalize(dir || __dirname);
+        let commands = this.commandCache.get(dir);
+        if (commands) {
+            return commands;
+        }
+        const files = await readdir(dir);
+        const luaFiles = files.filter((file) => path.extname(file) === '.lua');
+        if (luaFiles.length === 0) {
+            /**
+             * To prevent unclarified runtime error "updateDelayset is not a function
+             * @see https://github.com/OptimalBits/bull/issues/920
+             */
+            throw new ScriptLoaderError('No .lua files found!', dir, []);
+        }
+        commands = [];
+        cache = cache !== null && cache !== void 0 ? cache : new Map();
+        for (let i = 0; i < luaFiles.length; i++) {
+            const file = path.join(dir, luaFiles[i]);
+            const command = await this.loadCommand(file, cache);
+            commands.push(command);
+        }
+        this.commandCache.set(dir, commands);
+        return commands;
+    }
+    /**
+     * Attach all lua scripts in a given directory to a client instance
+     * @param client - redis client to attach script to
+     * @param pathname - the path to the directory containing the scripts
+     */
+    async load(client, pathname, cache) {
+        let paths = this.clientScripts.get(client);
+        if (!paths) {
+            paths = new Set();
+            this.clientScripts.set(client, paths);
+        }
+        if (!paths.has(pathname)) {
+            paths.add(pathname);
+            const scripts = await this.loadScripts(pathname, cache !== null && cache !== void 0 ? cache : new Map());
+            scripts.forEach((command) => {
+                // Only define the command if not already defined
+                if (!client[command.name]) {
+                    client.defineCommand(command.name, command.options);
+                }
+            });
+        }
+    }
+    /**
+     * Clears the command cache
+     */
+    clearCache() {
+        this.commandCache.clear();
+    }
+}
+function ensureExt(filename, ext = 'lua') {
+    const foundExt = path.extname(filename);
+    if (foundExt && foundExt !== '.') {
+        return filename;
+    }
+    if (ext && ext[0] !== '.') {
+        ext = `.${ext}`;
+    }
+    return `${filename}${ext}`;
+}
+function splitFilename(filePath) {
+    const longName = path.basename(filePath, '.lua');
+    const [name, num] = longName.split('-');
+    const numberOfKeys = num ? parseInt(num, 10) : undefined;
+    return { name, numberOfKeys };
+}
+async function getFilenamesByPattern(pattern) {
+    return new Promise((resolve, reject) => {
+        glob(pattern, GlobOptions, (err, files) => {
+            return err ? reject(err) : resolve(files);
+        });
+    });
+}
+// Determine the project root
+// https://stackoverflow.com/a/18721515
+function getPkgJsonDir() {
+    for (const modPath of module.paths || []) {
+        try {
+            const prospectivePkgJsonDir = path.dirname(modPath);
+            fs.accessSync(modPath, fs.constants.F_OK);
+            return prospectivePkgJsonDir;
+            // eslint-disable-next-line no-empty
+        }
+        catch (e) { }
+    }
+    return '';
+}
+// https://stackoverflow.com/a/66842927
+// some dark magic here :-)
+// this version is preferred to the simpler version because of
+// https://github.com/facebook/jest/issues/5303 -
+// tldr: dont assume you're the only one with the doing something like this
+function getCallerFile() {
+    var _a, _b, _c;
+    const originalFunc = Error.prepareStackTrace;
+    let callerFile = '';
+    try {
+        Error.prepareStackTrace = (_, stack) => stack;
+        const sites = new Error().stack;
+        const currentFile = (_a = sites.shift()) === null || _a === void 0 ? void 0 : _a.getFileName();
+        while (sites.length) {
+            callerFile = (_c = (_b = sites.shift()) === null || _b === void 0 ? void 0 : _b.getFileName()) !== null && _c !== void 0 ? _c : '';
+            if (currentFile !== callerFile) {
+                break;
+            }
+        }
+        // eslint-disable-next-line no-empty
+    }
+    catch (e) {
+    }
+    finally {
+        Error.prepareStackTrace = originalFunc;
+    }
+    return callerFile;
+}
+function sha1(data) {
+    return createHash('sha1').update(data).digest('hex');
+}
+function getPathHash(normalizedPath) {
+    return `@@${sha1(normalizedPath)}`;
+}
+function replaceAll(str, find, replace) {
+    return str.replace(new RegExp(find, 'g'), replace);
+}
+function removeEmptyLines(str) {
+    return str.replace(EmptyLineRegex, '');
+}
+//# sourceMappingURL=script-loader.js.map