@keetanetwork/anchor 0.0.33 → 0.0.35

package/lib/queue/drivers/queue_redis.js

@@ -0,0 +1,359 @@
+import { MethodLogger, ManageStatusUpdates, ConvertStringToRequestID } from '../internal.js';
+import { Errors } from '../common.js';
+export default class KeetaAnchorQueueStorageDriverRedis {
+    logger;
+    redisInternal = null;
+    name = 'KeetaAnchorQueueStorageDriverRedis';
+    id;
+    path = [];
+    pathStr;
+    constructor(options) {
+        this.id = options?.id ?? crypto.randomUUID();
+        this.logger = options?.logger;
+        this.redisInternal = options.redis;
+        this.path = options.path ?? [];
+        this.pathStr = ['root', ...this.path].join('.');
+        Object.freeze(this.path);
+        this.methodLogger('new')?.debug('Initialized Redis queue storage driver');
+    }
+    methodLogger(method) {
+        return (MethodLogger(this.logger, {
+            class: 'KeetaAnchorQueueStorageDriverRedis',
+            file: 'src/lib/queue/drivers/queue_redis.ts',
+            method: method,
+            instanceID: this.id
+        }));
+    }
+    async getRedis() {
+        if (this.redisInternal === null) {
+            throw (new Error('Redis connection is not available'));
+        }
+        return (await this.redisInternal());
+    }
+    queueKey(id) {
+        return (`queue:${this.pathStr}:entry:${String(id)}`);
+    }
+    idempotentKey(idempotentID) {
+        return (`queue:${this.pathStr}:idempotent:${String(idempotentID)}`);
+    }
+    indexKey(status) {
+        if (status) {
+            return (`queue:${this.pathStr}:index:${status}`);
+        }
+        return (`queue:${this.pathStr}:index:all`);
+    }
+    async add(request, info) {
+        const redis = await this.getRedis();
+        const logger = this.methodLogger('add');
+        let entryID = ConvertStringToRequestID(info?.id);
+        if (entryID) {
+            const exists = await redis.exists(this.queueKey(entryID));
+            if (exists) {
+                logger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);
+                return (entryID);
+            }
+        }
+        const idempotentIDs = info?.idempotentKeys;
+        if (idempotentIDs) {
+            const matchingIdempotentEntries = new Set();
+            for (const idempotentID of idempotentIDs) {
+                const existingEntryID = await redis.get(this.idempotentKey(idempotentID));
+                if (existingEntryID) {
+                    matchingIdempotentEntries.add(idempotentID);
+                }
+            }
+            if (matchingIdempotentEntries.size !== 0) {
+                throw (new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', matchingIdempotentEntries));
+            }
+        }
+        entryID ??= ConvertStringToRequestID(crypto.randomUUID());
+        logger?.debug(`Enqueuing request with id ${String(entryID)}`);
+        const currentTime = Date.now();
+        const requestJSON = JSON.stringify(request);
+        /**
+         * The status to use for the new entry
+         */
+        const status = info?.status ?? 'pending';
+        const entryData = {
+            id: String(entryID),
+            request: requestJSON,
+            output: null,
+            lastError: null,
+            status: status,
+            created: currentTime,
+            updated: currentTime,
+            worker: null,
+            failures: 0
+        };
+        if (idempotentIDs && idempotentIDs.size > 0) {
+            entryData.idempotentKeys = Array.from(idempotentIDs).map(String);
+        }
+        if (idempotentIDs && idempotentIDs.size > 0) {
+            const idempotentKeysArr = Array.from(idempotentIDs).map(String);
+            const luaScript = `
+				local entryKey = KEYS[1]
+				local statusIndexKey = KEYS[2]
+				local allIndexKey = KEYS[3]
+				
+				local entryData = ARGV[1]
+				local score = ARGV[2]
+				local entryId = ARGV[3]
+				local numIdempotentKeys = tonumber(ARGV[4])
+				
+				-- Check if any idempotent keys already exist
+				for i = 1, numIdempotentKeys do
+					local idempotentKey = ARGV[4 + i]
+					if redis.call('EXISTS', idempotentKey) == 1 then
+						return redis.error_reply('IDEMPOTENT_EXISTS')
+					end
+				end
+				
+				-- Add the entry
+				redis.call('SET', entryKey, entryData)
+				redis.call('ZADD', statusIndexKey, score, entryId)
+				redis.call('ZADD', allIndexKey, score, entryId)
+				
+				-- Set idempotent keys
+				for i = 1, numIdempotentKeys do
+					local idempotentKey = ARGV[4 + i]
+					redis.call('SET', idempotentKey, entryId)
+				end
+				
+				return 'OK'
+			`;
+            const idempotentKeyPairs = idempotentKeysArr.map((idKey) => {
+                return (this.idempotentKey(ConvertStringToRequestID(idKey)));
+            });
+            try {
+                await redis.eval(luaScript, {
+                    keys: [
+                        this.queueKey(entryID),
+                        this.indexKey(status),
+                        this.indexKey()
+                    ],
+                    arguments: [
+                        JSON.stringify(entryData),
+                        String(currentTime),
+                        String(entryID),
+                        String(idempotentKeysArr.length),
+                        ...idempotentKeyPairs
+                    ]
+                });
+            }
+            catch (error) {
+                if (error instanceof Error && error.message.includes('IDEMPOTENT_EXISTS')) {
+                    throw (new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', idempotentIDs));
+                }
+                throw (error);
+            }
+        }
+        else {
+            const multi = redis.multi();
+            multi.set(this.queueKey(entryID), JSON.stringify(entryData));
+            multi.zAdd(this.indexKey(status), { score: currentTime, value: String(entryID) });
+            multi.zAdd(this.indexKey(), { score: currentTime, value: String(entryID) });
+            await multi.exec();
+        }
+        return (entryID);
+    }
+    async setStatus(id, status, ancillary) {
+        const { oldStatus } = ancillary ?? {};
+        const redis = await this.getRedis();
+        const logger = this.methodLogger('setStatus');
+        const entryJSON = await redis.get(this.queueKey(id));
+        if (!entryJSON) {
+            throw (new Error(`Request with ID ${String(id)} not found`));
+        }
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+        const existingEntry = JSON.parse(entryJSON);
+        if (oldStatus && existingEntry.status !== oldStatus) {
+            throw (new Errors.IncorrectStateAssertedError(id, oldStatus, existingEntry.status));
+        }
+        const newEntryRaw = ManageStatusUpdates(id, existingEntry, status, ancillary, logger);
+        const newEntry = {
+            status: newEntryRaw.status,
+            worker: newEntryRaw.worker,
+            failures: newEntryRaw.failures ?? existingEntry.failures,
+            updated: newEntryRaw.updated.getTime(),
+            lastError: newEntryRaw.lastError !== undefined ? newEntryRaw.lastError : existingEntry.lastError,
+            output: newEntryRaw.output !== undefined ? newEntryRaw.output ? JSON.stringify(newEntryRaw.output) : null : existingEntry.output
+        };
+        const updatedEntry = {
+            ...existingEntry,
+            ...newEntry
+        };
+        const currentTime = updatedEntry.updated;
+        if (oldStatus) {
+            const luaScript = `
+				local key = KEYS[1]
+				local expectedStatus = ARGV[1]
+				local newData = ARGV[2]
+				local oldIndexKey = ARGV[3]
+				local newIndexKey = ARGV[4]
+				local allIndexKey = ARGV[5]
+				local entryId = ARGV[6]
+				local score = ARGV[7]
+				
+				local current = redis.call('GET', key)
+				if not current then
+					return {err = 'NOT_FOUND'}
+				end
+				
+				local currentData = cjson.decode(current)
+				if currentData.status ~= expectedStatus then
+					return {err = 'STATUS_MISMATCH'}
+				end
+				
+				redis.call('SET', key, newData)
+				if oldIndexKey ~= newIndexKey then
+					redis.call('ZREM', oldIndexKey, entryId)
+				end
+				redis.call('ZADD', newIndexKey, score, entryId)
+				redis.call('ZADD', allIndexKey, score, entryId)
+				
+				return {ok = 'OK'}
+			`;
+            const result = await redis.eval(luaScript, {
+                keys: [this.queueKey(id)],
+                arguments: [
+                    oldStatus,
+                    JSON.stringify(updatedEntry),
+                    this.indexKey(oldStatus),
+                    this.indexKey(status),
+                    this.indexKey(),
+                    String(id),
+                    String(currentTime)
+                ]
+            });
+            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+            const resultObj = result;
+            if (resultObj.err === 'NOT_FOUND') {
+                throw (new Error(`Request with ID ${String(id)} not found`));
+            }
+            if (resultObj.err === 'STATUS_MISMATCH') {
+                throw (new Errors.IncorrectStateAssertedError(id, oldStatus, existingEntry.status));
+            }
+        }
+        else {
+            const multi = redis.multi();
+            multi.set(this.queueKey(id), JSON.stringify(updatedEntry));
+            if (existingEntry.status !== status) {
+                multi.zRem(this.indexKey(existingEntry.status), String(id));
+            }
+            multi.zAdd(this.indexKey(status), { score: currentTime, value: String(id) });
+            multi.zAdd(this.indexKey(), { score: currentTime, value: String(id) });
+            await multi.exec();
+        }
+    }
+    async get(id) {
+        const redis = await this.getRedis();
+        const entryJSON = await redis.get(this.queueKey(id));
+        if (!entryJSON) {
+            return (null);
+        }
+        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+        const entryData = JSON.parse(entryJSON);
+        const idempotentKeys = entryData.idempotentKeys && entryData.idempotentKeys.length > 0
+            ? new Set(entryData.idempotentKeys.map(function (key) {
+                return (ConvertStringToRequestID(key));
+            }))
+            : undefined;
+        return ({
+            id: ConvertStringToRequestID(entryData.id),
+            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+            request: JSON.parse(entryData.request),
+            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+            output: entryData.output ? JSON.parse(entryData.output) : null,
+            lastError: entryData.lastError,
+            status: entryData.status,
+            created: new Date(entryData.created),
+            updated: new Date(entryData.updated),
+            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
+            worker: entryData.worker,
+            failures: entryData.failures,
+            idempotentKeys: idempotentKeys
+        });
+    }
+    async query(filter) {
+        const redis = await this.getRedis();
+        const logger = this.methodLogger('query');
+        logger?.debug(`Querying queue with id ${this.id} with filter:`, filter);
+        let entryIDs;
+        if (filter?.status) {
+            const count = filter.limit ?? -1;
+            const allIDs = await redis.zRange(this.indexKey(filter.status), 0, -1);
+            if (filter.updatedBefore) {
+                const maxScore = filter.updatedBefore.getTime();
+                const filteredIDs = [];
+                for (const entryID of allIDs) {
+                    const score = await redis.zScore(this.indexKey(filter.status), entryID);
+                    if (score !== null && score < maxScore) {
+                        filteredIDs.push(entryID);
+                    }
+                }
+                entryIDs = filteredIDs;
+            }
+            else {
+                entryIDs = allIDs;
+            }
+            if (count !== -1) {
+                entryIDs = entryIDs.slice(0, count);
+            }
+        }
+        else {
+            const count = filter?.limit ?? -1;
+            const allIDs = await redis.zRange(this.indexKey(), 0, -1);
+            if (filter?.updatedBefore) {
+                const maxScore = filter.updatedBefore.getTime();
+                const filteredIDs = [];
+                for (const entryID of allIDs) {
+                    const score = await redis.zScore(this.indexKey(), entryID);
+                    if (score !== null && score < maxScore) {
+                        filteredIDs.push(entryID);
+                    }
+                }
+                entryIDs = filteredIDs;
+            }
+            else {
+                entryIDs = allIDs;
+            }
+            if (count !== -1) {
+                entryIDs = entryIDs.slice(0, count);
+            }
+        }
+        const entries = [];
+        for (const entryIDStr of entryIDs) {
+            const entryID = ConvertStringToRequestID(entryIDStr);
+            const entry = await this.get(entryID);
+            if (entry) {
+                if (filter?.status && entry.status !== filter.status) {
+                    continue;
+                }
+                entries.push(entry);
+            }
+        }
+        logger?.debug(`Queried queue with id ${this.id} with filter:`, filter, '-- found', entries.length, 'entries');
+        return (entries);
+    }
+    async partition(path) {
+        this.methodLogger('partition')?.debug(`Creating partitioned queue storage driver for path: ${path}`);
+        if (this.redisInternal === null) {
+            throw (new Error('Asked to partition but the instance has been destroyed'));
+        }
+        const retval = new KeetaAnchorQueueStorageDriverRedis({
+            id: `${this.id}::${path}`,
+            logger: this.logger,
+            redis: this.redisInternal,
+            path: [...this.path, path]
+        });
+        return (retval);
+    }
+    async destroy() {
+        this.methodLogger('destroy')?.debug('Destroying instance');
+        this.redisInternal = null;
+    }
+    async [Symbol.asyncDispose]() {
+        return (await this.destroy());
+    }
+}
+//# sourceMappingURL=queue_redis.js.map

package/lib/queue/drivers/queue_redis.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue_redis.js","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_redis.ts"],"names":[],"mappings":"AAYA,OAAO,EACN,YAAY,EACZ,mBAAmB,EACnB,wBAAwB,EACxB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAoBtC,MAAM,CAAC,OAAO,OAAO,kCAAkC;IACrC,MAAM,CAAqB;IACpC,aAAa,GAA4C,IAAI,CAAC;IAE7D,IAAI,GAAG,oCAAoC,CAAC;IAC5C,EAAE,CAAS;IACX,IAAI,GAAa,EAAE,CAAC;IACZ,OAAO,CAAS;IAEjC,YAAY,OAAgK;QAC3K,IAAI,CAAC,EAAE,GAAG,OAAO,EAAE,EAAE,IAAI,MAAM,CAAC,UAAU,EAAE,CAAC;QAC7C,IAAI,CAAC,MAAM,GAAG,OAAO,EAAE,MAAM,CAAA;QAC7B,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,KAAK,CAAC;QACnC,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAChD,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEzB,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,wCAAwC,CAAC,CAAC;IAC3E,CAAC;IAEO,YAAY,CAAC,MAAc;QAClC,OAAM,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,EAAE;YAChC,KAAK,EAAE,oCAAoC;YAC3C,IAAI,EAAE,sCAAsC;YAC5C,MAAM,EAAE,MAAM;YACd,UAAU,EAAE,IAAI,CAAC,EAAE;SACnB,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,QAAQ;QACrB,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YACjC,MAAK,CAAC,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAC,CAAC;QACvD,CAAC;QACD,OAAM,CAAC,MAAM,IAAI,CAAC,aAAa,EAAE,CAAC,CAAC;IACpC,CAAC;IAEO,QAAQ,CAAC,EAA6B;QAC7C,OAAM,CAAC,SAAS,IAAI,CAAC,OAAO,UAAU,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IAEO,aAAa,CAAC,YAAuC;QAC5D,OAAM,CAAC,SAAS,IAAI,CAAC,OAAO,eAAe,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IACpE,CAAC;IAEO,QAAQ,CAAC,MAA+B;QAC/C,IAAI,MAAM,EAAE,CAAC;YACZ,OAAM,CAAC,SAAS,IAAI,CAAC,OAAO,UAAU,MAAM,EAAE,CAAC,CAAC;QACjD,CAAC;QACD,OAAM,CAAC,SAAS,IAAI,CAAC,OAAO,YAAY,CAAC,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAA8C,EAAE,IAAiC;QAC1F,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC;QAExC,IAAI,OAAO,GAAG,wBAAwB,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACjD,IAAI,OAAO,EAAE,CAAC;YACb,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;YAC1D,IAAI,MAAM,EAAE,CAAC;gBACZ,MAAM,EAAE,KAAK,CAAC,mBAAmB,MAAM,CAAC,OAAO,CAAC,2BAA2B,CAAC,CAAC;gBAC7E,OAAM,CAAC,OAAO,CAAC,CAAC;YACjB,CAAC;QACF,CAAC;QAED,MAAM,aAAa,GAAG,IAAI,EAAE,cAAc,CAAC;QAC3C,IAAI,aAAa,EAAE,CAAC;YACnB,MAAM,yBAAyB,GAAG,IAAI,GAAG,EAA6B,CAAC;YACvE,KAAK,MAAM,YAAY,IAAI,aAAa,EAAE,CAAC;gBAC1C,MAAM,eAAe,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,YAAY,CAAC,CAAC,CAAC;gBAC1E,IAAI,eAAe,EAAE,CAAC;oBACrB,yBAAyB,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;gBAC7C,CAAC;YACF,CAAC;YAED,IAAI,yBAAyB,CAAC,IAAI,KAAK,CAAC,EAAE,CAAC;gBAC1C,MAAK,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,2DAA2D,EAAE,yBAAyB,CAAC,CAAC,CAAC;YACjI,CAAC;QACF,CAAC;QAED,OAAO,KAAK,wBAAwB,CAAC,MAAM,CAAC,UAAU,EAAE,CAAC,CAAC;QAE1D,MAAM,EAAE,KAAK,CAAC,6BAA6B,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAE9D,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC/B,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;QAE5C;;WAEG;QACH,MAAM,MAAM,GAAG,IAAI,EAAE,MAAM,IAAI,SAAS,CAAC;QAEzC,MAAM,SAAS,GAAmB;YACjC,EAAE,EAAE,MAAM,CAAC,OAAO,CAAC;YACnB,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,IAAI;YACZ,SAAS,EAAE,IAAI;YACf,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,WAAW;YACpB,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,IAAI;YACZ,QAAQ,EAAE,CAAC;SACX,CAAC;QAEF,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC7C,SAAS,CAAC,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAClE,CAAC;QAED,IAAI,aAAa,IAAI,aAAa,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC7C,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAChE,MAAM,SAAS,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA8BjB,CAAC;YAEF,MAAM,kBAAkB,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC1D,OAAM,CAAC,IAAI,CAAC,aAAa,CAAC,wBAAwB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;YAC7D,CAAC,CAAC,CAAC;YAEH,IAAI,CAAC;gBACJ,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE;oBAC3B,IAAI,EAAE;wBACL,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC;wBACtB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;wBACrB,IAAI,CAAC,QAAQ,EAAE;qBACf;oBACD,SAAS,EAAE;wBACV,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC;wBACzB,MAAM,CAAC,WAAW,CAAC;wBACnB,MAAM,CAAC,OAAO,CAAC;wBACf,MAAM,CAAC,iBAAiB,CAAC,MAAM,CAAC;wBAChC,GAAG,kBAAkB;qBACrB;iBACD,CAAC,CAAC;YACJ,CAAC;YAAC,OAAO,KAAc,EAAE,CAAC;gBACzB,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAC,EAAE,CAAC;oBAC3E,MAAK,CAAC,IAAI,MAAM,CAAC,qBAAqB,CAAC,2DAA2D,EAAE,aAAa,CAAC,CAAC,CAAC;gBACrH,CAAC;gBACD,MAAK,CAAC,KAAK,CAAC,CAAC;YACd,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAC5B,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;YAC7D,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAClF,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAE5E,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC;QAED,OAAM,CAAC,OAAO,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,EAA6B,EAAE,MAA8B,EAAE,SAA2D;QACzI,MAAM,EAAE,SAAS,EAAE,GAAG,SAAS,IAAI,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,CAAC;QAE9C,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;QAC7D,CAAC;QAED,yEAAyE;QACzE,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAmB,CAAC;QAE9D,IAAI,SAAS,IAAI,aAAa,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACrD,MAAK,CAAC,IAAI,MAAM,CAAC,2BAA2B,CAAC,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;QACpF,CAAC;QAED,MAAM,WAAW,GAAG,mBAAmB,CAAc,EAAE,EAAE,aAAa,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,CAAC,CAAC;QACnG,MAAM,QAAQ,GAA4B;YACzC,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ,IAAI,aAAa,CAAC,QAAQ;YACxD,OAAO,EAAE,WAAW,CAAC,OAAO,CAAC,OAAO,EAAE;YACtC,SAAS,EAAE,WAAW,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC,SAAS;YAChG,MAAM,EAAE,WAAW,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM;SAChI,CAAC;QAEF,MAAM,YAAY,GAAmB;YACpC,GAAG,aAAa;YAChB,GAAG,QAAQ;SACX,CAAC;QAEF,MAAM,WAAW,GAAG,YAAY,CAAC,OAAO,CAAC;QAEzC,IAAI,SAAS,EAAE,CAAC;YACf,MAAM,SAAS,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA4BjB,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE;gBAC1C,IAAI,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACzB,SAAS,EAAE;oBACV,SAAS;oBACT,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC;oBAC5B,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC;oBACxB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;oBACrB,IAAI,CAAC,QAAQ,EAAE;oBACf,MAAM,CAAC,EAAE,CAAC;oBACV,MAAM,CAAC,WAAW,CAAC;iBACnB;aACD,CAAC,CAAC;YAEH,yEAAyE;YACzE,MAAM,SAAS,GAAG,MAAuC,CAAC;YAC1D,IAAI,SAAS,CAAC,GAAG,KAAK,WAAW,EAAE,CAAC;gBACnC,MAAK,CAAC,IAAI,KAAK,CAAC,mBAAmB,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,IAAI,SAAS,CAAC,GAAG,KAAK,iBAAiB,EAAE,CAAC;gBACzC,MAAK,CAAC,IAAI,MAAM,CAAC,2BAA2B,CAAC,EAAE,EAAE,SAAS,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC;YACpF,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,EAAE,CAAC;YAC5B,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC,CAAC;YAE3D,IAAI,aAAa,CAAC,MAAM,KAAK,MAAM,EAAE,CAAC;gBACrC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAC7D,CAAC;YACD,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAC7E,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,KAAK,EAAE,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;YAEvE,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;QACpB,CAAC;IACF,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,EAA6B;QACtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QAEpC,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;QACrD,IAAI,CAAC,SAAS,EAAE,CAAC;YAChB,OAAM,CAAC,IAAI,CAAC,CAAC;QACd,CAAC;QAED,yEAAyE;QACzE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAmB,CAAC;QAE1D,MAAM,cAAc,GAAG,SAAS,CAAC,cAAc,IAAI,SAAS,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;YACrF,CAAC,CAAC,IAAI,GAAG,CAAC,SAAS,CAAC,cAAc,CAAC,GAAG,CAAC,UAAS,GAAW;gBAC1D,OAAM,CAAC,wBAAwB,CAAC,GAAG,CAAC,CAAC,CAAC;YACvC,CAAC,CAAC,CAAC;YACH,CAAC,CAAC,SAAS,CAAC;QAEb,OAAM,CAAC;YACN,EAAE,EAAE,wBAAwB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC1C,yEAAyE;YACzE,OAAO,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,OAAO,CAAiB;YACtD,yEAAyE;YACzE,MAAM,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAgB,CAAC,CAAC,CAAC,IAAI;YAC7E,SAAS,EAAE,SAAS,CAAC,SAAS;YAC9B,MAAM,EAAE,SAAS,CAAC,MAAM;YACxB,OAAO,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YACpC,OAAO,EAAE,IAAI,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;YACpC,yEAAyE;YACzE,MAAM,EAAE,SAAS,CAAC,MAAoD;YACtE,QAAQ,EAAE,SAAS,CAAC,QAAQ;YAC5B,cAAc,EAAE,cAAc;SAC9B,CAAC,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,MAA+B;QAC1C,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,QAAQ,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;QAE1C,MAAM,EAAE,KAAK,CAAC,0BAA0B,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,CAAC;QAExE,IAAI,QAAkB,CAAC;QAEvB,IAAI,MAAM,EAAE,MAAM,EAAE,CAAC;YACpB,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;YACjC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAEvE,IAAI,MAAM,CAAC,aAAa,EAAE,CAAC;gBAC1B,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;gBAChD,MAAM,WAAW,GAAa,EAAE,CAAC;gBACjC,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;oBAC9B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;oBACxE,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;wBACxC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC3B,CAAC;gBACF,CAAC;gBACD,QAAQ,GAAG,WAAW,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACP,QAAQ,GAAG,MAAM,CAAC;YACnB,CAAC;YAED,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;gBAClB,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACrC,CAAC;QACF,CAAC;aAAM,CAAC;YACP,MAAM,KAAK,GAAG,MAAM,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;YAClC,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;YAE1D,IAAI,MAAM,EAAE,aAAa,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,MAAM,CAAC,aAAa,CAAC,OAAO,EAAE,CAAC;gBAChD,MAAM,WAAW,GAAa,EAAE,CAAC;gBACjC,KAAK,MAAM,OAAO,IAAI,MAAM,EAAE,CAAC;oBAC9B,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,OAAO,CAAC,CAAC;oBAC3D,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,GAAG,QAAQ,EAAE,CAAC;wBACxC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;oBAC3B,CAAC;gBACF,CAAC;gBACD,QAAQ,GAAG,WAAW,CAAC;YACxB,CAAC;iBAAM,CAAC;gBACP,QAAQ,GAAG,MAAM,CAAC;YACnB,CAAC;YAED,IAAI,KAAK,KAAK,CAAC,CAAC,EAAE,CAAC;gBAClB,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACrC,CAAC;QACF,CAAC;QAED,MAAM,OAAO,GAAuD,EAAE,CAAC;QAEvE,KAAK,MAAM,UAAU,IAAI,QAAQ,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,wBAAwB,CAAC,UAAU,CAAC,CAAC;YACrD,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACtC,IAAI,KAAK,EAAE,CAAC;gBACX,IAAI,MAAM,EAAE,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;oBACtD,SAAS;gBACV,CAAC;gBACD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACF,CAAC;QAED,MAAM,EAAE,KAAK,CAAC,yBAAyB,IAAI,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAE9G,OAAM,CAAC,OAAO,CAAC,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,IAAY;QAC3B,IAAI,CAAC,YAAY,CAAC,WAAW,CAAC,EAAE,KAAK,CAAC,uDAAuD,IAAI,EAAE,CAAC,CAAC;QAErG,IAAI,IAAI,CAAC,aAAa,KAAK,IAAI,EAAE,CAAC;YACjC,MAAK,CAAC,IAAI,KAAK,CAAC,wDAAwD,CAAC,CAAC,CAAC;QAC5E,CAAC;QAED,MAAM,MAAM,GAAG,IAAI,kCAAkC,CAA4B;YAChF,EAAE,EAAE,GAAG,IAAI,CAAC,EAAE,KAAK,IAAI,EAAE;YACzB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,KAAK,EAAE,IAAI,CAAC,aAAa;YACzB,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC;SAC1B,CAAC,CAAC;QAEH,OAAM,CAAC,MAAM,CAAC,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,OAAO;QACZ,IAAI,CAAC,YAAY,CAAC,SAAS,CAAC,EAAE,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAE3D,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC;QAC1B,OAAM,CAAC,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;IAC9B,CAAC;CACD","sourcesContent":["import type {\n\tKeetaAnchorQueueStorageDriver,\n\tKeetaAnchorQueueStorageDriverConstructor,\n\tKeetaAnchorQueueRequest,\n\tKeetaAnchorQueueRequestID,\n\tKeetaAnchorQueueEntry,\n\tKeetaAnchorQueueEntryExtra,\n\tKeetaAnchorQueueEntryAncillaryData,\n\tKeetaAnchorQueueStatus,\n\tKeetaAnchorQueueFilter,\n\tKeetaAnchorQueueWorkerID\n} from '../index.ts';\nimport {\n\tMethodLogger,\n\tManageStatusUpdates,\n\tConvertStringToRequestID\n} from '../internal.js';\nimport { Errors } from '../common.js';\n\nimport type { Logger } from '../../log/index.ts';\nimport type { JSONSerializable } from '../../utils/json.js';\n\nimport type { RedisClientType } from 'redis';\n\ntype QueueEntryData = {\n\tid: string;\n\trequest: string;\n\toutput: string | null;\n\tlastError: string | null;\n\tstatus: KeetaAnchorQueueStatus;\n\tcreated: number;\n\tupdated: number;\n\tworker: number | null;\n\tfailures: number;\n\tidempotentKeys?: string[];\n};\n\nexport default class KeetaAnchorQueueStorageDriverRedis<QueueRequest extends JSONSerializable = JSONSerializable, QueueResult extends JSONSerializable = JSONSerializable> implements KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult> {\n\tprivate readonly logger: Logger | undefined;\n\tprivate redisInternal: (() => Promise<RedisClientType>) | null = null;\n\n\treadonly name = 'KeetaAnchorQueueStorageDriverRedis';\n\treadonly id: string;\n\treadonly path: string[] = [];\n\tprivate readonly pathStr: string;\n\n\tconstructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & { redis: () => Promise<RedisClientType>; }) {\n\t\tthis.id = options?.id ?? crypto.randomUUID();\n\t\tthis.logger = options?.logger\n\t\tthis.redisInternal = options.redis;\n\t\tthis.path = options.path ?? [];\n\t\tthis.pathStr = ['root', ...this.path].join('.');\n\t\tObject.freeze(this.path);\n\n\t\tthis.methodLogger('new')?.debug('Initialized Redis queue storage driver');\n\t}\n\n\tprivate methodLogger(method: string): Logger | undefined {\n\t\treturn(MethodLogger(this.logger, {\n\t\t\tclass: 'KeetaAnchorQueueStorageDriverRedis',\n\t\t\tfile: 'src/lib/queue/drivers/queue_redis.ts',\n\t\t\tmethod: method,\n\t\t\tinstanceID: this.id\n\t\t}));\n\t}\n\n\tprivate async getRedis(): Promise<RedisClientType> {\n\t\tif (this.redisInternal === null) {\n\t\t\tthrow(new Error('Redis connection is not available'));\n\t\t}\n\t\treturn(await this.redisInternal());\n\t}\n\n\tprivate queueKey(id: KeetaAnchorQueueRequestID): string {\n\t\treturn(`queue:${this.pathStr}:entry:${String(id)}`);\n\t}\n\n\tprivate idempotentKey(idempotentID: KeetaAnchorQueueRequestID): string {\n\t\treturn(`queue:${this.pathStr}:idempotent:${String(idempotentID)}`);\n\t}\n\n\tprivate indexKey(status?: KeetaAnchorQueueStatus): string {\n\t\tif (status) {\n\t\t\treturn(`queue:${this.pathStr}:index:${status}`);\n\t\t}\n\t\treturn(`queue:${this.pathStr}:index:all`);\n\t}\n\n\tasync add(request: KeetaAnchorQueueRequest<QueueRequest>, info?: KeetaAnchorQueueEntryExtra): Promise<KeetaAnchorQueueRequestID> {\n\t\tconst redis = await this.getRedis();\n\t\tconst logger = this.methodLogger('add');\n\n\t\tlet entryID = ConvertStringToRequestID(info?.id);\n\t\tif (entryID) {\n\t\t\tconst exists = await redis.exists(this.queueKey(entryID));\n\t\t\tif (exists) {\n\t\t\t\tlogger?.debug(`Request with id ${String(entryID)} already exists, ignoring`);\n\t\t\t\treturn(entryID);\n\t\t\t}\n\t\t}\n\n\t\tconst idempotentIDs = info?.idempotentKeys;\n\t\tif (idempotentIDs) {\n\t\t\tconst matchingIdempotentEntries = new Set<KeetaAnchorQueueRequestID>();\n\t\t\tfor (const idempotentID of idempotentIDs) {\n\t\t\t\tconst existingEntryID = await redis.get(this.idempotentKey(idempotentID));\n\t\t\t\tif (existingEntryID) {\n\t\t\t\t\tmatchingIdempotentEntries.add(idempotentID);\n\t\t\t\t}\n\t\t\t}\n\n\t\t\tif (matchingIdempotentEntries.size !== 0) {\n\t\t\t\tthrow(new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', matchingIdempotentEntries));\n\t\t\t}\n\t\t}\n\n\t\tentryID ??= ConvertStringToRequestID(crypto.randomUUID());\n\n\t\tlogger?.debug(`Enqueuing request with id ${String(entryID)}`);\n\n\t\tconst currentTime = Date.now();\n\t\tconst requestJSON = JSON.stringify(request);\n\n\t\t/**\n\t\t * The status to use for the new entry\n\t\t */\n\t\tconst status = info?.status ?? 'pending';\n\n\t\tconst entryData: QueueEntryData = {\n\t\t\tid: String(entryID),\n\t\t\trequest: requestJSON,\n\t\t\toutput: null,\n\t\t\tlastError: null,\n\t\t\tstatus: status,\n\t\t\tcreated: currentTime,\n\t\t\tupdated: currentTime,\n\t\t\tworker: null,\n\t\t\tfailures: 0\n\t\t};\n\n\t\tif (idempotentIDs && idempotentIDs.size > 0) {\n\t\t\tentryData.idempotentKeys = Array.from(idempotentIDs).map(String);\n\t\t}\n\n\t\tif (idempotentIDs && idempotentIDs.size > 0) {\n\t\t\tconst idempotentKeysArr = Array.from(idempotentIDs).map(String);\n\t\t\tconst luaScript = `\n\t\t\t\tlocal entryKey = KEYS[1]\n\t\t\t\tlocal statusIndexKey = KEYS[2]\n\t\t\t\tlocal allIndexKey = KEYS[3]\n\t\t\t\t\n\t\t\t\tlocal entryData = ARGV[1]\n\t\t\t\tlocal score = ARGV[2]\n\t\t\t\tlocal entryId = ARGV[3]\n\t\t\t\tlocal numIdempotentKeys = tonumber(ARGV[4])\n\t\t\t\t\n\t\t\t\t-- Check if any idempotent keys already exist\n\t\t\t\tfor i = 1, numIdempotentKeys do\n\t\t\t\t\tlocal idempotentKey = ARGV[4 + i]\n\t\t\t\t\tif redis.call('EXISTS', idempotentKey) == 1 then\n\t\t\t\t\t\treturn redis.error_reply('IDEMPOTENT_EXISTS')\n\t\t\t\t\tend\n\t\t\t\tend\n\t\t\t\t\n\t\t\t\t-- Add the entry\n\t\t\t\tredis.call('SET', entryKey, entryData)\n\t\t\t\tredis.call('ZADD', statusIndexKey, score, entryId)\n\t\t\t\tredis.call('ZADD', allIndexKey, score, entryId)\n\t\t\t\t\n\t\t\t\t-- Set idempotent keys\n\t\t\t\tfor i = 1, numIdempotentKeys do\n\t\t\t\t\tlocal idempotentKey = ARGV[4 + i]\n\t\t\t\t\tredis.call('SET', idempotentKey, entryId)\n\t\t\t\tend\n\t\t\t\t\n\t\t\t\treturn 'OK'\n\t\t\t`;\n\n\t\t\tconst idempotentKeyPairs = idempotentKeysArr.map((idKey) => {\n\t\t\t\treturn(this.idempotentKey(ConvertStringToRequestID(idKey)));\n\t\t\t});\n\n\t\t\ttry {\n\t\t\t\tawait redis.eval(luaScript, {\n\t\t\t\t\tkeys: [\n\t\t\t\t\t\tthis.queueKey(entryID),\n\t\t\t\t\t\tthis.indexKey(status),\n\t\t\t\t\t\tthis.indexKey()\n\t\t\t\t\t],\n\t\t\t\t\targuments: [\n\t\t\t\t\t\tJSON.stringify(entryData),\n\t\t\t\t\t\tString(currentTime),\n\t\t\t\t\t\tString(entryID),\n\t\t\t\t\t\tString(idempotentKeysArr.length),\n\t\t\t\t\t\t...idempotentKeyPairs\n\t\t\t\t\t]\n\t\t\t\t});\n\t\t\t} catch (error: unknown) {\n\t\t\t\tif (error instanceof Error && error.message.includes('IDEMPOTENT_EXISTS')) {\n\t\t\t\t\tthrow(new Errors.IdempotentExistsError('One or more idempotent entries already exist in the queue', idempotentIDs));\n\t\t\t\t}\n\t\t\t\tthrow(error);\n\t\t\t}\n\t\t} else {\n\t\t\tconst multi = redis.multi();\n\t\t\tmulti.set(this.queueKey(entryID), JSON.stringify(entryData));\n\t\t\tmulti.zAdd(this.indexKey(status), { score: currentTime, value: String(entryID) });\n\t\t\tmulti.zAdd(this.indexKey(), { score: currentTime, value: String(entryID) });\n\n\t\t\tawait multi.exec();\n\t\t}\n\n\t\treturn(entryID);\n\t}\n\n\tasync setStatus(id: KeetaAnchorQueueRequestID, status: KeetaAnchorQueueStatus, ancillary?: KeetaAnchorQueueEntryAncillaryData<QueueResult>): Promise<void> {\n\t\tconst { oldStatus } = ancillary ?? {};\n\t\tconst redis = await this.getRedis();\n\t\tconst logger = this.methodLogger('setStatus');\n\n\t\tconst entryJSON = await redis.get(this.queueKey(id));\n\t\tif (!entryJSON) {\n\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t}\n\n\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\tconst existingEntry = JSON.parse(entryJSON) as QueueEntryData;\n\n\t\tif (oldStatus && existingEntry.status !== oldStatus) {\n\t\t\tthrow(new Errors.IncorrectStateAssertedError(id, oldStatus, existingEntry.status));\n\t\t}\n\n\t\tconst newEntryRaw = ManageStatusUpdates<QueueResult>(id, existingEntry, status, ancillary, logger);\n\t\tconst newEntry: Partial<QueueEntryData> = {\n\t\t\tstatus: newEntryRaw.status,\n\t\t\tworker: newEntryRaw.worker,\n\t\t\tfailures: newEntryRaw.failures ?? existingEntry.failures,\n\t\t\tupdated: newEntryRaw.updated.getTime(),\n\t\t\tlastError: newEntryRaw.lastError !== undefined ? newEntryRaw.lastError : existingEntry.lastError,\n\t\t\toutput: newEntryRaw.output !== undefined ? newEntryRaw.output ? JSON.stringify(newEntryRaw.output) : null : existingEntry.output\n\t\t};\n\n\t\tconst updatedEntry: QueueEntryData = {\n\t\t\t...existingEntry,\n\t\t\t...newEntry\n\t\t};\n\n\t\tconst currentTime = updatedEntry.updated;\n\n\t\tif (oldStatus) {\n\t\t\tconst luaScript = `\n\t\t\t\tlocal key = KEYS[1]\n\t\t\t\tlocal expectedStatus = ARGV[1]\n\t\t\t\tlocal newData = ARGV[2]\n\t\t\t\tlocal oldIndexKey = ARGV[3]\n\t\t\t\tlocal newIndexKey = ARGV[4]\n\t\t\t\tlocal allIndexKey = ARGV[5]\n\t\t\t\tlocal entryId = ARGV[6]\n\t\t\t\tlocal score = ARGV[7]\n\t\t\t\t\n\t\t\t\tlocal current = redis.call('GET', key)\n\t\t\t\tif not current then\n\t\t\t\t\treturn {err = 'NOT_FOUND'}\n\t\t\t\tend\n\t\t\t\t\n\t\t\t\tlocal currentData = cjson.decode(current)\n\t\t\t\tif currentData.status ~= expectedStatus then\n\t\t\t\t\treturn {err = 'STATUS_MISMATCH'}\n\t\t\t\tend\n\t\t\t\t\n\t\t\t\tredis.call('SET', key, newData)\n\t\t\t\tif oldIndexKey ~= newIndexKey then\n\t\t\t\t\tredis.call('ZREM', oldIndexKey, entryId)\n\t\t\t\tend\n\t\t\t\tredis.call('ZADD', newIndexKey, score, entryId)\n\t\t\t\tredis.call('ZADD', allIndexKey, score, entryId)\n\t\t\t\t\n\t\t\t\treturn {ok = 'OK'}\n\t\t\t`;\n\n\t\t\tconst result = await redis.eval(luaScript, {\n\t\t\t\tkeys: [this.queueKey(id)],\n\t\t\t\targuments: [\n\t\t\t\t\toldStatus,\n\t\t\t\t\tJSON.stringify(updatedEntry),\n\t\t\t\t\tthis.indexKey(oldStatus),\n\t\t\t\t\tthis.indexKey(status),\n\t\t\t\t\tthis.indexKey(),\n\t\t\t\t\tString(id),\n\t\t\t\t\tString(currentTime)\n\t\t\t\t]\n\t\t\t});\n\n\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\tconst resultObj = result as { err?: string; ok?: string };\n\t\t\tif (resultObj.err === 'NOT_FOUND') {\n\t\t\t\tthrow(new Error(`Request with ID ${String(id)} not found`));\n\t\t\t}\n\t\t\tif (resultObj.err === 'STATUS_MISMATCH') {\n\t\t\t\tthrow(new Errors.IncorrectStateAssertedError(id, oldStatus, existingEntry.status));\n\t\t\t}\n\t\t} else {\n\t\t\tconst multi = redis.multi();\n\t\t\tmulti.set(this.queueKey(id), JSON.stringify(updatedEntry));\n\n\t\t\tif (existingEntry.status !== status) {\n\t\t\t\tmulti.zRem(this.indexKey(existingEntry.status), String(id));\n\t\t\t}\n\t\t\tmulti.zAdd(this.indexKey(status), { score: currentTime, value: String(id) });\n\t\t\tmulti.zAdd(this.indexKey(), { score: currentTime, value: String(id) });\n\n\t\t\tawait multi.exec();\n\t\t}\n\t}\n\n\tasync get(id: KeetaAnchorQueueRequestID): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null> {\n\t\tconst redis = await this.getRedis();\n\n\t\tconst entryJSON = await redis.get(this.queueKey(id));\n\t\tif (!entryJSON) {\n\t\t\treturn(null);\n\t\t}\n\n\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\tconst entryData = JSON.parse(entryJSON) as QueueEntryData;\n\n\t\tconst idempotentKeys = entryData.idempotentKeys && entryData.idempotentKeys.length > 0\n\t\t\t? new Set(entryData.idempotentKeys.map(function(key: string) {\n\t\t\t\treturn(ConvertStringToRequestID(key));\n\t\t\t}))\n\t\t\t: undefined;\n\n\t\treturn({\n\t\t\tid: ConvertStringToRequestID(entryData.id),\n\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\trequest: JSON.parse(entryData.request) as QueueRequest,\n\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\toutput: entryData.output ? JSON.parse(entryData.output) as QueueResult : null,\n\t\t\tlastError: entryData.lastError,\n\t\t\tstatus: entryData.status,\n\t\t\tcreated: new Date(entryData.created),\n\t\t\tupdated: new Date(entryData.updated),\n\t\t\t// eslint-disable-next-line @typescript-eslint/consistent-type-assertions\n\t\t\tworker: entryData.worker as unknown as KeetaAnchorQueueWorkerID | null,\n\t\t\tfailures: entryData.failures,\n\t\t\tidempotentKeys: idempotentKeys\n\t\t});\n\t}\n\n\tasync query(filter?: KeetaAnchorQueueFilter): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]> {\n\t\tconst redis = await this.getRedis();\n\t\tconst logger = this.methodLogger('query');\n\n\t\tlogger?.debug(`Querying queue with id ${this.id} with filter:`, filter);\n\n\t\tlet entryIDs: string[];\n\n\t\tif (filter?.status) {\n\t\t\tconst count = filter.limit ?? -1;\n\t\t\tconst allIDs = await redis.zRange(this.indexKey(filter.status), 0, -1);\n\n\t\t\tif (filter.updatedBefore) {\n\t\t\t\tconst maxScore = filter.updatedBefore.getTime();\n\t\t\t\tconst filteredIDs: string[] = [];\n\t\t\t\tfor (const entryID of allIDs) {\n\t\t\t\t\tconst score = await redis.zScore(this.indexKey(filter.status), entryID);\n\t\t\t\t\tif (score !== null && score < maxScore) {\n\t\t\t\t\t\tfilteredIDs.push(entryID);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tentryIDs = filteredIDs;\n\t\t\t} else {\n\t\t\t\tentryIDs = allIDs;\n\t\t\t}\n\n\t\t\tif (count !== -1) {\n\t\t\t\tentryIDs = entryIDs.slice(0, count);\n\t\t\t}\n\t\t} else {\n\t\t\tconst count = filter?.limit ?? -1;\n\t\t\tconst allIDs = await redis.zRange(this.indexKey(), 0, -1);\n\n\t\t\tif (filter?.updatedBefore) {\n\t\t\t\tconst maxScore = filter.updatedBefore.getTime();\n\t\t\t\tconst filteredIDs: string[] = [];\n\t\t\t\tfor (const entryID of allIDs) {\n\t\t\t\t\tconst score = await redis.zScore(this.indexKey(), entryID);\n\t\t\t\t\tif (score !== null && score < maxScore) {\n\t\t\t\t\t\tfilteredIDs.push(entryID);\n\t\t\t\t\t}\n\t\t\t\t}\n\t\t\t\tentryIDs = filteredIDs;\n\t\t\t} else {\n\t\t\t\tentryIDs = allIDs;\n\t\t\t}\n\n\t\t\tif (count !== -1) {\n\t\t\t\tentryIDs = entryIDs.slice(0, count);\n\t\t\t}\n\t\t}\n\n\t\tconst entries: KeetaAnchorQueueEntry<QueueRequest, QueueResult>[] = [];\n\n\t\tfor (const entryIDStr of entryIDs) {\n\t\t\tconst entryID = ConvertStringToRequestID(entryIDStr);\n\t\t\tconst entry = await this.get(entryID);\n\t\t\tif (entry) {\n\t\t\t\tif (filter?.status && entry.status !== filter.status) {\n\t\t\t\t\tcontinue;\n\t\t\t\t}\n\t\t\t\tentries.push(entry);\n\t\t\t}\n\t\t}\n\n\t\tlogger?.debug(`Queried queue with id ${this.id} with filter:`, filter, '-- found', entries.length, 'entries');\n\n\t\treturn(entries);\n\t}\n\n\tasync partition(path: string): Promise<KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult>> {\n\t\tthis.methodLogger('partition')?.debug(`Creating partitioned queue storage driver for path: ${path}`);\n\n\t\tif (this.redisInternal === null) {\n\t\t\tthrow(new Error('Asked to partition but the instance has been destroyed'));\n\t\t}\n\n\t\tconst retval = new KeetaAnchorQueueStorageDriverRedis<QueueRequest, QueueResult>({\n\t\t\tid: `${this.id}::${path}`,\n\t\t\tlogger: this.logger,\n\t\t\tredis: this.redisInternal,\n\t\t\tpath: [...this.path, path]\n\t\t});\n\n\t\treturn(retval);\n\t}\n\n\tasync destroy(): Promise<void> {\n\t\tthis.methodLogger('destroy')?.debug('Destroying instance');\n\n\t\tthis.redisInternal = null;\n\t}\n\n\tasync [Symbol.asyncDispose](): Promise<void> {\n\t\treturn(await this.destroy());\n\t}\n}\n"]}

package/lib/queue/drivers/queue_sqlite3.d.ts

@@ -0,0 +1,28 @@
+import type { KeetaAnchorQueueStorageDriver, KeetaAnchorQueueStorageDriverConstructor, KeetaAnchorQueueRequest, KeetaAnchorQueueRequestID, KeetaAnchorQueueEntry, KeetaAnchorQueueEntryExtra, KeetaAnchorQueueEntryAncillaryData, KeetaAnchorQueueStatus, KeetaAnchorQueueFilter } from '../index.ts';
+import type { JSONSerializable } from '../../utils/json.js';
+import type * as sqlite from 'sqlite';
+export default class KeetaAnchorQueueStorageDriverSQLite3<QueueRequest extends JSONSerializable = JSONSerializable, QueueResult extends JSONSerializable = JSONSerializable> implements KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult> {
+    private readonly logger;
+    private dbInternal;
+    private dbInitializationPromise?;
+    readonly name = "KeetaAnchorQueueStorageDriverSQLite3";
+    readonly id: string;
+    readonly path: string[];
+    private readonly pathStr;
+    constructor(options: NonNullable<ConstructorParameters<KeetaAnchorQueueStorageDriverConstructor<QueueRequest, QueueResult>>[0]> & {
+        db: () => Promise<sqlite.Database>;
+    });
+    private initializeDBConnection;
+    private methodLogger;
+    private runWithBusyHandler;
+    private newDBConnection;
+    private dbTransaction;
+    add(request: KeetaAnchorQueueRequest<QueueRequest>, info?: KeetaAnchorQueueEntryExtra): Promise<KeetaAnchorQueueRequestID>;
+    setStatus(id: KeetaAnchorQueueRequestID, status: KeetaAnchorQueueStatus, ancillary?: KeetaAnchorQueueEntryAncillaryData<QueueResult>): Promise<void>;
+    get(id: KeetaAnchorQueueRequestID): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult> | null>;
+    query(filter?: KeetaAnchorQueueFilter): Promise<KeetaAnchorQueueEntry<QueueRequest, QueueResult>[]>;
+    partition(path: string): Promise<KeetaAnchorQueueStorageDriver<QueueRequest, QueueResult>>;
+    destroy(): Promise<void>;
+    [Symbol.asyncDispose](): Promise<void>;
+}
+//# sourceMappingURL=queue_sqlite3.d.ts.map

package/lib/queue/drivers/queue_sqlite3.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"queue_sqlite3.d.ts","sourceRoot":"","sources":["../../../../src/lib/queue/drivers/queue_sqlite3.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EACX,6BAA6B,EAC7B,wCAAwC,EACxC,uBAAuB,EACvB,yBAAyB,EACzB,qBAAqB,EACrB,0BAA0B,EAC1B,kCAAkC,EAClC,sBAAsB,EACtB,sBAAsB,EAEtB,MAAM,aAAa,CAAC;AAWrB,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,OAAO,KAAK,KAAK,MAAM,MAAM,QAAQ,CAAC;AAkBtC,MAAM,CAAC,OAAO,OAAO,oCAAoC,CAAC,YAAY,SAAS,gBAAgB,GAAG,gBAAgB,EAAE,WAAW,SAAS,gBAAgB,GAAG,gBAAgB,CAAE,YAAW,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC;IAC/O,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAqB;IAC5C,OAAO,CAAC,UAAU,CAAiD;IACnE,OAAO,CAAC,uBAAuB,CAAC,CAAmB;IAEnD,QAAQ,CAAC,IAAI,0CAA0C;IACvD,QAAQ,CAAC,EAAE,EAAE,MAAM,CAAC;IACpB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAM;IAC7B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,OAAO,EAAE,WAAW,CAAC,qBAAqB,CAAC,wCAAwC,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG;QAAE,EAAE,EAAE,MAAM,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;KAAE;YAW3J,sBAAsB;IAqDpC,OAAO,CAAC,YAAY;YASN,kBAAkB;IA8ChC,OAAO,CAAC,eAAe;YAyBT,aAAa;IAkCrB,GAAG,CAAC,OAAO,EAAE,uBAAuB,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,EAAE,0BAA0B,GAAG,OAAO,CAAC,yBAAyB,CAAC;IAqE1H,SAAS,CAAC,EAAE,EAAE,yBAAyB,EAAE,MAAM,EAAE,sBAAsB,EAAE,SAAS,CAAC,EAAE,kCAAkC,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;IAwDpJ,GAAG,CAAC,EAAE,EAAE,yBAAyB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,GAAG,IAAI,CAAC;IAyCpG,KAAK,CAAC,MAAM,CAAC,EAAE,sBAAsB,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,EAAE,CAAC;IAsEnG,SAAS,CAAC,IAAI,EAAE,MAAM,GAAI,OAAO,CAAC,6BAA6B,CAAC,YAAY,EAAE,WAAW,CAAC,CAAC;IAiB3F,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAMxB,CAAC,MAAM,CAAC,YAAY,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC;CAG5C"}