@keetanetwork/anchor 0.0.38 → 0.0.40

package/lib/queue/index.js

@@ -1,8 +1,10 @@
 import { __addDisposableResource, __disposeResources } from "tslib";
+import { assertNever } from '../utils/never.js';
 import { asleep } from '../utils/asleep.js';
 import { Errors } from './common.js';
 import { MethodLogger, ManageStatusUpdates, ConvertStringToRequestID } from './internal.js';
 import { AsyncDisposableStack } from '../utils/defer.js';
+const keetaAnchorPipeableQueueStatuses = ['completed', 'failed_permanently'];
 /**
  * An in-memory implementation of the KeetaAnchorQueueStorageDriver
  */
@@ -214,9 +216,74 @@ export class KeetaAnchorQueueRunner {
     /**
      * Configuration for this queue
      */
+    /**
+     * The maximum number of times to retry a failed job before giving up
+     * and marking it as permanently failed
+     */
     maxRetries = 5;
+    /**
+     * The amount of time to allow a job to run before considering it
+     * timed out and marking it as aborted, the `aborted` processor
+     * (if provided) will be responsible for determining what to do with
+     * the job at that point (e.g. retry it, mark it as failed, etc)
+     * because some work from the job may have been completed
+     * before the timeout was reached -- it could even still be on-going
+     * after the timeout is reached, so the `aborted` processor should be
+     * prepared to handle that situation as well.
+     */
     processTimeout = 300_000; /* 5 minutes */
+    /**
+     * When piping a batch of jobs to another runner, this is the number
+     * of jobs to include in each batch (max).
+     */
     batchSize = 100;
+    /**
+     * The amount of time to wait before retrying a failed job -- this should
+     * be long enough to allow any transient issues to be resolved (e.g. a downstream
+     * service to come back up, etc) but not so long that it causes excessive
+     * delays in processing.
+     *
+     * Keep in mind that after the {@link maxRetries} is reached, the job will
+     * be marked as permanently failed, so if the retry delay is too low then
+     * it could cause jobs to be marked as permanently failed before transient
+     * issues have a chance to be resolved.
+     */
+    get retryDelay() {
+        if (this.internalRetryDelay !== undefined) {
+            return (this.internalRetryDelay);
+        }
+        return (this.processTimeout * 10);
+    }
+    set retryDelay(value) {
+        this.internalRetryDelay = value;
+    }
+    internalRetryDelay;
+    /**
+     * The number of {@link processTimeout} intervals to wait before
+     * considering a job to be stuck -- this is for jobs that are
+     * still in the 'processing' state but have not updated their
+     * timestamp in a long time, which likely indicates that the worker
+     * processing the job has died or is otherwise no longer making
+     * progress on the job.
+     *
+     * Like the `aborted` status, the `stuck` status means the job is in
+     * an indeterminate state where it may have done some of the work of
+     * the processor but we don't know how much (if any) of it was
+     * completed, so the `stuck` processor should be prepared to handle
+     * that situation.  It is unlikely that the job is still being actively
+     * processed by a worker at this point, but it is possible, so the
+     * `stuck` processor should be prepared to handle that as well.
+     */
+    get stuckMultiplier() {
+        if (this.internalStuckMultiplier !== undefined) {
+            return (this.internalStuckMultiplier);
+        }
+        return (10);
+    }
+    set stuckMultiplier(value) {
+        this.internalStuckMultiplier = value;
+    }
+    internalStuckMultiplier;
     /**
      * How many runners can process this queue in parallel
      */
@@ -278,6 +345,16 @@ export class KeetaAnchorQueueRunner {
         })();
         return (await this.initializePromise);
     }
+    setConfiguration(parameters) {
+        const parameterNames = ['batchSize', 'maxRetries', 'processTimeout', 'retryDelay', 'stuckMultiplier'];
+        for (const parameterName of parameterNames) {
+            const value = parameters[parameterName];
+            if (value === undefined) {
+                continue;
+            }
+            this[parameterName] = value;
+        }
+    }
     methodLogger(method) {
         return (MethodLogger(this.logger, {
             class: 'KeetaAnchorQueueRunner',
@@ -292,12 +369,11 @@ export class KeetaAnchorQueueRunner {
             throw (new Error('This is a testing only method'));
         }
         return ({
-            setParams: (maxBatchSize, processTimeout, maxRetries, maxWorkers) => {
-                this.batchSize = maxBatchSize;
-                this.processTimeout = processTimeout;
-                this.maxRetries = maxRetries;
-                if (maxWorkers !== undefined) {
-                    this.maxRunners = maxWorkers;
+            setParams: (params) => {
+                const { maxRunners, ...configParams } = params;
+                this.setConfiguration(configParams);
+                if (maxRunners !== undefined) {
+                    this.maxRunners = maxRunners;
                 }
             },
             queue: () => {
@@ -446,7 +522,7 @@ export class KeetaAnchorQueueRunner {
                 return;
             }
             const lockAge = moment.getTime() - lockEntry.updated.getTime();
-            if (lockAge > this.processTimeout * 10) {
+            if (lockAge > this.processTimeout * this.stuckMultiplier) {
                 logger?.warn('Processing lock is stale, taking over lock for worker ID', this.workerID);
                 await this.queue.setStatus(this.runnerLockKey, '@internal', {
                     oldStatus: 'processing',
@@ -574,6 +650,9 @@ export class KeetaAnchorQueueRunner {
              */
             const pipes = [...this.pipes];
             for (const pipe of pipes) {
+                if (pipe.exclusiveTarget === false) {
+                    continue;
+                }
                 let remainingTime = undefined;
                 if (timeout !== undefined) {
                     const elapsed = Date.now() - startTime;
@@ -634,7 +713,7 @@ export class KeetaAnchorQueueRunner {
         }
     }
     async markStuckRequestsAsStuck() {
-        const stuckThreshold = this.processTimeout * 10;
+        const stuckThreshold = this.processTimeout * this.stuckMultiplier;
         const logger = this.methodLogger('markStuckRequestsAsStuck');
         const now = Date.now();
         const requests = await this.queue.query({ status: 'processing', limit: 100, updatedBefore: new Date(now - stuckThreshold) });
@@ -655,7 +734,7 @@ export class KeetaAnchorQueueRunner {
         }
     }
     async requeueFailedRequests() {
-        const retryDelay = this.processTimeout * 10;
+        const retryDelay = this.retryDelay;
         const maxRetries = this.maxRetries;
         const logger = this.methodLogger('requeueFailedRequests');
         const now = Date.now();
@@ -675,19 +754,30 @@ export class KeetaAnchorQueueRunner {
             }
         }
     }
-    async moveCompletedToNextStage() {
+    async moveCompletedToNextStage(statusTarget) {
         const logger = this.methodLogger('moveCompletedToNextStage');
-        const pipes = [...this.pipes];
+        const pipes = [...this.pipes].filter(function (pipe) {
+            return (pipe.acceptStatus === statusTarget);
+        });
         if (pipes.length === 0) {
             return;
         }
-        const allRequests = await this.queue.query({ status: 'completed', limit: 100 });
+        const allRequests = await this.queue.query({ status: statusTarget, limit: 100 });
         let requests = allRequests;
         const RequestSentToPipes = new Map();
         function IncrRequestSentToPipes(requestID) {
             const sentCount = RequestSentToPipes.get(requestID) ?? 0;
             RequestSentToPipes.set(requestID, sentCount + 1);
         }
+        const getNextPipeRequestInput = (entry) => {
+            if (statusTarget === 'completed') {
+                return (this.decodeResponse(entry.output));
+            }
+            else if (statusTarget === 'failed_permanently') {
+                return (this.decodeRequest(entry.request));
+            }
+            assertNever(statusTarget);
+        };
         for (const pipe of pipes) {
             logger?.debug('Processing pipe to target', pipe.target.id, pipe.isBatchPipe ? '(batch pipe)' : '(single item pipe)');
             if (pipe.isBatchPipe) {
@@ -726,14 +816,14 @@ export class KeetaAnchorQueueRunner {
                     return (!iterationTargetSeenRequestIDs.has(entry.id) && !skipRequestIDs.has(entry.id));
                 })) {
                     iterationTargetSeenRequestIDs.clear();
-                    logger?.debug(`Preparing to move completed requests to next stage ${pipe.target.id} (min=${pipe.minBatchSize}, max=${pipe.maxBatchSize}), have ${requests.length} completed requests available`);
+                    logger?.debug(`Preparing to move ${statusTarget} requests to next stage ${pipe.target.id} (min=${pipe.minBatchSize}, max=${pipe.maxBatchSize}), have ${requests.length} ${statusTarget} requests available`);
                     /**
                      * Compute a batch of entries to send to the next stage,
                      * constrained to the max batch size of the pipe and
                      * the entries which have non-null outputs
                      */
                     const batchRaw = requests.map((entry) => {
-                        return ({ output: this.decodeResponse(entry.output), id: entry.id });
+                        return ({ output: getNextPipeRequestInput(entry), id: entry.id });
                     }).filter(function (entry) {
                         if (entry === null) {
                             return (false);
@@ -747,10 +837,10 @@ export class KeetaAnchorQueueRunner {
                     if (batchRaw.length < pipe.minBatchSize) {
                         sequentialFailureCount++;
                         if (sequentialFailureCount >= 3) {
-                            logger?.debug(`Not enough completed requests to move to next stage ${pipe.target.id}, stopping batch processing`);
+                            logger?.debug(`Not enough ${statusTarget} requests to move to next stage ${pipe.target.id}, stopping batch processing`);
                             break;
                         }
-                        logger?.debug(`Not moving completed requests to next stage ${pipe.target.id} because batch size ${batchRaw.length} is less than minimum size ${pipe.minBatchSize}`);
+                        logger?.debug(`Not moving ${statusTarget} requests to next stage ${pipe.target.id} because batch size ${batchRaw.length} is less than minimum size ${pipe.minBatchSize}`);
                         continue;
                     }
                     sequentialFailureCount = 0;
@@ -769,8 +859,10 @@ export class KeetaAnchorQueueRunner {
                     const batchOutput = batchRaw.map(function (entry) {
                         return (entry.output);
                     });
-                    logger?.debug(`Moving batch of ${batchOutput.length} completed requests to next pipe`, pipe.target.id, '(input entry IDs:', Array.from(batchLocalIDs), '->', `${pipe.target.id}:${String(batchID)})`);
+                    logger?.debug(`Moving batch of ${batchOutput.length} ${statusTarget} requests to next pipe`, pipe.target.id, '(input entry IDs:', Array.from(batchLocalIDs), '->', `${pipe.target.id}:${String(batchID)})`);
                     try {
+                        // There is no way for typescript to cleanly infer the type here, so we assert it.
+                        // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
                         await pipe.target.add(batchOutput, {
                             id: batchID,
                             /* Use the set of IDs as the idempotent IDs for the batch */
@@ -816,12 +908,14 @@ export class KeetaAnchorQueueRunner {
                 for (const request of requests) {
                     let shouldMarkAsMoved = true;
                     try {
-                        const output = this.decodeResponse(request.output);
+                        const output = getNextPipeRequestInput(request);
                         if (output === null) {
                             logger?.debug(`Completed request with id ${String(request.id)} has no output -- next stage will not be run`);
                         }
                         else {
-                            logger?.debug(`Moving completed request with id ${String(request.id)} to next pipe`, pipe.target.id);
+                            logger?.debug(`Moving ${statusTarget} request with id ${String(request.id)} to next pipe`, pipe.target.id);
+                            // There is no way for typescript to cleanly infer the type here, so we assert it.
+                            // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
                             await pipe.target.add(output, { id: request.id });
                         }
                     }
@@ -839,11 +933,11 @@ export class KeetaAnchorQueueRunner {
         for (const request of allRequests) {
             const sentCount = RequestSentToPipes.get(request.id) ?? 0;
             if (sentCount !== TotalPipes) {
-                logger?.debug(`Completed request with id ${String(request.id)} was only moved to ${sentCount} out of ${TotalPipes} pipes -- not marking as moved`);
+                logger?.debug(`${statusTarget} request with id ${String(request.id)} was only moved to ${sentCount} out of ${TotalPipes} pipes -- not marking as moved`);
                 continue;
             }
-            logger?.debug(`Marking completed request with id ${String(request.id)} as moved`);
-            await this.queue.setStatus(request.id, 'moved', { oldStatus: 'completed', by: this.workerID });
+            logger?.debug(`Marking ${statusTarget} request with id ${String(request.id)} as moved`);
+            await this.queue.setStatus(request.id, 'moved', { oldStatus: statusTarget, by: this.workerID });
         }
     }
     async maintain() {
@@ -876,14 +970,19 @@ export class KeetaAnchorQueueRunner {
         catch (error) {
             logger?.debug('Failed to requeue failed requests:', error);
         }
-        try {
-            await this.moveCompletedToNextStage();
-        }
-        catch (error) {
-            logger?.debug('Failed to move completed requests to next stage:', error);
+        for (const pipeStatus of keetaAnchorPipeableQueueStatuses) {
+            try {
+                await this.moveCompletedToNextStage(pipeStatus);
+            }
+            catch (error) {
+                logger?.debug(`Failed to move ${pipeStatus} requests to next stage:`, error);
+            }
         }
         for (const pipe of this.pipes) {
             try {
+                if (pipe.exclusiveTarget === false) {
+                    continue;
+                }
                 await pipe.target.maintain();
             }
             catch (error) {
@@ -902,22 +1001,46 @@ export class KeetaAnchorQueueRunner {
     /**
      * Pipe the the completed entries of this runner to another runner
      */
-    pipe(target) {
+    pipe(target, options) {
         this.pipes.push({
+            ...options,
             isBatchPipe: false,
-            target: target
+            target: target,
+            acceptStatus: 'completed'
+        });
+        return (target);
+    }
+    pipeFailed(target, options) {
+        this.pipes.push({
+            ...options,
+            isBatchPipe: false,
+            target: target,
+            acceptStatus: 'failed_permanently'
         });
         return (target);
     }
     /**
      * Pipe batches of completed entries from this runner to another runner
      */
-    pipeBatch(target, maxBatchSize = 100, minBatchSize = 1) {
+    pipeBatch(target, maxBatchSize = 100, minBatchSize = 1, options) {
+        this.pipes.push({
+            ...options,
+            isBatchPipe: true,
+            target: target,
+            minBatchSize: minBatchSize,
+            maxBatchSize: maxBatchSize,
+            acceptStatus: 'completed'
+        });
+        return (target);
+    }
+    pipeBatchFailed(target, maxBatchSize = 100, minBatchSize = 1, options) {
         this.pipes.push({
+            ...options,
             isBatchPipe: true,
             target: target,
             minBatchSize: minBatchSize,
-            maxBatchSize: maxBatchSize
+            maxBatchSize: maxBatchSize,
+            acceptStatus: 'failed_permanently'
         });
         return (target);
     }
@@ -956,13 +1079,15 @@ export class KeetaAnchorQueueRunnerJSONConfigProc extends KeetaAnchorQueueRunner
     processor;
     constructor(config) {
         super(config);
-        this.processor = config.processor;
-        if (config.processorStuck) {
-            this.processorStuck = config.processorStuck;
+        const { processor, processorStuck, processorAborted, ...parameters } = config;
+        this.processor = processor;
+        if (processorStuck) {
+            this.processorStuck = processorStuck;
         }
-        if (config.processorAborted) {
-            this.processorAborted = config.processorAborted;
+        if (processorAborted) {
+            this.processorAborted = processorAborted;
         }
+        this.setConfiguration(parameters);
     }
 }
 //# sourceMappingURL=index.js.map