@hotmeshio/hotmesh 0.0.1

package/build/services/serializer/index.d.ts

@@ -0,0 +1,36 @@
+import { StringStringType, StringAnyType, SymbolMap, SymbolMaps, SymbolSets, Symbols } from '../../types/serializer';
+export declare const MDATA_SYMBOLS: {
+    SLOTS: number;
+    ACTIVITY: {
+        KEYS: string[];
+    };
+    ACTIVITY_UPDATE: {
+        KEYS: string[];
+    };
+    JOB: {
+        KEYS: string[];
+    };
+    JOB_UPDATE: {
+        KEYS: string[];
+    };
+};
+export declare class SerializerService {
+    symKeys: SymbolMaps;
+    symReverseKeys: SymbolMaps;
+    symValMaps: SymbolMap;
+    symValReverseMaps: SymbolMap;
+    constructor();
+    resetSymbols(symKeys: SymbolSets, symVals: Symbols): void;
+    getReverseKeyMap(keyMap: SymbolMap, id?: string): SymbolMap;
+    getReverseValueMap(valueMap: SymbolMap): SymbolMap;
+    static filterSymVals(startIndex: number, maxIndex: number, existingSymbolValues: Symbols, proposedValues: Set<string>): Symbols;
+    compress(document: StringStringType, ids: string[]): StringStringType;
+    decompress(document: StringStringType, ids: string[]): StringStringType;
+    stringify(document: Record<string, any>): StringStringType;
+    parse(document: StringStringType): any;
+    toString(value: any): string | undefined;
+    fromString(value: string | undefined): any;
+    package(multiDimensionalDocument: StringAnyType, ids: string[]): StringStringType;
+    unpackage(document: StringStringType, ids: string[]): StringAnyType;
+    export(): SymbolSets;
+}

package/build/services/serializer/index.js

@@ -0,0 +1,222 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SerializerService = exports.MDATA_SYMBOLS = void 0;
+const utils_1 = require("../../modules/utils");
+const dateReg = /^"\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z)?"$/;
+exports.MDATA_SYMBOLS = {
+    SLOTS: 26,
+    ACTIVITY: {
+        KEYS: ['aid', 'dad', 'as', 'atp', 'stp', 'ac', 'au', 'err', 'l1s', 'l2s']
+    },
+    ACTIVITY_UPDATE: {
+        KEYS: ['au', 'err', 'l2s']
+    },
+    JOB: {
+        KEYS: ['ngn', 'tpc', 'pj', 'pd', 'pa', 'key', 'app', 'vrs', 'jid', 'aid', 'ts', 'jc', 'ju', 'js', 'err', 'trc']
+    },
+    JOB_UPDATE: {
+        KEYS: ['ju', 'err']
+    }
+};
+class SerializerService {
+    constructor() {
+        this.resetSymbols({}, {});
+    }
+    resetSymbols(symKeys, symVals) {
+        this.symKeys = new Map();
+        this.symReverseKeys = new Map();
+        for (const id in symKeys) {
+            this.symKeys.set(id, new Map(Object.entries(symKeys[id])));
+        }
+        this.symValMaps = new Map(Object.entries(symVals));
+        this.symValReverseMaps = this.getReverseValueMap(this.symValMaps);
+    }
+    getReverseKeyMap(keyMap, id) {
+        let map = this.symReverseKeys.get(id);
+        if (!map) {
+            map = new Map();
+            for (let [key, val] of keyMap.entries()) {
+                map.set(val, key);
+            }
+            this.symReverseKeys.set(id, map);
+        }
+        return map;
+    }
+    getReverseValueMap(valueMap) {
+        const map = new Map();
+        for (let [key, val] of valueMap.entries()) {
+            map.set(val, key);
+        }
+        return map;
+    }
+    static filterSymVals(startIndex, maxIndex, existingSymbolValues, proposedValues) {
+        let newSymbolValues = {};
+        let currentSymbolValues = { ...existingSymbolValues };
+        let currentValuesSet = new Set(Object.values(currentSymbolValues));
+        for (let value of proposedValues) {
+            if (!currentValuesSet.has(value)) {
+                if (startIndex > maxIndex) {
+                    return newSymbolValues;
+                }
+                const symbol = (0, utils_1.getSymVal)(startIndex);
+                startIndex++;
+                newSymbolValues[symbol] = value;
+                currentValuesSet.add(value);
+            }
+        }
+        return newSymbolValues;
+    }
+    compress(document, ids) {
+        if (this.symKeys.size === 0) {
+            return document;
+        }
+        let result = { ...document };
+        const compressWithMap = (abbreviationMap) => {
+            for (let key in result) {
+                let safeKey = abbreviationMap.get(key) || key;
+                let value = result[key];
+                let safeValue = abbreviationMap.get(value) || value;
+                if (safeKey !== key || safeValue !== value) {
+                    result[safeKey] = safeValue;
+                    if (safeKey !== key) {
+                        delete result[key];
+                    }
+                }
+            }
+        };
+        for (let id of ids) {
+            const abbreviationMap = this.symKeys.get(id);
+            if (abbreviationMap) {
+                compressWithMap(abbreviationMap);
+            }
+        }
+        return result;
+    }
+    decompress(document, ids) {
+        if (this.symKeys.size === 0) {
+            return document;
+        }
+        let result = { ...document };
+        const inflateWithMap = (abbreviationMap, id) => {
+            const reversedAbbreviationMap = this.getReverseKeyMap(abbreviationMap, id);
+            for (let key in result) {
+                let safeKey = reversedAbbreviationMap.get(key) || key;
+                let value = result[key];
+                let safeValue = reversedAbbreviationMap.get(value) || value;
+                if (safeKey !== key || safeValue !== value) {
+                    result[safeKey] = safeValue;
+                    if (safeKey !== key) {
+                        delete result[key];
+                    }
+                }
+            }
+        };
+        for (let id of ids) {
+            const abbreviationMap = this.symKeys.get(id);
+            if (abbreviationMap) {
+                inflateWithMap(abbreviationMap, id);
+            }
+        }
+        return result;
+    }
+    //stringify: convert a multi-dimensional document to a 2-d hash
+    stringify(document) {
+        let result = {};
+        for (let key in document) {
+            let value = this.toString(document[key]);
+            if (value) {
+                if (/^:*[a-zA-Z]{2}$/.test(value)) {
+                    value = ':' + value;
+                }
+                else if (this.symValReverseMaps.has(value)) {
+                    value = this.symValReverseMaps.get(value);
+                }
+                result[key] = value;
+            }
+        }
+        return result;
+    }
+    //parse: convert a 2-d hash to a multi-dimensional document
+    parse(document) {
+        let result = {};
+        for (let [key, value] of Object.entries(document)) {
+            if (value === undefined || value === null)
+                continue;
+            if (/^:+[a-zA-Z]{2}$/.test(value)) {
+                result[key] = value.slice(1);
+            }
+            else {
+                if (value?.length === 2 && this.symValMaps.has(value)) {
+                    value = this.symValMaps.get(value);
+                }
+                result[key] = this.fromString(value);
+            }
+        }
+        return result;
+    }
+    toString(value) {
+        switch (typeof value) {
+            case 'string':
+                break;
+            case 'boolean':
+                value = value ? '/t' : '/f';
+                break;
+            case 'number':
+                value = '/d' + value.toString();
+                break;
+            case 'undefined':
+                return undefined;
+            case 'object':
+                if (value === null) {
+                    value = '/n';
+                }
+                else {
+                    value = '/s' + JSON.stringify(value);
+                }
+                break;
+        }
+        return value;
+    }
+    fromString(value) {
+        if (typeof value !== 'string')
+            return undefined;
+        const prefix = value.slice(0, 2);
+        const rest = value.slice(2);
+        switch (prefix) {
+            case '/t': // boolean true
+                return true;
+            case '/f': // boolean false
+                return false;
+            case '/d': // number
+                return Number(rest);
+            case '/n': // null
+                return null;
+            case '/s': // object (JSON string)
+                if (dateReg.exec(rest)) {
+                    return new Date(JSON.parse(rest));
+                }
+                return JSON.parse(rest);
+            default: // string
+                return value;
+        }
+    }
+    package(multiDimensionalDocument, ids) {
+        const flatDocument = this.stringify(multiDimensionalDocument);
+        return this.compress(flatDocument, ids);
+    }
+    unpackage(document, ids) {
+        const multiDimensionalDocument = this.decompress(document, ids);
+        return this.parse(multiDimensionalDocument);
+    }
+    export() {
+        const obj = {};
+        for (const [id, map] of this.symKeys.entries()) {
+            obj[id] = {};
+            for (const [key, value] of map.entries()) {
+                obj[id][key] = value;
+            }
+        }
+        return obj;
+    }
+}
+exports.SerializerService = SerializerService;

package/build/services/signaler/store.d.ts

@@ -0,0 +1,15 @@
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { HookRule } from '../../types/hook';
+import { JobState } from '../../types/job';
+import { RedisClient, RedisMulti } from '../../types/redis';
+declare class StoreSignaler {
+    store: StoreService<RedisClient, RedisMulti>;
+    logger: ILogger;
+    constructor(store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
+    getHookRule(topic: string): Promise<HookRule | undefined>;
+    registerWebHook(topic: string, context: JobState, multi?: RedisMulti): Promise<string>;
+    processWebHookSignal(topic: string, data: Record<string, unknown>): Promise<string>;
+    deleteWebHookSignal(topic: string, data: Record<string, unknown>): Promise<number>;
+}
+export { StoreSignaler };

package/build/services/signaler/store.js

@@ -0,0 +1,53 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StoreSignaler = void 0;
+class StoreSignaler {
+    constructor(store, logger) {
+        this.store = store;
+        this.logger = logger;
+    }
+    async getHookRule(topic) {
+        const rules = await this.store.getHookRules();
+        return rules?.[topic]?.[0];
+    }
+    async registerWebHook(topic, context, multi) {
+        const hookRule = await this.getHookRule(topic);
+        if (hookRule) {
+            const jobId = context.metadata.jid;
+            const hook = {
+                topic,
+                resolved: jobId,
+                jobId,
+            };
+            await this.store.setHookSignal(hook, multi);
+            return jobId;
+        }
+        else {
+            throw new Error('signaler.registerWebHook:error: hook rule not found');
+        }
+    }
+    async processWebHookSignal(topic, data) {
+        const hookRule = await this.getHookRule(topic);
+        if (hookRule) {
+            //todo: use the rule to generate `resolved`
+            const resolved = data.id;
+            const jobId = await this.store.getHookSignal(topic, resolved);
+            return jobId;
+        }
+        else {
+            throw new Error('signaler.process:error: hook rule not found');
+        }
+    }
+    async deleteWebHookSignal(topic, data) {
+        const hookRule = await this.getHookRule(topic);
+        if (hookRule) {
+            //todo: use the rule to generate `resolved`
+            const resolved = data.id;
+            return await this.store.deleteHookSignal(topic, resolved);
+        }
+        else {
+            throw new Error('signaler.process:error: hook rule not found');
+        }
+    }
+}
+exports.StoreSignaler = StoreSignaler;

package/build/services/signaler/stream.d.ts

@@ -0,0 +1,43 @@
+/// <reference types="node" />
+import { ILogger } from '../logger';
+import { StoreService } from '../store';
+import { StreamService } from '../stream';
+import { RedisClient, RedisMulti } from '../../types/redis';
+import { ReclaimedMessageType, StreamConfig, StreamData, StreamDataResponse, StreamRole } from '../../types/stream';
+declare class StreamSignaler {
+    static signalers: Set<StreamSignaler>;
+    appId: string;
+    guid: string;
+    role: StreamRole;
+    topic: string | undefined;
+    store: StoreService<RedisClient, RedisMulti>;
+    stream: StreamService<RedisClient, RedisMulti>;
+    reclaimDelay: number;
+    reclaimCount: number;
+    logger: ILogger;
+    throttle: number;
+    errorCount: number;
+    currentTimerId: NodeJS.Timeout | null;
+    shouldConsume: boolean;
+    constructor(config: StreamConfig, stream: StreamService<RedisClient, RedisMulti>, store: StoreService<RedisClient, RedisMulti>, logger: ILogger);
+    createGroup(stream: string, group: string): Promise<void>;
+    publishMessage(topic: string, streamData: StreamData | StreamDataResponse, multi?: RedisMulti): Promise<string | RedisMulti>;
+    consumeMessages(stream: string, group: string, consumer: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
+    isStreamMessage(result: any): boolean;
+    consumeOne(stream: string, group: string, id: string, message: string[], callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<void>;
+    execStreamLeg(input: StreamData, stream: string, id: string, callback: (streamData: StreamData) => Promise<StreamDataResponse | void>): Promise<StreamDataResponse>;
+    ackAndDelete(stream: string, group: string, id: string): Promise<void>;
+    publishResponse(input: StreamData, output: StreamDataResponse | void): Promise<string>;
+    shouldRetry(input: StreamData, output: StreamDataResponse): [boolean, number];
+    structureUnhandledError(input: StreamData, err: Error): StreamDataResponse;
+    structureUnacknowledgedError(input: StreamData): StreamDataResponse;
+    structureError(input: StreamData, output: StreamDataResponse): StreamDataResponse;
+    static stopConsuming(): Promise<void>;
+    stopConsuming(): Promise<void>;
+    cancelThrottle(): void;
+    setThrottle(delayInMillis: number): void;
+    claimUnacknowledged(stream: string, group: string, consumer: string, idleTimeMs?: number, limit?: number): Promise<[string, [string, string]][]>;
+    expireUnacknowledged(reclaimedMessage: ReclaimedMessageType, stream: string, group: string, consumer: string, id: string, count: number): Promise<void>;
+    parseStreamData(str: string): [undefined, StreamData] | [Error];
+}
+export { StreamSignaler };

package/build/services/signaler/stream.js

@@ -0,0 +1,317 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamSignaler = void 0;
+const key_1 = require("../../modules/key");
+const utils_1 = require("../../modules/utils");
+const telemetry_1 = require("../telemetry");
+const stream_1 = require("../../types/stream");
+const MAX_RETRIES = 4; //max delay (10s using exponential backoff);
+const MAX_TIMEOUT_MS = 60000;
+const GRADUATED_INTERVAL_MS = 5000;
+const BLOCK_DURATION = 15000; //Set to `15` so SIGINT/SIGTERM can interrupt; set to `0` to BLOCK indefinitely
+const TEST_BLOCK_DURATION = 1000; //Set to `1000` so tests can interrupt quickly
+const BLOCK_TIME_MS = process.env.NODE_ENV === 'test' ? TEST_BLOCK_DURATION : BLOCK_DURATION;
+const SYSTEM_STATUS_CODE = 999;
+const UNKNOWN_STATUS_CODE = 500;
+const UNKNOWN_STATUS_MESSAGE = 'unknown';
+const XCLAIM_DELAY_MS = 1000 * 60; //max time a message can be unacked before it is claimed by another
+const XCLAIM_COUNT = 3; //max number of times a message can be claimed by another before it is dead-lettered
+const XPENDING_COUNT = 10;
+class StreamSignaler {
+    constructor(config, stream, store, logger) {
+        this.throttle = 0;
+        this.errorCount = 0;
+        this.currentTimerId = null;
+        this.appId = config.appId;
+        this.guid = config.guid;
+        this.role = config.role;
+        this.topic = config.topic;
+        this.stream = stream;
+        this.store = store;
+        this.reclaimDelay = config.reclaimDelay || XCLAIM_DELAY_MS;
+        this.reclaimCount = config.reclaimCount || XCLAIM_COUNT;
+        this.logger = logger;
+    }
+    async createGroup(stream, group) {
+        try {
+            await this.store.xgroup('CREATE', stream, group, '$', 'MKSTREAM');
+        }
+        catch (err) {
+            this.logger.info('consumer-group-exists', { stream, group });
+        }
+    }
+    async publishMessage(topic, streamData, multi) {
+        const stream = this.store.mintKey(key_1.KeyType.STREAMS, { appId: this.store.appId, topic });
+        return await this.store.xadd(stream, '*', 'message', JSON.stringify(streamData), multi);
+    }
+    async consumeMessages(stream, group, consumer, callback) {
+        this.logger.info(`stream-consumer-starting`, { group, consumer, stream });
+        StreamSignaler.signalers.add(this);
+        this.shouldConsume = true;
+        await this.createGroup(stream, group);
+        let lastCheckedPendingMessagesAt = Date.now();
+        async function consume() {
+            let sleep = (0, utils_1.XSleepFor)(this.throttle);
+            this.currentTimerId = sleep.timerId;
+            await sleep.promise;
+            if (!this.shouldConsume) {
+                this.logger.info(`stream-consumer-stopping`, { group, consumer, stream });
+                return;
+            }
+            try {
+                const result = await this.stream.xreadgroup('GROUP', group, consumer, 'BLOCK', BLOCK_TIME_MS, 'STREAMS', stream, '>');
+                if (this.isStreamMessage(result)) {
+                    const [[, messages]] = result;
+                    for (const [id, message] of messages) {
+                        await this.consumeOne(stream, group, id, message, callback);
+                    }
+                }
+                // Check for pending messages (note: Redis 6.2 simplifies)
+                const now = Date.now();
+                if (now - lastCheckedPendingMessagesAt > this.reclaimDelay) {
+                    lastCheckedPendingMessagesAt = now;
+                    const pendingMessages = await this.claimUnacknowledged(stream, group, consumer);
+                    for (const [id, message] of pendingMessages) {
+                        await this.consumeOne(stream, group, id, message, callback);
+                    }
+                }
+                setImmediate(consume.bind(this));
+            }
+            catch (err) {
+                if (this.shouldConsume && process.env.NODE_ENV !== 'test') {
+                    this.logger.error(`stream-consume-message-error`, { err, stream, group, consumer });
+                    this.errorCount++;
+                    const timeout = Math.min(GRADUATED_INTERVAL_MS * (2 ** this.errorCount), MAX_TIMEOUT_MS);
+                    setTimeout(consume.bind(this), timeout);
+                }
+            }
+        }
+        consume.call(this);
+    }
+    isStreamMessage(result) {
+        return Array.isArray(result) && Array.isArray(result[0]);
+    }
+    async consumeOne(stream, group, id, message, callback) {
+        this.logger.debug(`stream-consume-one`, { group, stream, id });
+        const [err, input] = this.parseStreamData(message[1]);
+        let output;
+        let telemetry;
+        try {
+            telemetry = new telemetry_1.TelemetryService(this.appId);
+            telemetry.startStreamSpan(input, this.role);
+            output = await this.execStreamLeg(input, stream, id, callback.bind(this));
+            if (output?.status === stream_1.StreamStatus.ERROR) {
+                telemetry.setStreamError(`Function Status Code ${output.code || UNKNOWN_STATUS_CODE}`);
+            }
+            this.errorCount = 0;
+        }
+        catch (err) {
+            this.logger.error(`stream-consume-one-error`, { group, stream, id, err });
+            telemetry.setStreamError(err.message);
+        }
+        const messageId = await this.publishResponse(input, output);
+        telemetry.setStreamAttributes({ 'app.worker.mid': messageId });
+        await this.ackAndDelete(stream, group, id);
+        telemetry.endStreamSpan();
+        this.logger.debug(`stream-consume-one-end`, { group, stream, id });
+    }
+    async execStreamLeg(input, stream, id, callback) {
+        let output;
+        try {
+            output = await callback(input);
+        }
+        catch (err) {
+            this.logger.error(`stream-call-function-error`, { stream, id, err });
+            output = this.structureUnhandledError(input, err);
+        }
+        return output;
+    }
+    async ackAndDelete(stream, group, id) {
+        const multi = this.stream.getMulti();
+        await this.stream.xack(stream, group, id, multi);
+        await this.stream.xdel(stream, id, multi);
+        await multi.exec();
+    }
+    async publishResponse(input, output) {
+        if (output && typeof output === 'object') {
+            if (output.status === 'error') {
+                const [shouldRetry, timeout] = this.shouldRetry(input, output);
+                if (shouldRetry) {
+                    await (0, utils_1.sleepFor)(timeout);
+                    return await this.publishMessage(input.metadata.topic, {
+                        data: input.data,
+                        metadata: { ...input.metadata, try: (input.metadata.try || 0) + 1 },
+                        policies: input.policies,
+                    });
+                }
+                else {
+                    output = this.structureError(input, output);
+                }
+            }
+            output.type = stream_1.StreamDataType.RESPONSE;
+            return await this.publishMessage(null, output);
+        }
+    }
+    shouldRetry(input, output) {
+        const policies = input.policies?.retry;
+        const errorCode = output.code.toString();
+        const policy = policies?.[errorCode];
+        const maxRetries = policy?.[0];
+        const tryCount = Math.min(input.metadata.try || 0, MAX_RETRIES);
+        if (maxRetries >= tryCount) {
+            return [true, Math.pow(10, tryCount)];
+        }
+        return [false, 0];
+    }
+    structureUnhandledError(input, err) {
+        let error = {};
+        if (typeof err.message === 'string') {
+            error.message = err.message;
+        }
+        else {
+            error.message = UNKNOWN_STATUS_MESSAGE;
+        }
+        if (typeof err.stack === 'string') {
+            error.stack = err.stack;
+        }
+        if (typeof err.name === 'string') {
+            error.name = err.name;
+        }
+        return {
+            status: 'error',
+            code: UNKNOWN_STATUS_CODE,
+            metadata: { ...input.metadata },
+            data: error
+        };
+    }
+    structureUnacknowledgedError(input) {
+        const message = 'stream message max delivery count exceeded';
+        const code = SYSTEM_STATUS_CODE;
+        const data = { message, code };
+        const output = {
+            metadata: { ...input.metadata },
+            status: stream_1.StreamStatus.ERROR,
+            code,
+            data,
+        };
+        //send unacknowleded errors to the engine (it has no topic)
+        delete output.metadata.topic;
+        return output;
+    }
+    structureError(input, output) {
+        const message = output.data?.message ? output.data?.message.toString() : UNKNOWN_STATUS_MESSAGE;
+        const statusCode = output.code || output.data?.code;
+        const code = isNaN(statusCode) ? UNKNOWN_STATUS_CODE : parseInt(statusCode.toString());
+        const data = { message, code };
+        if (typeof output.data?.error === 'object') {
+            data.error = { ...output.data.error };
+        }
+        return {
+            status: stream_1.StreamStatus.ERROR,
+            code,
+            metadata: { ...input.metadata },
+            data
+        };
+    }
+    static async stopConsuming() {
+        for (const instance of [...StreamSignaler.signalers]) {
+            instance.stopConsuming();
+        }
+        await (0, utils_1.sleepFor)(BLOCK_TIME_MS);
+    }
+    async stopConsuming() {
+        this.shouldConsume = false;
+        this.logger.info(`stream-consumer-stopping`, this.topic ? { topic: this.topic } : undefined);
+        this.cancelThrottle();
+        await (0, utils_1.sleepFor)(BLOCK_TIME_MS);
+    }
+    cancelThrottle() {
+        if (this.currentTimerId !== undefined) {
+            clearTimeout(this.currentTimerId);
+            this.currentTimerId = undefined;
+        }
+    }
+    setThrottle(delayInMillis) {
+        if (!Number.isInteger(delayInMillis) || delayInMillis < 0) {
+            throw new Error('Throttle must be a non-negative integer');
+        }
+        this.throttle = delayInMillis;
+        this.logger.info(`stream-throttle-reset`, { delay: this.throttle, topic: this.topic });
+    }
+    async claimUnacknowledged(stream, group, consumer, idleTimeMs = this.reclaimDelay, limit = XPENDING_COUNT) {
+        let pendingMessages = [];
+        const pendingMessagesInfo = await this.stream.xpending(stream, group, '-', '+', limit); //[[ '1688768134881-0', 'testConsumer1', 1017, 1 ]]
+        for (const pendingMessageInfo of pendingMessagesInfo) {
+            if (Array.isArray(pendingMessageInfo)) {
+                const [id, , elapsedTimeMs, deliveryCount] = pendingMessageInfo;
+                if (elapsedTimeMs > idleTimeMs) {
+                    const reclaimedMessage = await this.stream.xclaim(stream, group, consumer, idleTimeMs, id);
+                    if (reclaimedMessage.length) {
+                        if (deliveryCount <= this.reclaimCount) {
+                            pendingMessages = pendingMessages.concat(reclaimedMessage);
+                        }
+                        else {
+                            await this.expireUnacknowledged(reclaimedMessage, stream, group, consumer, id, deliveryCount);
+                        }
+                    }
+                }
+            }
+        }
+        return pendingMessages;
+    }
+    async expireUnacknowledged(reclaimedMessage, stream, group, consumer, id, count) {
+        //The stream activity was not processed within established limits. Possibilities Include:
+        // 1) user error: the workers were not properly configured and are timing out
+        // 2a) system error: JSON is corrupt
+        //     i) bad/unwitting actor
+        //     ii) corrupt hardware/network/transport/etc
+        // 3b) system error: Redis unable to accept `xadd` request
+        // 4c) system error: Redis unable to accept `xdel`/`xack` request
+        this.logger.error('stream-message-max-delivery-count-exceeded', { id, stream, group, consumer, code: SYSTEM_STATUS_CODE, count });
+        const streamData = reclaimedMessage[0]?.[1]?.[1];
+        //fatal risk point 1 of 3): json is corrupt
+        const [err, input] = this.parseStreamData(streamData);
+        if (err) {
+            return this.logger.error('expire-unacknowledged-parse-fatal-error', { id, err });
+        }
+        else if (!input || !input.metadata) {
+            return this.logger.error('expire-unacknowledged-parse-fatal-error', { id });
+        }
+        let telemetry;
+        let messageId;
+        try {
+            telemetry = new telemetry_1.TelemetryService(this.appId);
+            telemetry.startStreamSpan(input, stream_1.StreamRole.SYSTEM);
+            telemetry.setStreamError(`Stream Message Max Delivery Count Exceeded`);
+            //fatal risk point 2 of 3): unable to publish error message (to notify the parent job)
+            const output = this.structureUnacknowledgedError(input);
+            messageId = await this.publishResponse(input, output);
+            telemetry.setStreamAttributes({ 'app.worker.mid': messageId });
+            //fatal risk point 3 of 3): unable to ack and delete stream message
+            await this.ackAndDelete(stream, group, id);
+        }
+        catch (err) {
+            if (messageId) {
+                this.logger.error('expire-unacknowledged-pub-fatal-error', { id, err, ...input.metadata });
+                telemetry.setStreamAttributes({ 'app.system.fatal': 'expire-unacknowledged-pub-fatal-error' });
+            }
+            else {
+                this.logger.error('expire-unacknowledged-ack-fatal-error', { id, err, ...input.metadata });
+                telemetry.setStreamAttributes({ 'app.system.fatal': 'expire-unacknowledged-ack-fatal-error' });
+            }
+        }
+        finally {
+            telemetry.endStreamSpan();
+        }
+    }
+    parseStreamData(str) {
+        try {
+            return [, JSON.parse(str)];
+        }
+        catch (e) {
+            return [e];
+        }
+    }
+}
+exports.StreamSignaler = StreamSignaler;
+StreamSignaler.signalers = new Set();