@agentica/core 0.32.2 → 0.32.3-dev.2

package/lib/index.mjs

@@ -808,93 +808,30 @@ function isAgenticaContext(ctx) {
     return typeof ctx.initialize === "function";
 }
 
-class AsyncQueueClosedError extends Error {
-    constructor(message) {
-        super(message);
-        this.name = "AsyncQueueClosedError";
-    }
+function __get_retry(limit) {
+    const retryFn = async (fn, prevError, attempt = 0) => {
+        try {
+            return await fn(prevError);
+        } catch (error) {
+            if (attempt >= limit - 1) {
+                throw error;
+            }
+            return retryFn(fn, error, attempt + 1);
+        }
+    };
+    return retryFn;
 }
 
-class AsyncQueue {
+class AssistantMessageEmptyError extends Error {
     constructor() {
-        this.queue = [];
-        this.resolvers = [];
-        this.closeResolvers = [];
-        this.emptyResolvers = [];
-        this.closed = false;
-    }
-    enqueue(item) {
-        if (this.closed) {
-            console.error(new AsyncQueueClosedError("Cannot enqueue item: queue is closed."));
-            return;
-        }
-        this.queue.push(item);
-        if (this.resolvers.length > 0) {
-            this.resolvers.shift()?.({
-                value: this.queue.shift(),
-                done: false
-            });
-        }
+        super();
     }
-    async dequeue() {
-        const item = (() => {
-            if (!this.isEmpty()) {
-                return {
-                    value: this.queue.shift(),
-                    done: false
-                };
-            }
-            if (this.isClosed()) {
-                return {
-                    value: undefined,
-                    done: true
-                };
-            }
-            return null;
-        })();
-        if (this.isEmpty() && this.emptyResolvers.length !== 0) {
-            this.emptyResolvers.forEach((resolve => resolve()));
-            this.emptyResolvers = [];
-        }
-        if (item !== null) {
-            return item;
-        }
-        return new Promise((resolve => this.resolvers.push(resolve)));
-    }
-    isEmpty() {
-        return this.queue.length === 0;
-    }
-    isClosed() {
-        return this.closed;
-    }
-    done() {
-        return this.isClosed() && this.isEmpty();
-    }
-    close() {
-        this.closed = true;
-        while (this.resolvers.length > 0) {
-            this.resolvers.shift()?.({
-                value: undefined,
-                done: true
-            });
-        }
-        this.closeResolvers.forEach((resolve => resolve()));
-    }
-    async waitUntilEmpty() {
-        if (this.isEmpty()) {
-            return Promise.resolve();
-        }
-        return new Promise((resolve => {
-            this.emptyResolvers.push(resolve);
-        }));
-    }
-    async waitClosed() {
-        if (this.isClosed()) {
-            return Promise.resolve();
-        }
-        return new Promise((resolve => {
-            this.closeResolvers.push(resolve);
-        }));
+}
+
+class AssistantMessageEmptyWithReasoningError extends AssistantMessageEmptyError {
+    constructor(reasoning) {
+        super();
+        this.reasoning = reasoning;
     }
 }
 
@@ -1100,17 +1037,109 @@ const ChatGptCompletionMessageUtil = {
     mergeToolCalls
 };
 
+class AsyncQueueClosedError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "AsyncQueueClosedError";
+    }
+}
+
+class AsyncQueue {
+    constructor() {
+        this.queue = [];
+        this.resolvers = [];
+        this.closeResolvers = [];
+        this.emptyResolvers = [];
+        this.closed = false;
+    }
+    enqueue(item) {
+        if (this.closed) {
+            console.error(new AsyncQueueClosedError("Cannot enqueue item: queue is closed."));
+            return;
+        }
+        this.queue.push(item);
+        if (this.resolvers.length > 0) {
+            this.resolvers.shift()?.({
+                value: this.queue.shift(),
+                done: false
+            });
+        }
+    }
+    async dequeue() {
+        const item = (() => {
+            if (!this.isEmpty()) {
+                return {
+                    value: this.queue.shift(),
+                    done: false
+                };
+            }
+            if (this.isClosed()) {
+                return {
+                    value: undefined,
+                    done: true
+                };
+            }
+            return null;
+        })();
+        if (this.isEmpty() && this.emptyResolvers.length !== 0) {
+            this.emptyResolvers.forEach((resolve => resolve()));
+            this.emptyResolvers = [];
+        }
+        if (item !== null) {
+            return item;
+        }
+        return new Promise((resolve => this.resolvers.push(resolve)));
+    }
+    isEmpty() {
+        return this.queue.length === 0;
+    }
+    isClosed() {
+        return this.closed;
+    }
+    done() {
+        return this.isClosed() && this.isEmpty();
+    }
+    close() {
+        this.closed = true;
+        while (this.resolvers.length > 0) {
+            this.resolvers.shift()?.({
+                value: undefined,
+                done: true
+            });
+        }
+        this.closeResolvers.forEach((resolve => resolve()));
+    }
+    async waitUntilEmpty() {
+        if (this.isEmpty()) {
+            return Promise.resolve();
+        }
+        return new Promise((resolve => {
+            this.emptyResolvers.push(resolve);
+        }));
+    }
+    async waitClosed() {
+        if (this.isClosed()) {
+            return Promise.resolve();
+        }
+        return new Promise((resolve => {
+            this.closeResolvers.push(resolve);
+        }));
+    }
+}
+
 class MPSC {
     constructor() {
         this.queue = new AsyncQueue;
         this.consumer = new ReadableStream({
-            pull: async controller => {
-                const {value, done} = await this.queue.dequeue();
-                if (done === true) {
-                    controller.close();
-                    return;
+            start: async controller => {
+                while (true) {
+                    const {value, done} = await this.queue.dequeue();
+                    if (done === true) {
+                        controller.close();
+                        return;
+                    }
+                    controller.enqueue(value);
                 }
-                controller.enqueue(value);
             }
         });
     }
@@ -1158,10 +1187,10 @@ async function reduce(stream, reducer, initial) {
     return acc;
 }
 
-function from(value) {
+function from(...value) {
     const stream = new ReadableStream({
         start: controller => {
-            controller.enqueue(value);
+            value.forEach((v => controller.enqueue(v)));
             controller.close();
         }
     });
@@ -1214,6 +1243,70 @@ var index$2 = Object.freeze({
     toAsyncGenerator
 });
 
+async function reduceStreamingWithDispatch(stream, eventProcessor) {
+    const streamContext = new Map;
+    const nullableCompletion = await StreamUtil.reduce(stream, (async (accPromise, chunk) => {
+        const acc = await accPromise;
+        const registerContext = choices => {
+            for (const choice of choices) {
+                if (choice.delta.content != null && choice.delta.content !== "") {
+                    if (streamContext.has(choice.index)) {
+                        const context = streamContext.get(choice.index);
+                        context.content += choice.delta.content;
+                        context.mpsc.produce(choice.delta.content);
+                    } else {
+                        const mpsc = new MPSC;
+                        streamContext.set(choice.index, {
+                            content: choice.delta.content,
+                            mpsc
+                        });
+                        mpsc.produce(choice.delta.content);
+                        eventProcessor({
+                            stream: streamDefaultReaderToAsyncGenerator(mpsc.consumer.getReader()),
+                            done: () => mpsc.done(),
+                            get: () => streamContext.get(choice.index)?.content ?? "",
+                            join: async () => {
+                                await mpsc.waitClosed();
+                                return streamContext.get(choice.index).content;
+                            }
+                        });
+                    }
+                }
+                if (choice.finish_reason != null) {
+                    const context = streamContext.get(choice.index);
+                    if (context != null) {
+                        context.mpsc.close();
+                    }
+                }
+            }
+        };
+        if (acc.object === "chat.completion.chunk") {
+            registerContext([ acc, chunk ].flatMap((v => v.choices)));
+            return ChatGptCompletionMessageUtil.merge([ acc, chunk ]);
+        }
+        registerContext(chunk.choices);
+        return ChatGptCompletionMessageUtil.accumulate(acc, chunk);
+    }));
+    if (nullableCompletion == null) {
+        throw new Error("StreamUtil.reduce did not produce a ChatCompletion. Possible causes: the input stream was empty, invalid, or closed prematurely. " + "To debug: check that the stream is properly initialized and contains valid ChatCompletionChunk data. " + "You may also enable verbose logging upstream to inspect the stream contents. " + `Stream locked: ${stream.locked}.`);
+    }
+    if (nullableCompletion.object === "chat.completion.chunk") {
+        const completion = ChatGptCompletionMessageUtil.merge([ nullableCompletion ]);
+        completion.choices.forEach((choice => {
+            if (choice.message.content != null && choice.message.content !== "") {
+                eventProcessor({
+                    stream: toAsyncGenerator(choice.message.content),
+                    done: () => true,
+                    get: () => choice.message.content,
+                    join: async () => choice.message.content
+                });
+            }
+        }));
+        return completion;
+    }
+    return nullableCompletion;
+}
+
 function createOperationSelection(props) {
     return {
         operation: props.operation,
@@ -1242,87 +1335,66 @@ function cancelFunctionFromContext(ctx, reference) {
 }
 
 async function call(ctx, operations) {
-    const stream = await ctx.request("call", {
-        messages: [ {
-            role: "system",
-            content: AgenticaDefaultPrompt.write(ctx.config)
-        }, ...ctx.histories.map(decodeHistory).flat(), {
-            role: "user",
-            content: ctx.prompt.contents.map(decodeUserMessageContent)
-        }, ...ctx.config?.systemPrompt?.execute === null ? [] : [ {
-            role: "system",
-            content: ctx.config?.systemPrompt?.execute?.(ctx.histories) ?? AgenticaSystemPrompt.EXECUTE
-        } ] ],
-        tools: operations.map((s => ({
-            type: "function",
-            function: {
-                name: s.name,
-                description: s.function.description,
-                parameters: "separated" in s.function && s.function.separated !== undefined ? s.function.separated.llm ?? {
-                    type: "object",
-                    properties: {},
-                    required: [],
-                    additionalProperties: false,
-                    $defs: {}
-                } : s.function.parameters
-            }
-        }))),
-        tool_choice: "auto"
-    });
-    const selectContext = [];
-    const nullableCompletion = await StreamUtil.reduce(stream, (async (accPromise, chunk) => {
-        const acc = await accPromise;
-        const registerContext = choices => {
-            for (const choice of choices) {
-                if (choice.finish_reason != null) {
-                    selectContext[choice.index]?.mpsc.close();
-                    continue;
-                }
-                if (choice.delta.content == null || choice.delta.content === "") {
-                    continue;
-                }
-                if (selectContext[choice.index] != null) {
-                    selectContext[choice.index].content += choice.delta.content;
-                    selectContext[choice.index].mpsc.produce(choice.delta.content);
-                    continue;
-                }
-                const mpsc = new MPSC;
-                selectContext[choice.index] = {
-                    content: choice.delta.content,
-                    mpsc
-                };
-                mpsc.produce(choice.delta.content);
-                const event = createAssistantMessageEvent({
-                    stream: streamDefaultReaderToAsyncGenerator(mpsc.consumer.getReader()),
-                    done: () => mpsc.done(),
-                    get: () => selectContext[choice.index]?.content ?? "",
-                    join: async () => {
-                        await mpsc.waitClosed();
-                        return selectContext[choice.index].content;
-                    }
-                });
-                ctx.dispatch(event);
-            }
-        };
-        if (acc.object === "chat.completion.chunk") {
-            registerContext([ acc, chunk ].flatMap((v => v.choices)));
-            return ChatGptCompletionMessageUtil.merge([ acc, chunk ]);
+    const _retryFn = __get_retry(ctx.config?.retry ?? AgenticaConstant.RETRY);
+    const retryFn = async fn => _retryFn(fn).catch((e => {
+        if (e instanceof AssistantMessageEmptyError) {
+            return Symbol("emptyAssistantMessage");
         }
-        registerContext(chunk.choices);
-        return ChatGptCompletionMessageUtil.accumulate(acc, chunk);
+        throw e;
     }));
-    const completion = nullableCompletion;
-    const emptyAssistantMessages = completion.choices.filter((v => v.message.tool_calls == null && v.message.content === ""));
-    if (emptyAssistantMessages.length > 0) {
-        emptyAssistantMessages.forEach((v => {
-            const event = createAssistantMessageEvent({
-                stream: toAsyncGenerator(v.message.content ?? ""),
-                done: () => true,
-                get: () => v.message.content ?? "",
-                join: async () => v.message.content ?? ""
-            });
+    const completion = await retryFn((async prevError => {
+        const stream = await ctx.request("call", {
+            messages: [ {
+                role: "system",
+                content: AgenticaDefaultPrompt.write(ctx.config)
+            }, ...ctx.histories.map(decodeHistory).flat(), {
+                role: "user",
+                content: ctx.prompt.contents.map(decodeUserMessageContent)
+            }, ...prevError instanceof AssistantMessageEmptyWithReasoningError ? [ {
+                role: "assistant",
+                content: prevError.reasoning
+            } ] : [], ...ctx.config?.systemPrompt?.execute === null ? [] : [ {
+                role: "system",
+                content: ctx.config?.systemPrompt?.execute?.(ctx.histories) ?? AgenticaSystemPrompt.EXECUTE
+            } ] ],
+            tools: operations.map((s => ({
+                type: "function",
+                function: {
+                    name: s.name,
+                    description: s.function.description,
+                    parameters: "separated" in s.function && s.function.separated !== undefined ? s.function.separated.llm ?? {
+                        type: "object",
+                        properties: {},
+                        required: [],
+                        additionalProperties: false,
+                        $defs: {}
+                    } : s.function.parameters
+                }
+            }))),
+            tool_choice: "auto"
+        });
+        const completion = await reduceStreamingWithDispatch(stream, (props => {
+            const event = createAssistantMessageEvent(props);
             ctx.dispatch(event);
         }));
+        const allAssistantMessagesEmpty = completion.choices.every((v => v.message.tool_calls == null && v.message.content === ""));
+        if (allAssistantMessagesEmpty) {
+            const firstChoice = completion.choices.at(0);
+            if (firstChoice?.message?.reasoning != null) {
+                throw new AssistantMessageEmptyWithReasoningError(firstChoice?.message?.reasoning ?? "");
+            }
+            throw new AssistantMessageEmptyError;
+        }
+        return completion;
+    }));
+    if (typeof completion === "symbol") {
+        const event = createAssistantMessageEvent({
+            stream: toAsyncGenerator(""),
+            done: () => true,
+            get: () => "",
+            join: async () => ""
+        });
+        ctx.dispatch(event);
         return [];
     }
     const executes = [];
@@ -1872,48 +1944,12 @@ async function describe(ctx, histories) {
             content: ctx.config?.systemPrompt?.describe?.(histories) ?? AgenticaSystemPrompt.DESCRIBE
         } ]
     });
-    const describeContext = [];
-    await StreamUtil.reduce(completionStream, (async (accPromise, chunk) => {
-        const acc = await accPromise;
-        const registerContext = choices => {
-            for (const choice of choices) {
-                if (choice.finish_reason != null) {
-                    describeContext[choice.index].mpsc.close();
-                    continue;
-                }
-                if (choice.delta.content == null) {
-                    continue;
-                }
-                if (describeContext[choice.index] != null) {
-                    describeContext[choice.index].content += choice.delta.content;
-                    describeContext[choice.index].mpsc.produce(choice.delta.content);
-                    continue;
-                }
-                const mpsc = new MPSC;
-                describeContext[choice.index] = {
-                    content: choice.delta.content,
-                    mpsc
-                };
-                mpsc.produce(choice.delta.content);
-                const event = createDescribeEvent({
-                    executes: histories,
-                    stream: streamDefaultReaderToAsyncGenerator(mpsc.consumer.getReader()),
-                    done: () => mpsc.done(),
-                    get: () => describeContext[choice.index]?.content ?? "",
-                    join: async () => {
-                        await mpsc.waitClosed();
-                        return describeContext[choice.index].content;
-                    }
-                });
-                ctx.dispatch(event);
-            }
-        };
-        if (acc.object === "chat.completion.chunk") {
-            registerContext([ acc, chunk ].flatMap((v => v.choices)));
-            return ChatGptCompletionMessageUtil.merge([ acc, chunk ]);
-        }
-        registerContext(chunk.choices);
-        return ChatGptCompletionMessageUtil.accumulate(acc, chunk);
+    await reduceStreamingWithDispatch(completionStream, (props => {
+        const event = createDescribeEvent({
+            executes: histories,
+            ...props
+        });
+        ctx.dispatch(event);
     }));
 }
 
@@ -2578,47 +2614,9 @@ async function initialize(ctx) {
         } ],
         tool_choice: "auto"
     });
-    const textContext = [];
-    const completion = await StreamUtil.reduce(completionStream, (async (accPromise, chunk) => {
-        const acc = await accPromise;
-        const registerContext = choices => {
-            for (const choice of choices) {
-                if (choice.finish_reason != null) {
-                    textContext[choice.index]?.mpsc.close();
-                    continue;
-                }
-                if (choice.delta.content == null || choice.delta.content.length === 0) {
-                    continue;
-                }
-                if (textContext[choice.index] != null) {
-                    textContext[choice.index].content += choice.delta.content;
-                    textContext[choice.index].mpsc.produce(choice.delta.content);
-                    continue;
-                }
-                const mpsc = new MPSC;
-                textContext[choice.index] = {
-                    content: choice.delta.content,
-                    mpsc
-                };
-                mpsc.produce(choice.delta.content);
-                const event = createAssistantMessageEvent({
-                    stream: streamDefaultReaderToAsyncGenerator(mpsc.consumer.getReader()),
-                    done: () => mpsc.done(),
-                    get: () => textContext[choice.index].content,
-                    join: async () => {
-                        await mpsc.waitClosed();
-                        return textContext[choice.index].content;
-                    }
-                });
-                ctx.dispatch(event);
-            }
-        };
-        if (acc.object === "chat.completion.chunk") {
-            registerContext([ acc, chunk ].flatMap((v => v.choices)));
-            return ChatGptCompletionMessageUtil.merge([ acc, chunk ]);
-        }
-        registerContext(chunk.choices);
-        return ChatGptCompletionMessageUtil.accumulate(acc, chunk);
+    const completion = await reduceStreamingWithDispatch(completionStream, (props => {
+        const event = createAssistantMessageEvent(props);
+        ctx.dispatch(event);
     }));
     if (completion === null) {
         throw new Error("No completion received");
@@ -2766,103 +2764,83 @@ async function select(ctx) {
 }
 
 async function step(ctx, operations, retry, failures) {
-    const completionStream = await ctx.request("select", {
-        messages: [ {
-            role: "system",
-            content: AgenticaDefaultPrompt.write(ctx.config)
-        }, {
-            role: "assistant",
-            tool_calls: [ {
+    const _retryFn = __get_retry(ctx.config?.retry ?? AgenticaConstant.RETRY);
+    const retryFn = async fn => _retryFn(fn).catch((e => {
+        if (e instanceof AssistantMessageEmptyError) {
+            return Symbol("emptyAssistantMessage");
+        }
+        throw e;
+    }));
+    const completion = await retryFn((async prevError => {
+        const stream = await ctx.request("select", {
+            messages: [ {
+                role: "system",
+                content: AgenticaDefaultPrompt.write(ctx.config)
+            }, {
+                role: "assistant",
+                tool_calls: [ {
+                    type: "function",
+                    id: "getApiFunctions",
+                    function: {
+                        name: "getApiFunctions",
+                        arguments: JSON.stringify({})
+                    }
+                } ]
+            }, {
+                role: "tool",
+                tool_call_id: "getApiFunctions",
+                content: JSON.stringify(operations.map((op => ({
+                    name: op.name,
+                    description: op.function.description,
+                    ...op.protocol === "http" ? {
+                        method: op.function.method,
+                        path: op.function.path,
+                        tags: op.function.tags
+                    } : {}
+                }))))
+            }, ...ctx.histories.map(decodeHistory).flat(), {
+                role: "user",
+                content: ctx.prompt.contents.map(decodeUserMessageContent)
+            }, ...prevError instanceof AssistantMessageEmptyWithReasoningError ? [ {
+                role: "assistant",
+                content: prevError.reasoning
+            } ] : [], {
+                role: "system",
+                content: ctx.config?.systemPrompt?.select?.(ctx.histories) ?? AgenticaSystemPrompt.SELECT
+            }, ...emendMessages(failures ?? []) ],
+            tools: [ {
                 type: "function",
-                id: "getApiFunctions",
                 function: {
-                    name: "getApiFunctions",
-                    arguments: JSON.stringify({})
-                }
-            } ]
-        }, {
-            role: "tool",
-            tool_call_id: "getApiFunctions",
-            content: JSON.stringify(operations.map((op => ({
-                name: op.name,
-                description: op.function.description,
-                ...op.protocol === "http" ? {
-                    method: op.function.method,
-                    path: op.function.path,
-                    tags: op.function.tags
-                } : {}
-            }))))
-        }, ...ctx.histories.map(decodeHistory).flat(), {
-            role: "user",
-            content: ctx.prompt.contents.map(decodeUserMessageContent)
-        }, {
-            role: "system",
-            content: ctx.config?.systemPrompt?.select?.(ctx.histories) ?? AgenticaSystemPrompt.SELECT
-        }, ...emendMessages(failures ?? []) ],
-        tools: [ {
-            type: "function",
-            function: {
-                name: CONTAINER.functions[0].name,
-                description: CONTAINER.functions[0].description,
-                parameters: CONTAINER.functions[0].parameters
-            }
-        } ],
-        tool_choice: retry === 0 ? "auto" : "required"
-    });
-    const selectContext = [];
-    const nullableCompletion = await StreamUtil.reduce(completionStream, (async (accPromise, chunk) => {
-        const acc = await accPromise;
-        const registerContext = choices => {
-            for (const choice of choices) {
-                if (choice.finish_reason != null) {
-                    selectContext[choice.index]?.mpsc.close();
-                    continue;
+                    name: CONTAINER.functions[0].name,
+                    description: CONTAINER.functions[0].description,
+                    parameters: CONTAINER.functions[0].parameters
                 }
-                if (choice.delta.content == null || choice.delta.content === "") {
-                    continue;
-                }
-                if (selectContext[choice.index] != null) {
-                    selectContext[choice.index].content += choice.delta.content;
-                    selectContext[choice.index].mpsc.produce(choice.delta.content);
-                    continue;
-                }
-                const mpsc = new MPSC;
-                selectContext[choice.index] = {
-                    content: choice.delta.content,
-                    mpsc
-                };
-                mpsc.produce(choice.delta.content);
-                const event = createAssistantMessageEvent({
-                    stream: streamDefaultReaderToAsyncGenerator(mpsc.consumer.getReader()),
-                    done: () => mpsc.done(),
-                    get: () => selectContext[choice.index]?.content ?? "",
-                    join: async () => {
-                        await mpsc.waitClosed();
-                        return selectContext[choice.index].content;
-                    }
-                });
-                ctx.dispatch(event);
+            } ],
+            tool_choice: retry === 0 ? "auto" : "required"
+        });
+        const completion = await reduceStreamingWithDispatch(stream, (props => {
+            const event = createAssistantMessageEvent(props);
+            ctx.dispatch(event);
+        }));
+        const allAssistantMessagesEmpty = completion.choices.every((v => v.message.tool_calls == null && v.message.content === ""));
+        if (allAssistantMessagesEmpty) {
+            const firstChoice = completion.choices.at(0);
+            if (firstChoice?.message?.reasoning != null) {
+                throw new AssistantMessageEmptyWithReasoningError(firstChoice?.message?.reasoning ?? "");
             }
-        };
-        if (acc.object === "chat.completion.chunk") {
-            registerContext([ acc, chunk ].flatMap((v => v.choices)));
-            return ChatGptCompletionMessageUtil.merge([ acc, chunk ]);
+            throw new AssistantMessageEmptyError;
         }
-        registerContext(chunk.choices);
-        return ChatGptCompletionMessageUtil.accumulate(acc, chunk);
+        return completion;
     }));
-    const completion = nullableCompletion;
-    const emptyAssistantMessages = completion.choices.filter((v => v.message.tool_calls == null && v.message.content === ""));
-    if (emptyAssistantMessages.length > 0) {
-        emptyAssistantMessages.forEach((v => {
-            const event = createAssistantMessageEvent({
-                stream: toAsyncGenerator(v.message.content ?? ""),
-                done: () => true,
-                get: () => v.message.content ?? "",
-                join: async () => v.message.content ?? ""
-            });
-            ctx.dispatch(event);
-        }));
+    if (typeof completion === "symbol") {
+        const event = createAssistantMessageEvent({
+            stream: toAsyncGenerator(""),
+            done: () => true,
+            get: () => "",
+            join: async () => ""
+        });
+        ctx.dispatch(event);
+        return;
     }
     if (retry++ < (ctx.config?.retry ?? AgenticaConstant.RETRY)) {
         const failures = [];