@adminforth/agent 1.34.2 → 1.36.0

package/dist/index.js

@@ -47,6 +47,19 @@ function formatAgentError(error) {
     }
     return String(error);
 }
+function formatAgentResponseError(error) {
+    if (isAggregateErrorLike(error)) {
+        const nestedErrors = error.errors.map(formatAgentResponseError);
+        if (nestedErrors.length) {
+            return nestedErrors.join("\n");
+        }
+        return error.message || "Agent response failed";
+    }
+    if (error instanceof Error) {
+        return error.toString();
+    }
+    return String(error);
+}
 function formatAdminUserPrompt(adminUser, usernameField) {
     const dbUser = adminUser.dbUser;
     const adminUserContext = {
@@ -59,16 +72,6 @@ function formatAdminUserPrompt(adminUser, usernameField) {
         "Use this admin user email when the user asks to send information to themselves, the current admin, or the logged-in user.",
     ].join("\n");
 }
-function formatCurrentPagePrompt(currentPage) {
-    if (!currentPage) {
-        return null;
-    }
-    return [
-        "Current user page context for the latest message:",
-        JSON.stringify(currentPage, null, 2),
-        "When the user says here, this page, current page, or opened page, treat it as this page.",
-    ].join("\n");
-}
 function assertRequiredApiTool(apiBasedTools, toolName) {
     if (toolName in apiBasedTools) {
         return;
@@ -191,6 +194,7 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                     modes: this.options.modes.map((mode) => ({ name: mode.name })),
                     defaultModeName: this.options.modes[0].name,
                     stickByDefault: (_b = this.options.stickByDefault) !== null && _b !== void 0 ? _b : false,
+                    hasAudioAdapter: Boolean(this.options.audioAdapter),
                 }
             });
             if (!this.pluginOptions.sessionResource) {
@@ -199,12 +203,110 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
         });
     }
     validateConfigAfterDiscover(adminforth, resourceConfig) {
+        var _a;
+        (_a = this.options.audioAdapter) === null || _a === void 0 ? void 0 : _a.validate();
         this.agentSystemPromptPromise = buildAgentSystemPrompt(adminforth, this.getInternalAgentResourceIds())
             .then((systemPrompt) => appendCustomSystemPrompt(systemPrompt, this.options.systemPrompt));
     }
     instanceUniqueRepresentation(pluginOptions) {
         return `single`;
     }
+    runAgentTurn(input) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            var _d, _e, _f, _g;
+            let fullResponse = "";
+            const maxTokens = (_d = this.options.maxTokens) !== null && _d !== void 0 ? _d : 10000;
+            const selectedMode = (_e = this.options.modes.find((mode) => mode.name === input.modeName)) !== null && _e !== void 0 ? _e : this.options.modes[0];
+            const { model, summaryModel, modelMiddleware } = yield this.getModeModels(selectedMode, maxTokens);
+            const userLanguage = yield detectUserLanguage(selectedMode.completionAdapter, input.prompt)
+                .catch((error) => {
+                logger.warn(`Failed to detect user language: ${error instanceof Error ? error.message : String(error)}`);
+                return null;
+            });
+            const systemPrompt = [
+                yield this.agentSystemPromptPromise,
+                formatAdminUserPrompt(input.adminUser, this.adminforth.config.auth.usernameField),
+                formatLanguagePrompt(userLanguage),
+            ].join("\n\n");
+            const apiBasedTools = buildApiBasedTools(this.adminforth, this.getInternalAgentResourceIds());
+            for (const toolName of ALWAYS_AVAILABLE_API_TOOL_NAMES) {
+                assertRequiredApiTool(apiBasedTools, toolName);
+            }
+            assertRequiredApiTool(apiBasedTools, "update_record");
+            this.apiBasedTools = apiBasedTools;
+            const stream = yield callAgent({
+                name: `adminforth-agent-${this.pluginInstanceId}`,
+                model,
+                summaryModel,
+                modelMiddleware,
+                checkpointer: this.getCheckpointer(),
+                messages: [
+                    new SystemMessage(systemPrompt),
+                    new HumanMessage(input.prompt),
+                ],
+                adminUser: input.adminUser,
+                adminforth: this.adminforth,
+                apiBasedTools,
+                customComponentsDir: this.adminforth.config.customization.customComponentsDir,
+                sessionId: input.sessionId,
+                turnId: input.turnId,
+                currentPage: input.currentPage,
+                httpExtra: input.httpExtra,
+                userTimeZone: input.userTimeZone,
+                emitToolCallEvent: (event) => {
+                    var _a;
+                    input.sequenceDebugCollector.handleToolCallEvent(event);
+                    (_a = input.emitToolCallEvent) === null || _a === void 0 ? void 0 : _a.call(input, event);
+                },
+                sequenceDebugSink: input.sequenceDebugCollector,
+            });
+            try {
+                for (var _h = true, _j = __asyncValues(stream), _k; _k = yield _j.next(), _a = _k.done, !_a; _h = true) {
+                    _c = _k.value;
+                    _h = false;
+                    const rawChunk = _c;
+                    const [token, metadata] = rawChunk;
+                    const nodeName = typeof (metadata === null || metadata === void 0 ? void 0 : metadata.langgraph_node) === "string"
+                        ? metadata.langgraph_node
+                        : "";
+                    if (nodeName && !["model", "model_request"].includes(nodeName)) {
+                        continue;
+                    }
+                    const blocks = Array.isArray(token === null || token === void 0 ? void 0 : token.contentBlocks)
+                        ? token.contentBlocks
+                        : Array.isArray(token === null || token === void 0 ? void 0 : token.content)
+                            ? token.content
+                            : [];
+                    const reasoningDelta = blocks
+                        .filter((b) => (b === null || b === void 0 ? void 0 : b.type) === "reasoning")
+                        .map((b) => { var _a; return String((_a = b.reasoning) !== null && _a !== void 0 ? _a : ""); })
+                        .join("");
+                    const textDelta = blocks
+                        .filter((b) => (b === null || b === void 0 ? void 0 : b.type) === "text")
+                        .map((b) => { var _a; return String((_a = b.text) !== null && _a !== void 0 ? _a : ""); })
+                        .join("");
+                    if (reasoningDelta) {
+                        (_f = input.emitReasoningDelta) === null || _f === void 0 ? void 0 : _f.call(input, reasoningDelta);
+                    }
+                    if (textDelta) {
+                        fullResponse += textDelta;
+                        (_g = input.emitTextDelta) === null || _g === void 0 ? void 0 : _g.call(input, textDelta);
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_h && !_a && (_b = _j.return)) yield _b.call(_j);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            return {
+                text: fullResponse,
+            };
+        });
+    }
     setupEndpoints(server) {
         server.endpoint({
             method: 'POST',
@@ -234,13 +336,12 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
         server.endpoint({
             method: 'POST',
             path: `/agent/response`,
-            handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body, query, headers, cookies, adminUser, response, requestUrl, _raw_express_res }) {
-                var _b, e_1, _c, _d;
-                var _e, _f, _g;
+            handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body, query, headers, cookies, adminUser, response, requestUrl, _raw_express_res, abortSignal }) {
+                var _b;
                 const res = _raw_express_res;
                 const messageId = randomUUID();
                 const prompt = body.message;
-                const userTimeZone = (_e = body.timeZone) !== null && _e !== void 0 ? _e : 'UTC';
+                const userTimeZone = (_b = body.timeZone) !== null && _b !== void 0 ? _b : 'UTC';
                 const currentPage = body.currentPage;
                 const sessionId = body.sessionId || (adminUser === null || adminUser === void 0 ? void 0 : adminUser.pk) || (adminUser === null || adminUser === void 0 ? void 0 : adminUser.username) || 'default';
                 const turnId = yield this.createNewTurn(sessionId, prompt);
@@ -264,7 +365,6 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                     if (event.phase === "start") {
                         endActiveBlock();
                     }
-                    sequenceDebugCollector.handleToolCallEvent(event);
                     send({
                         type: "data-tool-call",
                         data: event,
@@ -311,43 +411,14 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                         type: 'start',
                         messageId,
                     });
-                    const maxTokens = (_f = this.options.maxTokens) !== null && _f !== void 0 ? _f : 10000;
-                    const selectedMode = (_g = this.options.modes.find((mode) => mode.name === body.mode)) !== null && _g !== void 0 ? _g : this.options.modes[0];
-                    const { model, summaryModel, modelMiddleware } = yield this.getModeModels(selectedMode, maxTokens);
-                    const userLanguage = yield detectUserLanguage(selectedMode.completionAdapter, prompt)
-                        .catch((error) => {
-                        logger.warn(`Failed to detect user language: ${error instanceof Error ? error.message : String(error)}`);
-                        return null;
-                    });
-                    const systemPrompt = [
-                        yield this.agentSystemPromptPromise,
-                        formatAdminUserPrompt(adminUser, this.adminforth.config.auth.usernameField),
-                        formatLanguagePrompt(userLanguage),
-                    ].join("\n\n");
-                    const apiBasedTools = buildApiBasedTools(this.adminforth, this.getInternalAgentResourceIds());
-                    for (const toolName of ALWAYS_AVAILABLE_API_TOOL_NAMES) {
-                        assertRequiredApiTool(apiBasedTools, toolName);
-                    }
-                    assertRequiredApiTool(apiBasedTools, "update_record");
-                    this.apiBasedTools = apiBasedTools;
-                    const currentPagePrompt = formatCurrentPagePrompt(currentPage);
-                    const stream = yield callAgent({
-                        name: `adminforth-agent-${this.pluginInstanceId}`,
-                        model,
-                        summaryModel,
-                        modelMiddleware,
-                        checkpointer: this.getCheckpointer(),
-                        messages: [
-                            new SystemMessage(systemPrompt),
-                            ...(currentPagePrompt ? [new SystemMessage(currentPagePrompt)] : []),
-                            new HumanMessage(prompt),
-                        ],
-                        adminUser,
-                        adminforth: this.adminforth,
-                        apiBasedTools,
-                        customComponentsDir: this.adminforth.config.customization.customComponentsDir,
+                    const agentResponse = yield this.runAgentTurn({
+                        prompt,
                         sessionId,
                         turnId,
+                        modeName: body.mode,
+                        userTimeZone,
+                        currentPage,
+                        adminUser,
                         httpExtra: {
                             body,
                             query,
@@ -356,70 +427,37 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                             requestUrl,
                             response,
                         },
-                        userTimeZone,
+                        sequenceDebugCollector,
                         emitToolCallEvent,
-                        sequenceDebugSink: sequenceDebugCollector,
+                        emitReasoningDelta: (reasoningDelta) => {
+                            const reasoningId = startBlock('reasoning');
+                            send({
+                                type: 'reasoning-delta',
+                                id: reasoningId,
+                                delta: reasoningDelta,
+                            });
+                        },
+                        emitTextDelta: (textDelta) => {
+                            const textId = startBlock('text');
+                            fullResponse += textDelta;
+                            send({
+                                type: 'text-delta',
+                                id: textId,
+                                delta: textDelta,
+                            });
+                        },
                     });
-                    try {
-                        for (var _h = true, _j = __asyncValues(stream), _k; _k = yield _j.next(), _b = _k.done, !_b; _h = true) {
-                            _d = _k.value;
-                            _h = false;
-                            const rawChunk = _d;
-                            const [token, metadata] = rawChunk;
-                            const nodeName = typeof (metadata === null || metadata === void 0 ? void 0 : metadata.langgraph_node) === "string"
-                                ? metadata.langgraph_node
-                                : "";
-                            if (nodeName && !["model", "model_request"].includes(nodeName)) {
-                                continue;
-                            }
-                            const blocks = Array.isArray(token === null || token === void 0 ? void 0 : token.contentBlocks)
-                                ? token.contentBlocks
-                                : Array.isArray(token === null || token === void 0 ? void 0 : token.content)
-                                    ? token.content
-                                    : [];
-                            const reasoningDelta = blocks
-                                .filter((b) => (b === null || b === void 0 ? void 0 : b.type) === "reasoning")
-                                .map((b) => { var _a; return String((_a = b.reasoning) !== null && _a !== void 0 ? _a : ""); })
-                                .join("");
-                            const textDelta = blocks
-                                .filter((b) => (b === null || b === void 0 ? void 0 : b.type) === "text")
-                                .map((b) => { var _a; return String((_a = b.text) !== null && _a !== void 0 ? _a : ""); })
-                                .join("");
-                            if (reasoningDelta) {
-                                const reasoningId = startBlock('reasoning');
-                                send({
-                                    type: 'reasoning-delta',
-                                    id: reasoningId,
-                                    delta: reasoningDelta,
-                                });
-                            }
-                            if (textDelta) {
-                                const textId = startBlock('text');
-                                fullResponse += textDelta;
-                                send({
-                                    type: 'text-delta',
-                                    id: textId,
-                                    delta: textDelta,
-                                });
-                            }
-                        }
-                    }
-                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                    finally {
-                        try {
-                            if (!_h && !_b && (_c = _j.return)) yield _c.call(_j);
-                        }
-                        finally { if (e_1) throw e_1.error; }
-                    }
+                    fullResponse = agentResponse.text;
                 }
                 catch (error) {
                     logger.error(`Agent response streaming failed:\n${formatAgentError(error)}`);
                     sequenceDebugCollector.flush();
+                    fullResponse = formatAgentResponseError(error);
                     const textId = startBlock('text');
                     send({
                         type: 'text-delta',
                         id: textId,
-                        delta: 'Agent response failed. Check server logs for details.',
+                        delta: fullResponse,
                     });
                 }
                 sequenceDebugCollector.flush();
@@ -434,6 +472,115 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                 return null;
             })
         });
+        server.endpoint({
+            method: 'POST',
+            path: `/agent/speech-response`,
+            handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body, query, headers, cookies, adminUser, response, requestUrl }) {
+                var _b;
+                const audioAdapter = this.options.audioAdapter;
+                if (!audioAdapter) {
+                    response.setStatus(400, undefined);
+                    return {
+                        error: "Audio adapter is not configured for AdminForth Agent",
+                    };
+                }
+                const speechBody = body;
+                let transcription;
+                try {
+                    transcription = yield audioAdapter.transcribe({
+                        buffer: Buffer.from(speechBody.audioBase64, "base64"),
+                        filename: speechBody.filename,
+                        mimeType: speechBody.mimeType,
+                        language: "auto",
+                        prompt: speechBody.prompt,
+                    });
+                }
+                catch (error) {
+                    logger.error(`Agent speech transcription failed:\n${formatAgentError(error)}`);
+                    response.setStatus(500, undefined);
+                    return {
+                        error: "Speech transcription failed. Check server logs for details.",
+                    };
+                }
+                const prompt = transcription.text;
+                if (!prompt) {
+                    response.setStatus(400, undefined);
+                    return {
+                        error: "Speech transcription is empty",
+                    };
+                }
+                const sessionId = speechBody.sessionId || (adminUser === null || adminUser === void 0 ? void 0 : adminUser.pk) || (adminUser === null || adminUser === void 0 ? void 0 : adminUser.username) || 'default';
+                const turnId = yield this.createNewTurn(sessionId, prompt);
+                yield this.updateSessionDate(sessionId);
+                const sequenceDebugCollector = createSequenceDebugCollector();
+                let fullResponse = "";
+                try {
+                    const agentResponse = yield this.runAgentTurn({
+                        prompt,
+                        sessionId,
+                        turnId,
+                        modeName: speechBody.mode,
+                        userTimeZone: (_b = speechBody.timeZone) !== null && _b !== void 0 ? _b : 'UTC',
+                        currentPage: speechBody.currentPage,
+                        adminUser,
+                        httpExtra: {
+                            body,
+                            query,
+                            headers,
+                            cookies,
+                            requestUrl,
+                            response,
+                        },
+                        sequenceDebugCollector,
+                        emitTextDelta: (textDelta) => {
+                            fullResponse += textDelta;
+                        },
+                    });
+                    fullResponse = agentResponse.text;
+                    const speech = yield audioAdapter.synthesize(Object.assign({ text: fullResponse }, speechBody.tts));
+                    sequenceDebugCollector.flush();
+                    const turnUpdates = {
+                        [this.options.turnResource.responseField]: fullResponse,
+                    };
+                    if (this.options.turnResource.debugField) {
+                        turnUpdates[this.options.turnResource.debugField] = sequenceDebugCollector.getHistory();
+                    }
+                    yield this.updateTurn(turnId, turnUpdates);
+                    return {
+                        transcript: {
+                            text: transcription.text,
+                            language: transcription.language,
+                        },
+                        response: {
+                            text: fullResponse,
+                        },
+                        audio: {
+                            base64: speech.audio.toString("base64"),
+                            mimeType: speech.mimeType,
+                            format: speech.format,
+                        },
+                        sessionId,
+                        turnId,
+                    };
+                }
+                catch (error) {
+                    logger.error(`Agent speech response failed:\n${formatAgentError(error)}`);
+                    sequenceDebugCollector.flush();
+                    fullResponse = formatAgentResponseError(error);
+                    const turnUpdates = {
+                        [this.options.turnResource.responseField]: fullResponse,
+                    };
+                    if (this.options.turnResource.debugField) {
+                        turnUpdates[this.options.turnResource.debugField] = sequenceDebugCollector.getHistory();
+                    }
+                    yield this.updateTurn(turnId, turnUpdates);
+                    response.setStatus(500, undefined);
+                    return {
+                        error: fullResponse,
+                    };
+                }
+            })
+        });
         server.endpoint({
             method: 'POST',
             path: `/agent/get-sessions`,
@@ -543,6 +690,18 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                     ok: true
                 };
             })
-        });
+        }),
+            server.endpoint({
+                method: 'POST',
+                path: `/agent/add-system-message-to-turns`,
+                handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body, adminUser, _raw_express_req }) {
+                    const sessionId = body.sessionId;
+                    const systemMessage = body.systemMessage;
+                    yield this.createNewTurn(sessionId, systemMessage);
+                    return {
+                        ok: true
+                    };
+                })
+            });
     }
 }