@adminforth/agent 1.34.2 → 1.35.0

package/agent/simpleAgent.ts

@@ -19,12 +19,14 @@ import {
 } from "./middleware/sequenceDebug.js";
 import type { ApiBasedTool } from "../apiBasedTools.js";
 import type { ToolCallEventSink } from "./toolCallEvents.js";
+import type { CurrentPageContext } from "./tools/getUserLocation.js";
 
 export const contextSchema = z.object({
   adminUser: z.custom<AdminUser>(),
   userTimeZone: z.string(),
   sessionId: z.string(),
   turnId: z.string(),
+  currentPage: z.custom<CurrentPageContext>().optional(),
   httpExtra: z.custom<Partial<HttpExtra>>().optional(),
   emitToolCallEvent: z.custom<ToolCallEventSink>(),
 });
@@ -231,6 +233,7 @@ export async function callAgent(params: {
   customComponentsDir: string;
   sessionId: string;
   turnId: string;
+  currentPage?: CurrentPageContext;
   httpExtra?: Partial<HttpExtra>;
   userTimeZone: string;
   emitToolCallEvent: ToolCallEventSink;
@@ -249,6 +252,7 @@ export async function callAgent(params: {
     customComponentsDir,
     sessionId,
     turnId,
+    currentPage,
     httpExtra,
     userTimeZone,
     emitToolCallEvent,
@@ -293,6 +297,7 @@ export async function callAgent(params: {
       userTimeZone,
       sessionId,
       turnId,
+      currentPage,
       httpExtra,
       emitToolCallEvent,
     },

package/agent/tools/getUserLocation.ts

@@ -0,0 +1,45 @@
+import { tool } from "langchain";
+import { z } from "zod";
+
+export type CurrentPageContext = {
+  path: string;
+  fullPath: string;
+  title: string;
+  url: string;
+};
+
+const getUserLocationSchema = z.object({});
+
+export function createGetUserLocationTool() {
+  return tool(
+    async (_input, runtime) => {
+      const currentPage = (runtime.context as { currentPage?: CurrentPageContext }).currentPage;
+
+      if (!currentPage) {
+        return JSON.stringify(
+          {
+            status: 404,
+            message: "Current user location is not available.",
+          },
+          null,
+          2,
+        );
+      }
+
+      return JSON.stringify(
+        {
+          status: 200,
+          location: currentPage,
+        },
+        null,
+        2,
+      );
+    },
+    {
+      name: "get_user_location",
+      description:
+        "Get the user's current location in the AdminForth UI, including current page path, full path, title, and URL. Call this tool when you do not understand what the user is referring to, especially when they say here, this page, current page, opened page, or otherwise rely on page context.",
+      schema: getUserLocationSchema,
+    },
+  );
+}

package/agent/tools/index.ts

@@ -3,6 +3,7 @@ import { createFetchSkillTool } from "./fetchSkill.js";
 import { createFetchToolSchemaTool } from "./fetchToolSchema.js";
 import type { ApiBasedTool } from "../../apiBasedTools.js";
 import { createApiTool } from "./apiTool.js";
+import { createGetUserLocationTool } from "./getUserLocation.js";
 
 export const ALWAYS_AVAILABLE_API_TOOL_NAMES = ["get_resource"] as const;
 
@@ -20,6 +21,7 @@ export async function createAgentTools(
 
       return createApiTool(toolName, apiBasedTool);
     }),
+    createGetUserLocationTool(),
     await createFetchSkillTool(customComponentsDir),
     await createFetchToolSchemaTool(apiBasedTools),
   ];

package/build.log

@@ -40,5 +40,5 @@ custom/skills/fetch_data/SKILL.md
 custom/skills/mutate_data/
 custom/skills/mutate_data/SKILL.md
 
-sent 210,653 bytes  received 585 bytes  422,476.00 bytes/sec
+sent 210,681 bytes  received 581 bytes  422,524.00 bytes/sec
 total size is 208,279  speedup is 0.99

package/dist/agent/simpleAgent.js

@@ -19,6 +19,7 @@ export const contextSchema = z.object({
     userTimeZone: z.string(),
     sessionId: z.string(),
     turnId: z.string(),
+    currentPage: z.custom().optional(),
     httpExtra: z.custom().optional(),
     emitToolCallEvent: z.custom(),
 });
@@ -131,7 +132,7 @@ export function createAgentChatModel(params) {
 }
 export function callAgent(params) {
     return __awaiter(this, void 0, void 0, function* () {
-        const { name, model, summaryModel, modelMiddleware = [], checkpointer, messages, adminUser, adminforth, apiBasedTools, customComponentsDir, sessionId, turnId, httpExtra, userTimeZone, emitToolCallEvent, sequenceDebugSink, } = params;
+        const { name, model, summaryModel, modelMiddleware = [], checkpointer, messages, adminUser, adminforth, apiBasedTools, customComponentsDir, sessionId, turnId, currentPage, httpExtra, userTimeZone, emitToolCallEvent, sequenceDebugSink, } = params;
         const tools = yield createAgentTools(customComponentsDir, apiBasedTools);
         const apiBasedToolsMiddleware = createApiBasedToolsMiddleware(apiBasedTools, adminforth);
         const sequenceDebugMiddleware = createSequenceDebugMiddleware(sequenceDebugSink);
@@ -165,6 +166,7 @@ export function callAgent(params) {
                 userTimeZone,
                 sessionId,
                 turnId,
+                currentPage,
                 httpExtra,
                 emitToolCallEvent,
             },

package/dist/agent/tools/getUserLocation.js

@@ -0,0 +1,31 @@
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+import { tool } from "langchain";
+import { z } from "zod";
+const getUserLocationSchema = z.object({});
+export function createGetUserLocationTool() {
+    return tool((_input, runtime) => __awaiter(this, void 0, void 0, function* () {
+        const currentPage = runtime.context.currentPage;
+        if (!currentPage) {
+            return JSON.stringify({
+                status: 404,
+                message: "Current user location is not available.",
+            }, null, 2);
+        }
+        return JSON.stringify({
+            status: 200,
+            location: currentPage,
+        }, null, 2);
+    }), {
+        name: "get_user_location",
+        description: "Get the user's current location in the AdminForth UI, including current page path, full path, title, and URL. Call this tool when you do not understand what the user is referring to, especially when they say here, this page, current page, opened page, or otherwise rely on page context.",
+        schema: getUserLocationSchema,
+    });
+}

package/dist/agent/tools/index.js

@@ -10,6 +10,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import { createFetchSkillTool } from "./fetchSkill.js";
 import { createFetchToolSchemaTool } from "./fetchToolSchema.js";
 import { createApiTool } from "./apiTool.js";
+import { createGetUserLocationTool } from "./getUserLocation.js";
 export const ALWAYS_AVAILABLE_API_TOOL_NAMES = ["get_resource"];
 export function createAgentTools(customComponentsDir, apiBasedTools) {
     return __awaiter(this, void 0, void 0, function* () {
@@ -21,6 +22,7 @@ export function createAgentTools(customComponentsDir, apiBasedTools) {
                 }
                 return createApiTool(toolName, apiBasedTool);
             }),
+            createGetUserLocationTool(),
             yield createFetchSkillTool(customComponentsDir),
             yield createFetchToolSchemaTool(apiBasedTools),
         ];

package/dist/index.js

@@ -47,6 +47,19 @@ function formatAgentError(error) {
     }
     return String(error);
 }
+function formatAgentResponseError(error) {
+    if (isAggregateErrorLike(error)) {
+        const nestedErrors = error.errors.map(formatAgentResponseError);
+        if (nestedErrors.length) {
+            return nestedErrors.join("\n");
+        }
+        return error.message || "Agent response failed";
+    }
+    if (error instanceof Error) {
+        return error.toString();
+    }
+    return String(error);
+}
 function formatAdminUserPrompt(adminUser, usernameField) {
     const dbUser = adminUser.dbUser;
     const adminUserContext = {
@@ -59,16 +72,6 @@ function formatAdminUserPrompt(adminUser, usernameField) {
         "Use this admin user email when the user asks to send information to themselves, the current admin, or the logged-in user.",
     ].join("\n");
 }
-function formatCurrentPagePrompt(currentPage) {
-    if (!currentPage) {
-        return null;
-    }
-    return [
-        "Current user page context for the latest message:",
-        JSON.stringify(currentPage, null, 2),
-        "When the user says here, this page, current page, or opened page, treat it as this page.",
-    ].join("\n");
-}
 function assertRequiredApiTool(apiBasedTools, toolName) {
     if (toolName in apiBasedTools) {
         return;
@@ -191,6 +194,7 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                     modes: this.options.modes.map((mode) => ({ name: mode.name })),
                     defaultModeName: this.options.modes[0].name,
                     stickByDefault: (_b = this.options.stickByDefault) !== null && _b !== void 0 ? _b : false,
+                    hasAudioAdapter: Boolean(this.options.audioAdapter),
                 }
             });
             if (!this.pluginOptions.sessionResource) {
@@ -199,12 +203,110 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
         });
     }
     validateConfigAfterDiscover(adminforth, resourceConfig) {
+        var _a;
+        (_a = this.options.audioAdapter) === null || _a === void 0 ? void 0 : _a.validate();
         this.agentSystemPromptPromise = buildAgentSystemPrompt(adminforth, this.getInternalAgentResourceIds())
             .then((systemPrompt) => appendCustomSystemPrompt(systemPrompt, this.options.systemPrompt));
     }
     instanceUniqueRepresentation(pluginOptions) {
         return `single`;
     }
+    runAgentTurn(input) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, e_1, _b, _c;
+            var _d, _e, _f, _g;
+            let fullResponse = "";
+            const maxTokens = (_d = this.options.maxTokens) !== null && _d !== void 0 ? _d : 10000;
+            const selectedMode = (_e = this.options.modes.find((mode) => mode.name === input.modeName)) !== null && _e !== void 0 ? _e : this.options.modes[0];
+            const { model, summaryModel, modelMiddleware } = yield this.getModeModels(selectedMode, maxTokens);
+            const userLanguage = yield detectUserLanguage(selectedMode.completionAdapter, input.prompt)
+                .catch((error) => {
+                logger.warn(`Failed to detect user language: ${error instanceof Error ? error.message : String(error)}`);
+                return null;
+            });
+            const systemPrompt = [
+                yield this.agentSystemPromptPromise,
+                formatAdminUserPrompt(input.adminUser, this.adminforth.config.auth.usernameField),
+                formatLanguagePrompt(userLanguage),
+            ].join("\n\n");
+            const apiBasedTools = buildApiBasedTools(this.adminforth, this.getInternalAgentResourceIds());
+            for (const toolName of ALWAYS_AVAILABLE_API_TOOL_NAMES) {
+                assertRequiredApiTool(apiBasedTools, toolName);
+            }
+            assertRequiredApiTool(apiBasedTools, "update_record");
+            this.apiBasedTools = apiBasedTools;
+            const stream = yield callAgent({
+                name: `adminforth-agent-${this.pluginInstanceId}`,
+                model,
+                summaryModel,
+                modelMiddleware,
+                checkpointer: this.getCheckpointer(),
+                messages: [
+                    new SystemMessage(systemPrompt),
+                    new HumanMessage(input.prompt),
+                ],
+                adminUser: input.adminUser,
+                adminforth: this.adminforth,
+                apiBasedTools,
+                customComponentsDir: this.adminforth.config.customization.customComponentsDir,
+                sessionId: input.sessionId,
+                turnId: input.turnId,
+                currentPage: input.currentPage,
+                httpExtra: input.httpExtra,
+                userTimeZone: input.userTimeZone,
+                emitToolCallEvent: (event) => {
+                    var _a;
+                    input.sequenceDebugCollector.handleToolCallEvent(event);
+                    (_a = input.emitToolCallEvent) === null || _a === void 0 ? void 0 : _a.call(input, event);
+                },
+                sequenceDebugSink: input.sequenceDebugCollector,
+            });
+            try {
+                for (var _h = true, _j = __asyncValues(stream), _k; _k = yield _j.next(), _a = _k.done, !_a; _h = true) {
+                    _c = _k.value;
+                    _h = false;
+                    const rawChunk = _c;
+                    const [token, metadata] = rawChunk;
+                    const nodeName = typeof (metadata === null || metadata === void 0 ? void 0 : metadata.langgraph_node) === "string"
+                        ? metadata.langgraph_node
+                        : "";
+                    if (nodeName && !["model", "model_request"].includes(nodeName)) {
+                        continue;
+                    }
+                    const blocks = Array.isArray(token === null || token === void 0 ? void 0 : token.contentBlocks)
+                        ? token.contentBlocks
+                        : Array.isArray(token === null || token === void 0 ? void 0 : token.content)
+                            ? token.content
+                            : [];
+                    const reasoningDelta = blocks
+                        .filter((b) => (b === null || b === void 0 ? void 0 : b.type) === "reasoning")
+                        .map((b) => { var _a; return String((_a = b.reasoning) !== null && _a !== void 0 ? _a : ""); })
+                        .join("");
+                    const textDelta = blocks
+                        .filter((b) => (b === null || b === void 0 ? void 0 : b.type) === "text")
+                        .map((b) => { var _a; return String((_a = b.text) !== null && _a !== void 0 ? _a : ""); })
+                        .join("");
+                    if (reasoningDelta) {
+                        (_f = input.emitReasoningDelta) === null || _f === void 0 ? void 0 : _f.call(input, reasoningDelta);
+                    }
+                    if (textDelta) {
+                        fullResponse += textDelta;
+                        (_g = input.emitTextDelta) === null || _g === void 0 ? void 0 : _g.call(input, textDelta);
+                    }
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (!_h && !_a && (_b = _j.return)) yield _b.call(_j);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            return {
+                text: fullResponse,
+            };
+        });
+    }
     setupEndpoints(server) {
         server.endpoint({
             method: 'POST',
@@ -235,12 +337,11 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
             method: 'POST',
             path: `/agent/response`,
             handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body, query, headers, cookies, adminUser, response, requestUrl, _raw_express_res }) {
-                var _b, e_1, _c, _d;
-                var _e, _f, _g;
+                var _b;
                 const res = _raw_express_res;
                 const messageId = randomUUID();
                 const prompt = body.message;
-                const userTimeZone = (_e = body.timeZone) !== null && _e !== void 0 ? _e : 'UTC';
+                const userTimeZone = (_b = body.timeZone) !== null && _b !== void 0 ? _b : 'UTC';
                 const currentPage = body.currentPage;
                 const sessionId = body.sessionId || (adminUser === null || adminUser === void 0 ? void 0 : adminUser.pk) || (adminUser === null || adminUser === void 0 ? void 0 : adminUser.username) || 'default';
                 const turnId = yield this.createNewTurn(sessionId, prompt);
@@ -264,7 +365,6 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                     if (event.phase === "start") {
                         endActiveBlock();
                     }
-                    sequenceDebugCollector.handleToolCallEvent(event);
                     send({
                         type: "data-tool-call",
                         data: event,
@@ -311,43 +411,14 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                         type: 'start',
                         messageId,
                     });
-                    const maxTokens = (_f = this.options.maxTokens) !== null && _f !== void 0 ? _f : 10000;
-                    const selectedMode = (_g = this.options.modes.find((mode) => mode.name === body.mode)) !== null && _g !== void 0 ? _g : this.options.modes[0];
-                    const { model, summaryModel, modelMiddleware } = yield this.getModeModels(selectedMode, maxTokens);
-                    const userLanguage = yield detectUserLanguage(selectedMode.completionAdapter, prompt)
-                        .catch((error) => {
-                        logger.warn(`Failed to detect user language: ${error instanceof Error ? error.message : String(error)}`);
-                        return null;
-                    });
-                    const systemPrompt = [
-                        yield this.agentSystemPromptPromise,
-                        formatAdminUserPrompt(adminUser, this.adminforth.config.auth.usernameField),
-                        formatLanguagePrompt(userLanguage),
-                    ].join("\n\n");
-                    const apiBasedTools = buildApiBasedTools(this.adminforth, this.getInternalAgentResourceIds());
-                    for (const toolName of ALWAYS_AVAILABLE_API_TOOL_NAMES) {
-                        assertRequiredApiTool(apiBasedTools, toolName);
-                    }
-                    assertRequiredApiTool(apiBasedTools, "update_record");
-                    this.apiBasedTools = apiBasedTools;
-                    const currentPagePrompt = formatCurrentPagePrompt(currentPage);
-                    const stream = yield callAgent({
-                        name: `adminforth-agent-${this.pluginInstanceId}`,
-                        model,
-                        summaryModel,
-                        modelMiddleware,
-                        checkpointer: this.getCheckpointer(),
-                        messages: [
-                            new SystemMessage(systemPrompt),
-                            ...(currentPagePrompt ? [new SystemMessage(currentPagePrompt)] : []),
-                            new HumanMessage(prompt),
-                        ],
-                        adminUser,
-                        adminforth: this.adminforth,
-                        apiBasedTools,
-                        customComponentsDir: this.adminforth.config.customization.customComponentsDir,
+                    const agentResponse = yield this.runAgentTurn({
+                        prompt,
                         sessionId,
                         turnId,
+                        modeName: body.mode,
+                        userTimeZone,
+                        currentPage,
+                        adminUser,
                         httpExtra: {
                             body,
                             query,
@@ -356,70 +427,37 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                             requestUrl,
                             response,
                         },
-                        userTimeZone,
+                        sequenceDebugCollector,
                         emitToolCallEvent,
-                        sequenceDebugSink: sequenceDebugCollector,
+                        emitReasoningDelta: (reasoningDelta) => {
+                            const reasoningId = startBlock('reasoning');
+                            send({
+                                type: 'reasoning-delta',
+                                id: reasoningId,
+                                delta: reasoningDelta,
+                            });
+                        },
+                        emitTextDelta: (textDelta) => {
+                            const textId = startBlock('text');
+                            fullResponse += textDelta;
+                            send({
+                                type: 'text-delta',
+                                id: textId,
+                                delta: textDelta,
+                            });
+                        },
                     });
-                    try {
-                        for (var _h = true, _j = __asyncValues(stream), _k; _k = yield _j.next(), _b = _k.done, !_b; _h = true) {
-                            _d = _k.value;
-                            _h = false;
-                            const rawChunk = _d;
-                            const [token, metadata] = rawChunk;
-                            const nodeName = typeof (metadata === null || metadata === void 0 ? void 0 : metadata.langgraph_node) === "string"
-                                ? metadata.langgraph_node
-                                : "";
-                            if (nodeName && !["model", "model_request"].includes(nodeName)) {
-                                continue;
-                            }
-                            const blocks = Array.isArray(token === null || token === void 0 ? void 0 : token.contentBlocks)
-                                ? token.contentBlocks
-                                : Array.isArray(token === null || token === void 0 ? void 0 : token.content)
-                                    ? token.content
-                                    : [];
-                            const reasoningDelta = blocks
-                                .filter((b) => (b === null || b === void 0 ? void 0 : b.type) === "reasoning")
-                                .map((b) => { var _a; return String((_a = b.reasoning) !== null && _a !== void 0 ? _a : ""); })
-                                .join("");
-                            const textDelta = blocks
-                                .filter((b) => (b === null || b === void 0 ? void 0 : b.type) === "text")
-                                .map((b) => { var _a; return String((_a = b.text) !== null && _a !== void 0 ? _a : ""); })
-                                .join("");
-                            if (reasoningDelta) {
-                                const reasoningId = startBlock('reasoning');
-                                send({
-                                    type: 'reasoning-delta',
-                                    id: reasoningId,
-                                    delta: reasoningDelta,
-                                });
-                            }
-                            if (textDelta) {
-                                const textId = startBlock('text');
-                                fullResponse += textDelta;
-                                send({
-                                    type: 'text-delta',
-                                    id: textId,
-                                    delta: textDelta,
-                                });
-                            }
-                        }
-                    }
-                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                    finally {
-                        try {
-                            if (!_h && !_b && (_c = _j.return)) yield _c.call(_j);
-                        }
-                        finally { if (e_1) throw e_1.error; }
-                    }
+                    fullResponse = agentResponse.text;
                 }
                 catch (error) {
                     logger.error(`Agent response streaming failed:\n${formatAgentError(error)}`);
                     sequenceDebugCollector.flush();
+                    fullResponse = formatAgentResponseError(error);
                     const textId = startBlock('text');
                     send({
                         type: 'text-delta',
                         id: textId,
-                        delta: 'Agent response failed. Check server logs for details.',
+                        delta: fullResponse,
                     });
                 }
                 sequenceDebugCollector.flush();
@@ -434,6 +472,115 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                 return null;
             })
         });
+        server.endpoint({
+            method: 'POST',
+            path: `/agent/speech-response`,
+            handler: (_a) => __awaiter(this, [_a], void 0, function* ({ body, query, headers, cookies, adminUser, response, requestUrl }) {
+                var _b;
+                const audioAdapter = this.options.audioAdapter;
+                if (!audioAdapter) {
+                    response.setStatus(400, undefined);
+                    return {
+                        error: "Audio adapter is not configured for AdminForth Agent",
+                    };
+                }
+                const speechBody = body;
+                let transcription;
+                try {
+                    transcription = yield audioAdapter.transcribe({
+                        buffer: Buffer.from(speechBody.audioBase64, "base64"),
+                        filename: speechBody.filename,
+                        mimeType: speechBody.mimeType,
+                        language: "auto",
+                        prompt: speechBody.prompt,
+                    });
+                }
+                catch (error) {
+                    logger.error(`Agent speech transcription failed:\n${formatAgentError(error)}`);
+                    response.setStatus(500, undefined);
+                    return {
+                        error: "Speech transcription failed. Check server logs for details.",
+                    };
+                }
+                const prompt = transcription.text;
+                if (!prompt) {
+                    response.setStatus(400, undefined);
+                    return {
+                        error: "Speech transcription is empty",
+                    };
+                }
+                const sessionId = speechBody.sessionId || (adminUser === null || adminUser === void 0 ? void 0 : adminUser.pk) || (adminUser === null || adminUser === void 0 ? void 0 : adminUser.username) || 'default';
+                const turnId = yield this.createNewTurn(sessionId, prompt);
+                yield this.updateSessionDate(sessionId);
+                const sequenceDebugCollector = createSequenceDebugCollector();
+                let fullResponse = "";
+                try {
+                    const agentResponse = yield this.runAgentTurn({
+                        prompt,
+                        sessionId,
+                        turnId,
+                        modeName: speechBody.mode,
+                        userTimeZone: (_b = speechBody.timeZone) !== null && _b !== void 0 ? _b : 'UTC',
+                        currentPage: speechBody.currentPage,
+                        adminUser,
+                        httpExtra: {
+                            body,
+                            query,
+                            headers,
+                            cookies,
+                            requestUrl,
+                            response,
+                        },
+                        sequenceDebugCollector,
+                        emitTextDelta: (textDelta) => {
+                            fullResponse += textDelta;
+                        },
+                    });
+                    fullResponse = agentResponse.text;
+                    const speech = yield audioAdapter.synthesize(Object.assign({ text: fullResponse }, speechBody.tts));
+                    sequenceDebugCollector.flush();
+                    const turnUpdates = {
+                        [this.options.turnResource.responseField]: fullResponse,
+                    };
+                    if (this.options.turnResource.debugField) {
+                        turnUpdates[this.options.turnResource.debugField] = sequenceDebugCollector.getHistory();
+                    }
+                    yield this.updateTurn(turnId, turnUpdates);
+                    return {
+                        transcript: {
+                            text: transcription.text,
+                            language: transcription.language,
+                        },
+                        response: {
+                            text: fullResponse,
+                        },
+                        audio: {
+                            base64: speech.audio.toString("base64"),
+                            mimeType: speech.mimeType,
+                            format: speech.format,
+                        },
+                        sessionId,
+                        turnId,
+                    };
+                }
+                catch (error) {
+                    logger.error(`Agent speech response failed:\n${formatAgentError(error)}`);
+                    sequenceDebugCollector.flush();
+                    fullResponse = formatAgentResponseError(error);
+                    const turnUpdates = {
+                        [this.options.turnResource.responseField]: fullResponse,
+                    };
+                    if (this.options.turnResource.debugField) {
+                        turnUpdates[this.options.turnResource.debugField] = sequenceDebugCollector.getHistory();
+                    }
+                    yield this.updateTurn(turnId, turnUpdates);
+                    response.setStatus(500, undefined);
+                    return {
+                        error: fullResponse,
+                    };
+                }
+            })
+        });
         server.endpoint({
             method: 'POST',
             path: `/agent/get-sessions`,

package/index.ts

@@ -1,8 +1,10 @@
 import type {
   AdminUser,
   AdminForthResource,
+  HttpExtra,
   IAdminForth,
-  IHttpServer
+  IHttpServer,
+  TextToSpeechInput,
 } from "adminforth";
 
 import { AdminForthPlugin, logger, Filters, Sorts } from "adminforth";
@@ -33,16 +35,36 @@ import {
 } from "./agent/systemPrompt.js";
 import { ALWAYS_AVAILABLE_API_TOOL_NAMES } from "./agent/tools/index.js";
 import type { ToolCallEvent } from "./agent/toolCallEvents.js";
+import type { CurrentPageContext } from "./agent/tools/getUserLocation.js";
 
 type CurrentPageRequestBody = {
   currentPage?: CurrentPageContext;
 };
 
-type CurrentPageContext = {
-  path: string;
-  fullPath: string;
-  title: string;
-  url: string;
+type SpeechResponseRequestBody = CurrentPageRequestBody & {
+  audioBase64: string;
+  filename: string;
+  mimeType: string;
+  prompt?: string;
+  sessionId?: string | null;
+  mode?: string | null;
+  timeZone?: string;
+  tts?: Omit<TextToSpeechInput, "text">;
+};
+
+type AgentTurnRunInput = {
+  prompt: string;
+  sessionId: string;
+  turnId: string;
+  modeName?: string | null;
+  userTimeZone: string;
+  currentPage?: CurrentPageContext;
+  adminUser: AdminUser;
+  httpExtra: HttpExtra;
+  sequenceDebugCollector: ReturnType<typeof createSequenceDebugCollector>;
+  emitReasoningDelta?: (delta: string) => void;
+  emitTextDelta?: (delta: string) => void;
+  emitToolCallEvent?: (event: ToolCallEvent) => void;
 };
 
 function isAggregateErrorLike(
@@ -73,6 +95,24 @@ function formatAgentError(error: unknown) {
   return String(error);
 }
 
+function formatAgentResponseError(error: unknown): string {
+  if (isAggregateErrorLike(error)) {
+    const nestedErrors = error.errors.map(formatAgentResponseError);
+
+    if (nestedErrors.length) {
+      return nestedErrors.join("\n");
+    }
+
+    return error.message || "Agent response failed";
+  }
+
+  if (error instanceof Error) {
+    return error.toString();
+  }
+
+  return String(error);
+}
+
 function formatAdminUserPrompt(adminUser: AdminUser, usernameField: string) {
   const dbUser = adminUser.dbUser as Record<string, unknown>;
   const adminUserContext = {
@@ -87,18 +127,6 @@ function formatAdminUserPrompt(adminUser: AdminUser, usernameField: string) {
   ].join("\n");
 }
 
-function formatCurrentPagePrompt(currentPage: CurrentPageContext | undefined) {
-  if (!currentPage) {
-    return null;
-  }
-
-  return [
-    "Current user page context for the latest message:",
-    JSON.stringify(currentPage, null, 2),
-    "When the user says here, this page, current page, or opened page, treat it as this page.",
-  ].join("\n");
-}
-
 function assertRequiredApiTool(
   apiBasedTools: Record<string, ApiBasedTool>,
   toolName: string,
@@ -245,6 +273,7 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
         modes: this.options.modes.map((mode) => ({ name: mode.name })),
         defaultModeName: this.options.modes[0].name,
         stickByDefault: this.options.stickByDefault ?? false,
+        hasAudioAdapter: Boolean(this.options.audioAdapter),
       }
     });
     if (!this.pluginOptions.sessionResource) {
@@ -253,6 +282,7 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
   }
   
   validateConfigAfterDiscover(adminforth: IAdminForth, resourceConfig: AdminForthResource) {
+    this.options.audioAdapter?.validate();
     this.agentSystemPromptPromise = buildAgentSystemPrompt(
       adminforth,
       this.getInternalAgentResourceIds(),
@@ -264,6 +294,101 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
     return `single`;
   }
 
+  private async runAgentTurn(input: AgentTurnRunInput) {
+    let fullResponse = "";
+    const maxTokens = this.options.maxTokens ?? 10000;
+    const selectedMode =
+      this.options.modes.find((mode) => mode.name === input.modeName) ??
+      this.options.modes[0];
+    const { model, summaryModel, modelMiddleware } =
+      await this.getModeModels(selectedMode, maxTokens);
+    const userLanguage = await detectUserLanguage(selectedMode.completionAdapter, input.prompt)
+      .catch((error) => {
+        logger.warn(`Failed to detect user language: ${error instanceof Error ? error.message : String(error)}`);
+        return null;
+      });
+    const systemPrompt = [
+      await this.agentSystemPromptPromise,
+      formatAdminUserPrompt(input.adminUser, this.adminforth.config.auth.usernameField),
+      formatLanguagePrompt(userLanguage),
+    ].join("\n\n");
+    const apiBasedTools = buildApiBasedTools(
+      this.adminforth,
+      this.getInternalAgentResourceIds(),
+    );
+    for (const toolName of ALWAYS_AVAILABLE_API_TOOL_NAMES) {
+      assertRequiredApiTool(apiBasedTools, toolName);
+    }
+    assertRequiredApiTool(apiBasedTools, "update_record");
+    this.apiBasedTools = apiBasedTools;
+    const stream = await callAgent({
+      name: `adminforth-agent-${this.pluginInstanceId}`,
+      model,
+      summaryModel,
+      modelMiddleware,
+      checkpointer: this.getCheckpointer(),
+      messages: [
+        new SystemMessage(systemPrompt),
+        new HumanMessage(input.prompt),
+      ],
+      adminUser: input.adminUser,
+      adminforth: this.adminforth,
+      apiBasedTools,
+      customComponentsDir: this.adminforth.config.customization.customComponentsDir,
+      sessionId: input.sessionId,
+      turnId: input.turnId,
+      currentPage: input.currentPage,
+      httpExtra: input.httpExtra,
+      userTimeZone: input.userTimeZone,
+      emitToolCallEvent: (event) => {
+        input.sequenceDebugCollector.handleToolCallEvent(event);
+        input.emitToolCallEvent?.(event);
+      },
+      sequenceDebugSink: input.sequenceDebugCollector,
+    });
+
+    for await (const rawChunk of stream as AsyncIterable<[any, any]>) {
+      const [token, metadata] = rawChunk;
+
+      const nodeName =
+        typeof metadata?.langgraph_node === "string"
+          ? metadata.langgraph_node
+          : "";
+
+      if (nodeName && !["model", "model_request"].includes(nodeName)) {
+        continue;
+      }
+
+      const blocks = Array.isArray(token?.contentBlocks)
+        ? token.contentBlocks
+        : Array.isArray(token?.content)
+          ? token.content
+          : [];
+      const reasoningDelta = blocks
+        .filter((b: any) => b?.type === "reasoning")
+        .map((b: any) => String(b.reasoning ?? ""))
+        .join("");
+
+      const textDelta = blocks
+        .filter((b: any) => b?.type === "text")
+        .map((b: any) => String(b.text ?? ""))
+        .join("");
+
+      if (reasoningDelta) {
+        input.emitReasoningDelta?.(reasoningDelta);
+      }
+
+      if (textDelta) {
+        fullResponse += textDelta;
+        input.emitTextDelta?.(textDelta);
+      }
+    }
+
+    return {
+      text: fullResponse,
+    };
+  }
+
   setupEndpoints(server: IHttpServer) {
     server.endpoint({
       method: 'POST',
@@ -327,8 +452,6 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
             endActiveBlock();
           }
 
-          sequenceDebugCollector.handleToolCallEvent(event);
-
           send({
             type: "data-tool-call",
             data: event,
@@ -389,47 +512,14 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
             messageId,
           });
 
-          const maxTokens = this.options.maxTokens ?? 10000;
-          const selectedMode = this.options.modes.find((mode) => mode.name === body.mode) ?? this.options.modes[0];
-          const { model, summaryModel, modelMiddleware } =
-            await this.getModeModels(selectedMode, maxTokens);
-          const userLanguage = await detectUserLanguage(selectedMode.completionAdapter, prompt)
-            .catch((error) => {
-              logger.warn(`Failed to detect user language: ${error instanceof Error ? error.message : String(error)}`);
-              return null;
-            });
-          const systemPrompt = [
-            await this.agentSystemPromptPromise,
-            formatAdminUserPrompt(adminUser, this.adminforth.config.auth.usernameField),
-            formatLanguagePrompt(userLanguage),
-          ].join("\n\n");
-          const apiBasedTools = buildApiBasedTools(
-            this.adminforth,
-            this.getInternalAgentResourceIds(),
-          );
-          for (const toolName of ALWAYS_AVAILABLE_API_TOOL_NAMES) {
-            assertRequiredApiTool(apiBasedTools, toolName);
-          }
-          assertRequiredApiTool(apiBasedTools, "update_record");
-          this.apiBasedTools = apiBasedTools;
-          const currentPagePrompt = formatCurrentPagePrompt(currentPage);
-          const stream = await callAgent({
-            name: `adminforth-agent-${this.pluginInstanceId}`,
-            model,
-            summaryModel,
-            modelMiddleware,
-            checkpointer: this.getCheckpointer(),
-            messages: [
-              new SystemMessage(systemPrompt),
-              ...(currentPagePrompt ? [new SystemMessage(currentPagePrompt)] : []),
-              new HumanMessage(prompt),
-            ],
-            adminUser,
-            adminforth: this.adminforth,
-            apiBasedTools,
-            customComponentsDir: this.adminforth.config.customization.customComponentsDir,
+          const agentResponse = await this.runAgentTurn({
+            prompt,
             sessionId,
             turnId,
+            modeName: body.mode,
+            userTimeZone,
+            currentPage,
+            adminUser,
             httpExtra: {
               body,
               query,
@@ -438,48 +528,17 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
               requestUrl,
               response,
             },
-            userTimeZone,
+            sequenceDebugCollector,
             emitToolCallEvent,
-            sequenceDebugSink: sequenceDebugCollector,
-          });
-
-          for await (const rawChunk of stream as AsyncIterable<[any, any]>) {
-            const [token, metadata] = rawChunk;
-
-            const nodeName =
-              typeof metadata?.langgraph_node === "string"
-                ? metadata.langgraph_node
-                : "";
-
-            if (nodeName && !["model", "model_request"].includes(nodeName)) {
-              continue;
-            }
-
-            const blocks = Array.isArray(token?.contentBlocks)
-              ? token.contentBlocks
-              : Array.isArray(token?.content)
-                ? token.content
-                : [];
-            const reasoningDelta = blocks
-              .filter((b: any) => b?.type === "reasoning")
-              .map((b: any) => String(b.reasoning ?? ""))
-              .join("");
-
-            const textDelta = blocks
-              .filter((b: any) => b?.type === "text")
-              .map((b: any) => String(b.text ?? ""))
-              .join("");
-
-            if (reasoningDelta) {
+            emitReasoningDelta: (reasoningDelta) => {
               const reasoningId = startBlock('reasoning');
-                send({
-                  type: 'reasoning-delta',
-                  id: reasoningId,
-                  delta: reasoningDelta,
-                });
-            }
-
-            if (textDelta) {
+              send({
+                type: 'reasoning-delta',
+                id: reasoningId,
+                delta: reasoningDelta,
+              });
+            },
+            emitTextDelta: (textDelta) => {
               const textId = startBlock('text');
               fullResponse += textDelta;
               send({
@@ -487,16 +546,18 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
                 id: textId,
                 delta: textDelta,
               });
-            }
-          }
+            },
+          });
+          fullResponse = agentResponse.text;
         } catch (error) {
           logger.error(`Agent response streaming failed:\n${formatAgentError(error)}`);
           sequenceDebugCollector.flush();
+          fullResponse = formatAgentResponseError(error);
           const textId = startBlock('text');
           send({
             type: 'text-delta',
             id: textId,
-            delta: 'Agent response failed. Check server logs for details.',
+            delta: fullResponse,
           });
         }
         sequenceDebugCollector.flush();
@@ -513,6 +574,125 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
         return null;
       }
     });
+    server.endpoint({
+      method: 'POST',
+      path: `/agent/speech-response`,
+      handler: async ({ body, query, headers, cookies, adminUser, response, requestUrl }) => {
+        const audioAdapter = this.options.audioAdapter;
+        if (!audioAdapter) {
+          response.setStatus(400, undefined);
+          return {
+            error: "Audio adapter is not configured for AdminForth Agent",
+          };
+        }
+
+        const speechBody = body as SpeechResponseRequestBody;
+        let transcription;
+
+        try {
+          transcription = await audioAdapter.transcribe({
+            buffer: Buffer.from(speechBody.audioBase64, "base64"),
+            filename: speechBody.filename,
+            mimeType: speechBody.mimeType,
+            language: "auto",
+            prompt: speechBody.prompt,
+          });
+        } catch (error) {
+          logger.error(`Agent speech transcription failed:\n${formatAgentError(error)}`);
+          response.setStatus(500, undefined);
+          return {
+            error: "Speech transcription failed. Check server logs for details.",
+          };
+        }
+
+        const prompt = transcription.text;
+        if (!prompt) {
+          response.setStatus(400, undefined);
+          return {
+            error: "Speech transcription is empty",
+          };
+        }
+
+        const sessionId = speechBody.sessionId || adminUser?.pk || adminUser?.username || 'default';
+        const turnId = await this.createNewTurn(sessionId, prompt);
+        await this.updateSessionDate(sessionId);
+        const sequenceDebugCollector = createSequenceDebugCollector();
+        let fullResponse = "";
+
+        try {
+          const agentResponse = await this.runAgentTurn({
+            prompt,
+            sessionId,
+            turnId,
+            modeName: speechBody.mode,
+            userTimeZone: speechBody.timeZone ?? 'UTC',
+            currentPage: speechBody.currentPage,
+            adminUser,
+            httpExtra: {
+              body,
+              query,
+              headers,
+              cookies,
+              requestUrl,
+              response,
+            },
+            sequenceDebugCollector,
+            emitTextDelta: (textDelta) => {
+              fullResponse += textDelta;
+            },
+          });
+          fullResponse = agentResponse.text;
+          const speech = await audioAdapter.synthesize({
+            text: fullResponse,
+            ...speechBody.tts,
+          });
+          sequenceDebugCollector.flush();
+          const turnUpdates: Record<string, unknown> = {
+            [this.options.turnResource.responseField]: fullResponse,
+          };
+
+          if (this.options.turnResource.debugField) {
+            turnUpdates[this.options.turnResource.debugField] = sequenceDebugCollector.getHistory();
+          }
+
+          await this.updateTurn(turnId, turnUpdates);
+
+          return {
+            transcript: {
+              text: transcription.text,
+              language: transcription.language,
+            },
+            response: {
+              text: fullResponse,
+            },
+            audio: {
+              base64: speech.audio.toString("base64"),
+              mimeType: speech.mimeType,
+              format: speech.format,
+            },
+            sessionId,
+            turnId,
+          };
+        } catch (error) {
+          logger.error(`Agent speech response failed:\n${formatAgentError(error)}`);
+          sequenceDebugCollector.flush();
+          fullResponse = formatAgentResponseError(error);
+          const turnUpdates: Record<string, unknown> = {
+            [this.options.turnResource.responseField]: fullResponse,
+          };
+
+          if (this.options.turnResource.debugField) {
+            turnUpdates[this.options.turnResource.debugField] = sequenceDebugCollector.getHistory();
+          }
+
+          await this.updateTurn(turnId, turnUpdates);
+          response.setStatus(500, undefined);
+          return {
+            error: fullResponse,
+          };
+        }
+      }
+    });
     server.endpoint({
       method: 'POST',
       path: `/agent/get-sessions`,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adminforth/agent",
-  "version": "1.34.2",
+  "version": "1.35.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",
@@ -33,7 +33,7 @@
     "@langchain/core": "^1.1.40",
     "@langchain/langgraph": "^1.2.8",
     "@langchain/langgraph-checkpoint": "^1.0.1",
-    "adminforth": "2.42.0",
+    "adminforth": "2.51.0",
     "dayjs": "^1.11.20",
     "langchain": "^1.3.3",
     "yaml": "^2.8.3",

package/types.ts

@@ -2,6 +2,7 @@ import {
   type PluginsCommonOptions,
   type AdminUser,
   type HttpExtra,
+  type AudioAdapter,
 } from "adminforth";
 import type { AgentModeCompletionAdapter } from "./agent/simpleAgent.js";
 
@@ -61,6 +62,11 @@ export interface PluginOptions extends PluginsCommonOptions {
     completionAdapter: AgentModeCompletionAdapter;
   }[];
 
+  /**
+   * Optional audio adapter for speech-to-text and text-to-speech flows.
+   */
+  audioAdapter?: AudioAdapter;
+
   /**
    * Max tokens for the generation.
    * Default is 1000