@adminforth/agent 1.45.1 → 1.46.0

package/build.log

@@ -63,5 +63,5 @@ custom/speech_recognition_frontend/voiceActivityDetection.ts
 custom/speech_recognition_frontend/types/
 custom/speech_recognition_frontend/types/voice-activity-detection.d.ts
 
-sent 1,671,664 bytes  received 936 bytes  3,345,200.00 bytes/sec
+sent 1,671,668 bytes  received 940 bytes  3,345,216.00 bytes/sec
 total size is 1,667,436  speedup is 1.00

package/chatSurfaceService.ts

@@ -5,12 +5,18 @@ import type {
   ChatSurfaceIncomingMessage,
   IAdminForth,
 } from "adminforth";
-import { Filters } from "adminforth";
+import { Filters, logger } from "adminforth";
 import { randomUUID } from "crypto";
 import type { AgentEventEmitter } from "./agentEvents.js";
-import type { HandleTurnInput } from "./agentTurnService.js";
+import type {
+  HandleTurnInput,
+  RunAndPersistAgentResponseInput,
+  RunAndPersistAgentResponseResult,
+} from "./agentTurnService.js";
 import type { PluginOptions } from "./types.js";
 import type { AgentSessionStore } from "./sessionStore.js";
+import { getErrorMessage, isAbortError } from "./errors.js";
+import { sanitizeSpeechText } from "./sanitizeSpeechText.js";
 
 type ChatSurfaceConnectAction = {
   type: "url";
@@ -18,6 +24,23 @@ type ChatSurfaceConnectAction = {
   url: string;
 };
 
+type ChatSurfaceIncomingMessageWithAudio = ChatSurfaceIncomingMessage & {
+  audio?: {
+    buffer: Buffer;
+    filename: string;
+    mimeType: string;
+  };
+};
+
+type ChatSurfaceEventSinkWithAudio = ChatSurfaceEventSink & {
+  emit(event: Parameters<ChatSurfaceEventSink["emit"]>[0] | {
+    type: "audio";
+    audio: Buffer;
+    filename: string;
+    mimeType: string;
+  }): void | Promise<void>;
+};
+
 export type ChatSurfaceAdapterWithConnectAction = ChatSurfaceAdapter & {
   createConnectAction?(input: {
     token: string;
@@ -41,6 +64,9 @@ export class ChatSurfaceService {
     private options: PluginOptions,
     private sessionStore: AgentSessionStore,
     private handleTurn: (input: HandleTurnInput) => Promise<unknown>,
+    private runAndPersistAgentResponse: (
+      input: RunAndPersistAgentResponseInput,
+    ) => Promise<RunAndPersistAgentResponseResult>,
   ) {}
 
   getConnectActionAdapters() {
@@ -108,10 +134,35 @@ export class ChatSurfaceService {
     incoming: ChatSurfaceIncomingMessage,
     sink: ChatSurfaceEventSink,
   ) {
-    if (typeof incoming.metadata?.startPayload !== "string") {
+    if (incoming.metadata?.isStartCommand !== true) {
       return false;
     }
 
+    const externalUserIdField = this.options.chatExternalIdsField ?? DEFAULT_ADMIN_USER_EXTERNAL_USER_ID_FIELD;
+    const adminforth = this.getAdminforth();
+    const authResourceId = adminforth.config.auth!.usersResourceId!;
+    const authResource = adminforth.config.resources.find((resource) => resource.resourceId === authResourceId)!;
+    const primaryKeyField = authResource.columns.find((column) => column.primaryKey)!.name!;
+    const linkedAdminUserRecord = (
+      await adminforth.resource(authResourceId).list(Filters.IS_NOT_EMPTY(externalUserIdField))
+    ).find((user) => user[externalUserIdField]?.[incoming.surface] === incoming.externalUserId);
+
+    if (linkedAdminUserRecord) {
+      await sink.emit({
+        type: "done",
+        text: `${incoming.surface} account is already connected to AdminForth.`,
+      });
+      return true;
+    }
+
+    if (typeof incoming.metadata?.startPayload !== "string") {
+      await sink.emit({
+        type: "done",
+        text: `Open AdminForth and connect your ${incoming.surface} account from Chat Surfaces settings.`,
+      });
+      return true;
+    }
+
     const payload = this.consumeLinkToken(incoming.surface, incoming.metadata.startPayload);
     if (!payload) {
       await sink.emit({
@@ -120,11 +171,7 @@ export class ChatSurfaceService {
       });
       return true;
     }
-    const externalUserIdField = this.options.chatExternalIdsField ?? DEFAULT_ADMIN_USER_EXTERNAL_USER_ID_FIELD;
-    const adminforth = this.getAdminforth();
-    const authResourceId = adminforth.config.auth!.usersResourceId!;
-    const authResource = adminforth.config.resources.find((resource) => resource.resourceId === authResourceId)!;
-    const primaryKeyField = authResource.columns.find((column) => column.primaryKey)!.name!;
+
     const adminUserRecord = await adminforth.resource(authResourceId).get([
       Filters.EQ(primaryKeyField, payload.adminUserId),
     ]);
@@ -143,6 +190,143 @@ export class ChatSurfaceService {
     return true;
   }
 
+  private async handleAudioMessage(
+    incoming: ChatSurfaceIncomingMessageWithAudio,
+    sink: ChatSurfaceEventSinkWithAudio,
+    adminUser: AdminUser,
+  ) {
+    const audioAdapter = this.options.audioAdapter;
+    if (!audioAdapter) {
+      await sink.emit({
+        type: "error",
+        message: "Audio adapter is not configured for AdminForth Agent.",
+      });
+      return;
+    }
+
+    let transcription;
+
+    try {
+      transcription = await audioAdapter.transcribe({
+        buffer: incoming.audio!.buffer,
+        filename: incoming.audio!.filename,
+        mimeType: incoming.audio!.mimeType,
+        language: "auto",
+      });
+    } catch (error) {
+      if (isAbortError(error)) {
+        logger.info(`Agent ${incoming.surface} surface speech transcription aborted`);
+        return;
+      }
+
+      logger.error(`Agent ${incoming.surface} surface speech transcription failed:\n${getErrorMessage(error)}`);
+      await sink.emit({
+        type: "error",
+        message: "Speech transcription failed. Check server logs for details.",
+      });
+      return;
+    }
+
+    if (!transcription.text) {
+      await sink.emit({
+        type: "error",
+        message: "Speech transcription is empty",
+      });
+      return;
+    }
+
+    const agentResponse = await this.handleAgentSurfaceResponse(
+      incoming,
+      sink,
+      adminUser,
+      transcription.text,
+      { emitDone: false },
+    );
+
+    if (!agentResponse || agentResponse.aborted || agentResponse.failed) {
+      return;
+    }
+
+    await sink.emit({
+      type: "done",
+      text: agentResponse.text,
+    });
+
+    try {
+      const speech = await audioAdapter.synthesize({
+        text: sanitizeSpeechText(agentResponse.text),
+        stream: false,
+        format: "opus",
+      });
+
+      await sink.emit({
+        type: "audio",
+        audio: speech.audio,
+        filename: "agent-response.ogg",
+        mimeType: speech.mimeType,
+      });
+    } catch (error) {
+      if (isAbortError(error)) {
+        logger.info(`Agent ${incoming.surface} surface speech synthesis aborted`);
+        return;
+      }
+
+      logger.error(`Agent ${incoming.surface} surface speech synthesis failed:\n${getErrorMessage(error)}`);
+      await sink.emit({
+        type: "error",
+        message: getErrorMessage(error),
+      });
+    }
+  }
+
+  private async handleAgentSurfaceResponse(
+    incoming: ChatSurfaceIncomingMessage,
+    sink: ChatSurfaceEventSink,
+    adminUser: AdminUser,
+    prompt: string,
+    options?: { emitDone?: boolean },
+  ) {
+    const emitDone = options?.emitDone ?? true;
+    const sessionId = await this.sessionStore.getOrCreateChatSurfaceSession(
+      { ...incoming, prompt },
+      adminUser,
+    );
+
+    if (emitDone) {
+      await this.handleTurn({
+        prompt,
+        sessionId,
+        modeName: incoming.modeName,
+        userTimeZone: incoming.userTimeZone ?? "UTC",
+        adminUser,
+        emit: this.createEventEmitter(sink),
+        failureLogMessage: `Agent ${incoming.surface} surface response failed`,
+        abortLogMessage: `Agent ${incoming.surface} surface response aborted`,
+      });
+      return null;
+    }
+
+    const agentResponse = await this.runAndPersistAgentResponse({
+      prompt,
+      sessionId,
+      modeName: incoming.modeName,
+      userTimeZone: incoming.userTimeZone ?? "UTC",
+      adminUser,
+      emit: this.createEventEmitter(sink),
+      failureLogMessage: `Agent ${incoming.surface} surface response failed`,
+      abortLogMessage: `Agent ${incoming.surface} surface response aborted`,
+    });
+
+    if (agentResponse.failed) {
+      await sink.emit({
+        type: "error",
+        message: agentResponse.text,
+      });
+    }
+
+    return agentResponse;
+  }
+
   async handleMessage(
     adapter: ChatSurfaceAdapter,
     incoming: ChatSurfaceIncomingMessage,
@@ -175,15 +359,12 @@ export class ChatSurfaceService {
       dbUser: adminUserRecord,
     };
 
-    await this.handleTurn({
-      prompt: incoming.prompt,
-      sessionId: await this.sessionStore.getOrCreateChatSurfaceSession(incoming, adminUser),
-      modeName: incoming.modeName,
-      userTimeZone: incoming.userTimeZone ?? "UTC",
-      adminUser,
-      emit: this.createEventEmitter(sink),
-      failureLogMessage: `Agent ${incoming.surface} surface response failed`,
-      abortLogMessage: `Agent ${incoming.surface} surface response aborted`,
-    });
+    const incomingWithAudio = incoming as ChatSurfaceIncomingMessageWithAudio;
+    if (incomingWithAudio.audio) {
+      await this.handleAudioMessage(incomingWithAudio, sink as ChatSurfaceEventSinkWithAudio, adminUser);
+      return;
+    }
+
+    await this.handleAgentSurfaceResponse(incoming, sink, adminUser, incoming.prompt);
   }
 }

package/dist/chatSurfaceService.d.ts

@@ -1,5 +1,5 @@
 import type { AdminUser, ChatSurfaceAdapter, ChatSurfaceEventSink, ChatSurfaceIncomingMessage, IAdminForth } from "adminforth";
-import type { HandleTurnInput } from "./agentTurnService.js";
+import type { HandleTurnInput, RunAndPersistAgentResponseInput, RunAndPersistAgentResponseResult } from "./agentTurnService.js";
 import type { PluginOptions } from "./types.js";
 import type { AgentSessionStore } from "./sessionStore.js";
 type ChatSurfaceConnectAction = {
@@ -17,13 +17,16 @@ export declare class ChatSurfaceService {
     private options;
     private sessionStore;
     private handleTurn;
+    private runAndPersistAgentResponse;
     private linkTokens;
-    constructor(getAdminforth: () => IAdminForth, options: PluginOptions, sessionStore: AgentSessionStore, handleTurn: (input: HandleTurnInput) => Promise<unknown>);
+    constructor(getAdminforth: () => IAdminForth, options: PluginOptions, sessionStore: AgentSessionStore, handleTurn: (input: HandleTurnInput) => Promise<unknown>, runAndPersistAgentResponse: (input: RunAndPersistAgentResponseInput) => Promise<RunAndPersistAgentResponseResult>);
     getConnectActionAdapters(): ChatSurfaceAdapterWithConnectAction[];
     createLinkToken(surface: string, adminUser: AdminUser): `${string}-${string}-${string}-${string}-${string}`;
     private consumeLinkToken;
     private createEventEmitter;
     private handleLink;
+    private handleAudioMessage;
+    private handleAgentSurfaceResponse;
     handleMessage(adapter: ChatSurfaceAdapter, incoming: ChatSurfaceIncomingMessage, sink: ChatSurfaceEventSink): Promise<void>;
 }
 export {};

package/dist/chatSurfaceService.js

@@ -7,16 +7,19 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import { Filters } from "adminforth";
+import { Filters, logger } from "adminforth";
 import { randomUUID } from "crypto";
+import { getErrorMessage, isAbortError } from "./errors.js";
+import { sanitizeSpeechText } from "./sanitizeSpeechText.js";
 const DEFAULT_ADMIN_USER_EXTERNAL_USER_ID_FIELD = "externalUserId";
 const CHAT_SURFACE_LINK_TOKEN_TTL_MS = 60 * 1000;
 export class ChatSurfaceService {
-    constructor(getAdminforth, options, sessionStore, handleTurn) {
+    constructor(getAdminforth, options, sessionStore, handleTurn, runAndPersistAgentResponse) {
         this.getAdminforth = getAdminforth;
         this.options = options;
         this.sessionStore = sessionStore;
         this.handleTurn = handleTurn;
+        this.runAndPersistAgentResponse = runAndPersistAgentResponse;
         this.linkTokens = new Map();
     }
     getConnectActionAdapters() {
@@ -73,10 +76,30 @@ export class ChatSurfaceService {
     }
     handleLink(incoming, sink) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b, _c;
-            if (typeof ((_a = incoming.metadata) === null || _a === void 0 ? void 0 : _a.startPayload) !== "string") {
+            var _a, _b, _c, _d;
+            if (((_a = incoming.metadata) === null || _a === void 0 ? void 0 : _a.isStartCommand) !== true) {
                 return false;
             }
+            const externalUserIdField = (_b = this.options.chatExternalIdsField) !== null && _b !== void 0 ? _b : DEFAULT_ADMIN_USER_EXTERNAL_USER_ID_FIELD;
+            const adminforth = this.getAdminforth();
+            const authResourceId = adminforth.config.auth.usersResourceId;
+            const authResource = adminforth.config.resources.find((resource) => resource.resourceId === authResourceId);
+            const primaryKeyField = authResource.columns.find((column) => column.primaryKey).name;
+            const linkedAdminUserRecord = (yield adminforth.resource(authResourceId).list(Filters.IS_NOT_EMPTY(externalUserIdField))).find((user) => { var _a; return ((_a = user[externalUserIdField]) === null || _a === void 0 ? void 0 : _a[incoming.surface]) === incoming.externalUserId; });
+            if (linkedAdminUserRecord) {
+                yield sink.emit({
+                    type: "done",
+                    text: `${incoming.surface} account is already connected to AdminForth.`,
+                });
+                return true;
+            }
+            if (typeof ((_c = incoming.metadata) === null || _c === void 0 ? void 0 : _c.startPayload) !== "string") {
+                yield sink.emit({
+                    type: "done",
+                    text: `Open AdminForth and connect your ${incoming.surface} account from Chat Surfaces settings.`,
+                });
+                return true;
+            }
             const payload = this.consumeLinkToken(incoming.surface, incoming.metadata.startPayload);
             if (!payload) {
                 yield sink.emit({
@@ -85,16 +108,11 @@ export class ChatSurfaceService {
                 });
                 return true;
             }
-            const externalUserIdField = (_b = this.options.chatExternalIdsField) !== null && _b !== void 0 ? _b : DEFAULT_ADMIN_USER_EXTERNAL_USER_ID_FIELD;
-            const adminforth = this.getAdminforth();
-            const authResourceId = adminforth.config.auth.usersResourceId;
-            const authResource = adminforth.config.resources.find((resource) => resource.resourceId === authResourceId);
-            const primaryKeyField = authResource.columns.find((column) => column.primaryKey).name;
             const adminUserRecord = yield adminforth.resource(authResourceId).get([
                 Filters.EQ(primaryKeyField, payload.adminUserId),
             ]);
             yield adminforth.resource(authResourceId).update(payload.adminUserId, {
-                [externalUserIdField]: Object.assign(Object.assign({}, ((_c = adminUserRecord[externalUserIdField]) !== null && _c !== void 0 ? _c : {})), { [incoming.surface]: incoming.externalUserId }),
+                [externalUserIdField]: Object.assign(Object.assign({}, ((_d = adminUserRecord[externalUserIdField]) !== null && _d !== void 0 ? _d : {})), { [incoming.surface]: incoming.externalUserId }),
             });
             yield sink.emit({
                 type: "done",
@@ -103,9 +121,118 @@ export class ChatSurfaceService {
             return true;
         });
     }
+    handleAudioMessage(incoming, sink, adminUser) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const audioAdapter = this.options.audioAdapter;
+            if (!audioAdapter) {
+                yield sink.emit({
+                    type: "error",
+                    message: "Audio adapter is not configured for AdminForth Agent.",
+                });
+                return;
+            }
+            let transcription;
+            try {
+                transcription = yield audioAdapter.transcribe({
+                    buffer: incoming.audio.buffer,
+                    filename: incoming.audio.filename,
+                    mimeType: incoming.audio.mimeType,
+                    language: "auto",
+                });
+            }
+            catch (error) {
+                if (isAbortError(error)) {
+                    logger.info(`Agent ${incoming.surface} surface speech transcription aborted`);
+                    return;
+                }
+                logger.error(`Agent ${incoming.surface} surface speech transcription failed:\n${getErrorMessage(error)}`);
+                yield sink.emit({
+                    type: "error",
+                    message: "Speech transcription failed. Check server logs for details.",
+                });
+                return;
+            }
+            if (!transcription.text) {
+                yield sink.emit({
+                    type: "error",
+                    message: "Speech transcription is empty",
+                });
+                return;
+            }
+            const agentResponse = yield this.handleAgentSurfaceResponse(incoming, sink, adminUser, transcription.text, { emitDone: false });
+            if (!agentResponse || agentResponse.aborted || agentResponse.failed) {
+                return;
+            }
+            yield sink.emit({
+                type: "done",
+                text: agentResponse.text,
+            });
+            try {
+                const speech = yield audioAdapter.synthesize({
+                    text: sanitizeSpeechText(agentResponse.text),
+                    stream: false,
+                    format: "opus",
+                });
+                yield sink.emit({
+                    type: "audio",
+                    audio: speech.audio,
+                    filename: "agent-response.ogg",
+                    mimeType: speech.mimeType,
+                });
+            }
+            catch (error) {
+                if (isAbortError(error)) {
+                    logger.info(`Agent ${incoming.surface} surface speech synthesis aborted`);
+                    return;
+                }
+                logger.error(`Agent ${incoming.surface} surface speech synthesis failed:\n${getErrorMessage(error)}`);
+                yield sink.emit({
+                    type: "error",
+                    message: getErrorMessage(error),
+                });
+            }
+        });
+    }
+    handleAgentSurfaceResponse(incoming, sink, adminUser, prompt, options) {
+        return __awaiter(this, void 0, void 0, function* () {
+            var _a, _b, _c;
+            const emitDone = (_a = options === null || options === void 0 ? void 0 : options.emitDone) !== null && _a !== void 0 ? _a : true;
+            const sessionId = yield this.sessionStore.getOrCreateChatSurfaceSession(Object.assign(Object.assign({}, incoming), { prompt }), adminUser);
+            if (emitDone) {
+                yield this.handleTurn({
+                    prompt,
+                    sessionId,
+                    modeName: incoming.modeName,
+                    userTimeZone: (_b = incoming.userTimeZone) !== null && _b !== void 0 ? _b : "UTC",
+                    adminUser,
+                    emit: this.createEventEmitter(sink),
+                    failureLogMessage: `Agent ${incoming.surface} surface response failed`,
+                    abortLogMessage: `Agent ${incoming.surface} surface response aborted`,
+                });
+                return null;
+            }
+            const agentResponse = yield this.runAndPersistAgentResponse({
+                prompt,
+                sessionId,
+                modeName: incoming.modeName,
+                userTimeZone: (_c = incoming.userTimeZone) !== null && _c !== void 0 ? _c : "UTC",
+                adminUser,
+                emit: this.createEventEmitter(sink),
+                failureLogMessage: `Agent ${incoming.surface} surface response failed`,
+                abortLogMessage: `Agent ${incoming.surface} surface response aborted`,
+            });
+            if (agentResponse.failed) {
+                yield sink.emit({
+                    type: "error",
+                    message: agentResponse.text,
+                });
+            }
+            return agentResponse;
+        });
+    }
     handleMessage(adapter, incoming, sink) {
         return __awaiter(this, void 0, void 0, function* () {
-            var _a, _b;
+            var _a;
             if (yield this.handleLink(incoming, sink)) {
                 return;
             }
@@ -127,16 +254,12 @@ export class ChatSurfaceService {
                 username: adminUserRecord[adminforth.config.auth.usernameField],
                 dbUser: adminUserRecord,
             };
-            yield this.handleTurn({
-                prompt: incoming.prompt,
-                sessionId: yield this.sessionStore.getOrCreateChatSurfaceSession(incoming, adminUser),
-                modeName: incoming.modeName,
-                userTimeZone: (_b = incoming.userTimeZone) !== null && _b !== void 0 ? _b : "UTC",
-                adminUser,
-                emit: this.createEventEmitter(sink),
-                failureLogMessage: `Agent ${incoming.surface} surface response failed`,
-                abortLogMessage: `Agent ${incoming.surface} surface response aborted`,
-            });
+            const incomingWithAudio = incoming;
+            if (incomingWithAudio.audio) {
+                yield this.handleAudioMessage(incomingWithAudio, sink, adminUser);
+                return;
+            }
+            yield this.handleAgentSurfaceResponse(incoming, sink, adminUser, incoming.prompt);
         });
     }
 }

package/dist/index.js

@@ -57,7 +57,7 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
             getInternalAgentResourceIds: this.getInternalAgentResourceIds.bind(this),
             getAgentSystemPrompt: () => this.agentSystemPromptPromise,
         });
-        this.chatSurfaceService = new ChatSurfaceService(() => this.adminforth, this.options, this.sessionStore, this.agentTurnService.handleTurn.bind(this.agentTurnService));
+        this.chatSurfaceService = new ChatSurfaceService(() => this.adminforth, this.options, this.sessionStore, this.agentTurnService.handleTurn.bind(this.agentTurnService), this.agentTurnService.runAndPersistAgentResponse.bind(this.agentTurnService));
         this.agentSystemPromptPromise = Promise.resolve(appendCustomSystemPrompt(DEFAULT_AGENT_SYSTEM_PROMPT, this.options.systemPrompt));
         this.shouldHaveSingleInstancePerWholeApp = () => false;
     }

package/index.ts

@@ -77,6 +77,7 @@ export default class AdminForthAgentPlugin extends AdminForthPlugin {
       this.options,
       this.sessionStore,
       this.agentTurnService.handleTurn.bind(this.agentTurnService),
+      this.agentTurnService.runAndPersistAgentResponse.bind(this.agentTurnService),
     );
     this.agentSystemPromptPromise = Promise.resolve(
       appendCustomSystemPrompt(DEFAULT_AGENT_SYSTEM_PROMPT, this.options.systemPrompt),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adminforth/agent",
-  "version": "1.45.1",
+  "version": "1.46.0",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
   "type": "module",