@adminforth/agent 1.50.0 → 1.51.0

package/agent/middleware/apiBasedTools.ts

@@ -11,6 +11,8 @@ import {
 } from "../toolCallEvents.js";
 import { ALWAYS_AVAILABLE_API_TOOL_NAMES } from "../tools/index.js";
 import { createApiTool } from "../tools/apiTool.js";
+import type { AgentEventEmitter } from "../../agentEvents.js";
+import type { SequenceDebugCollector } from "./sequenceDebug.js";
 
 function getEnabledApiToolNames(messages: unknown[]) {
   const enabledToolNames = new Set<string>();
@@ -80,11 +82,19 @@ export function createApiBasedToolsMiddleware(
     async wrapToolCall(request, handler) {
       const startedAt = Date.now();
       const toolInput = JSON.stringify(request.toolCall.args ?? {});
-      const { adminUser, emitToolCallEvent, userTimeZone } = request.runtime.context as {
+      const { adminUser, emit, sequenceDebugSink, userTimeZone } = request.runtime.context as {
         adminUser: AdminUser;
-        emitToolCallEvent: ToolCallEventSink;
+        emit?: AgentEventEmitter;
+        sequenceDebugSink: SequenceDebugCollector;
         userTimeZone: string;
       };
+      const emitToolCall: ToolCallEventSink = (event) => {
+        sequenceDebugSink.handleToolCallEvent(event);
+        void emit?.({
+          type: "tool-call",
+          data: event,
+        });
+      };
       const toolArgs = (request.toolCall.args ?? {}) as Record<string, unknown>;
       let toolInfo: string | undefined;
 
@@ -102,7 +112,7 @@ export function createApiBasedToolsMiddleware(
         });
       }
       const toolCallTracker = createToolCallTracker({
-        emit: emitToolCallEvent,
+        emit: emitToolCall,
         toolCallId: request.toolCall.id,
         toolName: request.toolCall.name,
         toolInfo,

package/agent/models/AgentModeResolver.ts

@@ -0,0 +1,9 @@
+import type { PluginOptions } from "../../types.js";
+
+export class AgentModeResolver {
+  constructor(private readonly options: PluginOptions) {}
+
+  resolve(modeName?: string | null) {
+    return this.options.modes.find((mode) => mode.name === modeName) ?? this.options.modes[0];
+  }
+}

package/agent/models/AgentModelFactory.ts

@@ -0,0 +1,28 @@
+import type { CompletionAdapter } from "adminforth";
+import { createAgentChatModel } from "../simpleAgent.js";
+import type { AgentTurnModels } from "../turn/turnTypes.js";
+
+export class AgentModelFactory {
+  constructor(private readonly maxTokens: number) {}
+
+  async create(completionAdapter: CompletionAdapter): Promise<AgentTurnModels> {
+    const [primaryModelSpec, summaryModelSpec] = await Promise.all([
+      createAgentChatModel({
+        adapter: completionAdapter,
+        maxTokens: this.maxTokens,
+        purpose: "primary",
+      }),
+      createAgentChatModel({
+        adapter: completionAdapter,
+        maxTokens: this.maxTokens,
+        purpose: "summary",
+      }),
+    ]);
+
+    return {
+      model: primaryModelSpec.model,
+      summaryModel: summaryModelSpec.model,
+      modelMiddleware: primaryModelSpec.middleware,
+    };
+  }
+}

package/agent/runtime/AgentContext.ts

@@ -0,0 +1,30 @@
+import type { AdminUser } from "adminforth";
+import { z } from "zod";
+import type { AgentEventEmitter } from "../../agentEvents.js";
+import type { SequenceDebugCollector } from "../middleware/sequenceDebug.js";
+import type { CurrentPageContext } from "../tools/getUserLocation.js";
+import type { AgentTurnContext } from "../turn/turnTypes.js";
+
+export const contextSchema = z.object({
+  adminUser: z.custom<AdminUser>(),
+  userTimeZone: z.string(),
+  sessionId: z.string(),
+  turnId: z.string(),
+  abortSignal: z.custom<AbortSignal>().optional(),
+  currentPage: z.custom<CurrentPageContext>().optional(),
+  chatSurface: z.string().optional(),
+  adminBaseUrl: z.string().optional(),
+  adminPublicOrigin: z.string().optional(),
+  emit: z.custom<AgentEventEmitter>().optional(),
+  sequenceDebugSink: z.custom<SequenceDebugCollector>(),
+});
+
+export function toLangchainAgentContext(
+  context: AgentTurnContext & {
+    adminBaseUrl: string;
+    emit?: AgentEventEmitter;
+    sequenceDebugSink: SequenceDebugCollector;
+  },
+) {
+  return context;
+}

package/agent/runtime/AgentRuntime.ts

@@ -0,0 +1,68 @@
+import type { IAdminForth } from "adminforth";
+import { createAgent, summarizationMiddleware } from "langchain";
+import type { BaseCheckpointSaver } from "@langchain/langgraph";
+import { createApiBasedToolsMiddleware } from "../middleware/apiBasedTools.js";
+import { createSequenceDebugMiddleware } from "../middleware/sequenceDebug.js";
+import { createAgentLlmMetricsLogger } from "../simpleAgent.js";
+import type { AgentToolProvider } from "../tools/AgentToolProvider.js";
+import type { AgentRuntimeRunInput } from "../turn/turnTypes.js";
+import { contextSchema, toLangchainAgentContext } from "./AgentContext.js";
+
+export type AgentRuntimeOptions = {
+  name: string;
+  getAdminforth: () => IAdminForth;
+  getCheckpointer: () => BaseCheckpointSaver;
+  toolProvider: AgentToolProvider;
+};
+
+export class AgentRuntime {
+  constructor(private readonly options: AgentRuntimeOptions) {}
+
+  async stream(input: AgentRuntimeRunInput) {
+    const apiBasedTools = this.options.toolProvider.getApiBasedTools();
+    const tools = await this.options.toolProvider.getTools(apiBasedTools);
+    const adminforth = this.options.getAdminforth();
+    const apiBasedToolsMiddleware = createApiBasedToolsMiddleware(
+      apiBasedTools,
+      adminforth,
+    );
+    const sequenceDebugMiddleware = createSequenceDebugMiddleware(
+      input.observability.sequenceDebugSink,
+    );
+    const middleware = [
+      apiBasedToolsMiddleware,
+      ...(input.models.modelMiddleware ?? []),
+      sequenceDebugMiddleware,
+      summarizationMiddleware({
+        model: input.models.summaryModel,
+        trigger: { tokens: 1024 * 64 },
+        keep: { messages: 10 },
+      }),
+    ] as const;
+
+    const agent = createAgent({
+      name: this.options.name,
+      model: input.models.model,
+      checkpointer: this.options.getCheckpointer(),
+      tools,
+      contextSchema,
+      middleware,
+    });
+
+    return agent.stream({ messages: input.messages } as any, {
+      streamMode: "messages",
+      recursionLimit: 100,
+      callbacks: [createAgentLlmMetricsLogger()],
+      signal: input.context.abortSignal,
+      configurable: {
+        thread_id: input.context.sessionId,
+      },
+      context: toLangchainAgentContext({
+        ...input.context,
+        adminBaseUrl: adminforth.config.baseUrlSlashed,
+        emit: input.observability.emit,
+        sequenceDebugSink: input.observability.sequenceDebugSink,
+      }),
+    });
+  }
+}

package/agent/simpleAgent.ts

@@ -1,39 +1,13 @@
 import type { BaseChatModel } from "@langchain/core/language_models/chat_models";
-import { createAgent, summarizationMiddleware } from "langchain";
 import {
   logger,
-  type AdminUser,
   type CompletionAdapter,
-  type IAdminForth,
 } from "adminforth";
 import { BaseCallbackHandler } from "@langchain/core/callbacks/base";
-import {type BaseCheckpointSaver, type Messages } from "@langchain/langgraph";
 import type { LLMResult } from "@langchain/core/outputs";
-import { z } from "zod";
-import { createAgentTools } from "./tools/index.js";
-import { createApiBasedToolsMiddleware } from "./middleware/apiBasedTools.js";
 import {
   createSequenceDebugMiddleware,
-  type SequenceDebugModelCallSink,
 } from "./middleware/sequenceDebug.js";
-import type { ApiBasedTool } from "../apiBasedTools.js";
-import type { ToolCallEventSink } from "./toolCallEvents.js";
-import type { CurrentPageContext } from "./tools/getUserLocation.js";
-import type { AgentEventEmitter } from "../agentEvents.js";
-
-export const contextSchema = z.object({
-  adminUser: z.custom<AdminUser>(),
-  userTimeZone: z.string(),
-  sessionId: z.string(),
-  turnId: z.string(),
-  abortSignal: z.custom<AbortSignal>().optional(),
-  currentPage: z.custom<CurrentPageContext>().optional(),
-  chatSurface: z.string().optional(),
-  adminBaseUrl: z.string().optional(),
-  adminPublicOrigin: z.string().optional(),
-  emitToolCallEvent: z.custom<ToolCallEventSink>(),
-  emitAgentEvent: z.custom<AgentEventEmitter>().optional(),
-});
 
 export type AgentChatModel = BaseChatModel<any, any>;
 export type AgentModelPurpose = "primary" | "summary";
@@ -50,7 +24,7 @@ export type AgentModeCompletionAdapter = CompletionAdapter & {
   };
 };
 
-type AgentMiddleware = ReturnType<typeof createSequenceDebugMiddleware>;
+export type AgentMiddleware = ReturnType<typeof createSequenceDebugMiddleware>;
 
 type AgentChatModelSpec = {
   model: AgentChatModel;
@@ -202,7 +176,7 @@ class AgentLlmMetricsLogger extends BaseCallbackHandler {
   }
 }
 
-function createAgentLlmMetricsLogger() {
+export function createAgentLlmMetricsLogger() {
   return new AgentLlmMetricsLogger();
 }
 
@@ -223,104 +197,3 @@ export async function createAgentChatModel(params: {
     purpose: params.purpose,
   });
 }
-
-export async function callAgent(params: {
-  name: string;
-  model: AgentChatModel;
-  summaryModel: AgentChatModel;
-  modelMiddleware?: AgentMiddleware[];
-  checkpointer?: BaseCheckpointSaver;
-  messages: Messages;
-  adminUser: AdminUser;
-  adminforth: IAdminForth;
-  apiBasedTools: Record<string, ApiBasedTool>;
-  customComponentsDir: string;
-  pluginCustomFolderPaths: string[];
-  sessionId: string;
-  turnId: string;
-  currentPage?: CurrentPageContext;
-  chatSurface?: string;
-  adminPublicOrigin?: string;
-  userTimeZone: string;
-  abortSignal?: AbortSignal;
-  emitToolCallEvent: ToolCallEventSink;
-  emitAgentEvent?: AgentEventEmitter;
-  sequenceDebugSink: SequenceDebugModelCallSink;
-}) {
-  const {
-    name,
-    model,
-    summaryModel,
-    modelMiddleware = [],
-    checkpointer,
-    messages,
-    adminUser,
-    adminforth,
-    apiBasedTools,
-    customComponentsDir,
-    pluginCustomFolderPaths,
-    sessionId,
-    turnId,
-    currentPage,
-    chatSurface,
-    adminPublicOrigin,
-    userTimeZone,
-    abortSignal,
-    emitToolCallEvent,
-    emitAgentEvent,
-    sequenceDebugSink,
-  } = params;
-
-  const tools = await createAgentTools(
-    customComponentsDir,
-    apiBasedTools,
-    pluginCustomFolderPaths,
-  );
-  const apiBasedToolsMiddleware = createApiBasedToolsMiddleware(apiBasedTools, adminforth);
-  const sequenceDebugMiddleware = createSequenceDebugMiddleware(
-    sequenceDebugSink,
-  );
-
-  const middleware = [
-    apiBasedToolsMiddleware,
-    ...modelMiddleware,
-    sequenceDebugMiddleware,
-    summarizationMiddleware({
-      model: summaryModel,
-      trigger: { tokens: 1024 * 64 },
-      keep: { messages: 10 },
-    }),
-  ] as const;
-
-  const agent = createAgent<undefined, typeof contextSchema, typeof middleware>({
-    name,
-    model,
-    checkpointer,
-    tools,
-    contextSchema,
-    middleware,
-  });
-
-  return await agent.stream({ messages } as any, {
-    streamMode: "messages",
-    recursionLimit: 100,
-    callbacks: [createAgentLlmMetricsLogger()],
-    signal: abortSignal,
-    configurable: {
-      thread_id: sessionId,
-    },
-    context: {
-      adminUser,
-      userTimeZone,
-      sessionId,
-      turnId,
-      abortSignal,
-      currentPage,
-      chatSurface,
-      adminBaseUrl: adminforth.config.baseUrlSlashed,
-      adminPublicOrigin,
-      emitToolCallEvent,
-      emitAgentEvent,
-    },
-  });
-}

package/agent/speech/SpeechTurnService.ts

@@ -0,0 +1,179 @@
+import { logger } from "adminforth";
+import { getErrorMessage, isAbortError } from "../../errors.js";
+import { sanitizeSpeechText } from "../../sanitizeSpeechText.js";
+import type {
+  RunAndPersistAgentResponseInput,
+  RunAndPersistAgentResponseResult,
+  SpeechAgentTurnInput,
+} from "../turn/turnTypes.js";
+
+export class SpeechTurnService {
+  constructor(
+    private readonly runAndPersistAgentResponse: (
+      input: RunAndPersistAgentResponseInput,
+    ) => Promise<RunAndPersistAgentResponseResult>,
+  ) {}
+
+  async handle(input: SpeechAgentTurnInput) {
+    let transcription;
+
+    try {
+      transcription = await input.audioAdapter.transcribe({
+        buffer: input.audio.buffer,
+        filename: input.audio.filename,
+        mimeType: input.audio.mimeType,
+        language: "auto",
+        abortSignal: input.abortSignal,
+      });
+    } catch (error) {
+      if (input.abortSignal?.aborted || isAbortError(error)) {
+        logger.info("Agent speech transcription aborted by the client");
+        await input.emit({ type: "finish" });
+        return null;
+      }
+
+      logger.error(`Agent speech transcription failed:\n${getErrorMessage(error)}`);
+      await input.emit({
+        type: "error",
+        error: "Speech transcription failed. Check server logs for details.",
+      });
+      await input.emit({ type: "finish" });
+      return null;
+    }
+
+    if (input.abortSignal?.aborted) {
+      await input.emit({ type: "finish" });
+      return null;
+    }
+
+    const prompt = transcription.text;
+    if (!prompt) {
+      await input.emit({
+        type: "error",
+        error: "Speech transcription is empty",
+      });
+      await input.emit({ type: "finish" });
+      return null;
+    }
+
+    await input.emit({
+      type: "transcript",
+      text: transcription.text,
+      language: transcription.language,
+    });
+
+    const agentResponse = await this.runAndPersistAgentResponse({
+      prompt,
+      sessionId: input.sessionId,
+      modeName: input.modeName,
+      userTimeZone: input.userTimeZone,
+      currentPage: input.currentPage,
+      chatSurface: input.chatSurface,
+      adminPublicOrigin: input.adminPublicOrigin,
+      abortSignal: input.abortSignal,
+      adminUser: input.adminUser,
+      emit: async (event) => {
+        if (event.type === "tool-call") {
+          await input.emit(event);
+        }
+      },
+      failureLogMessage: input.failureLogMessage ?? "Agent speech response failed",
+      abortLogMessage: input.abortLogMessage ?? "Agent speech response aborted by the client",
+    });
+
+    if (agentResponse.aborted) {
+      await input.emit({ type: "finish" });
+      return agentResponse;
+    }
+
+    if (agentResponse.failed) {
+      await input.emit({
+        type: "error",
+        error: agentResponse.text,
+      });
+      await input.emit({ type: "finish" });
+      return agentResponse;
+    }
+
+    try {
+      await input.emit({
+        type: "speech-response",
+        transcript: {
+          text: transcription.text,
+          language: transcription.language,
+        },
+        response: {
+          text: agentResponse.text,
+        },
+        sessionId: input.sessionId,
+        turnId: agentResponse.turnId,
+      });
+      const speech = await input.audioAdapter.synthesize({
+        text: sanitizeSpeechText(agentResponse.text),
+        stream: true,
+        streamFormat: "audio",
+        format: "pcm",
+        abortSignal: input.abortSignal,
+      });
+
+      await input.emit({
+        type: "audio-start",
+        mimeType: speech.mimeType,
+        format: speech.format,
+        sampleRate: 24000,
+        channelCount: 1,
+        bitsPerSample: 16,
+      });
+
+      const reader = speech.audioStream.getReader();
+      const cancelAudioStream = () => {
+        void reader.cancel().catch(() => undefined);
+      };
+
+      try {
+        input.abortSignal?.addEventListener("abort", cancelAudioStream, { once: true });
+
+        while (true) {
+          if (input.abortSignal?.aborted) {
+            await reader.cancel().catch(() => undefined);
+            break;
+          }
+
+          const { value, done } = await reader.read();
+
+          if (done) {
+            break;
+          }
+
+          if (input.abortSignal?.aborted) {
+            break;
+          }
+
+          await input.emit({
+            type: "audio-delta",
+            value,
+          });
+        }
+      } finally {
+        input.abortSignal?.removeEventListener("abort", cancelAudioStream);
+        reader.releaseLock();
+      }
+
+      await input.emit({ type: "audio-done" });
+      await input.emit({ type: "finish" });
+      return agentResponse;
+    } catch (error) {
+      if (input.abortSignal?.aborted || isAbortError(error)) {
+        logger.info("Agent speech audio streaming aborted by the client");
+      } else {
+        logger.error(`Agent speech audio streaming failed:\n${getErrorMessage(error)}`);
+        await input.emit({
+          type: "error",
+          error: getErrorMessage(error),
+        });
+      }
+      await input.emit({ type: "finish" });
+      return agentResponse;
+    }
+  }
+}

package/agent/tools/AgentToolProvider.ts

@@ -0,0 +1,28 @@
+import type { IAdminForth } from "adminforth";
+import { prepareApiBasedTools } from "../../apiBasedTools.js";
+import type { ApiBasedTool } from "../../apiBasedTools.js";
+import { createAgentTools } from "./index.js";
+
+export class AgentToolProvider {
+  constructor(
+    private readonly getAdminforth: () => IAdminForth,
+    private readonly getInternalAgentResourceIds: () => string[],
+  ) {}
+
+  getApiBasedTools(): Record<string, ApiBasedTool> {
+    return prepareApiBasedTools(
+      this.getAdminforth(),
+      this.getInternalAgentResourceIds(),
+    );
+  }
+
+  async getTools(apiBasedTools: Record<string, ApiBasedTool>) {
+    const adminforth = this.getAdminforth();
+
+    return createAgentTools(
+      adminforth.config.customization.customComponentsDir ?? "custom",
+      apiBasedTools,
+      adminforth.activatedPlugins.map((plugin) => plugin.customFolderPath),
+    );
+  }
+}

package/agent/tools/navigateUser.ts

@@ -159,7 +159,7 @@ export function createNavigateUserTool() {
         chatSurface?: string;
         adminBaseUrl?: string;
         adminPublicOrigin?: string;
-        emitAgentEvent?: AgentEventEmitter;
+        emit?: AgentEventEmitter;
       };
       const currentPage = context.currentPage;
       const basePath = input.targetPath
@@ -184,7 +184,7 @@ export function createNavigateUserTool() {
         );
       }
 
-      await context.emitAgentEvent?.({
+      await context.emit?.({
         type: "open-page",
         targetPath,
       });

package/agent/turn/TurnContextBuilder.ts

@@ -0,0 +1,36 @@
+import type { AdminUser, IAdminForth } from "adminforth";
+import type { AgentTurnContext, BaseAgentTurnInput } from "./turnTypes.js";
+
+export type UserContextProvider = {
+  getUserTimeZone(adminUser: AdminUser): Promise<string | null | undefined> | string | null | undefined;
+};
+
+export class TurnContextBuilder {
+  constructor(
+    private readonly getAdminforth: () => IAdminForth,
+    private readonly userContextProvider?: UserContextProvider,
+  ) {}
+
+  async build(input: {
+    base: BaseAgentTurnInput;
+    turnId: string;
+  }): Promise<AgentTurnContext> {
+    const adminforth = this.getAdminforth();
+
+    return {
+      adminUser: input.base.adminUser,
+      sessionId: input.base.sessionId,
+      turnId: input.turnId,
+      abortSignal: input.base.abortSignal,
+      currentPage: input.base.currentPage,
+      chatSurface: input.base.chatSurface,
+      userTimeZone:
+        input.base.userTimeZone
+        ?? await this.userContextProvider?.getUserTimeZone(input.base.adminUser)
+        ?? "UTC",
+      adminPublicOrigin:
+        input.base.adminPublicOrigin
+        ?? adminforth.config.baseUrlSlashed,
+    };
+  }
+}

package/agent/turn/TurnLifecycleService.ts

@@ -0,0 +1,29 @@
+import type { AgentSessionStore } from "../../sessionStore.js";
+import type { BaseAgentTurnInput } from "./turnTypes.js";
+import { TurnPersistenceService } from "./TurnPersistenceService.js";
+
+export class TurnLifecycleService {
+  constructor(
+    private readonly sessionStore: AgentSessionStore,
+    private readonly persistence: TurnPersistenceService,
+  ) {}
+
+  async start(input: BaseAgentTurnInput) {
+    const previousUserMessages = await this.sessionStore.getPreviousUserMessages(input.sessionId);
+    const turnId = await this.sessionStore.createNewTurn(input.sessionId, input.prompt);
+    await this.persistence.touchSession(input.sessionId);
+
+    return {
+      turnId,
+      previousUserMessages,
+    };
+  }
+
+  async finish(input: {
+    turnId: string;
+    responseText: string;
+    debugHistory?: unknown;
+  }) {
+    await this.persistence.saveTurnResponse(input);
+  }
+}

package/agent/turn/TurnPersistenceService.ts

@@ -0,0 +1,33 @@
+import type { IAdminForth } from "adminforth";
+import type { PluginOptions } from "../../types.js";
+
+export class TurnPersistenceService {
+  constructor(
+    private readonly getAdminforth: () => IAdminForth,
+    private readonly options: PluginOptions,
+  ) {}
+
+  async touchSession(sessionId: string) {
+    await this.getAdminforth().resource(this.options.sessionResource.resourceId).update(sessionId, {
+      [this.options.sessionResource.createdAtField]: new Date().toISOString(),
+    });
+  }
+
+  async saveTurnResponse(input: {
+    turnId: string;
+    responseText: string;
+    debugHistory?: unknown;
+  }) {
+    const turnUpdates: Record<string, unknown> = {
+      [this.options.turnResource.responseField]: input.responseText,
+    };
+
+    if (this.options.turnResource.debugField) {
+      turnUpdates[this.options.turnResource.debugField] = input.debugHistory;
+    }
+
+    await this.getAdminforth()
+      .resource(this.options.turnResource.resourceId)
+      .update(input.turnId, turnUpdates);
+  }
+}