@mastra/livekit 0.0.0 → 0.2.0-alpha.0

package/dist/worker-entry.js

@@ -0,0 +1,652 @@
+import { parseSessionMetadata, createWorkflowReplyGenerator, DEFAULT_LIVEKIT_AGENT_NAME } from './chunk-2E3MTAOA.js';
+import { InferenceRunner, defineAgent, voice, cli, ServerOptions, metrics, llm } from '@livekit/agents';
+import { RequestContext } from '@mastra/core/request-context';
+import { ReadableStream } from 'stream/web';
+import { getOrCreateSpan, SpanType } from '@mastra/core/observability';
+import { randomUUID } from 'crypto';
+import { fileURLToPath } from 'url';
+
+// src/messages.ts
+function textOfMessage(message) {
+  const parts = [];
+  for (const part of message.content) {
+    if (typeof part === "string") {
+      parts.push(part);
+    } else if (part.type === "instructions") {
+      parts.push(part.value);
+    } else if (part.type === "audio_content" && part.transcript) {
+      parts.push(part.transcript);
+    }
+  }
+  return parts.join("\n").trim();
+}
+function toVoiceTurnMessage(item) {
+  if (item.type !== "message") return void 0;
+  const content = textOfMessage(item);
+  if (!content) return void 0;
+  if (item.role === "user") return { role: "user", content };
+  if (item.role === "assistant") return { role: "assistant", content };
+  return { role: "system", content };
+}
+function extractNewTurnMessages(chatCtx) {
+  const items = chatCtx.items;
+  let lastAssistantIdx = -1;
+  for (let i = items.length - 1; i >= 0; i--) {
+    const item = items[i];
+    if (item?.type === "message" && item.role === "assistant") {
+      lastAssistantIdx = i;
+      break;
+    }
+  }
+  const messages = [];
+  for (const item of items.slice(lastAssistantIdx + 1)) {
+    const message = toVoiceTurnMessage(item);
+    if (message) messages.push(message);
+  }
+  return messages;
+}
+function chatContextToMessages(chatCtx) {
+  const withoutInstructions = chatCtx.copy({ excludeInstructions: true, excludeFunctionCall: true });
+  const messages = [];
+  for (const item of withoutInstructions.items) {
+    const message = toVoiceTurnMessage(item);
+    if (message) messages.push(message);
+  }
+  return messages;
+}
+
+// src/bridge.ts
+var DEFAULT_INSTRUCTIONS = "You are a helpful voice assistant powered by a Mastra agent.";
+function createAgentReplyGenerator(options) {
+  const { agent, streamOptions, toolFeedback, onTurnComplete } = options;
+  return (ctx) => {
+    if (ctx.messages.length === 0) return null;
+    const abortController = new AbortController();
+    const mergedOptions = {
+      ...streamOptions,
+      abortSignal: abortController.signal
+    };
+    if (ctx.memory) mergedOptions.memory = ctx.memory;
+    if (ctx.requestContext) mergedOptions.requestContext = ctx.requestContext;
+    let cancelled = false;
+    let replyText = "";
+    const toolCalls = [];
+    const emitTurnComplete = (interrupted) => {
+      if (!onTurnComplete) return;
+      const completeCtx = { ...ctx, result: { text: replyText, toolCalls, interrupted } };
+      Promise.resolve().then(() => onTurnComplete(completeCtx)).catch((error) => {
+        console.warn("@mastra/livekit: onTurnComplete hook threw", error);
+      });
+    };
+    return new ReadableStream({
+      start: async (controller) => {
+        try {
+          const result = await agent.stream(ctx.messages, mergedOptions);
+          for await (const chunk of result.fullStream) {
+            if (cancelled) break;
+            if (chunk.type === "text-delta") {
+              if (chunk.payload.text) {
+                replyText += chunk.payload.text;
+                controller.enqueue(chunk.payload.text);
+              }
+            } else if (chunk.type === "tool-call") {
+              const toolCall = {
+                toolCallId: chunk.payload.toolCallId,
+                toolName: chunk.payload.toolName,
+                args: chunk.payload.args
+              };
+              toolCalls.push(toolCall);
+              if (toolFeedback) {
+                const filler = toolFeedback(toolCall);
+                if (filler) controller.enqueue(filler.endsWith(" ") ? filler : `${filler} `);
+              }
+            } else if (chunk.type === "error") {
+              const error = chunk.payload.error;
+              throw error instanceof Error ? error : new Error(String(error));
+            }
+          }
+          if (!cancelled) controller.close();
+          emitTurnComplete(cancelled);
+        } catch (error) {
+          if (cancelled || abortController.signal.aborted) {
+            emitTurnComplete(true);
+            return;
+          }
+          controller.error(error);
+        }
+      },
+      cancel: () => {
+        cancelled = true;
+        abortController.abort();
+      }
+    });
+  };
+}
+function toRequestContext(value) {
+  if (!value) return void 0;
+  if (value instanceof RequestContext) return value;
+  return new RequestContext(Object.entries(value));
+}
+var MastraPlaceholderLLM = class extends llm.LLM {
+  label() {
+    return "mastra.MastraVoiceAgent";
+  }
+  get model() {
+    return "mastra-agent";
+  }
+  get provider() {
+    return "mastra";
+  }
+  chat() {
+    throw new Error(
+      "@mastra/livekit: reply generation runs through the Mastra agent via llmNode; the placeholder LLM cannot be used for inference."
+    );
+  }
+};
+var MastraVoiceAgent = class extends voice.Agent {
+  mastraAgent;
+  memory;
+  requestContext;
+  streamOptions;
+  replyGenerator;
+  constructor(options) {
+    if (options.agent && options.generate) {
+      throw new Error(
+        "@mastra/livekit: MastraVoiceAgent requires `agent` or `generate`, not both \u2014 they are mutually exclusive reply sources."
+      );
+    }
+    super({
+      id: options.id,
+      instructions: options.instructions ?? DEFAULT_INSTRUCTIONS,
+      stt: options.stt,
+      vad: options.vad,
+      llm: new MastraPlaceholderLLM(),
+      tts: options.tts,
+      turnHandling: options.turnHandling
+    });
+    this.memory = options.memory ?? false;
+    this.requestContext = toRequestContext(options.requestContext);
+    this.streamOptions = options.streamOptions;
+    if (options.generate) {
+      this.replyGenerator = options.generate;
+    } else if (options.agent) {
+      this.mastraAgent = options.agent;
+      this.replyGenerator = createAgentReplyGenerator({
+        agent: options.agent,
+        streamOptions: options.streamOptions,
+        toolFeedback: options.toolFeedback,
+        onTurnComplete: options.onTurnComplete
+      });
+    } else {
+      throw new Error("@mastra/livekit: MastraVoiceAgent requires `agent` or `generate`.");
+    }
+  }
+  async llmNode(chatCtx, _toolCtx, _modelSettings) {
+    const messages = this.memory === false ? chatContextToMessages(chatCtx) : extractNewTurnMessages(chatCtx);
+    if (messages.length === 0) return null;
+    return this.replyGenerator({
+      messages,
+      chatCtx,
+      memory: this.memory,
+      requestContext: this.requestContext,
+      tracingContext: this.streamOptions?.tracingContext
+    });
+  }
+};
+function createMastraVoiceAgent(options) {
+  return new MastraVoiceAgent(options);
+}
+function modelMeta(metadata) {
+  const out = {};
+  if (metadata?.modelProvider) out.modelProvider = metadata.modelProvider;
+  if (metadata?.modelName) out.modelName = metadata.modelName;
+  return out;
+}
+var ms = (n) => `${Math.round(n)}ms`;
+var secs = (n) => `${(n / 1e3).toFixed(1)}s`;
+function describeMetric(metric) {
+  switch (metric.type) {
+    case "eou_metrics":
+      return {
+        name: `eou ${ms(metric.endOfUtteranceDelayMs)}`,
+        data: {
+          endOfUtteranceDelayMs: metric.endOfUtteranceDelayMs,
+          transcriptionDelayMs: metric.transcriptionDelayMs,
+          onUserTurnCompletedDelayMs: metric.onUserTurnCompletedDelayMs
+        }
+      };
+    case "stt_metrics":
+      return {
+        name: `stt ${secs(metric.audioDurationMs)}`,
+        data: {
+          audioDurationMs: metric.audioDurationMs,
+          durationMs: metric.durationMs,
+          streamed: metric.streamed,
+          ...modelMeta(metric.metadata)
+        }
+      };
+    case "llm_metrics":
+      return {
+        name: `llm ttft ${ms(metric.ttftMs)}`,
+        data: {
+          ttftMs: metric.ttftMs,
+          durationMs: metric.durationMs,
+          tokensPerSecond: metric.tokensPerSecond,
+          promptTokens: metric.promptTokens,
+          completionTokens: metric.completionTokens,
+          totalTokens: metric.totalTokens,
+          cancelled: metric.cancelled,
+          ...modelMeta(metric.metadata)
+        }
+      };
+    case "tts_metrics":
+      return {
+        name: `tts ttfb ${ms(metric.ttfbMs)}`,
+        data: {
+          ttfbMs: metric.ttfbMs,
+          durationMs: metric.durationMs,
+          audioDurationMs: metric.audioDurationMs,
+          charactersCount: metric.charactersCount,
+          cancelled: metric.cancelled,
+          streamed: metric.streamed,
+          ...modelMeta(metric.metadata)
+        }
+      };
+    case "vad_metrics":
+      return {
+        name: "vad",
+        data: {
+          idleTimeMs: metric.idleTimeMs,
+          inferenceCount: metric.inferenceCount,
+          inferenceDurationTotalMs: metric.inferenceDurationTotalMs
+        }
+      };
+    case "realtime_model_metrics":
+      return {
+        name: `realtime ttft ${ms(metric.ttftMs)}`,
+        data: {
+          ttftMs: metric.ttftMs,
+          durationMs: metric.durationMs,
+          tokensPerSecond: metric.tokensPerSecond,
+          promptTokens: metric.inputTokens,
+          completionTokens: metric.outputTokens,
+          totalTokens: metric.totalTokens,
+          ...modelMeta(metric.metadata)
+        }
+      };
+    default:
+      return void 0;
+  }
+}
+function startVoiceCallObservability(options) {
+  const span = getOrCreateSpan({
+    mastra: options.mastra,
+    type: SpanType.GENERIC,
+    name: "voice call",
+    requestContext: options.requestContext,
+    metadata: {
+      agentId: options.agentId,
+      roomName: options.roomName,
+      ...options.metadata.threadId ? { threadId: options.metadata.threadId } : {},
+      ...options.metadata.resourceId ? { resourceId: options.metadata.resourceId } : {}
+    }
+  });
+  if (!span) return void 0;
+  const usage = new metrics.ModelUsageCollector();
+  let finalized = false;
+  const finalize = (opts) => {
+    if (finalized) return;
+    finalized = true;
+    const summary = usage.flatten();
+    if (opts?.error) {
+      span.error({
+        error: opts.error instanceof Error ? opts.error : new Error(String(opts.error)),
+        metadata: { usage: summary }
+      });
+    } else {
+      span.end({ output: { usage: summary } });
+    }
+  };
+  return {
+    span,
+    tracingContext: { currentSpan: span },
+    attach(session) {
+      session.on(voice.AgentSessionEventTypes.MetricsCollected, (event) => {
+        usage.collect(event.metrics);
+        const described = describeMetric(event.metrics);
+        if (!described) return;
+        span.createEventSpan({ type: SpanType.GENERIC, name: described.name, output: described.data });
+      });
+      session.on(voice.AgentSessionEventTypes.Close, (event) => {
+        finalize({ error: event?.error ?? void 0 });
+      });
+    },
+    finalize
+  };
+}
+async function ensureVoiceCallThread({
+  memory,
+  threadId,
+  resourceId,
+  roomName
+}) {
+  const existing = await memory.getThreadById({ threadId });
+  if (existing) return;
+  await memory.createThread({
+    threadId,
+    resourceId,
+    title: "Voice call",
+    metadata: { source: "livekit", roomName }
+  });
+}
+async function persistSpokenGreeting({
+  memory,
+  threadId,
+  resourceId,
+  greeting
+}) {
+  if (!greeting.trim()) return;
+  const message = {
+    id: randomUUID(),
+    role: "assistant",
+    type: "text",
+    createdAt: /* @__PURE__ */ new Date(),
+    threadId,
+    resourceId,
+    content: {
+      format: 2,
+      parts: [{ type: "text", text: greeting }],
+      metadata: { source: "voice", kind: "greeting" }
+    }
+  };
+  await memory.saveMessages({ messages: [message] });
+}
+
+// src/worker-setup.ts
+var pendingSetup = [];
+var requestedEouMethods = /* @__PURE__ */ new Set();
+function queueWorkerSetup(setup) {
+  pendingSetup.push(setup);
+}
+function workerSetupComplete() {
+  return Promise.allSettled(pendingSetup);
+}
+function requestEouMethod(method) {
+  requestedEouMethods.add(method);
+}
+function isEouMethodRequested(method) {
+  return requestedEouMethods.has(method);
+}
+
+// src/worker.ts
+var EOU_METHODS = {
+  english: "lk_end_of_utterance_en",
+  multilingual: "lk_end_of_utterance_multilingual"
+};
+async function loadSileroVad() {
+  let silero;
+  try {
+    silero = await import('@livekit/agents-plugin-silero');
+  } catch (error) {
+    throw new Error(
+      "@mastra/livekit: voice activity detection requires '@livekit/agents-plugin-silero'. Install it, pass your own `vad` instance, or set `vad: false`.",
+      { cause: error }
+    );
+  }
+  return silero.VAD.load();
+}
+async function loadTurnDetector(kind) {
+  let plugin;
+  try {
+    plugin = await import('@livekit/agents-plugin-livekit');
+  } catch (error) {
+    throw new Error(
+      `@mastra/livekit: turnDetection '${kind}' requires '@livekit/agents-plugin-livekit'. Install it or use a built-in mode like 'vad' or 'stt'.`,
+      { cause: error }
+    );
+  }
+  return kind === "english" ? new plugin.turnDetector.EnglishModel() : new plugin.turnDetector.MultilingualModel();
+}
+async function resolveMastraAgent(options, args) {
+  let ref = typeof options.agent === "function" ? await options.agent(args) : options.agent;
+  ref ??= args.metadata.agentId;
+  if (!ref) {
+    throw new Error(
+      "@mastra/livekit: no Mastra agent specified. Set `agent` on createLiveKitWorker or pass `agentId` in the dispatch metadata (e.g. via liveKitConnectionRoute)."
+    );
+  }
+  if (typeof ref !== "string") return ref;
+  try {
+    return options.mastra.getAgentById(ref);
+  } catch {
+    return options.mastra.getAgent(ref);
+  }
+}
+async function resolveWorkflow(options, args) {
+  const resolver = options.workflow;
+  const ref = typeof resolver === "function" ? await resolver(args) : resolver ?? "";
+  if (!ref) {
+    throw new Error("@mastra/livekit: no workflow specified. Set `workflow` on createLiveKitWorker.");
+  }
+  if (typeof ref !== "string") return { workflow: ref };
+  return { workflow: options.mastra.getWorkflowById(ref) };
+}
+function resolveMemory(options, mastraAgent, args, roomName) {
+  if (options.memory === false) return false;
+  if (typeof options.memory === "function") return options.memory({ ...args, roomName });
+  if (mastraAgent && !mastraAgent.hasOwnMemory()) return false;
+  const thread = args.metadata.threadId ?? roomName;
+  return { thread, resource: args.metadata.resourceId ?? thread };
+}
+async function resolveMemoryInstance(options, mastraAgent, args, requestContext) {
+  if (mastraAgent) return await mastraAgent.getMemory({ requestContext }) ?? null;
+  const resolver = options.memoryInstance;
+  if (!resolver) return null;
+  const instance = typeof resolver === "function" ? await resolver(args) : resolver;
+  if (!instance) return null;
+  instance.__registerMastra(options.mastra);
+  if (!instance.hasOwnStorage) {
+    const storage = options.mastra.getStorage();
+    if (storage) instance.setStorage(storage);
+  }
+  return instance;
+}
+function buildTurnHandling(options, turnDetection) {
+  return {
+    ...turnDetection ? { turnDetection } : {},
+    // Preemptive generation re-runs the Mastra agent on interim transcripts (up to 3
+    // times per turn), and every run persists the user message to the memory thread —
+    // duplicating and even saving partial transcripts. Off by default; opt back in via
+    // `turnHandling.preemptiveGeneration` if the latency win matters more than exact
+    // thread history.
+    preemptiveGeneration: { enabled: false },
+    ...options.turnHandling
+  };
+}
+async function resolveInstructions(mastraAgent, requestContext) {
+  try {
+    const instructions = await mastraAgent.getInstructions({ requestContext });
+    return typeof instructions === "string" ? instructions : void 0;
+  } catch {
+    return void 0;
+  }
+}
+function createLiveKitWorker(options) {
+  if (options.generate && (options.agent || options.workflow)) {
+    throw new Error(
+      "@mastra/livekit: set exactly one reply generator \u2014 `generate`, `agent`, or `workflow` \u2014 not a combination."
+    );
+  }
+  if (options.agent && options.workflow) {
+    throw new Error(
+      "@mastra/livekit: set `agent` or `workflow`, not both \u2014 they are mutually exclusive reply generators."
+    );
+  }
+  if (options.workflow && !options.workflowInput) {
+    throw new Error(
+      "@mastra/livekit: `workflowInput` is required when `workflow` is set. Map the turn into the workflow inputData, e.g. workflowInput: ({ chatCtx }) => ({ history: chatContextToMessages(chatCtx) })."
+    );
+  }
+  const wantsSileroVad = options.vad === void 0 || options.vad === "silero";
+  if (options.turnDetection === "multilingual" || options.turnDetection === "english") {
+    requestEouMethod(EOU_METHODS[options.turnDetection]);
+    queueWorkerSetup(
+      import('@livekit/agents-plugin-livekit').then(() => {
+        for (const method of Object.values(EOU_METHODS)) {
+          if (!isEouMethodRequested(method)) {
+            delete InferenceRunner.registeredRunners[method];
+          }
+        }
+      }).catch(() => {
+      })
+    );
+  }
+  return defineAgent({
+    prewarm: async (proc) => {
+      if (wantsSileroVad) {
+        proc.userData.vad = await loadSileroVad();
+      }
+    },
+    entry: async (ctx) => {
+      const logger = options.mastra.getLogger();
+      const metadata = parseSessionMetadata(ctx.job.metadata);
+      const args = { metadata, ctx };
+      const requestContext = metadata.requestContext ? new RequestContext(Object.entries(metadata.requestContext)) : void 0;
+      let mastraAgent;
+      let replyGenerator;
+      let agentLabel;
+      if (options.generate) {
+        replyGenerator = options.generate;
+        agentLabel = "mastra-voice";
+      } else if (options.workflow) {
+        const { workflow } = await resolveWorkflow(options, args);
+        agentLabel = workflow.id;
+        const mapInput = options.workflowInput;
+        replyGenerator = createWorkflowReplyGenerator({
+          workflow,
+          workflowInput: (turnCtx) => mapInput({ ...turnCtx, metadata }),
+          replyStep: options.replyStep,
+          resultText: options.resultText,
+          toolFeedback: options.toolFeedback,
+          onTurnComplete: options.onTurnComplete
+        });
+      } else {
+        mastraAgent = await resolveMastraAgent(options, args);
+        agentLabel = mastraAgent.id ?? mastraAgent.name;
+      }
+      await ctx.connect();
+      const roomName = ctx.room.name ?? "mastra-voice";
+      let vad;
+      if (options.vad && options.vad !== "silero") {
+        vad = options.vad;
+      } else if (wantsSileroVad) {
+        vad = ctx.proc.userData.vad ?? await loadSileroVad();
+      }
+      let turnDetection;
+      if (options.turnDetection === "multilingual" || options.turnDetection === "english") {
+        turnDetection = await loadTurnDetector(options.turnDetection);
+      } else {
+        turnDetection = options.turnDetection;
+      }
+      const memory = resolveMemory(options, mastraAgent, args, roomName);
+      const memoryInstance = memory ? await resolveMemoryInstance(options, mastraAgent, args, requestContext) : null;
+      if (memory && memoryInstance) {
+        try {
+          await ensureVoiceCallThread({
+            memory: memoryInstance,
+            threadId: memory.thread,
+            resourceId: memory.resource ?? memory.thread,
+            roomName
+          });
+        } catch (error) {
+          logger.warn("@mastra/livekit: failed to create the voice call thread", error);
+        }
+      }
+      const voiceObs = options.observability === false ? void 0 : startVoiceCallObservability({
+        mastra: options.mastra,
+        agentId: agentLabel,
+        roomName,
+        metadata,
+        requestContext
+      });
+      if (voiceObs) {
+        ctx.addShutdownCallback(async () => {
+          voiceObs.finalize();
+        });
+      }
+      if (options.onCallEnd) {
+        const onCallEnd = options.onCallEnd;
+        ctx.addShutdownCallback(async () => {
+          try {
+            await onCallEnd({ memory, memoryInstance, metadata, requestContext, roomName, ctx });
+          } catch (error) {
+            logger.warn("@mastra/livekit: onCallEnd hook threw", error);
+          }
+        });
+      }
+      const agent = createMastraVoiceAgent({
+        ...replyGenerator ? { generate: replyGenerator } : { agent: mastraAgent },
+        instructions: mastraAgent ? await resolveInstructions(mastraAgent, requestContext) : void 0,
+        memory,
+        requestContext,
+        toolFeedback: options.toolFeedback,
+        onTurnComplete: options.onTurnComplete,
+        streamOptions: voiceObs ? { tracingContext: voiceObs.tracingContext } : void 0
+      });
+      const session = new voice.AgentSession({
+        stt: options.stt,
+        tts: options.tts,
+        vad,
+        turnHandling: buildTurnHandling(options, turnDetection),
+        ...options.sessionOptions
+      });
+      voiceObs?.attach(session);
+      try {
+        await session.start({
+          agent,
+          room: ctx.room,
+          inputOptions: options.inputOptions,
+          outputOptions: options.outputOptions
+        });
+        if (options.greeting) {
+          session.say(options.greeting);
+          if (options.persistGreeting !== false && memory && memoryInstance) {
+            try {
+              await persistSpokenGreeting({
+                memory: memoryInstance,
+                threadId: memory.thread,
+                resourceId: memory.resource ?? memory.thread,
+                greeting: options.greeting
+              });
+            } catch (error) {
+              logger.warn("@mastra/livekit: failed to persist the greeting", error);
+            }
+          }
+        }
+        await options.onSessionStart?.({ session, ctx, agent, metadata });
+      } catch (error) {
+        voiceObs?.finalize({ error });
+        throw error;
+      }
+    }
+  });
+}
+function resolveWorkerEntryPath(entry) {
+  if (entry instanceof URL) return fileURLToPath(entry);
+  return entry.startsWith("file:") ? fileURLToPath(entry) : entry;
+}
+function runLiveKitWorker(options) {
+  void workerSetupComplete().then(() => {
+    cli.runApp(
+      new ServerOptions({
+        agent: resolveWorkerEntryPath(options.entry),
+        agentName: options.agentName ?? DEFAULT_LIVEKIT_AGENT_NAME,
+        ...options.serverOptions
+      })
+    );
+  });
+}
+
+export { chatContextToMessages, createLiveKitWorker, runLiveKitWorker };
+//# sourceMappingURL=worker-entry.js.map
+//# sourceMappingURL=worker-entry.js.map