@gakr-gakr/discord 0.1.0

package/src/voice/receive-recovery.ts

@@ -0,0 +1,159 @@
+import { formatErrorMessage } from "autobot/plugin-sdk/ssrf-runtime";
+
+const DECRYPT_FAILURE_WINDOW_MS = 30_000;
+const DECRYPT_FAILURE_RECONNECT_THRESHOLD = 3;
+const DECRYPT_FAILURE_MARKER = "DecryptionFailed(";
+const DAVE_PASSTHROUGH_DISABLED_MARKER = "UnencryptedWhenPassthroughDisabled";
+
+export const DAVE_RECEIVE_PASSTHROUGH_INITIAL_EXPIRY_SECONDS = 30;
+export const DAVE_RECEIVE_PASSTHROUGH_REARM_EXPIRY_SECONDS = 15;
+
+export type VoiceReceiveRecoveryState = {
+  decryptFailureCount: number;
+  lastDecryptFailureAt: number;
+  decryptRecoveryInFlight: boolean;
+};
+
+type VoiceReceiveErrorAnalysis = {
+  message: string;
+  isAbortLike: boolean;
+  shouldAttemptPassthrough: boolean;
+  countsAsDecryptFailure: boolean;
+};
+
+type DavePassthroughTarget = {
+  guildId: string;
+  channelId: string;
+  connection: {
+    state: {
+      status: unknown;
+      networking?: {
+        state?: {
+          code?: unknown;
+          dave?: {
+            session?: {
+              setPassthroughMode: (passthrough: boolean, expirySeconds: number) => void;
+            };
+          };
+        };
+      };
+    };
+  };
+};
+
+type DavePassthroughSdk = {
+  VoiceConnectionStatus: {
+    Ready: unknown;
+  };
+  NetworkingStatusCode: {
+    Ready: unknown;
+    Resuming: unknown;
+  };
+};
+
+export function createVoiceReceiveRecoveryState(): VoiceReceiveRecoveryState {
+  return {
+    decryptFailureCount: 0,
+    lastDecryptFailureAt: 0,
+    decryptRecoveryInFlight: false,
+  };
+}
+
+function isAbortLikeReceiveError(err: unknown): boolean {
+  if (!err || typeof err !== "object") {
+    return false;
+  }
+  const name =
+    "name" in err && typeof (err as { name?: unknown }).name === "string"
+      ? (err as { name: string }).name
+      : "";
+  const message =
+    "message" in err && typeof (err as { message?: unknown }).message === "string"
+      ? (err as { message: string }).message
+      : "";
+  return (
+    name === "AbortError" ||
+    message.includes("The operation was aborted") ||
+    message.includes("aborted")
+  );
+}
+
+export function analyzeVoiceReceiveError(err: unknown): VoiceReceiveErrorAnalysis {
+  const message = formatErrorMessage(err);
+  const shouldAttemptPassthrough = message.includes(DAVE_PASSTHROUGH_DISABLED_MARKER);
+  return {
+    message,
+    isAbortLike: isAbortLikeReceiveError(err),
+    shouldAttemptPassthrough,
+    countsAsDecryptFailure: message.includes(DECRYPT_FAILURE_MARKER) || shouldAttemptPassthrough,
+  };
+}
+
+export function noteVoiceDecryptFailure(
+  state: VoiceReceiveRecoveryState,
+  now: number = Date.now(),
+): {
+  firstFailure: boolean;
+  shouldRecover: boolean;
+} {
+  if (now - state.lastDecryptFailureAt > DECRYPT_FAILURE_WINDOW_MS) {
+    state.decryptFailureCount = 0;
+  }
+  state.lastDecryptFailureAt = now;
+  state.decryptFailureCount += 1;
+  const firstFailure = state.decryptFailureCount === 1;
+  if (
+    state.decryptFailureCount < DECRYPT_FAILURE_RECONNECT_THRESHOLD ||
+    state.decryptRecoveryInFlight
+  ) {
+    return { firstFailure, shouldRecover: false };
+  }
+  state.decryptRecoveryInFlight = true;
+  resetVoiceReceiveRecoveryState(state);
+  return { firstFailure, shouldRecover: true };
+}
+
+export function resetVoiceReceiveRecoveryState(state: VoiceReceiveRecoveryState): void {
+  state.decryptFailureCount = 0;
+  state.lastDecryptFailureAt = 0;
+}
+
+export function finishVoiceDecryptRecovery(state: VoiceReceiveRecoveryState): void {
+  state.decryptRecoveryInFlight = false;
+}
+
+export function enableDaveReceivePassthrough(params: {
+  target: DavePassthroughTarget;
+  sdk: DavePassthroughSdk;
+  reason: string;
+  expirySeconds: number;
+  onVerbose: (message: string) => void;
+  onWarn: (message: string) => void;
+}): boolean {
+  const { target, sdk, reason, expirySeconds, onVerbose, onWarn } = params;
+  const networkingState = target.connection.state.networking?.state;
+  if (
+    target.connection.state.status !== sdk.VoiceConnectionStatus.Ready ||
+    !networkingState ||
+    (networkingState.code !== sdk.NetworkingStatusCode.Ready &&
+      networkingState.code !== sdk.NetworkingStatusCode.Resuming)
+  ) {
+    return false;
+  }
+  const daveSession = networkingState.dave?.session;
+  if (!daveSession) {
+    return false;
+  }
+  try {
+    daveSession.setPassthroughMode(true, expirySeconds);
+    onVerbose(
+      `enabled DAVE receive passthrough: guild ${target.guildId} channel ${target.channelId} expiry=${expirySeconds}s reason=${reason}`,
+    );
+    return true;
+  } catch (err) {
+    onWarn(
+      `discord voice: failed to enable DAVE passthrough guild=${target.guildId} channel=${target.channelId} reason=${reason}: ${formatErrorMessage(err)}`,
+    );
+    return false;
+  }
+}

package/src/voice/sanitize.ts

@@ -0,0 +1,29 @@
+import { stripInlineDirectiveTagsForDisplay } from "autobot/plugin-sdk/text-chunking";
+import { escapeRegExp } from "autobot/plugin-sdk/text-utility-runtime";
+
+const SPEECH_EMOJI_RE =
+  /(?:\p{Extended_Pictographic}(?:\uFE0F|\u200D|\p{Extended_Pictographic}|\p{Emoji_Modifier})*)+/gu;
+
+function stripEmojiForSpeech(text: string): string {
+  return text
+    .replace(SPEECH_EMOJI_RE, " ")
+    .replace(/\s+([?!.,:;])/g, "$1")
+    .replace(/[ \t]{2,}/g, " ")
+    .replace(/ *\n */g, "\n")
+    .trim();
+}
+
+export function sanitizeVoiceReplyTextForSpeech(text: string, speakerLabel?: string): string {
+  let cleaned = stripInlineDirectiveTagsForDisplay(text).text.trim();
+  if (!cleaned) {
+    return "";
+  }
+
+  const label = speakerLabel?.trim();
+  if (label) {
+    const prefix = new RegExp(`^${escapeRegExp(label)}\\s*:\\s*`, "i");
+    cleaned = cleaned.replace(prefix, "").trim();
+  }
+
+  return stripEmojiForSpeech(cleaned);
+}

package/src/voice/sdk-runtime.ts

@@ -0,0 +1,14 @@
+import { createRequire } from "node:module";
+
+type DiscordVoiceSdk = typeof import("@discordjs/voice");
+
+let cachedDiscordVoiceSdk: DiscordVoiceSdk | null = null;
+
+export function loadDiscordVoiceSdk(): DiscordVoiceSdk {
+  if (cachedDiscordVoiceSdk) {
+    return cachedDiscordVoiceSdk;
+  }
+  const req = createRequire(import.meta.url);
+  cachedDiscordVoiceSdk = req("@discordjs/voice") as DiscordVoiceSdk;
+  return cachedDiscordVoiceSdk;
+}

package/src/voice/segment.ts

@@ -0,0 +1,160 @@
+import path from "node:path";
+import { Readable } from "node:stream";
+import type { DiscordAccountConfig, AutoBotConfig } from "autobot/plugin-sdk/config-contracts";
+import type { RuntimeEnv } from "autobot/plugin-sdk/runtime-env";
+import { createSubsystemLogger } from "autobot/plugin-sdk/runtime-env";
+import { resolveDiscordVoiceIngressContext, runDiscordVoiceAgentTurn } from "./ingress.js";
+import { formatVoiceIngressPrompt } from "./prompt.js";
+import { loadDiscordVoiceSdk } from "./sdk-runtime.js";
+import {
+  logVoiceVerbose,
+  PLAYBACK_READY_TIMEOUT_MS,
+  SPEAKING_READY_TIMEOUT_MS,
+  type VoiceSessionEntry,
+} from "./session.js";
+import type { DiscordVoiceSpeakerContextResolver } from "./speaker-context.js";
+import { synthesizeVoiceReplyAudio, transcribeVoiceAudio } from "./tts.js";
+
+const VOICE_TRANSCRIPT_LOG_PREVIEW_CHARS = 500;
+const logger = createSubsystemLogger("discord/voice");
+
+function formatVoiceTranscriptLogPreview(text: string): string {
+  const oneLine = text.replace(/\s+/g, " ").trim();
+  if (oneLine.length <= VOICE_TRANSCRIPT_LOG_PREVIEW_CHARS) {
+    return oneLine;
+  }
+  return `${oneLine.slice(0, VOICE_TRANSCRIPT_LOG_PREVIEW_CHARS)}...`;
+}
+
+export async function processDiscordVoiceSegment(params: {
+  entry: VoiceSessionEntry;
+  wavPath: string;
+  userId: string;
+  durationSeconds: number;
+  cfg: AutoBotConfig;
+  discordConfig: DiscordAccountConfig;
+  runtime: RuntimeEnv;
+  ownerAllowFrom?: string[];
+  fetchGuildName: (guildId: string) => Promise<string | undefined>;
+  speakerContext: DiscordVoiceSpeakerContextResolver;
+  enqueuePlayback: (entry: VoiceSessionEntry, task: () => Promise<void>) => void;
+}) {
+  const { entry, wavPath, userId, durationSeconds } = params;
+  logVoiceVerbose(
+    `segment processing (${durationSeconds.toFixed(2)}s): guild ${entry.guildId} channel ${entry.channelId}`,
+  );
+  const ingress = await resolveDiscordVoiceIngressContext({
+    entry,
+    userId,
+    cfg: params.cfg,
+    discordConfig: params.discordConfig,
+    ownerAllowFrom: params.ownerAllowFrom,
+    fetchGuildName: params.fetchGuildName,
+    speakerContext: params.speakerContext,
+  });
+  if (!ingress) {
+    logVoiceVerbose(
+      `segment unauthorized: guild ${entry.guildId} channel ${entry.channelId} user ${userId}`,
+    );
+    return;
+  }
+  const transcript = await transcribeVoiceAudio({
+    cfg: params.cfg,
+    agentId: entry.route.agentId,
+    filePath: wavPath,
+  });
+  if (!transcript) {
+    logVoiceVerbose(
+      `transcription empty: guild ${entry.guildId} channel ${entry.channelId} user ${userId}`,
+    );
+    return;
+  }
+  logVoiceVerbose(
+    `transcription ok (${transcript.length} chars): guild ${entry.guildId} channel ${entry.channelId}`,
+  );
+  logVoiceVerbose(
+    `transcript from ${ingress.speakerLabel} (${userId}) in guild ${entry.guildId} channel ${entry.channelId}: ${formatVoiceTranscriptLogPreview(transcript)}`,
+  );
+
+  const prompt = formatVoiceIngressPrompt(transcript, ingress.speakerLabel);
+  const turn = await runDiscordVoiceAgentTurn({
+    entry,
+    userId,
+    message: prompt,
+    cfg: params.cfg,
+    discordConfig: params.discordConfig,
+    runtime: params.runtime,
+    context: ingress,
+    ownerAllowFrom: params.ownerAllowFrom,
+    fetchGuildName: params.fetchGuildName,
+    speakerContext: params.speakerContext,
+  });
+  if (!turn) {
+    logVoiceVerbose(
+      `segment unauthorized before agent turn: guild ${entry.guildId} channel ${entry.channelId} user ${userId}`,
+    );
+    return;
+  }
+  const replyText = turn.text;
+
+  if (!replyText) {
+    logVoiceVerbose(
+      `reply empty: guild ${entry.guildId} channel ${entry.channelId} user ${userId}`,
+    );
+    return;
+  }
+  logVoiceVerbose(
+    `reply ok (${replyText.length} chars): guild ${entry.guildId} channel ${entry.channelId}`,
+  );
+
+  const voiceReplyAudio = await synthesizeVoiceReplyAudio({
+    cfg: params.cfg,
+    override: params.discordConfig.voice?.tts,
+    replyText,
+    speakerLabel: ingress.speakerLabel,
+  });
+  if (voiceReplyAudio.status === "empty") {
+    logVoiceVerbose(
+      `tts skipped (empty): guild ${entry.guildId} channel ${entry.channelId} user ${userId}`,
+    );
+    return;
+  }
+  if (voiceReplyAudio.status === "failed") {
+    logger.warn(`discord voice: TTS failed: ${voiceReplyAudio.error ?? "unknown error"}`);
+    return;
+  }
+  logVoiceVerbose(
+    `tts ok (${voiceReplyAudio.speakText.length} chars): guild ${entry.guildId} channel ${entry.channelId}`,
+  );
+
+  params.enqueuePlayback(entry, async () => {
+    const voiceSdk = loadDiscordVoiceSdk();
+    const releaseAudioStream =
+      voiceReplyAudio.mode === "stream" ? voiceReplyAudio.release : undefined;
+    try {
+      if (voiceReplyAudio.mode === "stream") {
+        logVoiceVerbose(`playback start: guild ${entry.guildId} channel ${entry.channelId} stream`);
+        const nodeStream = Readable.fromWeb(
+          voiceReplyAudio.audioStream as import("node:stream/web").ReadableStream<Uint8Array>,
+        );
+        const resource = voiceSdk.createAudioResource(nodeStream);
+        entry.player.play(resource);
+      } else {
+        logVoiceVerbose(
+          `playback start: guild ${entry.guildId} channel ${entry.channelId} file ${path.basename(voiceReplyAudio.audioPath)}`,
+        );
+        const resource = voiceSdk.createAudioResource(voiceReplyAudio.audioPath);
+        entry.player.play(resource);
+      }
+      await voiceSdk
+        .entersState(entry.player, voiceSdk.AudioPlayerStatus.Playing, PLAYBACK_READY_TIMEOUT_MS)
+        .catch(() => undefined);
+      await voiceSdk
+        .entersState(entry.player, voiceSdk.AudioPlayerStatus.Idle, SPEAKING_READY_TIMEOUT_MS)
+        .catch(() => undefined);
+      logVoiceVerbose(`playback done: guild ${entry.guildId} channel ${entry.channelId}`);
+    } finally {
+      await releaseAudioStream?.();
+    }
+  });
+}

package/src/voice/session.ts

@@ -0,0 +1,81 @@
+import type { resolveAgentRoute } from "autobot/plugin-sdk/routing";
+import { logVerbose } from "autobot/plugin-sdk/runtime-env";
+import { ChannelType } from "../internal/discord.js";
+import type { VoiceCaptureState } from "./capture-state.js";
+import type { VoiceReceiveRecoveryState } from "./receive-recovery.js";
+
+export const MIN_SEGMENT_SECONDS = 0.35;
+export const CAPTURE_FINALIZE_GRACE_MS = 2_500;
+export const VOICE_CONNECT_READY_TIMEOUT_MS = 30_000;
+export const VOICE_RECONNECT_GRACE_MS = 15_000;
+export const PLAYBACK_READY_TIMEOUT_MS = 60_000;
+export const SPEAKING_READY_TIMEOUT_MS = 60_000;
+
+export function resolveVoiceTimeoutMs(value: number | undefined, fallbackMs: number): number {
+  if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
+    return fallbackMs;
+  }
+  return Math.floor(value);
+}
+
+export type VoiceOperationResult = {
+  ok: boolean;
+  message: string;
+  channelId?: string;
+  guildId?: string;
+};
+
+export type VoiceRealtimeSpeakerContext = {
+  extraSystemPrompt?: string;
+  senderIsOwner: boolean;
+  speakerLabel: string;
+};
+
+export type VoiceRealtimeAgentTurnParams = {
+  context: VoiceRealtimeSpeakerContext;
+  message: string;
+  toolsAllow?: string[];
+  userId: string;
+};
+
+export type VoiceRealtimeSpeakerTurn = {
+  close: () => void;
+  sendInputAudio: (discordPcm48kStereo: Buffer) => void;
+};
+
+export type VoiceRealtimeSession = {
+  beginSpeakerTurn: (
+    context: VoiceRealtimeSpeakerContext,
+    userId: string,
+  ) => VoiceRealtimeSpeakerTurn;
+  close: () => void;
+  connect: () => Promise<void>;
+  handleBargeIn: (reason?: string) => void;
+  isBargeInEnabled: () => boolean;
+};
+
+export type VoiceSessionEntry = {
+  guildId: string;
+  guildName?: string;
+  channelId: string;
+  channelName?: string;
+  sessionChannelId: string;
+  voiceSessionKey: string;
+  route: ReturnType<typeof resolveAgentRoute>;
+  connection: import("@discordjs/voice").VoiceConnection;
+  player: import("@discordjs/voice").AudioPlayer;
+  playbackQueue: Promise<void>;
+  processingQueue: Promise<void>;
+  capture: VoiceCaptureState;
+  realtime?: VoiceRealtimeSession;
+  receiveRecovery: VoiceReceiveRecoveryState;
+  stop: () => void;
+};
+
+export function logVoiceVerbose(message: string): void {
+  logVerbose(`discord voice: ${message}`);
+}
+
+export function isVoiceChannel(type: ChannelType): boolean {
+  return type === ChannelType.GuildVoice || type === ChannelType.GuildStageVoice;
+}

package/src/voice/speaker-context.ts

@@ -0,0 +1,127 @@
+import type { Client } from "../internal/discord.js";
+import { resolveDiscordOwnerAccess } from "../monitor/allow-list.js";
+import { formatDiscordUserTag } from "../monitor/format.js";
+
+const SPEAKER_CONTEXT_CACHE_TTL_MS = 60_000;
+
+type VoiceSpeakerIdentity = {
+  id: string;
+  label: string;
+  name?: string;
+  tag?: string;
+  memberRoleIds: string[];
+};
+
+type VoiceSpeakerContext = Omit<VoiceSpeakerIdentity, "memberRoleIds"> & {
+  senderIsOwner: boolean;
+};
+
+export class DiscordVoiceSpeakerContextResolver {
+  private readonly cache = new Map<
+    string,
+    VoiceSpeakerContext & {
+      expiresAt: number;
+    }
+  >();
+
+  constructor(
+    private readonly params: {
+      client: Client;
+      ownerAllowFrom?: string[];
+    },
+  ) {}
+
+  async resolveContext(guildId: string, userId: string): Promise<VoiceSpeakerContext> {
+    const cached = this.getCachedContext(guildId, userId);
+    if (cached) {
+      return cached;
+    }
+    const identity = await this.resolveIdentity(guildId, userId);
+    const context = {
+      id: identity.id,
+      label: identity.label,
+      name: identity.name,
+      tag: identity.tag,
+      senderIsOwner: this.resolveIsOwner(identity),
+    };
+    this.setCachedContext(guildId, userId, context);
+    return context;
+  }
+
+  async resolveIdentity(guildId: string, userId: string): Promise<VoiceSpeakerIdentity> {
+    try {
+      const member = await this.params.client.fetchMember(guildId, userId);
+      const username = member.user?.username ?? undefined;
+      return {
+        id: userId,
+        label: member.nickname ?? member.user?.globalName ?? username ?? userId,
+        name: username,
+        tag: member.user ? formatDiscordUserTag(member.user) : undefined,
+        memberRoleIds: Array.isArray(member.roles)
+          ? member.roles
+              .map((role) =>
+                typeof role === "string" ? role : typeof role?.id === "string" ? role.id : "",
+              )
+              .filter(Boolean)
+          : [],
+      };
+    } catch {
+      try {
+        const user = await this.params.client.fetchUser(userId);
+        const username = user.username ?? undefined;
+        return {
+          id: userId,
+          label: user.globalName ?? username ?? userId,
+          name: username,
+          tag: formatDiscordUserTag(user),
+          memberRoleIds: [],
+        };
+      } catch {
+        return { id: userId, label: userId, memberRoleIds: [] };
+      }
+    }
+  }
+
+  private resolveIsOwner(identity: Pick<VoiceSpeakerIdentity, "id" | "name" | "tag">): boolean {
+    return resolveDiscordOwnerAccess({
+      allowFrom: this.params.ownerAllowFrom,
+      sender: {
+        id: identity.id,
+        name: identity.name,
+        tag: identity.tag,
+      },
+      allowNameMatching: false,
+    }).ownerAllowed;
+  }
+
+  private resolveCacheKey(guildId: string, userId: string): string {
+    return `${guildId}:${userId}`;
+  }
+
+  private getCachedContext(guildId: string, userId: string): VoiceSpeakerContext | undefined {
+    const key = this.resolveCacheKey(guildId, userId);
+    const cached = this.cache.get(key);
+    if (!cached) {
+      return undefined;
+    }
+    if (cached.expiresAt <= Date.now()) {
+      this.cache.delete(key);
+      return undefined;
+    }
+    return {
+      id: cached.id,
+      label: cached.label,
+      name: cached.name,
+      tag: cached.tag,
+      senderIsOwner: cached.senderIsOwner,
+    };
+  }
+
+  private setCachedContext(guildId: string, userId: string, context: VoiceSpeakerContext): void {
+    const key = this.resolveCacheKey(guildId, userId);
+    this.cache.set(key, {
+      ...context,
+      expiresAt: Date.now() + SPEAKER_CONTEXT_CACHE_TTL_MS,
+    });
+  }
+}