@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.2

package/src/routes/transcripts-routes.ts

@@ -0,0 +1,159 @@
+/**
+ * Transcript HTTP routes (#8789) — `/api/transcripts*`, served as rawPath plugin
+ * routes (on `runtime.routes`, dispatched by both the upstream agent server and
+ * app-core) so the Transcripts view + the recording pipeline have a backend.
+ * Audio is served by the existing content-addressed media store via each
+ * record's `audioUrl`, so no separate audio route is needed.
+ *
+ * Private routes: the host dispatcher answers 401 for unauthenticated callers.
+ */
+
+import type {
+	Route,
+	RouteHandlerContext,
+	RouteHandlerResult,
+	UUID,
+} from "@elizaos/core";
+import {
+	type Transcript,
+	type TranscriptScope,
+	type TranscriptSegment,
+	type TranscriptSource,
+	transcriptDurationMs,
+	transcriptSpeakerCount,
+} from "@elizaos/shared/transcripts";
+import {
+	TranscriptService,
+	type TranscriptServiceRuntime,
+} from "../services/voice/transcript-service.js";
+import { persistTranscriptAudioWav } from "./transcript-audio-store.js";
+
+function service(ctx: RouteHandlerContext): TranscriptService {
+	return new TranscriptService(
+		ctx.runtime as unknown as TranscriptServiceRuntime,
+	);
+}
+
+/** The body a recording session POSTs to create a transcript record. */
+export interface CreateTranscriptRequest {
+	/** Optional — the route derives these from the agent context when absent (the
+	 *  shell client doesn't carry world/room/entity ids). */
+	worldId?: UUID;
+	roomId?: UUID;
+	entityId?: UUID;
+	title?: string;
+	source?: TranscriptSource;
+	scope?: TranscriptScope;
+	segments: TranscriptSegment[];
+	audioUrl?: string;
+	audioContentType?: string;
+	/** Base64 WAV bytes — persisted to the media store; sets audioUrl. */
+	audioBase64?: string;
+	createdAt?: number;
+}
+
+/**
+ * Build a full {@link Transcript} from a create request — derives duration +
+ * speaker count from the segments, defaults title/scope/status. Pure (id + now
+ * injected) so it is unit-testable.
+ */
+export function buildTranscriptFromRequest(
+	body: CreateTranscriptRequest,
+	id: string,
+	now: number,
+): Transcript {
+	const segments = Array.isArray(body.segments) ? body.segments : [];
+	const createdAt = body.createdAt ?? now;
+	return {
+		id,
+		title: body.title?.trim() || defaultTitle(createdAt),
+		createdAt,
+		endedAt: now,
+		durationMs: transcriptDurationMs(segments),
+		audioUrl: body.audioUrl,
+		audioContentType: body.audioContentType,
+		segments,
+		source: body.source ?? "voice-session",
+		scope: body.scope ?? "owner-private",
+		status: "ready",
+		speakerCount: transcriptSpeakerCount(segments),
+	};
+}
+
+function defaultTitle(createdAt: number): string {
+	return `Recording ${new Date(createdAt).toLocaleString()}`;
+}
+
+const listRoute: Route = {
+	type: "GET",
+	path: "/api/transcripts",
+	rawPath: true,
+	routeHandler: async (ctx): Promise<RouteHandlerResult> => {
+		const roomId = (ctx.query.roomId as string | undefined) || undefined;
+		const transcripts = await service(ctx).list(roomId as UUID | undefined);
+		return { status: 200, body: { transcripts } };
+	},
+};
+
+const getRoute: Route = {
+	type: "GET",
+	path: "/api/transcripts/:id",
+	rawPath: true,
+	routeHandler: async (ctx): Promise<RouteHandlerResult> => {
+		const transcript = await service(ctx).get(ctx.params.id as UUID);
+		if (!transcript) return { status: 404, body: { error: "not found" } };
+		return { status: 200, body: { transcript } };
+	},
+};
+
+const deleteRoute: Route = {
+	type: "DELETE",
+	path: "/api/transcripts/:id",
+	rawPath: true,
+	routeHandler: async (ctx): Promise<RouteHandlerResult> => {
+		await service(ctx).delete(ctx.params.id as UUID);
+		return { status: 200, body: { ok: true } };
+	},
+};
+
+const createRoute: Route = {
+	type: "POST",
+	path: "/api/transcripts",
+	rawPath: true,
+	routeHandler: async (ctx): Promise<RouteHandlerResult> => {
+		const body = ctx.body as CreateTranscriptRequest | undefined;
+		if (!body || !Array.isArray(body.segments) || body.segments.length === 0) {
+			return { status: 400, body: { error: "segments are required" } };
+		}
+		// The shell client doesn't carry world/room/entity ids — default them to
+		// the agent context (single-user local) when not supplied.
+		const agentId = ctx.runtime.agentId as UUID;
+		// Persist the recorded session WAV into the served media store so the
+		// player has audio to scrub. The shell sends base64 (it can't write files).
+		if (body.audioBase64 && !body.audioUrl) {
+			body.audioUrl = persistTranscriptAudioWav(
+				Buffer.from(body.audioBase64, "base64"),
+			);
+			body.audioContentType = "audio/wav";
+		}
+		const transcript = buildTranscriptFromRequest(
+			body,
+			crypto.randomUUID(),
+			Date.now(),
+		);
+		const saved = await service(ctx).create({
+			worldId: (body.worldId ?? agentId) as UUID,
+			roomId: (body.roomId ?? agentId) as UUID,
+			entityId: (body.entityId ?? agentId) as UUID,
+			transcript,
+		});
+		return { status: 201, body: { transcript: saved } };
+	},
+};
+
+export const transcriptsRoutes: Route[] = [
+	listRoute,
+	createRoute,
+	getRoute,
+	deleteRoute,
+];

package/src/routes/voice-first-run-routes.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Voice first-run HTTP routes.
+ *
+ * The flow (R2-speaker.md §6):
+ *
+ *   POST /api/voice/first-run/profile/start
+ *     → { sessionId, script: ScriptStep[], embeddingModel }
+ *
+ *   POST /api/voice/first-run/profile/append?id=<sessionId>
+ *     Content-Type: application/octet-stream
+ *     Body: PCM (Float32, 16 kHz, mono) for one capture window
+ *     → { sessionId, samplesReceived, totalSamples, durationMs }
+ *
+ *   POST /api/voice/first-run/profile/finalize?id=<sessionId>&entityId=<id>
+ *     → { profileId, entityId, samples, durationMs }
+ *
+ *   POST /api/voice/first-run/complete
+ *     Body: { entityId: string }
+ *     → { ownerEntityId, settingsWritten: ["ELIZA_ADMIN_ENTITY_ID"] }
+ *
+ * Sessions are in-memory. They expire after 30 minutes of inactivity.
+ * The encoder is loaded lazily; the route handlers return a structured
+ * 503 if the native GGML library is missing.
+ *
+ * Audio storage: when the user grants `audioRefs` consent, sample WAVs
+ * land under `$ELIZA_STATE_DIR/voice-profiles/audio/<profileId>/...`.
+ * Otherwise nothing is written to disk except the centroid + variance.
+ */
+import type * as http from "node:http";
+import { VoiceProfileStore } from "../services/voice/profile-store";
+import { type SpeakerEncoder } from "../services/voice/speaker/encoder";
+/** Verbatim first-run script (R2-speaker.md §6.2). */
+export interface FirstRunScriptStep {
+    id: string;
+    role: "consent" | "calibration" | "phonetic" | "prosody" | "quiet" | "open";
+    prompt: string;
+    expectedDurationMs: number;
+    requiresUserSpeech: boolean;
+}
+export declare const FIRST_RUN_SCRIPT: ReadonlyArray<FirstRunScriptStep>;
+/**
+ * Encoder factory. By default the route handlers load the WeSpeaker
+ * ResNet34-LM speaker encoder through the fused `libelizainference`
+ * `eliza_inference_speaker_*` ABI — the sole on-device speaker runtime.
+ * Tests inject a fake encoder via `setVoiceFirstRunEncoderFactory()`.
+ */
+export type EncoderFactory = () => Promise<SpeakerEncoder>;
+export declare function setVoiceFirstRunEncoderFactory(factory: EncoderFactory | null): void;
+export declare function setVoiceFirstRunProfileStore(store: VoiceProfileStore | null): void;
+/** Settings-write hook. The runtime overrides this with `runtime.setSetting`. */
+type SettingsWriter = (key: string, value: string) => void | Promise<void>;
+export declare function setVoiceFirstRunSettingsWriter(writer: SettingsWriter | null): void;
+/**
+ * Mount-point: returns `true` if the request was handled, `false` if
+ * the path is not one of the voice-first-run routes (so the caller
+ * can fall through to the next handler).
+ */
+export declare function handleVoiceFirstRunRoutes(req: http.IncomingMessage, res: http.ServerResponse): Promise<boolean>;
+/** Test helper: clear in-memory sessions so a test starts clean. */
+export declare function __resetVoiceFirstRunSessions(): void;
+export {};
+//# sourceMappingURL=voice-first-run-routes.d.ts.map

package/src/routes/voice-first-run-routes.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"voice-first-run-routes.d.ts","sourceRoot":"","sources":["voice-first-run-routes.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AAGH,OAAO,KAAK,KAAK,IAAI,MAAM,WAAW,CAAC;AAWvC,OAAO,EAAE,iBAAiB,EAAE,MAAM,iCAAiC,CAAC;AACpE,OAAO,EAEN,KAAK,cAAc,EAMnB,MAAM,mCAAmC,CAAC;AAG3C,sDAAsD;AACtD,MAAM,WAAW,kBAAkB;IAClC,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,SAAS,GAAG,aAAa,GAAG,UAAU,GAAG,SAAS,GAAG,OAAO,GAAG,MAAM,CAAC;IAC5E,MAAM,EAAE,MAAM,CAAC;IACf,kBAAkB,EAAE,MAAM,CAAC;IAC3B,kBAAkB,EAAE,OAAO,CAAC;CAC5B;AAED,eAAO,MAAM,gBAAgB,EAAE,aAAa,CAAC,kBAAkB,CA4E9D,CAAC;AA2BF;;;;;GAKG;AACH,MAAM,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,cAAc,CAAC,CAAC;AAK3D,wBAAgB,8BAA8B,CAC7C,OAAO,EAAE,cAAc,GAAG,IAAI,GAC5B,IAAI,CAGN;AAyCD,wBAAgB,4BAA4B,CAC3C,KAAK,EAAE,iBAAiB,GAAG,IAAI,GAC7B,IAAI,CAEN;AAWD,iFAAiF;AACjF,KAAK,cAAc,GAAG,CAAC,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;AAG3E,wBAAgB,8BAA8B,CAC7C,MAAM,EAAE,cAAc,GAAG,IAAI,GAC3B,IAAI,CAEN;AAyCD;;;;GAIG;AACH,wBAAsB,yBAAyB,CAC9C,GAAG,EAAE,IAAI,CAAC,eAAe,EACzB,GAAG,EAAE,IAAI,CAAC,cAAc,GACtB,OAAO,CAAC,OAAO,CAAC,CAgOlB;AAED,oEAAoE;AACpE,wBAAgB,4BAA4B,IAAI,IAAI,CAEnD"}