@elizaos/plugin-local-inference 2.0.0-beta.1 → 2.0.3-beta.3

package/src/services/voice/transcript-knowledge.test.ts

@@ -0,0 +1,68 @@
+import type { Transcript } from "@elizaos/shared/transcripts";
+import { describe, expect, it } from "vitest";
+import {
+	TRANSCRIPT_DOCUMENT_TAG,
+	transcriptKnowledgePayload,
+} from "./transcript-knowledge";
+
+const transcript: Transcript = {
+	id: "t-123",
+	title: "Standup — June 20",
+	createdAt: 1000,
+	endedAt: 5000,
+	durationMs: 4000,
+	audioUrl: "/api/media/abc.wav",
+	source: "voice-session",
+	scope: "owner-private",
+	status: "ready",
+	speakerCount: 2,
+	segments: [
+		{
+			id: "s1",
+			speakerLabel: "Alice",
+			startMs: 0,
+			endMs: 1000,
+			text: "ship the build",
+			words: [],
+		},
+		{
+			id: "s2",
+			speakerLabel: "Bob",
+			startMs: 1200,
+			endMs: 2000,
+			text: "on it",
+			words: [],
+		},
+	],
+};
+
+describe("transcriptKnowledgePayload", () => {
+	it("builds a searchable text document linked back to the transcript", () => {
+		const p = transcriptKnowledgePayload(transcript);
+		// Body is the speaker-labeled plain text (what gets embedded + searched).
+		expect(p.content).toBe("Alice: ship the build\nBob: on it");
+		expect(p.contentType).toBe("text/plain");
+		expect(p.scope).toBe("owner-private");
+		// Slugified filename.
+		expect(p.filename).toBe("standup-june-20.txt");
+		// Tagged + linked back to the rich record.
+		expect(p.metadata.tags).toEqual([TRANSCRIPT_DOCUMENT_TAG]);
+		expect(p.metadata.source).toBe(TRANSCRIPT_DOCUMENT_TAG);
+		expect(p.metadata.transcriptId).toBe("t-123");
+		expect(p.metadata.title).toBe("Standup — June 20");
+		expect(p.metadata.audioUrl).toBe("/api/media/abc.wav");
+		expect(p.metadata.durationMs).toBe(4000);
+		expect(p.metadata.speakerCount).toBe(2);
+		expect(p.metadata.textBacked).toBe(true);
+	});
+
+	it("omits audioUrl when the transcript has no audio, and slugs an empty title", () => {
+		const p = transcriptKnowledgePayload({
+			...transcript,
+			title: "  ",
+			audioUrl: undefined,
+		});
+		expect(p.metadata.audioUrl).toBeUndefined();
+		expect(p.filename).toBe("transcript.txt");
+	});
+});

package/src/services/voice/transcript-knowledge.ts

@@ -0,0 +1,75 @@
+/**
+ * Transcript → knowledge (documents store) mapping (#8789 transcripts).
+ *
+ * A finished transcript is mirrored into the documents/knowledge store as a
+ * private, searchable text item so it is parsable/readable from the Knowledge
+ * view, surfaced by the documents provider, and found by the document `search`
+ * action — exactly as a user would expect "search my transcripts" to work. The
+ * structured record (audio + word timings + diarization) stays the source of
+ * truth in the transcript store; this is the denormalized search copy, linked
+ * back by `metadata.transcriptId`.
+ *
+ * Pure: it builds the documents-store *fields* from a shared `Transcript`; the
+ * service combines them with the runtime UUIDs (world/room/entity/client ids)
+ * and `addedFrom` to call `DocumentService.addDocument`.
+ */
+
+import type { Transcript, TranscriptScope } from "@elizaos/shared/transcripts";
+import { transcriptPlainText } from "@elizaos/shared/transcripts";
+
+/** The documents-store fields derived from a transcript (sans runtime UUIDs). */
+export interface TranscriptKnowledgePayload {
+	/** Plain, speaker-labeled transcript text — the searchable + chunked body. */
+	content: string;
+	/** Suggested filename (slugified title + `.txt`). */
+	filename: string;
+	contentType: string;
+	scope: TranscriptScope;
+	/** Metadata merged onto the document — tags + the link back to the record. */
+	metadata: Record<string, unknown>;
+}
+
+/** Tag every mirrored transcript carries so it's filterable as a transcript. */
+export const TRANSCRIPT_DOCUMENT_TAG = "transcript";
+
+/** Lowercase ascii slug for a filename (fallback "transcript"). */
+function slugify(title: string): string {
+	const slug = title
+		.toLowerCase()
+		.replace(/[^a-z0-9]+/g, "-")
+		.replace(/^-+|-+$/g, "")
+		.slice(0, 64);
+	return slug || "transcript";
+}
+
+/**
+ * Build the documents-store fields for mirroring a transcript into knowledge.
+ * The body is the speaker-labeled plain text (what gets embedded + searched);
+ * `metadata.transcriptId` links the knowledge item back to the rich record so a
+ * Knowledge-view consumer can open it in the player.
+ */
+export function transcriptKnowledgePayload(
+	transcript: Transcript,
+): TranscriptKnowledgePayload {
+	const metadata: Record<string, unknown> = {
+		source: TRANSCRIPT_DOCUMENT_TAG,
+		tags: [TRANSCRIPT_DOCUMENT_TAG],
+		transcriptId: transcript.id,
+		title: transcript.title,
+		durationMs: transcript.durationMs,
+		speakerCount: transcript.speakerCount,
+		createdAt: transcript.createdAt,
+		// Mark as text-backed so the documents UI treats it as editable/previewable
+		// text rather than an opaque binary upload.
+		textBacked: true,
+	};
+	if (transcript.audioUrl) metadata.audioUrl = transcript.audioUrl;
+
+	return {
+		content: transcriptPlainText(transcript.segments),
+		filename: `${slugify(transcript.title)}.txt`,
+		contentType: "text/plain",
+		scope: transcript.scope,
+		metadata,
+	};
+}

package/src/services/voice/transcript-service.test.ts

@@ -0,0 +1,195 @@
+import type { Memory, UUID } from "@elizaos/core";
+import type { Transcript } from "@elizaos/shared/transcripts";
+import { describe, expect, it, vi } from "vitest";
+import {
+	type CreateTranscriptInput,
+	TranscriptService,
+	type TranscriptServiceRuntime,
+} from "./transcript-service";
+
+const WORLD = "00000000-0000-0000-0000-0000000000ww" as UUID;
+const ROOM = "11111111-1111-1111-1111-111111111111" as UUID;
+const ENTITY = "22222222-2222-2222-2222-222222222222" as UUID;
+
+function makeTranscript(): Transcript {
+	return {
+		id: "aaaaaaaa-0000-0000-0000-000000000001",
+		title: "Standup",
+		createdAt: 1000,
+		durationMs: 2000,
+		audioUrl: "/api/media/x.wav",
+		source: "voice-session",
+		scope: "owner-private",
+		status: "ready",
+		speakerCount: 1,
+		segments: [
+			{
+				id: "s1",
+				speakerLabel: "Alice",
+				startMs: 0,
+				endMs: 2000,
+				text: "hi",
+				words: [],
+			},
+		],
+	};
+}
+
+function fakeRuntime(opts: {
+	withDocuments: boolean;
+}): TranscriptServiceRuntime & {
+	rows: Map<string, Memory>;
+	addDocument: ReturnType<typeof vi.fn>;
+} {
+	const rows = new Map<string, Memory>();
+	const addDocument = vi.fn(async () => ({
+		storedDocumentMemoryId: "dddddddd-0000-0000-0000-000000000001" as UUID,
+	}));
+	return {
+		rows,
+		addDocument,
+		agentId: "agent-1" as UUID,
+		async createMemory(memory) {
+			rows.set(memory.id as string, memory);
+			return memory.id as UUID;
+		},
+		async getMemories() {
+			return [...rows.values()];
+		},
+		async getMemoryById(id) {
+			return rows.get(id) ?? null;
+		},
+		async updateMemory(memory) {
+			const existing = rows.get(memory.id);
+			if (!existing) return false;
+			rows.set(memory.id, { ...existing, ...memory });
+			return true;
+		},
+		async deleteMemory(id) {
+			rows.delete(id);
+		},
+		getService<T>(name: string): T | null {
+			if (name === "documents" && opts.withDocuments) {
+				return { addDocument } as unknown as T;
+			}
+			return null;
+		},
+	};
+}
+
+const input = (transcript: Transcript): CreateTranscriptInput => ({
+	worldId: WORLD,
+	roomId: ROOM,
+	entityId: ENTITY,
+	transcript,
+});
+
+describe("TranscriptService", () => {
+	it("mirrors the transcript into knowledge and links the document id", async () => {
+		const rt = fakeRuntime({ withDocuments: true });
+		const svc = new TranscriptService(rt);
+		const t = makeTranscript();
+		const saved = await svc.create(input(t));
+
+		// Mirror called with the searchable text + transcript link metadata.
+		expect(rt.addDocument).toHaveBeenCalledTimes(1);
+		const opts = rt.addDocument.mock.calls[0][0];
+		expect(opts.content).toBe("Alice: hi");
+		expect(opts.scope).toBe("owner-private");
+		expect(opts.clientDocumentId).toBe(t.id);
+		expect((opts.metadata as { transcriptId: string }).transcriptId).toBe(t.id);
+
+		// The stored record carries the knowledge link.
+		expect(saved.knowledgeDocumentId).toBe(
+			"dddddddd-0000-0000-0000-000000000001",
+		);
+		const got = await svc.get(t.id as UUID);
+		expect(got?.knowledgeDocumentId).toBe(
+			"dddddddd-0000-0000-0000-000000000001",
+		);
+	});
+
+	it("still persists the record when no documents service is loaded", async () => {
+		const rt = fakeRuntime({ withDocuments: false });
+		const svc = new TranscriptService(rt);
+		const t = makeTranscript();
+		const saved = await svc.create(input(t));
+		expect(rt.addDocument).not.toHaveBeenCalled();
+		expect(saved.knowledgeDocumentId).toBeUndefined();
+		expect(await svc.get(t.id as UUID)).toEqual(t);
+	});
+
+	it("persists the record even if the mirror throws (recording is never lost)", async () => {
+		const rt = fakeRuntime({ withDocuments: true });
+		rt.addDocument.mockRejectedValueOnce(new Error("docs down"));
+		const svc = new TranscriptService(rt);
+		const t = makeTranscript();
+		const saved = await svc.create(input(t));
+		expect(saved.knowledgeDocumentId).toBeUndefined();
+		expect(await svc.get(t.id as UUID)).not.toBeNull();
+	});
+
+	it("removes the knowledge mirror on delete", async () => {
+		const rt = fakeRuntime({ withDocuments: true });
+		const svc = new TranscriptService(rt);
+		const t = makeTranscript();
+		await svc.create(input(t));
+		const docId = "dddddddd-0000-0000-0000-000000000001";
+		rt.rows.set(docId, { id: docId } as Memory); // stand-in for the doc row
+		await svc.delete(t.id as UUID);
+		expect(rt.rows.has(t.id)).toBe(false);
+		expect(rt.rows.has(docId)).toBe(false);
+	});
+
+	it("applies an edit, re-derives metadata, and re-mirrors to knowledge", async () => {
+		const rt = fakeRuntime({ withDocuments: true });
+		const svc = new TranscriptService(rt);
+		const t = makeTranscript();
+		await svc.create(input(t));
+		const firstDocId = "dddddddd-0000-0000-0000-000000000001";
+		rt.rows.set(firstDocId, { id: firstDocId } as Memory); // the create mirror row
+		rt.addDocument.mockClear();
+		rt.addDocument.mockResolvedValueOnce({
+			storedDocumentMemoryId: "dddddddd-0000-0000-0000-000000000002" as UUID,
+		});
+
+		const updated = await svc.update(t.id as UUID, {
+			worldId: WORLD,
+			roomId: ROOM,
+			entityId: ENTITY,
+			patch: {
+				title: "Edited title",
+				segments: [
+					{ ...t.segments[0], text: "corrected words here", words: [] },
+				],
+			},
+		});
+
+		expect(updated?.title).toBe("Edited title");
+		expect(updated?.editedAt).toBeGreaterThan(0);
+		expect(updated?.segments[0].text).toBe("corrected words here");
+		// Re-mirrored: the stale doc was removed and a fresh one created + linked.
+		expect(rt.rows.has(firstDocId)).toBe(false);
+		expect(rt.addDocument).toHaveBeenCalledTimes(1);
+		expect(updated?.knowledgeDocumentId).toBe(
+			"dddddddd-0000-0000-0000-000000000002",
+		);
+		// The store now round-trips the edited record.
+		expect((await svc.get(t.id as UUID))?.title).toBe("Edited title");
+	});
+
+	it("returns null when updating a transcript that does not exist", async () => {
+		const rt = fakeRuntime({ withDocuments: true });
+		const svc = new TranscriptService(rt);
+		const result = await svc.update(
+			"99999999-0000-0000-0000-000000000000" as UUID,
+			{
+				worldId: WORLD,
+				roomId: ROOM,
+				entityId: ENTITY,
+				patch: { title: "nope" },
+			},
+		);
+		expect(result).toBeNull();
+	});
+});

package/src/services/voice/transcript-service.ts

@@ -0,0 +1,205 @@
+/**
+ * TranscriptService (#8789) — orchestrates a finished recording into both homes:
+ * the rich {@link TranscriptStore} record (source of truth: audio + word-timed
+ * diarized segments) AND a searchable knowledge/documents mirror, linked by
+ * `knowledgeDocumentId`. Create mirrors first (best-effort — a search-index
+ * failure must not lose the recording), then persists the record carrying the
+ * link. Delete removes the mirror too.
+ */
+
+import { logger, type UUID } from "@elizaos/core";
+import type {
+	Transcript,
+	TranscriptScope,
+	TranscriptSegment,
+	TranscriptSummary,
+} from "@elizaos/shared/transcripts";
+import {
+	transcriptDurationMs,
+	transcriptSpeakerCount,
+} from "@elizaos/shared/transcripts";
+import { transcriptKnowledgePayload } from "./transcript-knowledge";
+import {
+	TranscriptStore,
+	type TranscriptStoreRuntime,
+} from "./transcript-store";
+
+/** The documents/knowledge service surface the mirror needs (structural). */
+interface DocumentsLike {
+	addDocument(options: {
+		worldId: UUID;
+		roomId: UUID;
+		entityId: UUID;
+		clientDocumentId: UUID;
+		contentType: string;
+		originalFilename: string;
+		content: string;
+		scope?: TranscriptScope;
+		addedFrom?: string;
+		metadata?: Record<string, unknown>;
+	}): Promise<{ storedDocumentMemoryId: UUID }>;
+}
+
+/** Store runtime + service resolution (the real IAgentRuntime satisfies it). */
+export interface TranscriptServiceRuntime extends TranscriptStoreRuntime {
+	getService<T>(name: string): T | null;
+}
+
+export interface CreateTranscriptInput {
+	worldId: UUID;
+	roomId: UUID;
+	/** The owner/speaker entity the recording is attributed to. */
+	entityId: UUID;
+	transcript: Transcript;
+}
+
+export interface UpdateTranscriptInput {
+	/** Ids the knowledge re-mirror is attributed to (default to the agent ctx). */
+	worldId: UUID;
+	roomId: UUID;
+	entityId: UUID;
+	/** The user edit: a new title and/or replacement segments. */
+	patch: { title?: string; segments?: TranscriptSegment[] };
+}
+
+export class TranscriptService {
+	private readonly store: TranscriptStore;
+
+	constructor(private readonly runtime: TranscriptServiceRuntime) {
+		this.store = new TranscriptStore(runtime);
+	}
+
+	/** Mirror the transcript text into knowledge, then persist the record. */
+	async create(input: CreateTranscriptInput): Promise<Transcript> {
+		const { worldId, roomId, entityId, transcript } = input;
+		const knowledgeDocumentId = await this.mirrorToKnowledge(
+			worldId,
+			roomId,
+			entityId,
+			transcript,
+		);
+		const record = knowledgeDocumentId
+			? { ...transcript, knowledgeDocumentId }
+			: transcript;
+		return this.store.create({ roomId, entityId, transcript: record });
+	}
+
+	list(roomId?: UUID, limit?: number): Promise<TranscriptSummary[]> {
+		return this.store.list(roomId, limit);
+	}
+
+	get(id: UUID): Promise<Transcript | null> {
+		return this.store.get(id);
+	}
+
+	/**
+	 * Apply a user edit to a transcript record (title and/or segment text) and
+	 * persist it, re-mirroring the new text into the knowledge index so search
+	 * reflects the correction. Returns the updated record, or null if no record
+	 * with that id exists. Timing/speaker metadata is re-derived from the edited
+	 * segments; the knowledge mirror is replaced (old doc removed best-effort).
+	 */
+	async update(
+		id: UUID,
+		input: UpdateTranscriptInput,
+	): Promise<Transcript | null> {
+		const existing = await this.store.get(id);
+		if (!existing) return null;
+
+		const segments = input.patch.segments ?? existing.segments;
+		const next: Transcript = {
+			...existing,
+			title: input.patch.title?.trim() || existing.title,
+			segments,
+			durationMs: transcriptDurationMs(segments),
+			speakerCount: transcriptSpeakerCount(segments),
+			editedAt: Date.now(),
+		};
+
+		// Replace the knowledge mirror so search reflects the corrected text.
+		if (existing.knowledgeDocumentId) {
+			try {
+				await this.runtime.deleteMemory(existing.knowledgeDocumentId as UUID);
+			} catch (err) {
+				logger.warn(
+					{
+						transcriptId: id,
+						knowledgeDocumentId: existing.knowledgeDocumentId,
+						error: err instanceof Error ? err.message : String(err),
+					},
+					"[TranscriptService] failed to remove stale knowledge mirror on update",
+				);
+			}
+		}
+		const knowledgeDocumentId = await this.mirrorToKnowledge(
+			input.worldId,
+			input.roomId,
+			input.entityId,
+			next,
+		);
+		next.knowledgeDocumentId = knowledgeDocumentId;
+
+		return this.store.update(next);
+	}
+
+	/** Delete the record + its knowledge mirror (mirror removal best-effort). */
+	async delete(id: UUID): Promise<void> {
+		const existing = await this.store.get(id);
+		if (existing?.knowledgeDocumentId) {
+			try {
+				await this.runtime.deleteMemory(existing.knowledgeDocumentId as UUID);
+			} catch (err) {
+				logger.warn(
+					{
+						transcriptId: id,
+						knowledgeDocumentId: existing.knowledgeDocumentId,
+						error: err instanceof Error ? err.message : String(err),
+					},
+					"[TranscriptService] failed to remove knowledge mirror",
+				);
+			}
+		}
+		await this.store.delete(id);
+	}
+
+	/**
+	 * Best-effort mirror into the documents store. Returns the stored document id
+	 * to link, or undefined when no documents service is loaded or the mirror
+	 * fails — the recording still persists either way (the link is a secondary
+	 * search index, never the source of truth).
+	 */
+	private async mirrorToKnowledge(
+		worldId: UUID,
+		roomId: UUID,
+		entityId: UUID,
+		transcript: Transcript,
+	): Promise<string | undefined> {
+		const documents = this.runtime.getService<DocumentsLike>("documents");
+		if (!documents) return undefined;
+		try {
+			const payload = transcriptKnowledgePayload(transcript);
+			const res = await documents.addDocument({
+				worldId,
+				roomId,
+				entityId,
+				clientDocumentId: transcript.id as UUID,
+				contentType: payload.contentType,
+				originalFilename: payload.filename,
+				content: payload.content,
+				scope: payload.scope,
+				addedFrom: "runtime-internal",
+				metadata: payload.metadata,
+			});
+			return res.storedDocumentMemoryId;
+		} catch (err) {
+			logger.warn(
+				{
+					transcriptId: transcript.id,
+					error: err instanceof Error ? err.message : String(err),
+				},
+				"[TranscriptService] knowledge mirror failed",
+			);
+			return undefined;
+		}
+	}
+}