dsh-aha-tutor 1.4.0 → 1.5.0

package/lib/index.js

@@ -1640,11 +1640,74 @@ async function ingestUrl(address, { fetchImpl = pageFetch, signal } = {}) {
 	if (!chunks.length) throw new MaterialError("page-no-text", "No readable text on this page");
 	return finish("url", title, chunks, { url: url.href });
 }
+const BYTES_PER_SECOND = 32e3;
+const MAX_SPEECH_BYTES = 3840044;
+const MAX_TEXT = 2e3;
+const USABLE = /* @__PURE__ */ new Set(["ready", "standby"]);
+const PREPARING = /* @__PURE__ */ new Set([
+	"checking",
+	"downloading",
+	"cancelling"
+]);
+function wavSeconds(bytes) {
+	const data = bytes instanceof Uint8Array ? bytes : null;
+	if (!data || data.length < 46) return null;
+	const view = new DataView(data.buffer, data.byteOffset, data.byteLength);
+	const tag = (at) => String.fromCharCode(...data.subarray(at, at + 4));
+	return tag(0) === "RIFF" && tag(8) === "WAVE" && tag(12) === "fmt " && tag(36) === "data" && view.getUint32(4, true) === data.length - 8 && view.getUint32(16, true) === 16 && view.getUint16(20, true) === 1 && view.getUint16(22, true) === 1 && view.getUint32(24, true) === 16e3 && view.getUint32(28, true) === BYTES_PER_SECOND && view.getUint16(32, true) === 2 && view.getUint16(34, true) === 16 && view.getUint32(40, true) === data.length - 44 && (data.length - 44) % 2 === 0 ? (data.length - 44) / BYTES_PER_SECOND : null;
+}
+function createSpeechInput({ getSpeech, logger }) {
+	function status() {
+		const speech = getSpeech();
+		let snapshot = null;
+		try {
+			snapshot = speech?.snapshot?.() ?? null;
+		} catch {
+			snapshot = null;
+		}
+		const chosen = snapshot?.selection?.providerId;
+		const provider = snapshot?.providers?.find((entry) => entry.id === chosen);
+		if (!provider) return {
+			state: "off",
+			maxSeconds: 120
+		};
+		const phase = provider.preparation?.phase ?? "ready";
+		return {
+			state: USABLE.has(phase) ? "ready" : PREPARING.has(phase) ? "preparing" : phase === "failed" ? "failed" : "unprepared",
+			maxSeconds: 120
+		};
+	}
+	async function transcribe({ audio, signal }) {
+		const { state } = status();
+		if (state === "off") throw new TutorError("speech-off", "Voice input is not enabled in DSH");
+		if (state !== "ready") throw new TutorError("speech-unprepared", "The speech recognizer is not ready yet");
+		const seconds = wavSeconds(audio);
+		if (seconds === null) throw new TutorError("speech-audio", "The recording could not be read");
+		if (seconds > 120) throw new TutorError("speech-too-long", `Recordings are limited to 120 seconds`);
+		const speech = getSpeech();
+		let result;
+		try {
+			result = await speech.transcribe(speech.resolve({ audio: Buffer.from(audio.buffer, audio.byteOffset, audio.byteLength) }), signal ?? new AbortController().signal);
+		} catch (error) {
+			if (signal?.aborted) throw error;
+			logger?.debug?.("aha-tutor speech recognition failed: %s", error?.message);
+			throw new TutorError("speech-failed", "Speech recognition failed");
+		}
+		const text = String(typeof result === "string" ? result : result?.text ?? "").replace(/\s+/gu, " ").trim().slice(0, MAX_TEXT);
+		if (!text) throw new TutorError("speech-empty", "No speech was heard");
+		return { text };
+	}
+	return Object.freeze({
+		status,
+		transcribe
+	});
+}
 //#endregion
 //#region src/host/http.js
 const UPLOAD_PATH = "/api/aha-tutor/upload";
 const TTS_PATH = "/api/aha-tutor/tts";
 const FONT_PATH = "/api/aha-tutor/font";
+const SPEECH_PATH = "/api/aha-tutor/speech";
 async function readCapped(stream, max, signal) {
 	const parts = [];
 	let size = 0;
@@ -1697,6 +1760,34 @@ function uploadRoute({ studio }) {
 		}
 	};
 }
+function speechRoute({ speechInput }) {
+	return {
+		path: SPEECH_PATH,
+		methods: ["POST"],
+		requestBody: "streaming",
+		fetch: async (request) => {
+			let audio;
+			try {
+				audio = await readCapped(request.body, MAX_SPEECH_BYTES, request.signal);
+			} catch {
+				return failure$1("aborted", "Upload interrupted", 400);
+			}
+			if (audio === null) return failure$1("speech-too-long", "The recording is too long", 413);
+			try {
+				return json({
+					ok: true,
+					value: await speechInput.transcribe({
+						audio: new Uint8Array(audio.buffer, audio.byteOffset, audio.byteLength),
+						signal: request.signal
+					})
+				});
+			} catch (error) {
+				const known = error?.name === "TutorError";
+				return failure$1(known ? publicCode(error) : "speech-failed", known ? error.message : "Speech recognition failed", known && error.code !== "speech-failed" ? 400 : 502);
+			}
+		}
+	};
+}
 function ttsRoute({ speak }) {
 	return {
 		path: TTS_PATH,
@@ -4215,7 +4306,7 @@ const STYLES = [
 	"simpler",
 	"deeper"
 ];
-function createRoutes({ courses, lessons, learner, studio, projects, jobs, settings, tts, plugin, info }) {
+function createRoutes({ courses, lessons, learner, studio, projects, jobs, settings, tts, plugin, info, speechInput }) {
 	return {
 		"app/info": () => info(),
 		"job/get": (payload) => jobs.get(need(payload, "id", "job"), { since: Math.max(0, Number(payload.since) || 0) }) ?? { status: "missing" },
@@ -4491,6 +4582,14 @@ function createRoutes({ courses, lessons, learner, studio, projects, jobs, setti
 				cached: result.cached === true
 			};
 		},
+		"speech/status": () => speechInput.status(),
+		"speech/transcribe": async (payload) => {
+			const encoded = typeof payload.base64 === "string" ? payload.base64 : "";
+			if (encoded.length > Math.ceil(3840044 / 3) * 4) throw new TutorError("speech-too-long", "The recording is too long");
+			const audio = Buffer.from(encoded, "base64");
+			if (!encoded || audio.toString("base64") !== encoded) throw new TutorError("speech-audio", "The recording could not be read");
+			return speechInput.transcribe({ audio: new Uint8Array(audio.buffer, audio.byteOffset, audio.byteLength) });
+		},
 		"settings/get": () => settings.get(),
 		"settings/save": (payload) => settings.save(payload.settings ?? {}),
 		"settings/set-key": (payload) => settings.setKey({
@@ -4998,6 +5097,7 @@ const PREF_DEFAULTS = Object.freeze({
 	voiceRate: 1,
 	boardTheme: "auto",
 	boardFont: "hand",
+	boardView: "canvas",
 	ttsProvider: "openai",
 	tts: Object.fromEntries(TTS_PROVIDERS.map((provider) => [provider, providerDefaults(provider)]))
 });
@@ -5028,6 +5128,7 @@ function cleanPrefs(input = {}, base = PREF_DEFAULTS) {
 		"dark"
 	].includes(input.boardTheme)) next.boardTheme = input.boardTheme;
 	if (["hand", "standard"].includes(input.boardFont)) next.boardFont = input.boardFont;
+	if (["canvas", "pages"].includes(input.boardView)) next.boardView = input.boardView;
 	if (TTS_PROVIDERS.includes(input.ttsProvider)) next.ttsProvider = input.ttsProvider;
 	for (const provider of TTS_PROVIDERS) {
 		const patch = input.tts?.[provider];
@@ -5802,6 +5903,10 @@ function apply(ctx) {
 		keys,
 		llmPort: llm
 	});
+	const speechInput = createSpeechInput({
+		getSpeech: () => ctx.get?.("speechToText"),
+		logger
+	});
 	async function speak({ text, provider, signal }) {
 		await ready;
 		const current = prefs.read();
@@ -5838,7 +5943,8 @@ function apply(ctx) {
 			settings,
 			tts: { speak },
 			plugin,
-			info
+			info,
+			speechInput
 		}),
 		"app/info": (payload) => info(payload),
 		"samples/list": () => samples.list(),
@@ -5853,6 +5959,7 @@ function apply(ctx) {
 		if (typeof fetchRoutes?.register !== "function") return;
 		connectionCtx.effect(() => fetchRoutes.register(uploadRoute({ studio })), "aha-tutor: upload route");
 		connectionCtx.effect(() => fetchRoutes.register(ttsRoute({ speak })), "aha-tutor: tts route");
+		connectionCtx.effect(() => fetchRoutes.register(speechRoute({ speechInput })), "aha-tutor: speech route");
 		connectionCtx.effect(() => fetchRoutes.register(fontRoute()), "aha-tutor: font route");
 	});
 	ctx.effect(() => publishRuntime({

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dsh-aha-tutor",
-  "version": "1.4.0",
+  "version": "1.5.0",
   "description": "开窍 Aha Tutor: a personal AI tutor for DeepSeek Harness that builds a course around you and teaches it on a narrated handwriting whiteboard",
   "type": "module",
   "main": "./lib/index.js",