@maheidem/pi-audio-transcribe 0.2.0

package/lib/json-store.ts

@@ -0,0 +1,92 @@
+/** pi-extension-builder JsonStore v0.1.0 — canonical source and vendored primitive. */
+import {
+  existsSync,
+  mkdirSync,
+  readFileSync,
+  renameSync,
+  unlinkSync,
+  writeFileSync,
+} from "node:fs";
+import { dirname } from "node:path";
+
+export interface JsonStoreOptions<T extends object> {
+  path: string;
+  defaults: T;
+  normalize(value: unknown, defaults: T): T;
+}
+
+/** Small synchronous JSON repository for user preferences. */
+export class JsonStore<T extends object> {
+  readonly path: string;
+  private readonly defaultsValue: T;
+  private readonly normalizeValue: JsonStoreOptions<T>["normalize"];
+
+  constructor(options: JsonStoreOptions<T>) {
+    this.path = options.path;
+    this.defaultsValue = this.clone(options.defaults);
+    this.normalizeValue = options.normalize;
+  }
+
+  defaults(): T {
+    return this.clone(this.defaultsValue);
+  }
+
+  load(): T {
+    if (!existsSync(this.path)) {
+      const value = this.defaults();
+      this.save(value);
+      return value;
+    }
+
+    try {
+      const raw = JSON.parse(readFileSync(this.path, "utf8")) as unknown;
+      const value = this.normalizeValue(raw, this.defaults());
+      if (JSON.stringify(raw) !== JSON.stringify(value)) this.save(value);
+      return this.clone(value);
+    } catch {
+      this.preserveCorruptFile();
+      const value = this.defaults();
+      this.save(value);
+      return value;
+    }
+  }
+
+  save(value: T): T {
+    const normalized = this.normalizeValue(value, this.defaults());
+    mkdirSync(dirname(this.path), { recursive: true, mode: 0o700 });
+    const temporary = `${this.path}.${process.pid}.${Date.now()}.tmp`;
+    try {
+      writeFileSync(temporary, `${JSON.stringify(normalized, null, 2)}\n`, {
+        encoding: "utf8",
+        mode: 0o600,
+      });
+      renameSync(temporary, this.path);
+    } catch (error) {
+      try {
+        if (existsSync(temporary)) unlinkSync(temporary);
+      } catch {
+        // Best-effort cleanup; preserve the original error.
+      }
+      throw error;
+    }
+    return this.clone(normalized);
+  }
+
+  reset(): T {
+    return this.save(this.defaults());
+  }
+
+  private preserveCorruptFile(): void {
+    if (!existsSync(this.path)) return;
+    const stamp = new Date().toISOString().replace(/[^0-9TZ]/g, "");
+    try {
+      renameSync(this.path, `${this.path}.corrupt-${stamp}`);
+    } catch {
+      // If preservation fails, save() will atomically replace the bad file.
+    }
+  }
+
+  private clone(value: T): T {
+    return JSON.parse(JSON.stringify(value)) as T;
+  }
+}

package/omlx.ts

@@ -0,0 +1,295 @@
+/**
+ * oMLX STT client.
+ *
+ * Verified against oMLX 0.6.4 / 192.168.31.152:8123:
+ *   POST /v1/audio/transcriptions  multipart(form-data: file, model [, language])
+ *     -> 200 { text, language, duration, segments[] }
+ *
+ * Server behaviours that this module compensates for:
+ *   - `duration` is INFERENCE LATENCY, not audio length. An 102.87 s clip
+ *     reported duration=1.72. Real duration comes from segments[].end
+ *     (reported 102.869, matching ffprobe 102.8677).
+ *   - `language` case is inconsistent: "Portuguese" on auto-detect vs
+ *     "portuguese" when the hint is sent. Always normalize.
+ *   - `word_timestamps=true` and `response_format` are accepted but NOT
+ *     implemented (docs say response_format is silently ignored; word
+ *     timestamps verified unchanged on real audio). Do not rely on either.
+ *   - Unsupported input returns HTTP 500 "unsupported file format" (not 4xx),
+ *     so callers must pre-flight the file type.
+ */
+
+export interface AsrSegment {
+	text: string;
+	language?: string | null;
+	start?: number;
+	end?: number;
+}
+
+export interface AsrResponse {
+	text: string;
+	language?: string | null;
+	/** INFERENCE LATENCY in seconds, not audio duration. */
+	duration?: number;
+	segments?: AsrSegment[];
+}
+
+export interface SttModel {
+	id: string;
+	loaded: boolean;
+	isLoading: boolean;
+	engineType?: string;
+	configModelType?: string;
+	estimatedSize?: number;
+	sourceType?: string;
+	realtimeStt?: boolean;
+}
+
+export interface ServerHealth {
+	ok: boolean;
+	version?: string;
+	loadedModels?: string[];
+	reachable: boolean;
+	error?: string;
+}
+
+export class OmnxError extends Error {
+	status?: number;
+	kind: "network" | "http" | "format" | "notfound" | "empty";
+	available?: string[];
+	constructor(message: string, kind: OmnxError["kind"], status?: number, available?: string[]) {
+		super(message);
+		this.name = "OmnxError";
+		this.kind = kind;
+		this.status = status;
+		this.available = available;
+	}
+}
+
+/** Language names sometimes come back capitalized, sometimes lowercased. */
+const LANG_NAMES: Record<string, string> = {
+	portuguese: "pt",
+	"pt-br": "pt",
+	"pt-pt": "pt",
+	brasil: "pt",
+	brazil: "pt",
+	english: "en",
+	spanish: "es",
+	french: "fr",
+	german: "de",
+	italian: "it",
+	dutch: "nl",
+	chinese: "zh",
+	japanese: "ja",
+	korean: "ko",
+};
+
+/** Normalize "Portuguese" / "portuguese" / "pt-BR" -> "pt". Returns null for None/unknown. */
+export function normalizeLanguage(raw: unknown): string | null {
+	if (typeof raw !== "string") return null;
+	const v = raw.trim().toLowerCase();
+	if (!v || v === "none" || v === "null" || v === "unknown") return null;
+	if (LANG_NAMES[v]) return LANG_NAMES[v];
+	if (/^[a-z]{2}(-[a-z]{2})?$/i.test(v)) return v.split("-")[0];
+	return v;
+}
+
+async function httpJson<T>(
+	url: string,
+	init: RequestInit,
+	timeoutMs: number,
+	signal?: AbortSignal,
+): Promise<T> {
+	const ctrl = new AbortController();
+	const timer = setTimeout(() => ctrl.abort(new Error("timeout")), timeoutMs);
+	// Chain the caller's signal so pi's cancel (ctx.signal) aborts in-flight requests.
+	const onAbort = () => ctrl.abort(signal?.reason ?? new Error("aborted"));
+	if (signal) {
+		if (signal.aborted) {
+			clearTimeout(timer);
+			throw new OmnxError("Cancelled", "network");
+		}
+		signal.addEventListener("abort", onAbort, { once: true });
+	}
+	try {
+		const res = await fetch(url, { ...init, signal: ctrl.signal });
+		const body = await res.text();
+		if (!res.ok) {
+			let message = body.slice(0, 400) || `HTTP ${res.status}`;
+			let available: string[] | undefined;
+			try {
+				const parsed = JSON.parse(body) as { error?: { message?: string }; message?: string };
+				const raw = typeof parsed.error === "string" ? parsed.error : (parsed.error?.message ?? parsed.message ?? message);
+				message = raw || message;
+				const m = /Available:\s*(.+)$/is.exec(message);
+				if (m) available = m[1].split(",").map((s) => s.trim()).filter(Boolean);
+			} catch {
+				/* non-JSON error body; keep raw text */
+			}
+			throw new OmnxError(message, res.status === 404 ? "notfound" : "http", res.status, available);
+		}
+		try {
+			return JSON.parse(body) as T;
+		} catch {
+			throw new OmnxError("Response was not valid JSON", "http", res.status);
+		}
+	} catch (err) {
+		if (err instanceof OmnxError) throw err;
+		const e = err as Error;
+		if (e.name === "AbortError" || /abort/i.test(e.message)) throw new OmnxError("Cancelled", "network");
+		throw new OmnxError(`Cannot reach ${new URL(url).host}: ${e.message}`, "network");
+	} finally {
+		clearTimeout(timer);
+		signal?.removeEventListener("abort", onAbort);
+	}
+}
+
+function authHeaders(apiKey?: string): Record<string, string> {
+	return apiKey ? { Authorization: `Bearer ${apiKey}` } : {};
+}
+
+/** Real audio duration from segments; falls back to the (untrusted) duration field. */
+export function audioDurationFromSegments(segments?: AsrSegment[]): number | null {
+	if (!Array.isArray(segments) || segments.length === 0) return null;
+	const ends = segments.map((s) => (typeof s?.end === "number" ? s.end : NaN)).filter((n) => Number.isFinite(n));
+	return ends.length ? Math.max(...ends) : null;
+}
+
+/** List STT-capable models. Prefers engine_type==="audio_stt" from /v1/models/status. */
+export async function listSttModels(baseUrl: string, apiKey?: string, timeoutMs = 15_000, signal?: AbortSignal): Promise<SttModel[]> {
+	const statusUrl = `${baseUrl.replace(/\/v1$/, "")}/v1/models/status`;
+	try {
+		const data = await httpJson<{ models?: Array<Record<string, unknown>> }>(statusUrl, { method: "GET", headers: authHeaders(apiKey) }, timeoutMs, signal);
+		const models = Array.isArray(data.models) ? data.models : [];
+		const stt = models
+			.filter((m) => {
+				const engine = String(m.engine_type ?? m.model_type ?? "").toLowerCase();
+				const cfg = String(m.config_model_type ?? "").toLowerCase();
+				return engine.includes("stt") || engine.includes("asr") || cfg.includes("asr") || cfg.includes("whisper");
+			})
+			.map((m) => ({
+				id: String(m.id),
+				loaded: Boolean(m.loaded),
+				isLoading: Boolean(m.is_loading),
+				engineType: String(m.engine_type ?? ""),
+				configModelType: String(m.config_model_type ?? ""),
+				estimatedSize: typeof m.estimated_size === "number" ? m.estimated_size : undefined,
+				sourceType: typeof m.source_type === "string" ? m.source_type : undefined,
+				realtimeStt: Boolean(m.realtime_stt),
+			}));
+		if (stt.length) return stt;
+	} catch {
+		/* fall through to /v1/models name sniffing */
+	}
+	// Fallback: /v1/models has no engine field, so sniff the id.
+	const data = await httpJson<{ data?: Array<{ id: string }> }>(`${baseUrl}/models`, { method: "GET", headers: authHeaders(apiKey) }, timeoutMs, signal);
+	return (data.data ?? [])
+		.filter((m) => /asr|whisper|stt|speech[-_]?rec/i.test(m.id))
+		.map((m) => ({ id: m.id, loaded: false, isLoading: false }));
+}
+
+/**
+ * Pick the best STT model: an already-loaded one wins (no cold start), then
+ * prefer a shorter canonical id over a duplicated `org--name` cache alias.
+ */
+export function pickSttModel(models: SttModel[]): SttModel | null {
+	if (!models.length) return null;
+	const score = (m: SttModel): number => {
+		let s = 0;
+		if (m.loaded) s += 100;
+		if (m.isLoading) s -= 50;
+		if (!m.id.includes("--")) s += 10; // canonical local id over "mlx-community--X" alias
+		if (/asr/i.test(m.id)) s += 5;
+		s -= m.id.length / 1000;
+		return s;
+	};
+	return [...models].sort((a, b) => score(b) - score(a))[0];
+}
+
+export interface TranscribeOptions {
+	baseUrl: string;
+	model: string;
+	filePath: string;
+	fileName?: string;
+	/** Omit for server auto-detect (the default). */
+	language?: string | null;
+	apiKey?: string;
+	timeoutMs?: number;
+	signal?: AbortSignal;
+	prompt?: string;
+}
+
+export async function transcribe(opts: TranscribeOptions): Promise<AsrResponse> {
+	const form = new FormData();
+	// A Blob from a byte buffer avoids relying on File constructor availability
+	// across Node versions, while still setting the multipart filename, which
+	// oMLX uses to sniff the container format.
+	const buf = await readFileBuffer(opts.filePath);
+	const blob = new Blob([new Uint8Array(buf)], { type: mimeFor(opts.fileName ?? opts.filePath) });
+	form.append("file", blob, opts.fileName ?? basename(opts.filePath));
+	form.append("model", opts.model);
+	if (opts.language) form.append("language", opts.language);
+	if (opts.prompt) form.append("prompt", opts.prompt);
+
+	const data = await httpJson<AsrResponse>(`${opts.baseUrl}/audio/transcriptions`, { method: "POST", headers: authHeaders(opts.apiKey), body: form }, opts.timeoutMs ?? 300_000, opts.signal);
+	if (typeof data?.text !== "string") throw new OmnxError("Transcription response missing `text`", "empty");
+	return data;
+}
+
+export async function serverHealth(baseUrl: string, apiKey?: string, timeoutMs = 8_000, signal?: AbortSignal): Promise<ServerHealth> {
+	try {
+		const data = await httpJson<{ status?: string; version?: string; loaded_models?: string[] }>(
+			`${baseUrl.replace(/\/v1$/, "")}/api/status`,
+			{ method: "GET", headers: authHeaders(apiKey) },
+			timeoutMs,
+			signal,
+		);
+		return { ok: data.status === "ok", version: data.version, loadedModels: data.loaded_models, reachable: true };
+	} catch (err) {
+		return { ok: false, reachable: false, error: err instanceof Error ? err.message : String(err) };
+	}
+}
+
+const EXT_MIME: Record<string, string> = {
+	// Verified accepted by oMLX 0.6.4 /v1/audio/transcriptions: wav, mp3,
+	// m4a, aac, opus. The remainder are advertised but untested here.
+	".3ga": "audio/3gpp",
+	".wav": "audio/wav",
+	".mp3": "audio/mpeg",
+	".m4a": "audio/mp4",
+	".mp4": "audio/mp4",
+	".aac": "audio/aac",
+	".opus": "audio/ogg",
+	".ogg": "audio/ogg",
+	".flac": "audio/flac",
+	".webm": "audio/webm",
+	".amr": "audio/amr",
+	".3gp": "audio/3gpp",
+};
+
+export function mimeFor(path: string): string {
+	const dot = path.lastIndexOf(".");
+	return EXT_MIME[path.slice(dot).toLowerCase()] ?? "application/octet-stream";
+}
+
+export const AUDIO_EXTENSIONS: string[] = Object.keys(EXT_MIME);
+
+/** Extensions confirmed working against the live server. */
+export const VERIFIED_AUDIO_EXTENSIONS = [".wav", ".mp3", ".m4a", ".aac", ".opus"];
+
+export function isAudioPath(path: string): boolean {
+	const dot = path.lastIndexOf(".");
+	if (dot < 0) return false;
+	return Object.keys(EXT_MIME).includes(path.slice(dot).toLowerCase());
+}
+
+function basename(path: string): string {
+	const i = Math.max(path.lastIndexOf("/"), path.lastIndexOf("\\"));
+	return i >= 0 ? path.slice(i + 1) : path;
+}
+
+async function readFileBuffer(path: string): Promise<ArrayBuffer> {
+	const { readFile } = await import("node:fs/promises");
+	const buf = await readFile(path);
+	// Node Buffers are ArrayBuffers but typed nominally; slice gives a clean copy.
+	return buf.buffer.slice(buf.byteOffset, buf.byteOffset + buf.byteLength) as ArrayBuffer;
+}

package/package.json

@@ -0,0 +1,69 @@
+{
+  "name": "@maheidem/pi-audio-transcribe",
+  "version": "0.2.0",
+  "type": "module",
+  "description": "Pi extension: transcribe audio with oMLX STT (Qwen3-ASR), validate the speech, and persist a JSON transcript sidecar beside each audio file.",
+  "keywords": [
+    "pi-package",
+    "extension",
+    "speech-to-text",
+    "stt",
+    "asr",
+    "omlx",
+    "qwen3-asr",
+    "mlx",
+    "transcription"
+  ],
+  "files": [
+    "config.ts",
+    "index.ts",
+    "omlx.ts",
+    "paths.ts",
+    "pipeline.ts",
+    "settings.ts",
+    "sidecar.ts",
+    "validate.ts",
+    "lib/json-store.ts",
+    "ui/audio-panel.ts",
+    "ui/settings-panel.ts",
+    "README.md",
+    "LICENSE"
+  ],
+  "license": "MIT",
+  "author": "maheidem",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Maheidem/pi-coder-management.git",
+    "directory": "custom-extensions/audio-transcribe"
+  },
+  "homepage": "https://github.com/Maheidem/pi-coder-management/tree/main/custom-extensions/audio-transcribe#readme",
+  "bugs": {
+    "url": "https://github.com/Maheidem/pi-coder-management/issues"
+  },
+  "scripts": {
+    "test": "node tests/run.mjs",
+    "typecheck": "tsc -p tsconfig.json",
+    "prepack": "npm run typecheck && npm test"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  },
+  "devDependencies": {
+    "@earendil-works/pi-coding-agent": "^0.84.4",
+    "@earendil-works/pi-tui": "^0.84.4",
+    "@types/node": "^22.0.0",
+    "tsx": "^4.20.0",
+    "typebox": "^1.0.17",
+    "typescript": "^5.9.3"
+  },
+  "pi": {
+    "extensions": [
+      "./index.ts"
+    ]
+  }
+}

package/paths.ts

@@ -0,0 +1,128 @@
+/**
+ * Audio path extraction from user input.
+ *
+ * Handles the forms that actually reach pi:
+ *   - interactive paste, which inserts a path with backslash-escaped spaces
+ *       /Users/me/Downloads/Medidas\ da\ gaveta.m4a
+ *   - quoted paths
+ *       "/Users/me/Downloads/Medidas da gaveta.m4a"
+ *   - @-prefixed paths (models often echo the @ from `pi -p @file`)
+ *   - plain paths without spaces
+ */
+
+import { AUDIO_EXTENSIONS, isAudioPath } from "./omlx.ts";
+
+/** Collapse a pasted path token into a real filesystem path. */
+export function unescapePath(token: string): string {
+	let t = token.trim();
+	// Strip surrounding quotes.
+	if ((t.startsWith('"') && t.endsWith('"')) || (t.startsWith("'") && t.endsWith("'"))) t = t.slice(1, -1);
+	// Leading @ used by pi's file attachment syntax.
+	if (t.startsWith("@")) t = t.slice(1);
+	// Backslash-escaped spaces (shell/paste form) -> real spaces.
+	t = t.replace(/\\ /g, " ");
+	// Collapse doubled slashes introduced by sloppy concatenation, but keep UNC-ish prefixes.
+	t = t.replace(/([^:])\/\/+/g, "$1/");
+	return t.trim();
+}
+
+function stripTrailingPunctuation(s: string): string {
+	return s.replace(/[),.;:!?'\]]+$/u, "").trim();
+}
+
+/**
+ * Find audio file paths referenced in free text, de-duplicated, in order.
+ * Deliberately conservative: a token must look like a path (contain a slash
+ * or start with ~ / . / a drive letter) AND carry an audio extension.
+ */
+export function extractAudioPaths(text: string): string[] {
+	if (!text) return [];
+	const found: string[] = [];
+	const push = (raw: string) => {
+		const p = unescapePath(stripTrailingPunctuation(raw));
+		if (!p) return;
+		if (!isAudioPath(p)) return;
+		if (!/[~/]/.test(p) && !/^\.{1,2}[\\/]/.test(p) && !/^[a-zA-Z]:[\\/]/.test(p)) return;
+		if (!found.includes(p)) found.push(p);
+	};
+
+	// 1. Quoted strings.
+	for (const m of text.matchAll(/"([^"]+)"|'([^']+)'/g)) push(m[1] ?? m[2] ?? "");
+
+	// 2. Backslash-escaped paths: greedy up to the audio extension.
+	const escapedRe = new RegExp(`(?:[^\\s"']|\\\\ )+\\.(?:${AUDIO_EXTENSIONS.map((e) => e.slice(1)).join("|")})\\b`, "gi");
+	for (const m of text.matchAll(escapedRe)) {
+		if (/\\\s/.test(m[0]) || m[0].includes("\\")) push(m[0]);
+	}
+
+	// 3. Plain whitespace-delimited tokens.
+	for (const tok of text.split(/\s+/)) {
+		if (!tok) continue;
+		if (tok.includes("\\ ")) continue; // handled by rule 2
+		push(tok);
+	}
+
+	return found;
+}
+
+/**
+ * Split a command argument string into tokens, honouring "quotes", 'quotes',
+ * and backslash-escaped spaces. Without this, `/transcribe "a b c.m4a"`
+ * shatters into three bogus paths — which matters because real recordings
+ * from phones are almost always named with spaces.
+ */
+export function tokenizeArgs(input: string): string[] {
+	const tokens: string[] = [];
+	let cur = "";
+	let quote: '"' | "'" | null = null;
+	let started = false;
+	for (let i = 0; i < input.length; i++) {
+		const ch = input[i]!;
+		if (quote) {
+			if (ch === quote) {
+				quote = null;
+				started = true;
+			}
+			// Inside single quotes everything is literal; inside double quotes
+			// a backslash still escapes the next char.
+			else if (ch === "\\" && quote === '"' && i + 1 < input.length) {
+				cur += input[++i];
+			}
+			// An escaped space typed inside quotes stays a space.
+			else cur += ch;
+			continue;
+		}
+		if (ch === '"' || ch === "'") {
+			quote = ch;
+			started = true;
+			continue;
+		}
+		if (ch === "\\" && i + 1 < input.length) {
+			cur += input[++i];
+			started = true;
+			continue;
+		}
+		if (/\s/.test(ch)) {
+			if (started) {
+				tokens.push(cur);
+				cur = "";
+				started = false;
+			}
+			continue;
+		}
+		cur += ch;
+		started = true;
+	}
+	if (started) tokens.push(cur);
+	return tokens;
+}
+
+/** Resolve a possibly-relative path against cwd, expanding a leading `~`. */
+export function expandPath(p: string, cwd: string): string {
+	let t = unescapePath(p);
+	if (t === "~") return process.env.HOME ?? t;
+	if (t.startsWith("~/")) return `${process.env.HOME ?? "~"}/${t.slice(2)}`;
+	if (t.startsWith("/")) return t;
+	if (/^[a-zA-Z]:[\\/]/.test(t)) return t;
+	return `${cwd.replace(/\/+$/, "")}/${t}`;
+}