@dxos/plugin-transformer 0.10.0 → 0.11.0

package/dist/lib/TransformerPlugin.mjs

@@ -0,0 +1,16 @@
+import { Plugin } from "@dxos/app-framework";
+import { AppPlugin } from "@dxos/app-toolkit";
+import { meta } from "#meta";
+import { translations } from "#translations";
+//#endregion
+//#region src/TransformerPlugin.tsx
+var TransformerPlugin = Plugin.define(meta).pipe(AppPlugin.addSchemaModule({ schema: [] }), AppPlugin.addTranslationsModule({ translations }), AppPlugin.addPluginAssetModule({ asset: {
+	pluginId: meta.profile.key,
+	path: "PLUGIN.mdl",
+	content: "---\nid: org.dxos.plugin.transformer\nname: TransformerPlugin\nversion: 0.1.0\n---\n\nA browser-based machine learning plugin for DXOS Composer that runs Hugging Face Transformers.js\nmodels entirely in-browser via WebAssembly and WebGPU. It provides automatic speech recognition\n(Whisper) through a React hook and component layer, enabling real-time voice transcription without\nany server-side inference infrastructure.\n\n## Extensions\n\nThe following extension dialects are used in this document.\nEach extension is defined in the Appendix or resolved via its URI.\n\n| Term        | URI                            |\n|-------------|--------------------------------|\n| `type`      | `org.dxos.mdl.type@1.0`       |\n| `feat`      | `org.dxos.mdl.feat@1.0`       |\n| `test`      | `org.dxos.mdl.test@1.0`       |\n| `component` | `org.dxos.mdl.component@1.0`  |\n| `op`        | `org.dxos.mdl.op@1.0`         |\n\n## Types\n\n```mdl\ntype PipelineConfig\n  fields:\n    model: string              # Hugging Face model id, e.g. 'Xenova/whisper-base'\n    active?: boolean           # whether to load and run the pipeline\n    debug?: boolean            # enable verbose logging\n```\n\n```mdl\ntype PipelineState\n  fields:\n    gpuInfo: string            # WebGPU adapter description, or fallback message\n    isLoaded: boolean          # true once the model is fully initialised\n    isLoading: boolean         # true while the model is being downloaded/compiled\n    error: string | null       # error message if loading failed\n```\n\n```mdl\ntype AudioStreamConfig\n  fields:\n    active?: boolean           # start/stop microphone capture\n    debug?: boolean\n    onAudioData?(audioData: Float32Array): Promise<void>\n```\n\n```mdl\ntype AudioStreamState\n  fields:\n    stream: MediaStream | null # live microphone stream, or null when inactive\n    error: string | null       # error from getUserMedia or audio processing\n    audioLevel: number         # 0-255 RMS level for visualisation\n```\n\n```mdl\ntype TranscriptionOptions\n  fields:\n    sampling_rate: number      # audio sample rate in Hz (16000)\n    chunk_length_s: number     # seconds of audio per chunk\n    stride_length_s: number    # overlap between adjacent chunks\n    return_timestamps: boolean # include word-level timestamps in output\n    language: string           # target language hint, e.g. 'english'\n```\n\n```mdl\ntype EmbeddingOutput\n  fields:\n    data: number[]             # embedding vector as a flat float array\n```\n\n## Components\n\n```mdl\ncomponent Voice\n  desc: |\n    Composite component that wires usePipeline and useAudioStream together to provide\n    live Whisper transcription from the microphone.  Accumulates the running transcript\n    in local state and renders a DebugInfo panel showing model status, audio level, and\n    the current transcription text.\n  props:\n    active?: boolean           # start/stop the entire pipeline\n    debug?: boolean\n    model?: string             # defaults to 'Xenova/whisper-base'\n  state:\n    isTranscribing: boolean    # true while a transcription call is in-flight\n    transcription: string      # accumulated transcript text\n  layout: |\n    ┌──────────────────────────────────────┐\n    │  [DebugInfo panel]                   │\n    │   model: Xenova/whisper-base         │\n    │   gpu: GPU Available                 │\n    │   stream: active / inactive          │\n    │   level: ███░░░░ (audioLevel bar)    │\n    │   ──────────────────────────────     │\n    │   [transcription text]               │\n    └──────────────────────────────────────┘\n```\n\n## Operations\n\n```mdl\nop LoadModel\n  desc: |\n    Loads a Transformers.js automatic-speech-recognition pipeline for the given model id.\n    Configures ONNX backend to use WebGPU if available, falling back to WASM/CPU.\n    Invoked automatically when the Voice component mounts with active = true.\n  input: void\n  output: void\n  effects: [http, fs]\n  note: Model weights are cached in ./.cache by the Transformers.js env settings.\n```\n\n```mdl\nop Transcribe\n  desc: |\n    Runs the loaded ASR pipeline against a Float32Array of 16 kHz mono audio.\n    Returns the transcribed text string from the pipeline result.\n  input: void\n  output: string\n  effects: [gpu]\n  note: Guarded by isTranscribing flag to prevent concurrent calls.\n```\n\n```mdl\nop CaptureAudio\n  desc: |\n    Requests microphone permission via getUserMedia and wires an AudioWorklet that\n    accumulates 2-second chunks (32000 samples at 16 kHz) before forwarding them to\n    the onAudioData callback.\n  input: void\n  output: void\n  effects: [audio]\n  note: Cleaned up automatically when the component unmounts or active becomes false.\n```\n\n## Features\n\n```mdl\nfeat F-1: In-Browser ASR Pipeline\n\n  req F-1.1:\n    when: Voice component mounts with active = true\n    then: usePipeline loads the Whisper model via Transformers.js; isLoading is true until complete\n\n  req F-1.2:\n    when: WebGPU is available in the browser\n    then: the ONNX backend is configured for WebGPU execution and gpuInfo reflects the adapter\n\n  req F-1.3:\n    when: WebGPU is not available\n    then: the pipeline falls back to WASM/CPU and gpuInfo is set to a 'not supported' message\n\n  req F-1.4:\n    when: model loading fails\n    then: PipelineState.error contains a human-readable message; isLoading is false\n```\n\n```mdl\nfeat F-2: Microphone Capture\n\n  req F-2.1:\n    when: useAudioStream is activated\n    then: getUserMedia is called with mono 16 kHz audio and echo/noise cancellation enabled\n\n  req F-2.2:\n    when: audio is captured\n    then: an AudioWorklet accumulates samples and emits 2-second chunks via the onAudioData callback\n\n  req F-2.3:\n    when: active is set to false or the component unmounts\n    then: all MediaStream tracks are stopped, AudioContext is closed, and AnimationFrame is cancelled\n\n  req F-2.4:\n    when: getUserMedia is denied\n    then: AudioStreamState.error is set with the underlying error message\n```\n\n```mdl\nfeat F-3: Live Transcription\n\n  req F-3.1:\n    when: a 2-second audio chunk arrives and the model is loaded\n    then: usePipeline.transcribe is called with 16 kHz mono options and the result appended to the transcript\n\n  req F-3.2:\n    when: transcription is already in progress (isTranscribing = true)\n    then: the incoming audio chunk is dropped to avoid concurrent inference\n\n  req F-3.3:\n    when: transcription returns an empty or whitespace-only string\n    then: the transcript state is not updated\n```\n\n```mdl\nfeat F-4: RAG Embedding Pipeline (Testing)\n\n  req F-4.1:\n    when: RagPipeline.generateCompletions is called with an input and a knowledge base\n    then: embeddings are generated for both the input and each knowledge-base entry\n\n  req F-4.2:\n    when: embeddings are available\n    then: cosine similarity is computed and the top-3 most similar contexts are selected\n```\n\n## Acceptance\n\n```mdl\ntest T-1: Model loads and transitions isLoaded\n  given: Voice component mounts with active = true and model = 'Xenova/whisper-base'\n  when: model download and ONNX compilation complete\n  then:\n    - isLoading transitions from true to false\n    - isLoaded is true\n    - error is null\n```\n\n```mdl\ntest T-2: Model load failure sets error state\n  given: the model download throws a network error\n  when: usePipeline attempts to load the model\n  then:\n    - isLoading is false\n    - error contains the error message string\n    - isLoaded is false\n```\n\n```mdl\ntest T-3: Microphone capture populates stream state\n  given: getUserMedia is granted\n  when: useAudioStream activates\n  then:\n    - AudioStreamState.stream is a non-null MediaStream\n    - AudioStreamState.error is null\n```\n\n```mdl\ntest T-4: Audio chunk triggers transcription\n  given: model is loaded and stream is active\n  when: AudioWorklet posts a 32000-sample chunk\n  then:\n    - isTranscribing becomes true during the pipeline call\n    - transcription string is updated with the returned text\n    - isTranscribing returns to false after completion\n```\n\n```mdl\ntest T-5: Concurrent transcription calls are dropped\n  given: isTranscribing is true\n  when: a second audio chunk arrives\n  then: the chunk is discarded and no second pipeline call is made\n```\n\n---\n\n## Appendix: Extension Definitions\n\nExtension block types used in this document are defined below using\nthe core `ext` primitive — the only construct the base language provides.\n\n```mdl\next type\n  uri: org.dxos.mdl.type@1.0\n  desc: A named data structure with typed fields and optional literals.\n  fields:\n    desc?: Prose\n    fields?: FieldMap        # name[?]: TypeExpr  (# inline comment)\n    literals?: UnionList     # a | b | c\n    extends?: TypeRef[]\n```\n\n```mdl\next feat\n  uri: org.dxos.mdl.feat@1.0\n  desc: A named feature grouping one or more requirements.\n  fields:\n    desc?: Prose\n    req: RequirementList\n  nesting: self              # feat blocks may contain feat blocks\n```\n\n```mdl\next test\n  uri: org.dxos.mdl.test@1.0\n  desc: An acceptance scenario expressed as given / when / then steps.\n  fields:\n    given?: Step | Step[]\n    when?: Step | Step[]\n    then: Step | Step[]\n    tags?: TagList\n```\n\n```mdl\next component\n  uri: org.dxos.mdl.component@1.0\n  desc: A UI component with props, internal state, slots, actions, and events.\n  fields:\n    desc?: Prose\n    props?: FieldMap            # external inputs (immutable inside component)\n    state?: FieldMap            # internal reactive state\n    slots?: FieldMap            # named ReactNode injection points\n    actions?: ActionMap         # methods the component exposes or handles\n    emits?: EventMap            # events the component raises to its parent\n    layout?: CodeBlock          # ASCII sketch of visual structure (non-normative)\n```\n\n```mdl\next op\n  uri: org.dxos.mdl.op@1.0\n  desc: |\n    A named operation with typed inputs, outputs, and declared errors.\n    Pure ops have no effects or requires. Effectful ops declare both.\n  fields:\n    desc?: Prose\n    input?: FieldMap            # named input parameters\n    output?: TypeExpr           # return type\n    errors?: ErrorMap           # name: Prose  (when this error occurs)\n    effects?: EffectList        # echo:read | echo:write | http | fs | ...\n    requires?: ServiceList      # injected service dependencies\n    note?: Prose                # implementation guidance (non-normative)\n```\n",
+	mimeType: "application/x-mdl"
+} }), Plugin.make);
+//#endregion
+export { TransformerPlugin, TransformerPlugin as default };
+
+//# sourceMappingURL=TransformerPlugin.mjs.map

package/dist/lib/TransformerPlugin.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"TransformerPlugin.mjs","names":[],"sources":["../../src/TransformerPlugin.tsx"],"sourcesContent":["//\n// Copyright 2024 DXOS.org\n//\n\nimport { Plugin } from '@dxos/app-framework';\nimport { AppPlugin } from '@dxos/app-toolkit';\n\n// import { IntentResolver } from './capabilities';\nimport { meta } from '#meta';\nimport { translations } from '#translations';\n\n// eslint-disable-next-line import/no-relative-packages\nimport pluginSpec from '../PLUGIN.mdl?raw';\n\nexport const TransformerPlugin = Plugin.define(meta).pipe(\n  AppPlugin.addSchemaModule({ schema: [] }),\n  AppPlugin.addTranslationsModule({ translations }),\n  // Plugin.addModule({\n  //   id: 'intent-resolver',\n  //   activatesOn: Events.SetupIntentResolver,\n  //   activate: IntentResolver,\n  // }),\n  AppPlugin.addPluginAssetModule({\n    asset: { pluginId: meta.profile.key, path: 'PLUGIN.mdl', content: pluginSpec, mimeType: 'application/x-mdl' },\n  }),\n  Plugin.make,\n);\n\nexport default TransformerPlugin;\n"],"mappings":";;;;;;AAcA,IAAa,oBAAoB,OAAO,OAAO,IAAI,CAAC,CAAC,KACnD,UAAU,gBAAgB,EAAE,QAAQ,CAAC,EAAE,CAAC,GACxC,UAAU,sBAAsB,EAAE,aAAa,CAAC,GAMhD,UAAU,qBAAqB,EAC7B,OAAO;CAAE,UAAU,KAAK,QAAQ;CAAK,MAAM;CAAc,SAAS;CAAY,UAAU;AAAoB,EAC9G,CAAC,GACD,OAAO,IACT"}

package/dist/lib/{neutral/chunk-2UJUBMOA.mjs → chunk-meta.mjs}

@@ -1,15 +1,12 @@
-// src/meta.ts
 import { Plugin } from "@dxos/app-framework";
-
-// dx.config.ts
 import { Config2 } from "@dxos/app-framework/config";
 import { trim } from "@dxos/util";
-var dx_config_default = Config2.make({
-  plugin: {
-    key: "org.dxos.plugin.transformer",
-    name: "Transformer",
-    author: "DXOS",
-    description: trim`
+//#region dx.config.ts
+var dx_config_default = Config2.make({ plugin: {
+	key: "org.dxos.plugin.transformer",
+	name: "Transformer",
+	author: "DXOS",
+	description: trim`
       Browser-based machine learning plugin that runs Hugging Face Transformers.js models
       entirely in-browser via WebAssembly and WebGPU — no server-side inference required.
 
@@ -25,21 +22,15 @@ var dx_config_default = Config2.make({
       experiments, with cosine similarity ranking for selecting the most relevant
       knowledge-base contexts before text generation.
     `,
-    source: "https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transformer",
-    icon: {
-      key: "ph--cpu--regular"
-    },
-    spec: "PLUGIN.mdl",
-    tags: [
-      "labs"
-    ]
-  }
-});
-
-// src/meta.ts
+	source: "https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transformer",
+	icon: { key: "ph--cpu--regular" },
+	spec: "PLUGIN.mdl",
+	tags: ["labs"]
+} });
+//#endregion
+//#region src/meta.ts
 var meta = Plugin.getMetaFromConfig(dx_config_default);
+//#endregion
+export { meta as t };
 
-export {
-  meta
-};
-//# sourceMappingURL=chunk-2UJUBMOA.mjs.map
+//# sourceMappingURL=chunk-meta.mjs.map

package/dist/lib/chunk-meta.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"chunk-meta.mjs","names":[],"sources":["../../dx.config.ts","../../src/meta.ts"],"sourcesContent":["//\n// Copyright 2023 DXOS.org\n//\n\nimport { Config2 } from '@dxos/app-framework/config';\nimport { trim } from '@dxos/util';\n\nexport default Config2.make({\n  plugin: {\n    key: 'org.dxos.plugin.transformer',\n    name: 'Transformer',\n    author: 'DXOS',\n    description: trim`\n      Browser-based machine learning plugin that runs Hugging Face Transformers.js models\n      entirely in-browser via WebAssembly and WebGPU — no server-side inference required.\n\n      Provides automatic speech recognition through a Whisper pipeline hook (usePipeline)\n      and a microphone capture hook (useAudioStream) that buffers 16 kHz mono audio into\n      2-second chunks before forwarding them to the model.\n\n      Exposes a Voice component that wires the two hooks together to deliver live\n      transcription, accumulating the running transcript in local state and rendering\n      a debug panel with model status, GPU info, and audio level visualisation.\n\n      Includes a RAG embedding pipeline base class for retrieval-augmented generation\n      experiments, with cosine similarity ranking for selecting the most relevant\n      knowledge-base contexts before text generation.\n    `,\n    source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transformer',\n    icon: { key: 'ph--cpu--regular' },\n    spec: 'PLUGIN.mdl',\n    tags: ['labs'],\n  },\n});\n","//\n// Copyright 2023 DXOS.org\n//\n\nimport { Plugin } from '@dxos/app-framework';\n\nimport config from '../dx.config';\n\nexport const meta = Plugin.getMetaFromConfig(config);\n"],"mappings":";;;;AAOA,IAAA,oBAAe,QAAQ,KAAK,EAC1B,QAAQ;CACN,KAAK;CACL,MAAM;CACN,QAAQ;CACR,aAAa,IAAI;;;;;;;;;;;;;;;;CAgBjB,QAAQ;CACR,MAAM,EAAE,KAAK,mBAAmB;CAChC,MAAM;CACN,MAAM,CAAC,MAAM;AACf,EACF,CAAC;;;ACzBD,IAAa,OAAO,OAAO,kBAAkB,iBAAM"}

package/dist/lib/components.mjs

@@ -0,0 +1,171 @@
+import { useCallback, useEffect, useState } from "react";
+import { jsx, jsxs } from "react/jsx-runtime";
+import { log } from "@dxos/log";
+import { useAudioStream, usePipeline } from "#hooks";
+//#region src/components/Voice/DebugInfo.tsx
+var DebugInfo = ({ error, isModelLoading, stream, isTranscribing, transcription, audioLevel, gpuInfo, model, debug = false }) => {
+	return /* @__PURE__ */ jsxs("div", {
+		className: "p-4",
+		children: [
+			error && /* @__PURE__ */ jsxs("div", {
+				className: "mb-4 text-red-600",
+				children: [
+					/* @__PURE__ */ jsx("strong", { children: "Error:" }),
+					" ",
+					error
+				]
+			}),
+			isModelLoading && /* @__PURE__ */ jsxs("div", {
+				className: "mb-4",
+				children: [/* @__PURE__ */ jsx("div", { children: "Loading model..." }), /* @__PURE__ */ jsx("div", {
+					className: "text-sm text-gray-500",
+					children: "This may take a few moments"
+				})]
+			}),
+			stream ? /* @__PURE__ */ jsxs("div", { children: [
+				/* @__PURE__ */ jsxs("div", {
+					className: "mb-2 text-green-600",
+					children: [
+						/* @__PURE__ */ jsx("strong", { children: "Status:" }),
+						" Microphone is active",
+						debug && audioLevel && /* @__PURE__ */ jsx("div", {
+							className: "mt-2 w-48 h-5 bg-gray-200 rounded-sm relative",
+							children: /* @__PURE__ */ jsx("div", {
+								className: "h-full bg-green-500 transition-all duration-100 rounded-sm",
+								style: { width: `${audioLevel / 255 * 100}%` }
+							})
+						})
+					]
+				}),
+				isTranscribing && /* @__PURE__ */ jsx("div", {
+					className: "mb-2 text-gray-500",
+					children: "Processing audio..."
+				}),
+				debug && /* @__PURE__ */ jsxs("div", {
+					className: "mb-4 text-sm text-gray-500 space-y-1",
+					children: [
+						/* @__PURE__ */ jsxs("div", { children: ["Model: ", model] }),
+						/* @__PURE__ */ jsx("div", { children: "Sample Rate: 16000 Hz" }),
+						/* @__PURE__ */ jsx("div", { children: "Format: audio/wav" }),
+						/* @__PURE__ */ jsx("div", { children: "Chunk Size: 10 seconds" }),
+						/* @__PURE__ */ jsxs("div", { children: ["GPU: ", gpuInfo || "Not available"] }),
+						/* @__PURE__ */ jsx("div", { children: "Backend: WebGPU" })
+					]
+				}),
+				transcription && /* @__PURE__ */ jsxs("div", {
+					className: "mt-4",
+					children: [/* @__PURE__ */ jsx("strong", { children: "Transcription:" }), /* @__PURE__ */ jsx("p", {
+						className: "mt-2 p-4 bg-gray-100 rounded-sm whitespace-pre-wrap",
+						children: transcription
+					})]
+				})
+			] }) : /* @__PURE__ */ jsx("div", { children: !isModelLoading && !error && /* @__PURE__ */ jsx("div", {
+				className: "text-gray-500",
+				children: "Microphone is inactive"
+			}) })
+		]
+	});
+};
+//#endregion
+//#region src/components/Voice/Voice.tsx
+var __dxlog_file = "/__w/dxos/dxos/packages/plugins/plugin-transformer/src/components/Voice/Voice.tsx";
+var Voice = ({ active, debug, model = "Xenova/whisper-base" }) => {
+	const [isTranscribing, setIsTranscribing] = useState(false);
+	const [transcription, setTranscription] = useState("");
+	const { transcribe, gpuInfo, isLoaded: isModelLoaded, isLoading: isModelLoading, error: pipelineError } = usePipeline({
+		active,
+		debug,
+		model
+	});
+	const { stream, error: audioError, audioLevel } = useAudioStream({
+		active,
+		debug
+	});
+	const handleAudioData = useCallback(async (audioData) => {
+		if (!isModelLoaded) return;
+		if (isTranscribing) return;
+		setIsTranscribing(true);
+		try {
+			const result = await transcribe(audioData, {
+				sampling_rate: 16e3,
+				chunk_length_s: 5,
+				stride_length_s: 1,
+				return_timestamps: false,
+				language: "english"
+			});
+			if (result?.text?.trim()) setTranscription((prev) => prev + " " + result.text);
+		} catch (err) {
+			log.error("transcription error", { err }, {
+				"~LogMeta": "~LogMeta",
+				F: __dxlog_file,
+				L: 65,
+				S: void 0
+			});
+			throw err;
+		} finally {
+			setIsTranscribing(false);
+		}
+	}, [transcribe, isTranscribing]);
+	log.info("handleAudioData", { handleAudioData }, {
+		"~LogMeta": "~LogMeta",
+		F: __dxlog_file,
+		L: 73,
+		S: void 0
+	});
+	useEffect(() => {
+		if (debug) log.info("audio state", {
+			hasStream: !!stream,
+			audioError,
+			audioLevel,
+			shouldBeActive: active && isModelLoaded
+		}, {
+			"~LogMeta": "~LogMeta",
+			F: __dxlog_file,
+			L: 77,
+			S: void 0
+		});
+	}, [
+		debug,
+		stream,
+		audioError,
+		audioLevel,
+		active,
+		isModelLoaded
+	]);
+	useEffect(() => {
+		if (debug) log.info("transcription state", {
+			active,
+			isModelLoaded,
+			isModelLoading,
+			isTranscribing,
+			pipelineError
+		}, {
+			"~LogMeta": "~LogMeta",
+			F: __dxlog_file,
+			L: 88,
+			S: void 0
+		});
+	}, [
+		active,
+		debug,
+		isModelLoaded,
+		isModelLoading,
+		pipelineError,
+		isTranscribing
+	]);
+	return /* @__PURE__ */ jsx(DebugInfo, {
+		error: audioError || pipelineError || void 0,
+		isModelLoading,
+		stream,
+		isTranscribing,
+		transcription,
+		audioLevel,
+		gpuInfo,
+		model,
+		debug
+	});
+};
+//#endregion
+export { DebugInfo, Voice };
+
+//# sourceMappingURL=components.mjs.map

package/dist/lib/components.mjs.map

@@ -0,0 +1 @@
+{"version":3,"file":"components.mjs","names":[],"sources":["../../src/components/Voice/DebugInfo.tsx","../../src/components/Voice/Voice.tsx"],"sourcesContent":["//\n// Copyright 2025 DXOS.org\n//\n\nimport React, { type FC } from 'react';\n\nexport type DebugInfoProps = {\n  error: string;\n  isModelLoading: boolean;\n  stream: MediaStream | null;\n  isTranscribing: boolean;\n  transcription: string;\n  audioLevel: number;\n  gpuInfo: string;\n  model: string;\n  debug: boolean;\n};\n\nexport const DebugInfo: FC<Partial<DebugInfoProps>> = ({\n  error,\n  isModelLoading,\n  stream,\n  isTranscribing,\n  transcription,\n  audioLevel,\n  gpuInfo,\n  model,\n  debug = false,\n}) => {\n  return (\n    <div className='p-4'>\n      {error && (\n        <div className='mb-4 text-red-600'>\n          <strong>Error:</strong> {error}\n        </div>\n      )}\n      {isModelLoading && (\n        <div className='mb-4'>\n          <div>Loading model...</div>\n          <div className='text-sm text-gray-500'>This may take a few moments</div>\n        </div>\n      )}\n      {stream ? (\n        <div>\n          <div className='mb-2 text-green-600'>\n            <strong>Status:</strong> Microphone is active\n            {debug && audioLevel && (\n              <div className='mt-2 w-48 h-5 bg-gray-200 rounded-sm relative'>\n                <div\n                  className='h-full bg-green-500 transition-all duration-100 rounded-sm'\n                  style={{ width: `${(audioLevel / 255) * 100}%` }}\n                />\n              </div>\n            )}\n          </div>\n          {isTranscribing && <div className='mb-2 text-gray-500'>Processing audio...</div>}\n          {debug && (\n            <div className='mb-4 text-sm text-gray-500 space-y-1'>\n              <div>Model: {model}</div>\n              <div>Sample Rate: 16000 Hz</div>\n              <div>Format: audio/wav</div>\n              <div>Chunk Size: 10 seconds</div>\n              <div>GPU: {gpuInfo || 'Not available'}</div>\n              <div>Backend: WebGPU</div>\n            </div>\n          )}\n          {transcription && (\n            <div className='mt-4'>\n              <strong>Transcription:</strong>\n              <p className='mt-2 p-4 bg-gray-100 rounded-sm whitespace-pre-wrap'>{transcription}</p>\n            </div>\n          )}\n        </div>\n      ) : (\n        <div>{!isModelLoading && !error && <div className='text-gray-500'>Microphone is inactive</div>}</div>\n      )}\n    </div>\n  );\n};\n","//\n// Copyright 2025 DXOS.org\n//\n\nimport React, { useCallback, useEffect, useState } from 'react';\n\nimport { log } from '@dxos/log';\n\nimport { useAudioStream, usePipeline } from '#hooks';\n\nimport { DebugInfo } from './DebugInfo';\n\nexport type VoiceProps = {\n  active?: boolean;\n  debug?: boolean;\n  model?: string;\n};\n\nexport const Voice = ({ active, debug, model = 'Xenova/whisper-base' }: VoiceProps) => {\n  const [isTranscribing, setIsTranscribing] = useState(false);\n  const [transcription, setTranscription] = useState<string>('');\n\n  const {\n    transcribe,\n    gpuInfo,\n    isLoaded: isModelLoaded,\n    isLoading: isModelLoading,\n    error: pipelineError,\n  } = usePipeline({ active, debug, model });\n\n  const {\n    stream,\n    error: audioError,\n    audioLevel,\n  } = useAudioStream({\n    active,\n    debug,\n    // onAudioData: handleAudioData\n  });\n\n  const handleAudioData = useCallback(\n    async (audioData: Float32Array) => {\n      if (!isModelLoaded) {\n        return;\n      }\n\n      if (isTranscribing) {\n        return;\n      }\n\n      setIsTranscribing(true);\n      try {\n        const result = await transcribe(audioData, {\n          sampling_rate: 16000,\n          chunk_length_s: 5,\n          stride_length_s: 1,\n          return_timestamps: false,\n          language: 'english',\n        });\n\n        if (result?.text?.trim()) {\n          setTranscription((prev) => prev + ' ' + result.text);\n        }\n      } catch (err) {\n        log.error('transcription error', { err });\n        throw err;\n      } finally {\n        setIsTranscribing(false);\n      }\n    },\n    [transcribe, isTranscribing],\n  );\n  log.info('handleAudioData', { handleAudioData });\n\n  useEffect(() => {\n    if (debug) {\n      log.info('audio state', {\n        hasStream: !!stream,\n        audioError,\n        audioLevel,\n        shouldBeActive: active && isModelLoaded,\n      });\n    }\n  }, [debug, stream, audioError, audioLevel, active, isModelLoaded]);\n\n  useEffect(() => {\n    if (debug) {\n      log.info('transcription state', {\n        active,\n        isModelLoaded,\n        isModelLoading,\n        isTranscribing,\n        pipelineError,\n      });\n    }\n  }, [active, debug, isModelLoaded, isModelLoading, pipelineError, isTranscribing]);\n\n  return (\n    <DebugInfo\n      error={audioError || pipelineError || undefined}\n      isModelLoading={isModelLoading}\n      stream={stream}\n      isTranscribing={isTranscribing}\n      transcription={transcription}\n      audioLevel={audioLevel}\n      gpuInfo={gpuInfo}\n      model={model}\n      debug={debug}\n    />\n  );\n};\n"],"mappings":";;;;;AAkBA,IAAa,aAA0C,EACrD,OACA,gBACA,QACA,gBACA,eACA,YACA,SACA,OACA,QAAQ,YACJ;CACJ,OACE,qBAAC,OAAD;EAAK,WAAU;YAAf;GACG,SACC,qBAAC,OAAD;IAAK,WAAU;cAAf;KACE,oBAAC,UAAD,EAAA,UAAQ,SAAc,CAAA;KAAC;KAAE;IACtB;;GAEN,kBACC,qBAAC,OAAD;IAAK,WAAU;cAAf,CACE,oBAAC,OAAD,EAAA,UAAK,mBAAqB,CAAA,GAC1B,oBAAC,OAAD;KAAK,WAAU;eAAwB;IAAgC,CAAA,CACpE;;GAEN,SACC,qBAAC,OAAD,EAAA,UAAA;IACE,qBAAC,OAAD;KAAK,WAAU;eAAf;MACE,oBAAC,UAAD,EAAA,UAAQ,UAAe,CAAA;MAAC;MACvB,SAAS,cACR,oBAAC,OAAD;OAAK,WAAU;iBACb,oBAAC,OAAD;QACE,WAAU;QACV,OAAO,EAAE,OAAO,GAAI,aAAa,MAAO,IAAI,GAAG;OAChD,CAAA;MACE,CAAA;KAEJ;;IACJ,kBAAkB,oBAAC,OAAD;KAAK,WAAU;eAAqB;IAAwB,CAAA;IAC9E,SACC,qBAAC,OAAD;KAAK,WAAU;eAAf;MACE,qBAAC,OAAD,EAAA,UAAA,CAAK,WAAQ,KAAW,EAAA,CAAA;MACxB,oBAAC,OAAD,EAAA,UAAK,wBAA0B,CAAA;MAC/B,oBAAC,OAAD,EAAA,UAAK,oBAAsB,CAAA;MAC3B,oBAAC,OAAD,EAAA,UAAK,yBAA2B,CAAA;MAChC,qBAAC,OAAD,EAAA,UAAA,CAAK,SAAM,WAAW,eAAqB,EAAA,CAAA;MAC3C,oBAAC,OAAD,EAAA,UAAK,kBAAoB,CAAA;KACtB;;IAEN,iBACC,qBAAC,OAAD;KAAK,WAAU;eAAf,CACE,oBAAC,UAAD,EAAA,UAAQ,iBAAsB,CAAA,GAC9B,oBAAC,KAAD;MAAG,WAAU;gBAAuD;KAAiB,CAAA,CAClF;;GAEJ,EAAA,CAAA,IAEL,oBAAC,OAAD,EAAA,UAAM,CAAC,kBAAkB,CAAC,SAAS,oBAAC,OAAD;IAAK,WAAU;cAAgB;GAA2B,CAAA,EAAO,CAAA;EAEnG;;AAET;;;;AC5DA,IAAa,SAAS,EAAE,QAAQ,OAAO,QAAQ,4BAAwC;CACrF,MAAM,CAAC,gBAAgB,qBAAqB,SAAS,KAAK;CAC1D,MAAM,CAAC,eAAe,oBAAoB,SAAiB,EAAE;CAE7D,MAAM,EACJ,YACA,SACA,UAAU,eACV,WAAW,gBACX,OAAO,kBACL,YAAY;EAAE;EAAQ;EAAO;CAAM,CAAC;CAExC,MAAM,EACJ,QACA,OAAO,YACP,eACE,eAAe;EACjB;EACA;CAEF,CAAC;CAED,MAAM,kBAAkB,YACtB,OAAO,cAA4B;EACjC,IAAI,CAAC,eACH;EAGF,IAAI,gBACF;EAGF,kBAAkB,IAAI;EACtB,IAAI;GACF,MAAM,SAAS,MAAM,WAAW,WAAW;IACzC,eAAe;IACf,gBAAgB;IAChB,iBAAiB;IACjB,mBAAmB;IACnB,UAAU;GACZ,CAAC;GAED,IAAI,QAAQ,MAAM,KAAK,GACrB,kBAAkB,SAAS,OAAO,MAAM,OAAO,IAAI;EAEvD,SAAS,KAAK;GACZ,IAAI,MAAM,uBAAuB,EAAE,IAAI,GAAA;IAAA,YAAA;IAAA,GAAA;IAAA,GAAA;IAAA,GAAA,KAAA;GAAA,CAAC;GACxC,MAAM;EACR,UAAU;GACR,kBAAkB,KAAK;EACzB;CACF,GACA,CAAC,YAAY,cAAc,CAC7B;CACA,IAAI,KAAK,mBAAmB,EAAE,gBAAgB,GAAA;EAAA,YAAA;EAAA,GAAA;EAAA,GAAA;EAAA,GAAA,KAAA;CAAA,CAAC;CAE/C,gBAAgB;EACd,IAAI,OACF,IAAI,KAAK,eAAe;GACtB,WAAW,CAAC,CAAC;GACb;GACA;GACA,gBAAgB,UAAU;EAC5B,GAAA;GAAA,YAAA;GAAA,GAAA;GAAA,GAAA;GAAA,GAAA,KAAA;EAAA,CAAC;CAEL,GAAG;EAAC;EAAO;EAAQ;EAAY;EAAY;EAAQ;CAAa,CAAC;CAEjE,gBAAgB;EACd,IAAI,OACF,IAAI,KAAK,uBAAuB;GAC9B;GACA;GACA;GACA;GACA;EACF,GAAA;GAAA,YAAA;GAAA,GAAA;GAAA,GAAA;GAAA,GAAA,KAAA;EAAA,CAAC;CAEL,GAAG;EAAC;EAAQ;EAAO;EAAe;EAAgB;EAAe;CAAc,CAAC;CAEhF,OACE,oBAAC,WAAD;EACE,OAAO,cAAc,iBAAiB,KAAA;EACtB;EACR;EACQ;EACD;EACH;EACH;EACF;EACA;CACR,CAAA;AAEL"}

package/dist/lib/hooks.mjs

@@ -0,0 +1,349 @@
+import { useCallback, useEffect, useRef, useState } from "react";
+import { log } from "@dxos/log";
+import { env, pipeline } from "@xenova/transformers";
+import { invariant } from "@dxos/invariant";
+//#region src/hooks/useAudioStream.ts
+var __dxlog_file$1 = "/__w/dxos/dxos/packages/plugins/plugin-transformer/src/hooks/useAudioStream.ts";
+var useAudioStream = ({ active, debug, onAudioData }) => {
+	const [state, setState] = useState({
+		stream: null,
+		error: null,
+		audioLevel: 0
+	});
+	const audioContextRef = useRef(null);
+	const analyserRef = useRef(null);
+	const animationFrameRef = useRef(void 0);
+	const workletNodeRef = useRef(null);
+	const isProcessingRef = useRef(false);
+	const mediaStreamRef = useRef(null);
+	const audioBufferRef = useRef([]);
+	const updateAudioLevel = useCallback(() => {
+		if (analyserRef.current) {
+			const dataArray = new Uint8Array(analyserRef.current.frequencyBinCount);
+			analyserRef.current.getByteFrequencyData(dataArray);
+			const average = dataArray.reduce((acc, val) => acc + val, 0) / dataArray.length;
+			setState((prev) => ({
+				...prev,
+				audioLevel: average
+			}));
+			animationFrameRef.current = requestAnimationFrame(updateAudioLevel);
+		}
+	}, []);
+	const cleanup = useCallback(() => {
+		log("cleaning up audio resources", void 0, {
+			"~LogMeta": "~LogMeta",
+			F: __dxlog_file$1,
+			L: 49,
+			S: void 0
+		});
+		if (mediaStreamRef.current) {
+			mediaStreamRef.current.getTracks().forEach((track) => {
+				track.stop();
+				track.enabled = false;
+			});
+			mediaStreamRef.current = null;
+		}
+		if (workletNodeRef.current) {
+			workletNodeRef.current.disconnect();
+			workletNodeRef.current = null;
+		}
+		if (analyserRef.current) {
+			analyserRef.current.disconnect();
+			analyserRef.current = null;
+		}
+		if (audioContextRef.current) {
+			audioContextRef.current.close();
+			audioContextRef.current = null;
+		}
+		if (animationFrameRef.current) {
+			cancelAnimationFrame(animationFrameRef.current);
+			animationFrameRef.current = void 0;
+		}
+		audioBufferRef.current = [];
+		setState({
+			stream: null,
+			error: null,
+			audioLevel: 0
+		});
+	}, [debug]);
+	useEffect(() => {
+		let mounted = true;
+		const startStream = async () => {
+			try {
+				if (active) {
+					cleanup();
+					log.info("initializing audio stream...", void 0, {
+						"~LogMeta": "~LogMeta",
+						F: __dxlog_file$1,
+						L: 93,
+						S: void 0
+					});
+					const stream = await navigator.mediaDevices.getUserMedia({
+						audio: {
+							channelCount: 1,
+							sampleRate: 16e3,
+							echoCancellation: true,
+							noiseSuppression: true,
+							autoGainControl: true
+						},
+						video: false
+					});
+					if (!mounted || !active) {
+						stream.getTracks().forEach((track) => {
+							track.stop();
+							track.enabled = false;
+						});
+						return;
+					}
+					mediaStreamRef.current = stream;
+					const context = new AudioContext({ sampleRate: 16e3 });
+					await context.audioWorklet.addModule(URL.createObjectURL(new Blob([`class AudioProcessor extends AudioWorkletProcessor {
+                    constructor() {
+                      super();
+                      this._buffer = [];
+                      this._samplesProcessed = 0;
+                    }
+
+                    process(inputs, outputs) {
+                      const input = inputs[0];
+                      const channel = input[0];
+                      
+                      if (channel) {
+                        this._buffer.push(new Float32Array(channel));
+                        this._samplesProcessed += channel.length;
+
+                        // Process every 2 seconds (32000 samples at 16kHz).
+                        if (this._samplesProcessed >= 32000) {
+                          const combinedLength = this._buffer.reduce((acc, curr) => acc + curr.length, 0);
+                          const combinedAudio = new Float32Array(combinedLength);
+                          let offset = 0;
+
+                          for (const buffer of this._buffer) {
+                            combinedAudio.set(buffer, offset);
+                            offset += buffer.length;
+                          }
+
+                          this.port.postMessage({ type: 'audio-data', data: combinedAudio });
+                          
+                          // Reset buffer and counter.
+                          this._buffer = [];
+                          this._samplesProcessed = 0;
+                        }
+                      }
+                      return true;
+                    }
+                  }
+
+                  registerProcessor('audio-processor', AudioProcessor);`], { type: "application/javascript" })));
+					const source = context.createMediaStreamSource(stream);
+					const analyser = context.createAnalyser();
+					analyserRef.current = analyser;
+					const workletNode = new AudioWorkletNode(context, "audio-processor");
+					workletNodeRef.current = workletNode;
+					workletNode.port.onmessage = async (event) => {
+						if (!mounted || !active) return;
+						if (event.data.type === "audio-data") {
+							isProcessingRef.current = true;
+							try {
+								log("processing audio", {
+									sampleRate: context.sampleRate,
+									length: event.data.data.length,
+									min: Math.min(...event.data.data),
+									max: Math.max(...event.data.data)
+								}, {
+									"~LogMeta": "~LogMeta",
+									F: __dxlog_file$1,
+									L: 183,
+									S: void 0
+								});
+								await onAudioData?.(event.data.data);
+							} catch (err) {
+								if (mounted) setState((prev) => ({
+									...prev,
+									error: "Error processing audio: " + err.message
+								}));
+								log.error("audio processing error", { err }, {
+									"~LogMeta": "~LogMeta",
+									F: __dxlog_file$1,
+									L: 198,
+									S: void 0
+								});
+							} finally {
+								isProcessingRef.current = false;
+							}
+						}
+					};
+					source.connect(analyser);
+					analyser.connect(workletNode);
+					workletNode.connect(context.destination);
+					if (debug) {
+						analyser.fftSize = 256;
+						updateAudioLevel();
+					}
+					audioContextRef.current = context;
+					if (mounted && active) setState({
+						stream,
+						error: null,
+						audioLevel: 0
+					});
+				}
+			} catch (err) {
+				if (mounted) setState((prev) => ({
+					...prev,
+					error: "Error accessing microphone: " + err.message,
+					stream: null
+				}));
+				log.error("microphone error", { err }, {
+					"~LogMeta": "~LogMeta",
+					F: __dxlog_file$1,
+					L: 232,
+					S: void 0
+				});
+				cleanup();
+			}
+		};
+		startStream();
+		return () => {
+			mounted = false;
+			cleanup();
+		};
+	}, [
+		active,
+		debug,
+		onAudioData,
+		updateAudioLevel,
+		cleanup
+	]);
+	useEffect(() => {
+		if (!active) cleanup();
+	}, [active, cleanup]);
+	return state;
+};
+//#endregion
+//#region src/hooks/usePipeline.ts
+var __dxlog_file = "/__w/dxos/dxos/packages/plugins/plugin-transformer/src/hooks/usePipeline.ts";
+env.cacheDir = "./.cache";
+env.allowLocalModels = true;
+env.backends.onnx.wasm.numThreads = 1;
+env.backends.onnx.provider = "webgpu";
+env.backends.onnx.webgpu = { profilingMode: true };
+var usePipeline = ({ active, model, debug }) => {
+	const [state, setState] = useState({
+		gpuInfo: "",
+		isLoaded: false,
+		isLoading: false,
+		error: null
+	});
+	const pipelineRef = useRef(null);
+	useEffect(() => {
+		const loadModel = async () => {
+			try {
+				setState((prev) => ({
+					...prev,
+					isLoading: true,
+					error: null
+				}));
+				if (!navigator.gpu) {
+					log.warn("WebGPU is not supported, falling back to CPU", void 0, {
+						"~LogMeta": "~LogMeta",
+						F: __dxlog_file,
+						L: 62,
+						S: void 0
+					});
+					setState((prev) => ({
+						...prev,
+						gpuInfo: "WebGPU not supported (using CPU)"
+					}));
+				} else try {
+					const adapter = await navigator.gpu.requestAdapter();
+					if (!adapter) throw new Error("No GPU adapter found");
+					const adapterInfo = adapter.info;
+					if (adapterInfo) setState((prev) => ({
+						...prev,
+						gpuInfo: `${adapterInfo.description || "GPU"} (${adapterInfo.vendor || "Unknown"})`
+					}));
+					else setState((prev) => ({
+						...prev,
+						gpuInfo: "GPU Available (details unknown)"
+					}));
+				} catch (err) {
+					log.warn("WebGPU initialization failed", { err }, {
+						"~LogMeta": "~LogMeta",
+						F: __dxlog_file,
+						L: 82,
+						S: void 0
+					});
+					setState((prev) => ({
+						...prev,
+						gpuInfo: "GPU initialization failed (using CPU)"
+					}));
+				}
+				const pipe = await pipeline("automatic-speech-recognition", model, {
+					quantized: true,
+					progress_callback: (progress) => {
+						if (debug) log(`loading model: ${Math.round(progress.progress * 100)}%`, void 0, {
+							"~LogMeta": "~LogMeta",
+							F: __dxlog_file,
+							L: 91,
+							S: void 0
+						});
+					}
+				});
+				pipelineRef.current = pipe;
+				setState((prev) => ({
+					...prev,
+					isLoaded: true,
+					isLoading: false
+				}));
+				log.info("model loaded successfully", void 0, {
+					"~LogMeta": "~LogMeta",
+					F: __dxlog_file,
+					L: 98,
+					S: void 0
+				});
+			} catch (err) {
+				log.error("error loading model", { err }, {
+					"~LogMeta": "~LogMeta",
+					F: __dxlog_file,
+					L: 100,
+					S: void 0
+				});
+				setState((prev) => ({
+					...prev,
+					isLoading: false,
+					error: "error loading model: " + err.message
+				}));
+			}
+		};
+		if (active) loadModel();
+		return () => {
+			pipelineRef.current = null;
+			setState((prev) => ({
+				...prev,
+				isLoaded: false
+			}));
+		};
+	}, [
+		active,
+		debug,
+		model
+	]);
+	const transcribe = async (audioData, options) => {
+		invariant(pipelineRef.current, "pipeline not initialized", {
+			"~LogMeta": "~LogMeta",
+			F: __dxlog_file,
+			L: 120,
+			S: void 0,
+			A: ["pipelineRef.current", "'pipeline not initialized'"]
+		});
+		return await pipelineRef.current(audioData, options);
+	};
+	return {
+		...state,
+		transcribe
+	};
+};
+//#endregion
+export { useAudioStream, usePipeline };
+
+//# sourceMappingURL=hooks.mjs.map