@dxos/plugin-transformer 0.8.4-main.d05673bc65 → 0.8.4-main.d9fc60f731

package/LICENSE

@@ -1,8 +1,105 @@
-MIT License 
-Copyright (c) 2022 DXOS
+# Functional Source License, Version 1.1, ALv2 Future License
 
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+## Abbreviation
 
-The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+FSL-1.1-Apache-2.0
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+## Notice
+
+Copyright 2026 DXOS
+
+## Terms and Conditions
+
+### Licensor ("We")
+
+The party offering the Software under these Terms and Conditions.
+
+### The Software
+
+The "Software" is each version of the software that we make available under
+these Terms and Conditions, as indicated by our inclusion of these Terms and
+Conditions with the Software.
+
+### License Grant
+
+Subject to your compliance with this License Grant and the Patents,
+Redistribution and Trademark clauses below, we hereby grant you the right to
+use, copy, modify, create derivative works, publicly perform, publicly display
+and redistribute the Software for any Permitted Purpose identified below.
+
+### Permitted Purpose
+
+A Permitted Purpose is any purpose other than a Competing Use. A Competing Use
+means making the Software available to others in a commercial product or
+service that:
+
+1. substitutes for the Software;
+
+2. substitutes for any other product or service we offer using the Software
+   that exists as of the date we make the Software available; or
+
+3. offers the same or substantially similar functionality as the Software.
+
+Permitted Purposes specifically include using the Software:
+
+1. for your internal use and access;
+
+2. for non-commercial education;
+
+3. for non-commercial research; and
+
+4. in connection with professional services that you provide to a licensee
+   using the Software in accordance with these Terms and Conditions.
+
+### Patents
+
+To the extent your use for a Permitted Purpose would necessarily infringe our
+patents, the license grant above includes a license under our patents. If you
+make a claim against any party that the Software infringes or contributes to
+the infringement of any patent, then your patent license to the Software ends
+immediately.
+
+### Redistribution
+
+The Terms and Conditions apply to all copies, modifications and derivatives of
+the Software.
+
+If you redistribute any copies, modifications or derivatives of the Software,
+you must include a copy of or a link to these Terms and Conditions and not
+remove any copyright notices provided in or with the Software.
+
+### Disclaimer
+
+THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR
+PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT.
+
+IN NO EVENT WILL WE HAVE ANY LIABILITY TO YOU ARISING OUT OF OR RELATED TO THE
+SOFTWARE, INCLUDING INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES,
+EVEN IF WE HAVE BEEN INFORMED OF THEIR POSSIBILITY IN ADVANCE.
+
+### Trademarks
+
+Except for displaying the License Details and identifying us as the origin of
+the Software, you have no right under these Terms and Conditions to use our
+trademarks, trade names, service marks or product names.
+
+## Grant of Future License
+
+We hereby irrevocably grant you an additional license to use the Software under
+the Apache License, Version 2.0 that is effective on the second anniversary of
+the date we make the Software available. On or after that date, you may use the
+Software under the Apache License, Version 2.0, in which case the following
+will apply:
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use
+this file except in compliance with the License.
+
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.

package/PLUGIN.mdl

@@ -0,0 +1,326 @@
+---
+id: org.dxos.plugin.transformer
+name: TransformerPlugin
+version: 0.1.0
+---
+
+A browser-based machine learning plugin for DXOS Composer that runs Hugging Face Transformers.js
+models entirely in-browser via WebAssembly and WebGPU. It provides automatic speech recognition
+(Whisper) through a React hook and component layer, enabling real-time voice transcription without
+any server-side inference infrastructure.
+
+## Extensions
+
+The following extension dialects are used in this document.
+Each extension is defined in the Appendix or resolved via its URI.
+
+| Term        | URI                            |
+|-------------|--------------------------------|
+| `type`      | `org.dxos.mdl.type@1.0`       |
+| `feat`      | `org.dxos.mdl.feat@1.0`       |
+| `test`      | `org.dxos.mdl.test@1.0`       |
+| `component` | `org.dxos.mdl.component@1.0`  |
+| `op`        | `org.dxos.mdl.op@1.0`         |
+
+## Types
+
+```mdl
+type PipelineConfig
+  fields:
+    model: string              # Hugging Face model id, e.g. 'Xenova/whisper-base'
+    active?: boolean           # whether to load and run the pipeline
+    debug?: boolean            # enable verbose logging
+```
+
+```mdl
+type PipelineState
+  fields:
+    gpuInfo: string            # WebGPU adapter description, or fallback message
+    isLoaded: boolean          # true once the model is fully initialised
+    isLoading: boolean         # true while the model is being downloaded/compiled
+    error: string | null       # error message if loading failed
+```
+
+```mdl
+type AudioStreamConfig
+  fields:
+    active?: boolean           # start/stop microphone capture
+    debug?: boolean
+    onAudioData?(audioData: Float32Array): Promise<void>
+```
+
+```mdl
+type AudioStreamState
+  fields:
+    stream: MediaStream | null # live microphone stream, or null when inactive
+    error: string | null       # error from getUserMedia or audio processing
+    audioLevel: number         # 0-255 RMS level for visualisation
+```
+
+```mdl
+type TranscriptionOptions
+  fields:
+    sampling_rate: number      # audio sample rate in Hz (16000)
+    chunk_length_s: number     # seconds of audio per chunk
+    stride_length_s: number    # overlap between adjacent chunks
+    return_timestamps: boolean # include word-level timestamps in output
+    language: string           # target language hint, e.g. 'english'
+```
+
+```mdl
+type EmbeddingOutput
+  fields:
+    data: number[]             # embedding vector as a flat float array
+```
+
+## Components
+
+```mdl
+component Voice
+  desc: |
+    Composite component that wires usePipeline and useAudioStream together to provide
+    live Whisper transcription from the microphone.  Accumulates the running transcript
+    in local state and renders a DebugInfo panel showing model status, audio level, and
+    the current transcription text.
+  props:
+    active?: boolean           # start/stop the entire pipeline
+    debug?: boolean
+    model?: string             # defaults to 'Xenova/whisper-base'
+  state:
+    isTranscribing: boolean    # true while a transcription call is in-flight
+    transcription: string      # accumulated transcript text
+  layout: |
+    ┌──────────────────────────────────────┐
+    │  [DebugInfo panel]                   │
+    │   model: Xenova/whisper-base         │
+    │   gpu: GPU Available                 │
+    │   stream: active / inactive          │
+    │   level: ███░░░░ (audioLevel bar)    │
+    │   ──────────────────────────────     │
+    │   [transcription text]               │
+    └──────────────────────────────────────┘
+```
+
+## Operations
+
+```mdl
+op LoadModel
+  desc: |
+    Loads a Transformers.js automatic-speech-recognition pipeline for the given model id.
+    Configures ONNX backend to use WebGPU if available, falling back to WASM/CPU.
+    Invoked automatically when the Voice component mounts with active = true.
+  input: void
+  output: void
+  effects: [http, fs]
+  note: Model weights are cached in ./.cache by the Transformers.js env settings.
+```
+
+```mdl
+op Transcribe
+  desc: |
+    Runs the loaded ASR pipeline against a Float32Array of 16 kHz mono audio.
+    Returns the transcribed text string from the pipeline result.
+  input: void
+  output: string
+  effects: [gpu]
+  note: Guarded by isTranscribing flag to prevent concurrent calls.
+```
+
+```mdl
+op CaptureAudio
+  desc: |
+    Requests microphone permission via getUserMedia and wires an AudioWorklet that
+    accumulates 2-second chunks (32000 samples at 16 kHz) before forwarding them to
+    the onAudioData callback.
+  input: void
+  output: void
+  effects: [audio]
+  note: Cleaned up automatically when the component unmounts or active becomes false.
+```
+
+## Features
+
+```mdl
+feat F-1: In-Browser ASR Pipeline
+
+  req F-1.1:
+    when: Voice component mounts with active = true
+    then: usePipeline loads the Whisper model via Transformers.js; isLoading is true until complete
+
+  req F-1.2:
+    when: WebGPU is available in the browser
+    then: the ONNX backend is configured for WebGPU execution and gpuInfo reflects the adapter
+
+  req F-1.3:
+    when: WebGPU is not available
+    then: the pipeline falls back to WASM/CPU and gpuInfo is set to a 'not supported' message
+
+  req F-1.4:
+    when: model loading fails
+    then: PipelineState.error contains a human-readable message; isLoading is false
+```
+
+```mdl
+feat F-2: Microphone Capture
+
+  req F-2.1:
+    when: useAudioStream is activated
+    then: getUserMedia is called with mono 16 kHz audio and echo/noise cancellation enabled
+
+  req F-2.2:
+    when: audio is captured
+    then: an AudioWorklet accumulates samples and emits 2-second chunks via the onAudioData callback
+
+  req F-2.3:
+    when: active is set to false or the component unmounts
+    then: all MediaStream tracks are stopped, AudioContext is closed, and AnimationFrame is cancelled
+
+  req F-2.4:
+    when: getUserMedia is denied
+    then: AudioStreamState.error is set with the underlying error message
+```
+
+```mdl
+feat F-3: Live Transcription
+
+  req F-3.1:
+    when: a 2-second audio chunk arrives and the model is loaded
+    then: usePipeline.transcribe is called with 16 kHz mono options and the result appended to the transcript
+
+  req F-3.2:
+    when: transcription is already in progress (isTranscribing = true)
+    then: the incoming audio chunk is dropped to avoid concurrent inference
+
+  req F-3.3:
+    when: transcription returns an empty or whitespace-only string
+    then: the transcript state is not updated
+```
+
+```mdl
+feat F-4: RAG Embedding Pipeline (Testing)
+
+  req F-4.1:
+    when: RagPipeline.generateCompletions is called with an input and a knowledge base
+    then: embeddings are generated for both the input and each knowledge-base entry
+
+  req F-4.2:
+    when: embeddings are available
+    then: cosine similarity is computed and the top-3 most similar contexts are selected
+```
+
+## Acceptance
+
+```mdl
+test T-1: Model loads and transitions isLoaded
+  given: Voice component mounts with active = true and model = 'Xenova/whisper-base'
+  when: model download and ONNX compilation complete
+  then:
+    - isLoading transitions from true to false
+    - isLoaded is true
+    - error is null
+```
+
+```mdl
+test T-2: Model load failure sets error state
+  given: the model download throws a network error
+  when: usePipeline attempts to load the model
+  then:
+    - isLoading is false
+    - error contains the error message string
+    - isLoaded is false
+```
+
+```mdl
+test T-3: Microphone capture populates stream state
+  given: getUserMedia is granted
+  when: useAudioStream activates
+  then:
+    - AudioStreamState.stream is a non-null MediaStream
+    - AudioStreamState.error is null
+```
+
+```mdl
+test T-4: Audio chunk triggers transcription
+  given: model is loaded and stream is active
+  when: AudioWorklet posts a 32000-sample chunk
+  then:
+    - isTranscribing becomes true during the pipeline call
+    - transcription string is updated with the returned text
+    - isTranscribing returns to false after completion
+```
+
+```mdl
+test T-5: Concurrent transcription calls are dropped
+  given: isTranscribing is true
+  when: a second audio chunk arrives
+  then: the chunk is discarded and no second pipeline call is made
+```
+
+---
+
+## Appendix: Extension Definitions
+
+Extension block types used in this document are defined below using
+the core `ext` primitive — the only construct the base language provides.
+
+```mdl
+ext type
+  uri: org.dxos.mdl.type@1.0
+  desc: A named data structure with typed fields and optional literals.
+  fields:
+    desc?: Prose
+    fields?: FieldMap        # name[?]: TypeExpr  (# inline comment)
+    literals?: UnionList     # a | b | c
+    extends?: TypeRef[]
+```
+
+```mdl
+ext feat
+  uri: org.dxos.mdl.feat@1.0
+  desc: A named feature grouping one or more requirements.
+  fields:
+    desc?: Prose
+    req: RequirementList
+  nesting: self              # feat blocks may contain feat blocks
+```
+
+```mdl
+ext test
+  uri: org.dxos.mdl.test@1.0
+  desc: An acceptance scenario expressed as given / when / then steps.
+  fields:
+    given?: Step | Step[]
+    when?: Step | Step[]
+    then: Step | Step[]
+    tags?: TagList
+```
+
+```mdl
+ext component
+  uri: org.dxos.mdl.component@1.0
+  desc: A UI component with props, internal state, slots, actions, and events.
+  fields:
+    desc?: Prose
+    props?: FieldMap            # external inputs (immutable inside component)
+    state?: FieldMap            # internal reactive state
+    slots?: FieldMap            # named ReactNode injection points
+    actions?: ActionMap         # methods the component exposes or handles
+    emits?: EventMap            # events the component raises to its parent
+    layout?: CodeBlock          # ASCII sketch of visual structure (non-normative)
+```
+
+```mdl
+ext op
+  uri: org.dxos.mdl.op@1.0
+  desc: |
+    A named operation with typed inputs, outputs, and declared errors.
+    Pure ops have no effects or requires. Effectful ops declare both.
+  fields:
+    desc?: Prose
+    input?: FieldMap            # named input parameters
+    output?: TypeExpr           # return type
+    errors?: ErrorMap           # name: Prose  (when this error occurs)
+    effects?: EffectList        # echo:read | echo:write | http | fs | ...
+    requires?: ServiceList      # injected service dependencies
+    note?: Prose                # implementation guidance (non-normative)
+```

package/README.md

@@ -12,4 +12,4 @@ Run local transformers.
 
 Your ideas, issues, and code are most welcome. Please take a look at our [community code of conduct](https://github.com/dxos/dxos/blob/main/CODE_OF_CONDUCT.md), the [issue guide](https://github.com/dxos/dxos/blob/main/CONTRIBUTING.md#submitting-issues), and the [PR contribution guide](https://github.com/dxos/dxos/blob/main/CONTRIBUTING.md#submitting-prs).
 
-License: [MIT](./LICENSE) Copyright 2023 © DXOS
+License: [FSL-1.1-Apache-2.0](./LICENSE) Copyright 2023 © DXOS

package/dist/lib/neutral/TransformerPlugin.mjs

@@ -0,0 +1,38 @@
+// src/TransformerPlugin.tsx
+import { Plugin } from "@dxos/app-framework";
+import { AppPlugin } from "@dxos/app-toolkit";
+import { meta } from "#meta";
+import { translations } from "#translations";
+
+// raw-loader:/__w/dxos/dxos/packages/plugins/plugin-transformer/PLUGIN.mdl?raw
+var PLUGIN_default = "---\nid: org.dxos.plugin.transformer\nname: TransformerPlugin\nversion: 0.1.0\n---\n\nA browser-based machine learning plugin for DXOS Composer that runs Hugging Face Transformers.js\nmodels entirely in-browser via WebAssembly and WebGPU. It provides automatic speech recognition\n(Whisper) through a React hook and component layer, enabling real-time voice transcription without\nany server-side inference infrastructure.\n\n## Extensions\n\nThe following extension dialects are used in this document.\nEach extension is defined in the Appendix or resolved via its URI.\n\n| Term        | URI                            |\n|-------------|--------------------------------|\n| `type`      | `org.dxos.mdl.type@1.0`       |\n| `feat`      | `org.dxos.mdl.feat@1.0`       |\n| `test`      | `org.dxos.mdl.test@1.0`       |\n| `component` | `org.dxos.mdl.component@1.0`  |\n| `op`        | `org.dxos.mdl.op@1.0`         |\n\n## Types\n\n```mdl\ntype PipelineConfig\n  fields:\n    model: string              # Hugging Face model id, e.g. 'Xenova/whisper-base'\n    active?: boolean           # whether to load and run the pipeline\n    debug?: boolean            # enable verbose logging\n```\n\n```mdl\ntype PipelineState\n  fields:\n    gpuInfo: string            # WebGPU adapter description, or fallback message\n    isLoaded: boolean          # true once the model is fully initialised\n    isLoading: boolean         # true while the model is being downloaded/compiled\n    error: string | null       # error message if loading failed\n```\n\n```mdl\ntype AudioStreamConfig\n  fields:\n    active?: boolean           # start/stop microphone capture\n    debug?: boolean\n    onAudioData?(audioData: Float32Array): Promise<void>\n```\n\n```mdl\ntype AudioStreamState\n  fields:\n    stream: MediaStream | null # live microphone stream, or null when inactive\n    error: string | null       # error from getUserMedia or audio processing\n    audioLevel: number         # 0-255 RMS level for visualisation\n```\n\n```mdl\ntype TranscriptionOptions\n  fields:\n    sampling_rate: number      # audio sample rate in Hz (16000)\n    chunk_length_s: number     # seconds of audio per chunk\n    stride_length_s: number    # overlap between adjacent chunks\n    return_timestamps: boolean # include word-level timestamps in output\n    language: string           # target language hint, e.g. 'english'\n```\n\n```mdl\ntype EmbeddingOutput\n  fields:\n    data: number[]             # embedding vector as a flat float array\n```\n\n## Components\n\n```mdl\ncomponent Voice\n  desc: |\n    Composite component that wires usePipeline and useAudioStream together to provide\n    live Whisper transcription from the microphone.  Accumulates the running transcript\n    in local state and renders a DebugInfo panel showing model status, audio level, and\n    the current transcription text.\n  props:\n    active?: boolean           # start/stop the entire pipeline\n    debug?: boolean\n    model?: string             # defaults to 'Xenova/whisper-base'\n  state:\n    isTranscribing: boolean    # true while a transcription call is in-flight\n    transcription: string      # accumulated transcript text\n  layout: |\n    \u250C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n    \u2502  [DebugInfo panel]                   \u2502\n    \u2502   model: Xenova/whisper-base         \u2502\n    \u2502   gpu: GPU Available                 \u2502\n    \u2502   stream: active / inactive          \u2502\n    \u2502   level: \u2588\u2588\u2588\u2591\u2591\u2591\u2591 (audioLevel bar)    \u2502\n    \u2502   \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500     \u2502\n    \u2502   [transcription text]               \u2502\n    \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n```\n\n## Operations\n\n```mdl\nop LoadModel\n  desc: |\n    Loads a Transformers.js automatic-speech-recognition pipeline for the given model id.\n    Configures ONNX backend to use WebGPU if available, falling back to WASM/CPU.\n    Invoked automatically when the Voice component mounts with active = true.\n  input: void\n  output: void\n  effects: [http, fs]\n  note: Model weights are cached in ./.cache by the Transformers.js env settings.\n```\n\n```mdl\nop Transcribe\n  desc: |\n    Runs the loaded ASR pipeline against a Float32Array of 16 kHz mono audio.\n    Returns the transcribed text string from the pipeline result.\n  input: void\n  output: string\n  effects: [gpu]\n  note: Guarded by isTranscribing flag to prevent concurrent calls.\n```\n\n```mdl\nop CaptureAudio\n  desc: |\n    Requests microphone permission via getUserMedia and wires an AudioWorklet that\n    accumulates 2-second chunks (32000 samples at 16 kHz) before forwarding them to\n    the onAudioData callback.\n  input: void\n  output: void\n  effects: [audio]\n  note: Cleaned up automatically when the component unmounts or active becomes false.\n```\n\n## Features\n\n```mdl\nfeat F-1: In-Browser ASR Pipeline\n\n  req F-1.1:\n    when: Voice component mounts with active = true\n    then: usePipeline loads the Whisper model via Transformers.js; isLoading is true until complete\n\n  req F-1.2:\n    when: WebGPU is available in the browser\n    then: the ONNX backend is configured for WebGPU execution and gpuInfo reflects the adapter\n\n  req F-1.3:\n    when: WebGPU is not available\n    then: the pipeline falls back to WASM/CPU and gpuInfo is set to a 'not supported' message\n\n  req F-1.4:\n    when: model loading fails\n    then: PipelineState.error contains a human-readable message; isLoading is false\n```\n\n```mdl\nfeat F-2: Microphone Capture\n\n  req F-2.1:\n    when: useAudioStream is activated\n    then: getUserMedia is called with mono 16 kHz audio and echo/noise cancellation enabled\n\n  req F-2.2:\n    when: audio is captured\n    then: an AudioWorklet accumulates samples and emits 2-second chunks via the onAudioData callback\n\n  req F-2.3:\n    when: active is set to false or the component unmounts\n    then: all MediaStream tracks are stopped, AudioContext is closed, and AnimationFrame is cancelled\n\n  req F-2.4:\n    when: getUserMedia is denied\n    then: AudioStreamState.error is set with the underlying error message\n```\n\n```mdl\nfeat F-3: Live Transcription\n\n  req F-3.1:\n    when: a 2-second audio chunk arrives and the model is loaded\n    then: usePipeline.transcribe is called with 16 kHz mono options and the result appended to the transcript\n\n  req F-3.2:\n    when: transcription is already in progress (isTranscribing = true)\n    then: the incoming audio chunk is dropped to avoid concurrent inference\n\n  req F-3.3:\n    when: transcription returns an empty or whitespace-only string\n    then: the transcript state is not updated\n```\n\n```mdl\nfeat F-4: RAG Embedding Pipeline (Testing)\n\n  req F-4.1:\n    when: RagPipeline.generateCompletions is called with an input and a knowledge base\n    then: embeddings are generated for both the input and each knowledge-base entry\n\n  req F-4.2:\n    when: embeddings are available\n    then: cosine similarity is computed and the top-3 most similar contexts are selected\n```\n\n## Acceptance\n\n```mdl\ntest T-1: Model loads and transitions isLoaded\n  given: Voice component mounts with active = true and model = 'Xenova/whisper-base'\n  when: model download and ONNX compilation complete\n  then:\n    - isLoading transitions from true to false\n    - isLoaded is true\n    - error is null\n```\n\n```mdl\ntest T-2: Model load failure sets error state\n  given: the model download throws a network error\n  when: usePipeline attempts to load the model\n  then:\n    - isLoading is false\n    - error contains the error message string\n    - isLoaded is false\n```\n\n```mdl\ntest T-3: Microphone capture populates stream state\n  given: getUserMedia is granted\n  when: useAudioStream activates\n  then:\n    - AudioStreamState.stream is a non-null MediaStream\n    - AudioStreamState.error is null\n```\n\n```mdl\ntest T-4: Audio chunk triggers transcription\n  given: model is loaded and stream is active\n  when: AudioWorklet posts a 32000-sample chunk\n  then:\n    - isTranscribing becomes true during the pipeline call\n    - transcription string is updated with the returned text\n    - isTranscribing returns to false after completion\n```\n\n```mdl\ntest T-5: Concurrent transcription calls are dropped\n  given: isTranscribing is true\n  when: a second audio chunk arrives\n  then: the chunk is discarded and no second pipeline call is made\n```\n\n---\n\n## Appendix: Extension Definitions\n\nExtension block types used in this document are defined below using\nthe core `ext` primitive \u2014 the only construct the base language provides.\n\n```mdl\next type\n  uri: org.dxos.mdl.type@1.0\n  desc: A named data structure with typed fields and optional literals.\n  fields:\n    desc?: Prose\n    fields?: FieldMap        # name[?]: TypeExpr  (# inline comment)\n    literals?: UnionList     # a | b | c\n    extends?: TypeRef[]\n```\n\n```mdl\next feat\n  uri: org.dxos.mdl.feat@1.0\n  desc: A named feature grouping one or more requirements.\n  fields:\n    desc?: Prose\n    req: RequirementList\n  nesting: self              # feat blocks may contain feat blocks\n```\n\n```mdl\next test\n  uri: org.dxos.mdl.test@1.0\n  desc: An acceptance scenario expressed as given / when / then steps.\n  fields:\n    given?: Step | Step[]\n    when?: Step | Step[]\n    then: Step | Step[]\n    tags?: TagList\n```\n\n```mdl\next component\n  uri: org.dxos.mdl.component@1.0\n  desc: A UI component with props, internal state, slots, actions, and events.\n  fields:\n    desc?: Prose\n    props?: FieldMap            # external inputs (immutable inside component)\n    state?: FieldMap            # internal reactive state\n    slots?: FieldMap            # named ReactNode injection points\n    actions?: ActionMap         # methods the component exposes or handles\n    emits?: EventMap            # events the component raises to its parent\n    layout?: CodeBlock          # ASCII sketch of visual structure (non-normative)\n```\n\n```mdl\next op\n  uri: org.dxos.mdl.op@1.0\n  desc: |\n    A named operation with typed inputs, outputs, and declared errors.\n    Pure ops have no effects or requires. Effectful ops declare both.\n  fields:\n    desc?: Prose\n    input?: FieldMap            # named input parameters\n    output?: TypeExpr           # return type\n    errors?: ErrorMap           # name: Prose  (when this error occurs)\n    effects?: EffectList        # echo:read | echo:write | http | fs | ...\n    requires?: ServiceList      # injected service dependencies\n    note?: Prose                # implementation guidance (non-normative)\n```\n";
+
+// src/TransformerPlugin.tsx
+var TransformerPlugin = Plugin.define(meta).pipe(
+  AppPlugin.addSchemaModule({
+    schema: []
+  }),
+  AppPlugin.addTranslationsModule({
+    translations
+  }),
+  // Plugin.addModule({
+  //   id: 'intent-resolver',
+  //   activatesOn: Events.SetupIntentResolver,
+  //   activate: IntentResolver,
+  // }),
+  AppPlugin.addPluginAssetModule({
+    asset: {
+      pluginId: meta.id,
+      path: "PLUGIN.mdl",
+      content: PLUGIN_default,
+      mimeType: "application/x-mdl"
+    }
+  }),
+  Plugin.make
+);
+var TransformerPlugin_default = TransformerPlugin;
+export {
+  TransformerPlugin,
+  TransformerPlugin_default as default
+};
+//# sourceMappingURL=TransformerPlugin.mjs.map

package/dist/lib/neutral/TransformerPlugin.mjs.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/TransformerPlugin.tsx", "raw-loader:/__w/dxos/dxos/packages/plugins/plugin-transformer/PLUGIN.mdl?raw"],
+  "sourcesContent": ["//\n// Copyright 2024 DXOS.org\n//\n\nimport { Plugin } from '@dxos/app-framework';\nimport { AppPlugin } from '@dxos/app-toolkit';\n\n// import { IntentResolver } from './capabilities';\nimport { meta } from '#meta';\nimport { translations } from '#translations';\n\n// eslint-disable-next-line import/no-relative-packages\nimport pluginSpec from '../PLUGIN.mdl?raw';\n\nexport const TransformerPlugin = Plugin.define(meta).pipe(\n  AppPlugin.addSchemaModule({ schema: [] }),\n  AppPlugin.addTranslationsModule({ translations }),\n  // Plugin.addModule({\n  //   id: 'intent-resolver',\n  //   activatesOn: Events.SetupIntentResolver,\n  //   activate: IntentResolver,\n  // }),\n  AppPlugin.addPluginAssetModule({\n    asset: { pluginId: meta.id, path: 'PLUGIN.mdl', content: pluginSpec, mimeType: 'application/x-mdl' },\n  }),\n  Plugin.make,\n);\n\nexport default TransformerPlugin;\n", "---\nid: org.dxos.plugin.transformer\nname: TransformerPlugin\nversion: 0.1.0\n---\n\nA browser-based machine learning plugin for DXOS Composer that runs Hugging Face Transformers.js\nmodels entirely in-browser via WebAssembly and WebGPU. It provides automatic speech recognition\n(Whisper) through a React hook and component layer, enabling real-time voice transcription without\nany server-side inference infrastructure.\n\n## Extensions\n\nThe following extension dialects are used in this document.\nEach extension is defined in the Appendix or resolved via its URI.\n\n| Term        | URI                            |\n|-------------|--------------------------------|\n| `type`      | `org.dxos.mdl.type@1.0`       |\n| `feat`      | `org.dxos.mdl.feat@1.0`       |\n| `test`      | `org.dxos.mdl.test@1.0`       |\n| `component` | `org.dxos.mdl.component@1.0`  |\n| `op`        | `org.dxos.mdl.op@1.0`         |\n\n## Types\n\n```mdl\ntype PipelineConfig\n  fields:\n    model: string              # Hugging Face model id, e.g. 'Xenova/whisper-base'\n    active?: boolean           # whether to load and run the pipeline\n    debug?: boolean            # enable verbose logging\n```\n\n```mdl\ntype PipelineState\n  fields:\n    gpuInfo: string            # WebGPU adapter description, or fallback message\n    isLoaded: boolean          # true once the model is fully initialised\n    isLoading: boolean         # true while the model is being downloaded/compiled\n    error: string | null       # error message if loading failed\n```\n\n```mdl\ntype AudioStreamConfig\n  fields:\n    active?: boolean           # start/stop microphone capture\n    debug?: boolean\n    onAudioData?(audioData: Float32Array): Promise<void>\n```\n\n```mdl\ntype AudioStreamState\n  fields:\n    stream: MediaStream | null # live microphone stream, or null when inactive\n    error: string | null       # error from getUserMedia or audio processing\n    audioLevel: number         # 0-255 RMS level for visualisation\n```\n\n```mdl\ntype TranscriptionOptions\n  fields:\n    sampling_rate: number      # audio sample rate in Hz (16000)\n    chunk_length_s: number     # seconds of audio per chunk\n    stride_length_s: number    # overlap between adjacent chunks\n    return_timestamps: boolean # include word-level timestamps in output\n    language: string           # target language hint, e.g. 'english'\n```\n\n```mdl\ntype EmbeddingOutput\n  fields:\n    data: number[]             # embedding vector as a flat float array\n```\n\n## Components\n\n```mdl\ncomponent Voice\n  desc: |\n    Composite component that wires usePipeline and useAudioStream together to provide\n    live Whisper transcription from the microphone.  Accumulates the running transcript\n    in local state and renders a DebugInfo panel showing model status, audio level, and\n    the current transcription text.\n  props:\n    active?: boolean           # start/stop the entire pipeline\n    debug?: boolean\n    model?: string             # defaults to 'Xenova/whisper-base'\n  state:\n    isTranscribing: boolean    # true while a transcription call is in-flight\n    transcription: string      # accumulated transcript text\n  layout: |\n    \u250C\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2510\n    \u2502  [DebugInfo panel]                   \u2502\n    \u2502   model: Xenova/whisper-base         \u2502\n    \u2502   gpu: GPU Available                 \u2502\n    \u2502   stream: active / inactive          \u2502\n    \u2502   level: \u2588\u2588\u2588\u2591\u2591\u2591\u2591 (audioLevel bar)    \u2502\n    \u2502   \u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500     \u2502\n    \u2502   [transcription text]               \u2502\n    \u2514\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2500\u2518\n```\n\n## Operations\n\n```mdl\nop LoadModel\n  desc: |\n    Loads a Transformers.js automatic-speech-recognition pipeline for the given model id.\n    Configures ONNX backend to use WebGPU if available, falling back to WASM/CPU.\n    Invoked automatically when the Voice component mounts with active = true.\n  input: void\n  output: void\n  effects: [http, fs]\n  note: Model weights are cached in ./.cache by the Transformers.js env settings.\n```\n\n```mdl\nop Transcribe\n  desc: |\n    Runs the loaded ASR pipeline against a Float32Array of 16 kHz mono audio.\n    Returns the transcribed text string from the pipeline result.\n  input: void\n  output: string\n  effects: [gpu]\n  note: Guarded by isTranscribing flag to prevent concurrent calls.\n```\n\n```mdl\nop CaptureAudio\n  desc: |\n    Requests microphone permission via getUserMedia and wires an AudioWorklet that\n    accumulates 2-second chunks (32000 samples at 16 kHz) before forwarding them to\n    the onAudioData callback.\n  input: void\n  output: void\n  effects: [audio]\n  note: Cleaned up automatically when the component unmounts or active becomes false.\n```\n\n## Features\n\n```mdl\nfeat F-1: In-Browser ASR Pipeline\n\n  req F-1.1:\n    when: Voice component mounts with active = true\n    then: usePipeline loads the Whisper model via Transformers.js; isLoading is true until complete\n\n  req F-1.2:\n    when: WebGPU is available in the browser\n    then: the ONNX backend is configured for WebGPU execution and gpuInfo reflects the adapter\n\n  req F-1.3:\n    when: WebGPU is not available\n    then: the pipeline falls back to WASM/CPU and gpuInfo is set to a 'not supported' message\n\n  req F-1.4:\n    when: model loading fails\n    then: PipelineState.error contains a human-readable message; isLoading is false\n```\n\n```mdl\nfeat F-2: Microphone Capture\n\n  req F-2.1:\n    when: useAudioStream is activated\n    then: getUserMedia is called with mono 16 kHz audio and echo/noise cancellation enabled\n\n  req F-2.2:\n    when: audio is captured\n    then: an AudioWorklet accumulates samples and emits 2-second chunks via the onAudioData callback\n\n  req F-2.3:\n    when: active is set to false or the component unmounts\n    then: all MediaStream tracks are stopped, AudioContext is closed, and AnimationFrame is cancelled\n\n  req F-2.4:\n    when: getUserMedia is denied\n    then: AudioStreamState.error is set with the underlying error message\n```\n\n```mdl\nfeat F-3: Live Transcription\n\n  req F-3.1:\n    when: a 2-second audio chunk arrives and the model is loaded\n    then: usePipeline.transcribe is called with 16 kHz mono options and the result appended to the transcript\n\n  req F-3.2:\n    when: transcription is already in progress (isTranscribing = true)\n    then: the incoming audio chunk is dropped to avoid concurrent inference\n\n  req F-3.3:\n    when: transcription returns an empty or whitespace-only string\n    then: the transcript state is not updated\n```\n\n```mdl\nfeat F-4: RAG Embedding Pipeline (Testing)\n\n  req F-4.1:\n    when: RagPipeline.generateCompletions is called with an input and a knowledge base\n    then: embeddings are generated for both the input and each knowledge-base entry\n\n  req F-4.2:\n    when: embeddings are available\n    then: cosine similarity is computed and the top-3 most similar contexts are selected\n```\n\n## Acceptance\n\n```mdl\ntest T-1: Model loads and transitions isLoaded\n  given: Voice component mounts with active = true and model = 'Xenova/whisper-base'\n  when: model download and ONNX compilation complete\n  then:\n    - isLoading transitions from true to false\n    - isLoaded is true\n    - error is null\n```\n\n```mdl\ntest T-2: Model load failure sets error state\n  given: the model download throws a network error\n  when: usePipeline attempts to load the model\n  then:\n    - isLoading is false\n    - error contains the error message string\n    - isLoaded is false\n```\n\n```mdl\ntest T-3: Microphone capture populates stream state\n  given: getUserMedia is granted\n  when: useAudioStream activates\n  then:\n    - AudioStreamState.stream is a non-null MediaStream\n    - AudioStreamState.error is null\n```\n\n```mdl\ntest T-4: Audio chunk triggers transcription\n  given: model is loaded and stream is active\n  when: AudioWorklet posts a 32000-sample chunk\n  then:\n    - isTranscribing becomes true during the pipeline call\n    - transcription string is updated with the returned text\n    - isTranscribing returns to false after completion\n```\n\n```mdl\ntest T-5: Concurrent transcription calls are dropped\n  given: isTranscribing is true\n  when: a second audio chunk arrives\n  then: the chunk is discarded and no second pipeline call is made\n```\n\n---\n\n## Appendix: Extension Definitions\n\nExtension block types used in this document are defined below using\nthe core `ext` primitive \u2014 the only construct the base language provides.\n\n```mdl\next type\n  uri: org.dxos.mdl.type@1.0\n  desc: A named data structure with typed fields and optional literals.\n  fields:\n    desc?: Prose\n    fields?: FieldMap        # name[?]: TypeExpr  (# inline comment)\n    literals?: UnionList     # a | b | c\n    extends?: TypeRef[]\n```\n\n```mdl\next feat\n  uri: org.dxos.mdl.feat@1.0\n  desc: A named feature grouping one or more requirements.\n  fields:\n    desc?: Prose\n    req: RequirementList\n  nesting: self              # feat blocks may contain feat blocks\n```\n\n```mdl\next test\n  uri: org.dxos.mdl.test@1.0\n  desc: An acceptance scenario expressed as given / when / then steps.\n  fields:\n    given?: Step | Step[]\n    when?: Step | Step[]\n    then: Step | Step[]\n    tags?: TagList\n```\n\n```mdl\next component\n  uri: org.dxos.mdl.component@1.0\n  desc: A UI component with props, internal state, slots, actions, and events.\n  fields:\n    desc?: Prose\n    props?: FieldMap            # external inputs (immutable inside component)\n    state?: FieldMap            # internal reactive state\n    slots?: FieldMap            # named ReactNode injection points\n    actions?: ActionMap         # methods the component exposes or handles\n    emits?: EventMap            # events the component raises to its parent\n    layout?: CodeBlock          # ASCII sketch of visual structure (non-normative)\n```\n\n```mdl\next op\n  uri: org.dxos.mdl.op@1.0\n  desc: |\n    A named operation with typed inputs, outputs, and declared errors.\n    Pure ops have no effects or requires. Effectful ops declare both.\n  fields:\n    desc?: Prose\n    input?: FieldMap            # named input parameters\n    output?: TypeExpr           # return type\n    errors?: ErrorMap           # name: Prose  (when this error occurs)\n    effects?: EffectList        # echo:read | echo:write | http | fs | ...\n    requires?: ServiceList      # injected service dependencies\n    note?: Prose                # implementation guidance (non-normative)\n```\n"],
+  "mappings": ";AAIA,SAASA,cAAc;AACvB,SAASC,iBAAiB;AAG1B,SAASC,YAAY;AACrB,SAASC,oBAAoB;;;ACT7B;;;ADcO,IAAMC,oBAAoBC,OAAOC,OAAOC,IAAAA,EAAMC;EACnDC,UAAUC,gBAAgB;IAAEC,QAAQ,CAAA;EAAG,CAAA;EACvCF,UAAUG,sBAAsB;IAAEC;EAAa,CAAA;;;;;;EAM/CJ,UAAUK,qBAAqB;IAC7BC,OAAO;MAAEC,UAAUT,KAAKU;MAAIC,MAAM;MAAcC,SAASC;MAAYC,UAAU;IAAoB;EACrG,CAAA;EACAhB,OAAOiB;AAAI;AAGb,IAAA,4BAAelB;",
+  "names": ["Plugin", "AppPlugin", "meta", "translations", "TransformerPlugin", "Plugin", "define", "meta", "pipe", "AppPlugin", "addSchemaModule", "schema", "addTranslationsModule", "translations", "addPluginAssetModule", "asset", "pluginId", "id", "path", "content", "pluginSpec", "mimeType", "make"]
+}

package/dist/lib/neutral/chunk-SFEKXMAC.mjs

@@ -0,0 +1,36 @@
+// src/meta.ts
+import { Plugin } from "@dxos/app-framework";
+import { DXN } from "@dxos/keys";
+import { trim } from "@dxos/util";
+var meta = Plugin.makeMeta({
+  key: DXN.make("org.dxos.plugin.transformer"),
+  name: "Transformer",
+  author: "DXOS",
+  spec: "PLUGIN.mdl",
+  description: trim`
+    Browser-based machine learning plugin that runs Hugging Face Transformers.js models
+    entirely in-browser via WebAssembly and WebGPU — no server-side inference required.
+
+    Provides automatic speech recognition through a Whisper pipeline hook (usePipeline)
+    and a microphone capture hook (useAudioStream) that buffers 16 kHz mono audio into
+    2-second chunks before forwarding them to the model.
+
+    Exposes a Voice component that wires the two hooks together to deliver live
+    transcription, accumulating the running transcript in local state and rendering
+    a debug panel with model status, GPU info, and audio level visualisation.
+
+    Includes a RAG embedding pipeline base class for retrieval-augmented generation
+    experiments, with cosine similarity ranking for selecting the most relevant
+    knowledge-base contexts before text generation.
+  `,
+  icon: "ph--cpu--regular",
+  source: "https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transformer",
+  tags: [
+    "labs"
+  ]
+});
+
+export {
+  meta
+};
+//# sourceMappingURL=chunk-SFEKXMAC.mjs.map

package/dist/lib/neutral/chunk-SFEKXMAC.mjs.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/meta.ts"],
+  "sourcesContent": ["//\n// Copyright 2023 DXOS.org\n//\n\nimport { Plugin } from '@dxos/app-framework';\nimport { DXN } from '@dxos/keys';\nimport { trim } from '@dxos/util';\n\nexport const meta = Plugin.makeMeta({\n  key: DXN.make('org.dxos.plugin.transformer'),\n  name: 'Transformer',\n  author: 'DXOS',\n  spec: 'PLUGIN.mdl',\n  description: trim`\n    Browser-based machine learning plugin that runs Hugging Face Transformers.js models\n    entirely in-browser via WebAssembly and WebGPU — no server-side inference required.\n\n    Provides automatic speech recognition through a Whisper pipeline hook (usePipeline)\n    and a microphone capture hook (useAudioStream) that buffers 16 kHz mono audio into\n    2-second chunks before forwarding them to the model.\n\n    Exposes a Voice component that wires the two hooks together to deliver live\n    transcription, accumulating the running transcript in local state and rendering\n    a debug panel with model status, GPU info, and audio level visualisation.\n\n    Includes a RAG embedding pipeline base class for retrieval-augmented generation\n    experiments, with cosine similarity ranking for selecting the most relevant\n    knowledge-base contexts before text generation.\n  `,\n  icon: 'ph--cpu--regular',\n  source: 'https://github.com/dxos/dxos/tree/main/packages/plugins/plugin-transformer',\n  tags: ['labs'],\n});\n"],
+  "mappings": ";AAIA,SAASA,cAAc;AACvB,SAASC,WAAW;AACpB,SAASC,YAAY;AAEd,IAAMC,OAAOH,OAAOI,SAAS;EAClCC,KAAKJ,IAAIK,KAAK,6BAAA;EACdC,MAAM;EACNC,QAAQ;EACRC,MAAM;EACNC,aAAaR;;;;;;;;;;;;;;;;EAgBbS,MAAM;EACNC,QAAQ;EACRC,MAAM;IAAC;;AACT,CAAA;",
+  "names": ["Plugin", "DXN", "trim", "meta", "makeMeta", "key", "make", "name", "author", "spec", "description", "icon", "source", "tags"]
+}