@mlx-node/asr 0.0.9

package/README.md

@@ -0,0 +1,144 @@
+# @mlx-node/asr
+
+Local Qwen3-ASR transcription and realtime meeting capture on Apple Silicon.
+
+## Convert the Hugging Face checkpoint
+
+For the fastest tested decoder path, pack the Qwen text model as MXFP4. The
+audio encoder and multimodal projector deliberately remain BF16 so speech
+features are not quantized:
+
+```bash
+yarn mlx convert \
+  -i .cache/models/qwen3-asr-1.7b-hf \
+  -o .cache/models/qwen3-asr-1.7b-mlx-mxfp4 \
+  -d bfloat16 \
+  -q --q-mode mxfp4
+```
+
+Use a dense conversion when weight fidelity or quantization comparisons matter
+more than decode throughput:
+
+```bash
+yarn mlx convert \
+  -i .cache/models/qwen3-asr-1.7b-hf \
+  -o .cache/models/qwen3-asr-1.7b-mlx \
+  -d bfloat16
+```
+
+The converter detects `model_type: "qwen3_asr"`, canonicalizes the checkpoint
+keys, and converts the three audio convolutions to MLX layout. Packed
+conversions support uniform affine, MXFP4, and MXFP8 text weights; recipe-based
+or per-layer quantization is rejected.
+
+## Offline transcription
+
+```typescript
+import { Qwen3AsrModel } from '@mlx-node/asr';
+
+const model = await Qwen3AsrModel.load('.cache/models/qwen3-asr-1.7b-mlx-mxfp4');
+const pcm = new Float32Array(/* mono PCM samples */);
+const result = await model.transcribe(pcm, {
+  sampleRate: 16_000,
+  language: 'en', // omit for language detection
+});
+
+console.log(result.text, result.realTimeFactor);
+```
+
+`transcribe()` accepts mono floating-point PCM at any positive sample rate and
+resamples it to the model's native 16 kHz input.
+
+## Streaming manually supplied audio
+
+```typescript
+const stream = await model.createStream({
+  sampleRate: 48_000,
+  chunkSeconds: 2,
+  provisionalTokens: 5,
+  unfixedChunks: 2,
+  maxTokens: 32,
+});
+
+for await (const pcmChunk of yourAudioSource) {
+  const revision = await stream.feed(pcmChunk);
+  if (revision) {
+    process.stdout.write(`\r${revision.stableText}\x1b[2m${revision.provisionalText}\x1b[0m`);
+  }
+}
+
+const final = await stream.finish();
+console.log(`\n${final.text}`);
+```
+
+Streaming follows Qwen's official rolling policy: every 2 seconds it feeds the
+previous raw transcript minus the last 5 tokens back to the model. The first 2
+chunks are decoded without transcript conditioning. `stableText` is the
+current fixed frontier; render each result as a complete revision because a
+later hypothesis can still move that frontier. The next revision may replace
+`provisionalText`.
+
+The audio tower itself is not causal. To keep long meetings realtime, completed
+8-second local-attention windows are encoded once and cached. The stream keeps
+the latest four completed windows plus the current partial window (less than 40
+seconds total), bounds the decoder's transcript prefix to 150 tokens, and
+reuses KV state through the longest unchanged audio prefix. These bounds keep
+memory and per-revision work approximately constant. If `reachedMaxTokens` is
+true, the continuation exhausted its 32-token budget and the current
+provisional suffix is worth flagging for review. Repeated-token and stalled
+decode guards automatically discard a degenerate provisional tail and
+re-anchor the next chunk with fresh bounded audio context.
+
+## Realtime meeting capture
+
+`startMeetingTranscription()` captures the local microphone and the Mac's
+system/output audio by default. Revisions are kept on separate source-tagged
+tracks because the two devices have independent clocks and represent different
+speakers.
+
+```typescript
+import { Qwen3AsrModel, qwen3AsrAudioDevices, startMeetingTranscription } from '@mlx-node/asr';
+
+console.table(qwen3AsrAudioDevices());
+
+const model = await Qwen3AsrModel.load('.cache/models/qwen3-asr-1.7b-mlx-mxfp4');
+const meeting = await startMeetingTranscription(model, {
+  stream: { chunkSeconds: 2, provisionalTokens: 5, unfixedChunks: 2, maxTokens: 32 },
+  microphone: { feedMilliseconds: 100, ringSeconds: 10 },
+  systemAudio: {
+    feedMilliseconds: 100,
+    ringSeconds: 10,
+    // Optional: capture selected apps instead of all system output.
+    // applicationBundleIds: ['us.zoom.xos', 'com.microsoft.teams2'],
+  },
+  onResult({ source, result }) {
+    console.log(`[${source}] ${result.stableText}${result.provisionalText}`);
+  },
+  onError({ source, error }) {
+    console.error(`[${source}]`, error);
+  },
+});
+
+process.once('SIGINT', async () => {
+  const final = await meeting.stop();
+  console.log('local:', final.microphone?.result.text);
+  console.log('remote:', final.systemAudio?.result.text);
+});
+```
+
+Set either `microphone: false` or `systemAudio: false` for a single-track
+session. The lower-level `startRealtimeTranscription()` API remains available
+when you want to own exactly one source; its `capture.source` defaults to
+`Qwen3AsrCaptureSource.Microphone`.
+
+The native Core Audio callbacks only write packed mono float samples into
+bounded single-producer/single-consumer rings. Resampling and MLX inference run
+outside the realtime callbacks. Each capture automatically binds its ASR stream
+to the selected device's actual sample rate. `feedMilliseconds` controls how
+often capture drains into the model buffer; `chunkSeconds` controls the
+transcription update cadence.
+
+Packaged macOS hosts must include both `NSMicrophoneUsageDescription` and
+`NSAudioCaptureUsageDescription` in `Info.plist`. macOS prompts separately for
+microphone and system-audio permission. System capture uses a private Core
+Audio tap and does not mute normal speaker playback.

package/dist/index.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Qwen3-ASR inference and low-latency Core Audio capture on Apple Silicon.
+ */
+import { Qwen3AsrCapture, Qwen3AsrCaptureSource, Qwen3AsrModel, Qwen3AsrStream, qwen3AsrAudioDevices, qwen3AsrInputDevices, type Qwen3AsrAudioDevice, type Qwen3AsrCaptureOptions, type Qwen3AsrCaptureStats, type Qwen3AsrInputDevice, type Qwen3AsrResult, type Qwen3AsrStreamOptions, type Qwen3AsrTranscribeOptions } from '@mlx-node/core';
+export { Qwen3AsrCapture, Qwen3AsrCaptureSource, Qwen3AsrModel, Qwen3AsrStream, qwen3AsrAudioDevices, qwen3AsrInputDevices, type Qwen3AsrAudioDevice, type Qwen3AsrCaptureOptions, type Qwen3AsrCaptureStats, type Qwen3AsrInputDevice, type Qwen3AsrResult, type Qwen3AsrStreamOptions, type Qwen3AsrTranscribeOptions, };
+export interface Qwen3AsrRealtimeOptions {
+    /** Rolling decode cadence, language, and prompting options. */
+    stream?: Qwen3AsrStreamOptions;
+    /** Core Audio source, device, application filter, and callback-ring options. */
+    capture?: Qwen3AsrCaptureOptions;
+    /** Called for every rolling transcription revision. */
+    onResult: (result: Qwen3AsrResult) => void;
+    /** Called for asynchronous capture or model-worker errors. */
+    onError?: (error: Error) => void;
+}
+export interface Qwen3AsrRealtimeFinal {
+    result: Qwen3AsrResult;
+    capture: Qwen3AsrCaptureStats;
+}
+/**
+ * Owns one model stream and one Core Audio source. Call `stop()` to drain the
+ * lock-free capture ring and receive the final, non-provisional transcript.
+ */
+export declare class Qwen3AsrRealtimeSession {
+    #private;
+    readonly stream: Qwen3AsrStream;
+    readonly capture: Qwen3AsrCapture;
+    private readonly getLastError;
+    private constructor();
+    static start(model: Qwen3AsrModel, options: Qwen3AsrRealtimeOptions): Promise<Qwen3AsrRealtimeSession>;
+    get deviceName(): string;
+    get source(): Qwen3AsrCaptureSource;
+    get sampleRate(): number;
+    get lastError(): Error | undefined;
+    pause(): void;
+    resume(): void;
+    stop(): Promise<Qwen3AsrRealtimeFinal>;
+}
+export declare function startRealtimeTranscription(model: Qwen3AsrModel, options: Qwen3AsrRealtimeOptions): Promise<Qwen3AsrRealtimeSession>;
+type CaptureTimingOptions = Pick<Qwen3AsrCaptureOptions, 'feedMilliseconds' | 'ringSeconds'>;
+export interface Qwen3AsrMicrophoneOptions extends CaptureTimingOptions {
+    /** Stable input-device UID from `qwen3AsrAudioDevices()`. */
+    deviceId?: string;
+    /** Input-device name. Prefer `deviceId` when persisting a selection. */
+    deviceName?: string;
+}
+export interface Qwen3AsrSystemAudioOptions extends CaptureTimingOptions {
+    /** Stable output-device UID from `qwen3AsrAudioDevices()`. */
+    deviceId?: string;
+    /** Output-device name. Prefer `deviceId` when persisting a selection. */
+    deviceName?: string;
+    /** Capture only these applications. Omit to capture all system output. */
+    applicationBundleIds?: string[];
+}
+export type Qwen3AsrMeetingSource = 'microphone' | 'systemAudio';
+export interface Qwen3AsrMeetingResultEvent {
+    source: Qwen3AsrMeetingSource;
+    result: Qwen3AsrResult;
+}
+export interface Qwen3AsrMeetingErrorEvent {
+    source: Qwen3AsrMeetingSource;
+    error: Error;
+}
+export interface Qwen3AsrMeetingOptions {
+    /** Shared rolling decode cadence, language, and prompting options. */
+    stream?: Qwen3AsrStreamOptions;
+    /** Microphone capture options. `false` disables this track. Default enabled. */
+    microphone?: false | Qwen3AsrMicrophoneOptions;
+    /** System/output audio options. `false` disables this track. Default enabled. */
+    systemAudio?: false | Qwen3AsrSystemAudioOptions;
+    /** Called for every rolling revision, tagged with its audio source. */
+    onResult: (event: Qwen3AsrMeetingResultEvent) => void;
+    /** Called for asynchronous capture or model-worker errors. */
+    onError?: (event: Qwen3AsrMeetingErrorEvent) => void;
+}
+export interface Qwen3AsrMeetingFinal {
+    microphone?: Qwen3AsrRealtimeFinal;
+    systemAudio?: Qwen3AsrRealtimeFinal;
+}
+/**
+ * A meeting owns one independently clocked transcription track per enabled
+ * source. Results stay source-tagged so speaker-mic audio and remote/system
+ * audio are never silently mixed or ordered by unrelated device clocks.
+ */
+export declare class Qwen3AsrMeetingSession {
+    #private;
+    readonly microphone?: Qwen3AsrRealtimeSession;
+    readonly systemAudio?: Qwen3AsrRealtimeSession;
+    private constructor();
+    static start(model: Qwen3AsrModel, options: Qwen3AsrMeetingOptions): Promise<Qwen3AsrMeetingSession>;
+    pause(): void;
+    resume(): void;
+    stop(): Promise<Qwen3AsrMeetingFinal>;
+    private forEachTrack;
+}
+/** Start microphone and system-audio transcription as one meeting session. */
+export declare function startMeetingTranscription(model: Qwen3AsrModel, options: Qwen3AsrMeetingOptions): Promise<Qwen3AsrMeetingSession>;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,aAAa,EACb,cAAc,EACd,oBAAoB,EACpB,oBAAoB,EACpB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,yBAAyB,EAC/B,MAAM,gBAAgB,CAAC;AAExB,OAAO,EACL,eAAe,EACf,qBAAqB,EACrB,aAAa,EACb,cAAc,EACd,oBAAoB,EACpB,oBAAoB,EACpB,KAAK,mBAAmB,EACxB,KAAK,sBAAsB,EAC3B,KAAK,oBAAoB,EACzB,KAAK,mBAAmB,EACxB,KAAK,cAAc,EACnB,KAAK,qBAAqB,EAC1B,KAAK,yBAAyB,GAC/B,CAAC;AAEF,MAAM,WAAW,uBAAuB;IACtC,+DAA+D;IAC/D,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,gFAAgF;IAChF,OAAO,CAAC,EAAE,sBAAsB,CAAC;IACjC,uDAAuD;IACvD,QAAQ,EAAE,CAAC,MAAM,EAAE,cAAc,KAAK,IAAI,CAAC;IAC3C,8DAA8D;IAC9D,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,KAAK,KAAK,IAAI,CAAC;CAClC;AAED,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,cAAc,CAAC;IACvB,OAAO,EAAE,oBAAoB,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,uBAAuB;;IAClC,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,QAAQ,CAAC,OAAO,EAAE,eAAe,CAAC;IAGlC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA0B;IAEvD,OAAO,eAIN;IAED,OAAa,KAAK,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,uBAAuB,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAkB3G;IAED,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,MAAM,IAAI,qBAAqB,CAElC;IAED,IAAI,UAAU,IAAI,MAAM,CAEvB;IAED,IAAI,SAAS,IAAI,KAAK,GAAG,SAAS,CAEjC;IAED,KAAK,IAAI,IAAI,CAEZ;IAED,MAAM,IAAI,IAAI,CAEb;IAED,IAAI,IAAI,OAAO,CAAC,qBAAqB,CAAC,CAerC;CACF;AAED,wBAAgB,0BAA0B,CACxC,KAAK,EAAE,aAAa,EACpB,OAAO,EAAE,uBAAuB,GAC/B,OAAO,CAAC,uBAAuB,CAAC,CAElC;AAED,KAAK,oBAAoB,GAAG,IAAI,CAAC,sBAAsB,EAAE,kBAAkB,GAAG,aAAa,CAAC,CAAC;AAE7F,MAAM,WAAW,yBAA0B,SAAQ,oBAAoB;IACrE,6DAA6D;IAC7D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,wEAAwE;IACxE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,0BAA2B,SAAQ,oBAAoB;IACtE,8DAA8D;IAC9D,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,yEAAyE;IACzE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,0EAA0E;IAC1E,oBAAoB,CAAC,EAAE,MAAM,EAAE,CAAC;CACjC;AAED,MAAM,MAAM,qBAAqB,GAAG,YAAY,GAAG,aAAa,CAAC;AAEjE,MAAM,WAAW,0BAA0B;IACzC,MAAM,EAAE,qBAAqB,CAAC;IAC9B,MAAM,EAAE,cAAc,CAAC;CACxB;AAED,MAAM,WAAW,yBAAyB;IACxC,MAAM,EAAE,qBAAqB,CAAC;IAC9B,KAAK,EAAE,KAAK,CAAC;CACd;AAED,MAAM,WAAW,sBAAsB;IACrC,sEAAsE;IACtE,MAAM,CAAC,EAAE,qBAAqB,CAAC;IAC/B,gFAAgF;IAChF,UAAU,CAAC,EAAE,KAAK,GAAG,yBAAyB,CAAC;IAC/C,iFAAiF;IACjF,WAAW,CAAC,EAAE,KAAK,GAAG,0BAA0B,CAAC;IACjD,uEAAuE;IACvE,QAAQ,EAAE,CAAC,KAAK,EAAE,0BAA0B,KAAK,IAAI,CAAC;IACtD,8DAA8D;IAC9D,OAAO,CAAC,EAAE,CAAC,KAAK,EAAE,yBAAyB,KAAK,IAAI,CAAC;CACtD;AAED,MAAM,WAAW,oBAAoB;IACnC,UAAU,CAAC,EAAE,qBAAqB,CAAC;IACnC,WAAW,CAAC,EAAE,qBAAqB,CAAC;CACrC;AAED;;;;GAIG;AACH,qBAAa,sBAAsB;;IACjC,QAAQ,CAAC,UAAU,CAAC,EAAE,uBAAuB,CAAC;IAC9C,QAAQ,CAAC,WAAW,CAAC,EAAE,uBAAuB,CAAC;IAI/C,OAAO,eAGN;IAED,OAAa,KAAK,CAAC,KAAK,EAAE,aAAa,EAAE,OAAO,EAAE,sBAAsB,GAAG,OAAO,CAAC,sBAAsB,CAAC,CA+CzG;IAED,KAAK,IAAI,IAAI,CAEZ;IAED,MAAM,IAAI,IAAI,CAEb;IAED,IAAI,IAAI,OAAO,CAAC,oBAAoB,CAAC,CAiBpC;IAED,OAAO,CAAC,YAAY;CAYrB;AAED,8EAA8E;AAC9E,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,aAAa,EACpB,OAAO,EAAE,sBAAsB,GAC9B,OAAO,CAAC,sBAAsB,CAAC,CAEjC"}

package/dist/index.js

@@ -0,0 +1,175 @@
+/**
+ * Qwen3-ASR inference and low-latency Core Audio capture on Apple Silicon.
+ */
+import { Qwen3AsrCapture, Qwen3AsrModel, Qwen3AsrStream, qwen3AsrAudioDevices, qwen3AsrInputDevices, } from '@mlx-node/core';
+export { Qwen3AsrCapture, Qwen3AsrModel, Qwen3AsrStream, qwen3AsrAudioDevices, qwen3AsrInputDevices, };
+/**
+ * Owns one model stream and one Core Audio source. Call `stop()` to drain the
+ * lock-free capture ring and receive the final, non-provisional transcript.
+ */
+export class Qwen3AsrRealtimeSession {
+    stream;
+    capture;
+    #stopPromise;
+    getLastError;
+    constructor(stream, capture, getLastError) {
+        this.stream = stream;
+        this.capture = capture;
+        this.getLastError = getLastError;
+    }
+    static async start(model, options) {
+        const stream = await model.createStream(options.stream);
+        let lastError;
+        let capture;
+        try {
+            capture = stream.startCapture(options.capture, (error, result) => {
+                if (error) {
+                    lastError = error;
+                    options.onError?.(error);
+                    return;
+                }
+                options.onResult(result);
+            });
+        }
+        catch (error) {
+            await stream.finish().catch(() => undefined);
+            throw error;
+        }
+        return new Qwen3AsrRealtimeSession(stream, capture, () => lastError);
+    }
+    get deviceName() {
+        return this.capture.deviceName;
+    }
+    get source() {
+        return this.capture.source;
+    }
+    get sampleRate() {
+        return this.capture.sampleRate;
+    }
+    get lastError() {
+        return this.getLastError();
+    }
+    pause() {
+        this.capture.pause();
+    }
+    resume() {
+        this.capture.resume();
+    }
+    stop() {
+        this.#stopPromise ??= (async () => {
+            let capture;
+            try {
+                capture = await this.capture.stop();
+            }
+            catch (error) {
+                await this.stream.finish().catch(() => undefined);
+                throw error;
+            }
+            const result = await this.stream.finish();
+            const error = this.lastError;
+            if (error)
+                throw error;
+            return { result, capture };
+        })();
+        return this.#stopPromise;
+    }
+}
+export function startRealtimeTranscription(model, options) {
+    return Qwen3AsrRealtimeSession.start(model, options);
+}
+/**
+ * A meeting owns one independently clocked transcription track per enabled
+ * source. Results stay source-tagged so speaker-mic audio and remote/system
+ * audio are never silently mixed or ordered by unrelated device clocks.
+ */
+export class Qwen3AsrMeetingSession {
+    microphone;
+    systemAudio;
+    #stopPromise;
+    constructor(tracks) {
+        this.microphone = tracks.microphone;
+        this.systemAudio = tracks.systemAudio;
+    }
+    static async start(model, options) {
+        if (options.microphone === false && options.systemAudio === false) {
+            throw new Error('At least one meeting audio source must be enabled');
+        }
+        const tracks = {};
+        const started = [];
+        const startTrack = async (source, capture) => {
+            const session = await Qwen3AsrRealtimeSession.start(model, {
+                stream: options.stream,
+                capture,
+                onResult: (result) => options.onResult({ source, result }),
+                onError: (error) => options.onError?.({ source, error }),
+            });
+            started.push(session);
+            return session;
+        };
+        try {
+            // Ask for system-audio permission before opening the microphone. This
+            // avoids leaving a live mic running while the system permission sheet is
+            // waiting for a response.
+            if (options.systemAudio !== false) {
+                tracks.systemAudio = await startTrack('systemAudio', {
+                    ...options.systemAudio,
+                    source: 'systemAudio',
+                });
+            }
+            if (options.microphone !== false) {
+                tracks.microphone = await startTrack('microphone', {
+                    ...options.microphone,
+                    source: 'microphone',
+                });
+            }
+        }
+        catch (error) {
+            await Promise.allSettled(started.map((session) => session.stop()));
+            throw error;
+        }
+        return new Qwen3AsrMeetingSession(tracks);
+    }
+    pause() {
+        this.forEachTrack((track) => track.pause());
+    }
+    resume() {
+        this.forEachTrack((track) => track.resume());
+    }
+    stop() {
+        this.#stopPromise ??= (async () => {
+            const microphone = this.microphone?.stop();
+            const systemAudio = this.systemAudio?.stop();
+            const settled = await Promise.allSettled([microphone, systemAudio].filter((promise) => promise !== undefined));
+            const failure = settled.find((result) => result.status === 'rejected');
+            if (failure)
+                throw failure.reason;
+            const final = {};
+            let index = 0;
+            if (microphone)
+                final.microphone = settled[index++].value;
+            if (systemAudio)
+                final.systemAudio = settled[index].value;
+            return final;
+        })();
+        return this.#stopPromise;
+    }
+    forEachTrack(action) {
+        let firstError;
+        for (const track of [this.microphone, this.systemAudio]) {
+            if (!track)
+                continue;
+            try {
+                action(track);
+            }
+            catch (error) {
+                firstError ??= error;
+            }
+        }
+        if (firstError)
+            throw firstError;
+    }
+}
+/** Start microphone and system-audio transcription as one meeting session. */
+export function startMeetingTranscription(model, options) {
+    return Qwen3AsrMeetingSession.start(model, options);
+}

package/package.json

@@ -0,0 +1,35 @@
+{
+  "name": "@mlx-node/asr",
+  "version": "0.0.9",
+  "homepage": "https://github.com/mlx-node/mlx-node",
+  "bugs": {
+    "url": "https://github.com/mlx-node/mlx-node/issues"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/mlx-node/mlx-node.git",
+    "directory": "packages/asr"
+  },
+  "files": [
+    "dist"
+  ],
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc -b"
+  },
+  "dependencies": {
+    "@mlx-node/core": "workspace:*"
+  },
+  "devDependencies": {
+    "@types/node": "^26.4.0"
+  }
+}