@gjsify/media-capture 0.0.0-stage → 0.64.14

package/README.md

@@ -1,3 +1,53 @@
-# Temporary Holding Version
+# @gjsify/media-capture
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+The GStreamer capture chain for GJS: pick a microphone or camera source that actually opens, bind a device to it, express constraints as caps, and read 16 kHz mono PCM for speech recognition. [`@gjsify/webrtc`](../webrtc) builds `getUserMedia` on it; extracted from there per [ADR 0127](../../../docs/adr/0127-speech-recognition-on-local-engines.md). Tier 3 (experimental, [ADR 0003](../../../docs/adr/0003-package-tiering.md)) until promoted. GJS only.
+
+Part of the [gjsify](https://github.com/gjsify/gjsify) project — Node.js and Web APIs for GJS (GNOME JavaScript).
+
+## Installation
+
+```bash
+gjsify install @gjsify/media-capture
+```
+
+Needs GStreamer with `gst-plugins-base` (`GstApp` typelib).
+
+## PCM for speech recognition
+
+```typescript
+import { openPcmSource } from '@gjsify/media-capture';
+
+const mic = openPcmSource({ sampleRate: 16000, format: 'S16LE' });
+if (mic.synthetic) console.warn('no microphone opened; this is a test tone');
+
+for await (const chunk of mic) {
+    // chunk: Uint8Array of interleaved little-endian mono samples
+    recognizer.feed(chunk);
+    if (done) break; // leaving the loop stops the pipeline
+}
+```
+
+`source ! audioconvert ! audioresample ! capsfilter ! appsink`. The default source is the capture chain's audio source; `deviceId` selects another, and `source` swaps in any `Gst.Element` (a file, a test tone). `read()` returns the next chunk or `null` at end of stream and rejects on a pipeline error; `stop()` is idempotent. The appsink is polled, never driven by `new-sample`, because that signal fires on GStreamer's streaming thread where GJS refuses to run JS. A reader that falls behind blocks the source (200 buffers deep) instead of losing audio.
+
+## The capture chain
+
+```typescript
+import { openCapture } from '@gjsify/media-capture';
+
+const { source, pipeline } = openCapture('video', { width: 640, height: 480, frameRate: 30 });
+```
+
+| Export | Does |
+|---|---|
+| `openCapture(kind, constraints)` | A source in a fresh NULL pipeline, with a capsfilter when a constraint maps to caps |
+| `createAudioSource` / `createVideoSource` | The bare source element |
+| `bindDevice(kind, element, deviceId)` | Writes `deviceId` to whichever of `device`, `target-object`, `path` the element declares; warns once when none |
+| `isTestSource(element)` | True for `audiotestsrc` / `videotestsrc` |
+| `buildAudioCaps` / `buildVideoCaps` | W3C constraints to a caps string |
+| `ensureGstInit`, `excludeUnboundedAutodetectCandidates` | Lazy `Gst.init`; keeps `openalsrc`/`openalsink` out of autodetect |
+
+Source order: `pipewiresrc`, `pulsesrc`, `autoaudiosrc` for audio; `pipewiresrc`, `v4l2src`, `autovideosrc` for video; then `audiotestsrc` / `videotestsrc`. A candidate is used only if it starts. The verdict is cached per process, so a device plugged in later is not seen until restart; elements are still made fresh per call.
+
+## License
+
+MIT

package/lib/esm/_virtual/_rolldown/runtime.js

@@ -0,0 +1 @@
+var e=Object.defineProperty,__name=(t,n)=>e(t,`name`,{value:n,configurable:!0});export{__name};

package/lib/esm/capture-source.js

@@ -0,0 +1 @@
+import"./_virtual/_rolldown/runtime.js";import{Gst as e,ensureGstInit as t}from"./gst-init.js";function openCapture(n,r={}){t();let i=n===`audio`?createAudioSource(r):createVideoSource(r),a=new e.Pipeline;a.add(i);let o=n===`audio`?buildAudioCaps(r):buildVideoCaps(r);if(o){let t=e.ElementFactory.make(`capsfilter`,null);t.caps=e.Caps.from_string(o),a.add(t),i.link(t)}return{source:i,pipeline:a}}function isTestSource(e){let t=e.get_factory()?.get_name();return t===`audiotestsrc`||t===`videotestsrc`}function buildAudioCaps(e){let t=[];return e.sampleRate!=null&&t.push(`rate=${Math.trunc(e.sampleRate)}`),e.channelCount!=null&&t.push(`channels=${Math.trunc(e.channelCount)}`),t.length===0?null:`audio/x-raw,${t.join(`,`)}`}function buildVideoCaps(e){let t=[];return e.width!=null&&t.push(`width=${Math.trunc(e.width)}`),e.height!=null&&t.push(`height=${Math.trunc(e.height)}`),e.frameRate!=null&&t.push(`framerate=${Math.trunc(e.frameRate)}/1`),t.length===0?null:`video/x-raw,${t.join(`,`)}`}function probeSource(t,n){let r=e.ElementFactory.make(t,null);if(!r)return null;let i=e.ElementFactory.make(n,null),a=e.ElementFactory.make(`fakesink`,null);if(!i||!a)return r;let o=new e.Pipeline;o.add(r),o.add(i),o.add(a),r.link(i),i.link(a);let s=o.set_state(e.State.PLAYING);return s!==e.StateChangeReturn.FAILURE&&(s=o.get_state(500*Number(e.MSECOND))[0]),o.set_state(e.State.NULL),r.unlink(i),o.remove(r),s===e.StateChangeReturn.FAILURE?null:r}const n=new Map;function chooseSource(t,r,i,a){let o=n.get(t);if(o)return e.ElementFactory.make(o,null);for(let e of r){let r=probeSource(e,i);if(r)return n.set(t,e),r}return n.set(t,a),e.ElementFactory.make(a,null)}const r=[`device`,`target_object`,`path`],i=new Set;function bindDevice(e,t,n){if(n){for(let e of r)if(e in t){t[e]=n;return}i.has(e)||(i.add(e),console.warn(`getUserMedia: deviceId ${JSON.stringify(n)} ignored — ${t.name} has no device property, so the ${e} source is the host default. getSupportedConstraints() reports deviceId: true.`))}}function createAudioSource(e={}){let t=chooseSource(`audio`,[`pipewiresrc`,`pulsesrc`,`autoaudiosrc`],`audioconvert`,`audiotestsrc`);try{t.is_live=!0}catch{}return n.get(`audio`)===`audiotestsrc`&&(t.wave=0),bindDevice(`audio`,t,e.deviceId),t}function createVideoSource(e={}){let t=chooseSource(`video`,[`pipewiresrc`,`v4l2src`,`autovideosrc`],`videoconvert`,`videotestsrc`);return n.get(`video`)===`videotestsrc`&&(t.is_live=!0,t.pattern=0),bindDevice(`video`,t,e.deviceId),t}export{bindDevice,buildAudioCaps,buildVideoCaps,createAudioSource,createVideoSource,isTestSource,openCapture};

package/lib/esm/gst-init.js

@@ -0,0 +1 @@
+import"./_virtual/_rolldown/runtime.js";import e from"gi://Gst?version=1.0";import t from"gi://GLib?version=2.0";let n=!1;const r=[`openalsrc`,`openalsink`];function excludeUnboundedAutodetectCandidates(n=t.getenv(`GJSIFY_GST_KEEP_OPENAL`)===`1`){if(n)return;let i=e.Registry.get();for(let t of r)i.lookup_feature(t)?.set_rank(e.Rank.NONE)}function ensureGstInit(){n||=(e.init(null),excludeUnboundedAutodetectCandidates(),!0)}export{e as Gst,ensureGstInit,excludeUnboundedAutodetectCandidates};

package/lib/esm/index.js

@@ -0,0 +1 @@
+import{Gst as e,ensureGstInit as t,excludeUnboundedAutodetectCandidates as n}from"./gst-init.js";import{bindDevice as r,buildAudioCaps as i,buildVideoCaps as a,createAudioSource as o,createVideoSource as s,isTestSource as c,openCapture as l}from"./capture-source.js";import{openPcmSource as u}from"./pcm-source.js";export{e as Gst,r as bindDevice,i as buildAudioCaps,a as buildVideoCaps,o as createAudioSource,s as createVideoSource,t as ensureGstInit,n as excludeUnboundedAutodetectCandidates,c as isTestSource,l as openCapture,u as openPcmSource};

package/lib/esm/pcm-source.js

@@ -0,0 +1 @@
+import"./_virtual/_rolldown/runtime.js";import{Gst as e,ensureGstInit as t}from"./gst-init.js";import{createAudioSource as n,isTestSource as r}from"./capture-source.js";import i from"gi://GLib?version=2.0";import a from"gi://GstApp?version=1.0";function ensureGstAppLoaded(){if(a.AppSink===void 0)throw Error(`GstApp typelib is unavailable — install gstreamer1-plugins-base (gst-plugins-base)`)}function sleep(e){return new Promise(t=>{i.timeout_add(i.PRIORITY_DEFAULT,e,()=>(t(),i.SOURCE_REMOVE))})}function openPcmSource(i={}){t(),ensureGstAppLoaded();let a=Math.trunc(i.sampleRate??16e3),o=i.format??`S16LE`,s=i.pollIntervalMs??10;if(!(a>0))throw RangeError(`openPcmSource: invalid sampleRate ${i.sampleRate}`);if(o!==`S16LE`&&o!==`F32LE`)throw TypeError(`openPcmSource: unsupported format ${o}`);let c=i.source??n({deviceId:i.deviceId}),l=e.ElementFactory.make(`audioconvert`,null),u=e.ElementFactory.make(`audioresample`,null),d=e.ElementFactory.make(`capsfilter`,null),f=e.ElementFactory.make(`appsink`,null);if(!l||!u||!d||!f)throw Error(`openPcmSource: GStreamer audioconvert, audioresample or appsink is missing (gst-plugins-base)`);d.caps=e.Caps.from_string(`audio/x-raw,format=${o},rate=${a},channels=1,layout=interleaved`);let p=f;p.emit_signals=!1,p.sync=!1,p.max_buffers=200;let m=new e.Pipeline,h=[c,l,u,d,f];for(let e of h)m.add(e);for(let e=0;e<h.length-1;e++)if(!h[e].link(h[e+1]))throw Error(`openPcmSource: cannot link ${h[e].name} to ${h[e+1].name}`);let g=m.get_bus(),_=f,v=!1,y=!1,failIfBusError=()=>{let t=g.pop_filtered(e.MessageType.ERROR);if(!t)return;let[n,r]=t.parse_error();throw Error(`openPcmSource: ${n.message}${r?` (${r})`:``}`)},b={sampleRate:a,format:o,channels:1,label:c.get_factory()?.get_name()??c.name,synthetic:r(c),start(){if(!(v||y)&&(v=!0,m.set_state(e.State.PLAYING)===e.StateChangeReturn.FAILURE))throw failIfBusError(),Error(`openPcmSource: ${b.label} failed to start`)},async read(){for(b.start();!y;){failIfBusError();let t=_.try_pull_sample(0);if(t){let n=t.get_buffer();if(n){let[t,r]=n.map(e.MapFlags.READ);if(t){let e=new Uint8Array(r.data);if(n.unmap(r),e.length>0)return e}}continue}if(_.is_eos())return null;await sleep(s)}return null},stop(){y||(y=!0,m.set_state(e.State.NULL))},async*[Symbol.asyncIterator](){try{for(let e=await b.read();e;e=await b.read())yield e}finally{b.stop()}}};return b}export{openPcmSource};

package/lib/types/capture-source.d.ts

@@ -0,0 +1,35 @@
+import { Gst } from './gst-init.js';
+export interface CaptureConstraints {
+    deviceId?: string;
+    sampleRate?: number;
+    channelCount?: number;
+    width?: number;
+    height?: number;
+    frameRate?: number;
+}
+export type CaptureKind = 'audio' | 'video';
+/** A capture source inside its own pipeline, constraints applied, not yet started. */
+export interface Capture {
+    source: Gst.Element;
+    pipeline: Gst.Pipeline;
+}
+/**
+ * Open a capture source for `kind` in a fresh pipeline, with the constraints
+ * mapped to a capsfilter behind it. The pipeline stays in NULL; the caller
+ * decides when to start it.
+ */
+export declare function openCapture(kind: CaptureKind, constraints?: CaptureConstraints): Capture;
+/** True when `source` is a synthetic test source, i.e. no capture device opened on this host. */
+export declare function isTestSource(source: Gst.Element): boolean;
+/** Build a GStreamer caps string for audio constraints. */
+export declare function buildAudioCaps(c: CaptureConstraints): string | null;
+/** Build a GStreamer caps string for video constraints. */
+export declare function buildVideoCaps(c: CaptureConstraints): string | null;
+/**
+ * Bind a W3C `deviceId` to a source element, reporting when the element cannot
+ * express one. A caller that asked for a specific device and silently receives
+ * another is the defect #1513 reports, so the unbindable case is never quiet.
+ */
+export declare function bindDevice(kind: string, el: Gst.Element, deviceId: string | undefined): void;
+export declare function createAudioSource(constraints?: CaptureConstraints): Gst.Element;
+export declare function createVideoSource(constraints?: CaptureConstraints): Gst.Element;

package/lib/types/capture-source.spec.d.ts

@@ -0,0 +1,2 @@
+declare const _default: () => Promise<void>;
+export default _default;

package/lib/types/gst-init.d.ts

@@ -0,0 +1,5 @@
+import Gst from 'gi://Gst?version=1.0';
+/** TEST SEAM (exported) — see the `keepOpenal` parameter doc above. */
+export declare function excludeUnboundedAutodetectCandidates(keepOpenal?: boolean): void;
+export declare function ensureGstInit(): void;
+export { Gst };

package/lib/types/index.d.ts

@@ -0,0 +1,5 @@
+export { Gst, ensureGstInit, excludeUnboundedAutodetectCandidates } from './gst-init.js';
+export { openCapture, createAudioSource, createVideoSource, bindDevice, isTestSource, buildAudioCaps, buildVideoCaps, } from './capture-source.js';
+export type { Capture, CaptureKind, CaptureConstraints } from './capture-source.js';
+export { openPcmSource } from './pcm-source.js';
+export type { PcmFormat, PcmSource, PcmSourceOptions } from './pcm-source.js';

package/lib/types/pcm-source.d.ts

@@ -0,0 +1,34 @@
+import { Gst } from './gst-init.js';
+export type PcmFormat = 'S16LE' | 'F32LE';
+export interface PcmSourceOptions {
+    /** Samples per second. Default 16000, what the recognition engines take. */
+    sampleRate?: number;
+    /** Sample format. Default `S16LE`. */
+    format?: PcmFormat;
+    /** A device id as `enumerateDevices()` reports it. Default: the host's default input. */
+    deviceId?: string;
+    /** Use this element instead of the capture chain — a file, a test tone, a fake. */
+    source?: Gst.Element;
+    /** How often an empty appsink is polled, in ms. Default 10. */
+    pollIntervalMs?: number;
+}
+export interface PcmSource extends AsyncIterable<Uint8Array> {
+    readonly sampleRate: number;
+    readonly format: PcmFormat;
+    readonly channels: 1;
+    /** Factory name of the source element, e.g. `pipewiresrc`. */
+    readonly label: string;
+    /** True when no capture device opened and a test tone stands in. */
+    readonly synthetic: boolean;
+    /** Start the pipeline. Idempotent; `read()` calls it. Throws if the source cannot start. */
+    start(): void;
+    /** The next chunk of interleaved little-endian samples, or `null` at end of stream. Rejects on a pipeline error. */
+    read(): Promise<Uint8Array | null>;
+    /** Stop and release the pipeline. Idempotent; a pending `read()` resolves `null`. */
+    stop(): void;
+}
+/**
+ * Open a mono PCM source: `source ! audioconvert ! audioresample ! capsfilter ! appsink`.
+ * The pipeline is built but not started.
+ */
+export declare function openPcmSource(options?: PcmSourceOptions): PcmSource;

package/lib/types/pcm-source.spec.d.ts

@@ -0,0 +1,2 @@
+declare const _default: () => Promise<void>;
+export default _default;

package/package.json

@@ -1,6 +1,72 @@
 {
-  "name": "@gjsify/media-capture",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+    "name": "@gjsify/media-capture",
+    "version": "0.64.14",
+    "description": "GStreamer capture chain for GJS (pipewiresrc / pulsesrc / v4l2src with probed fallback) and a 16 kHz mono PCM audio source",
+    "type": "module",
+    "module": "lib/esm/index.js",
+    "types": "lib/types/index.d.ts",
+    "exports": {
+        ".": {
+            "types": "./lib/types/index.d.ts",
+            "default": "./lib/esm/index.js"
+        }
+    },
+    "files": [
+        "lib"
+    ],
+    "sideEffects": false,
+    "scripts": {
+        "clear": "gjsify clear lib tsconfig.tsbuildinfo tsconfig.types.tsbuildinfo test.gjs.mjs",
+        "check": "gjsify tsc --noEmit",
+        "build": "gjsify run build:gjsify && gjsify run build:types",
+        "build:gjsify": "gjsify build --library 'src/**/*.{ts,js}' --exclude 'src/**/*.spec.{mts,ts}' 'src/test*.{mts,ts}'",
+        "build:types": "gjsify tsc",
+        "build:test": "gjsify run build:test:gjs",
+        "build:test:gjs": "gjsify build src/test.mts --app gjs --outfile test.gjs.mjs",
+        "test": "gjsify run build:gjsify && gjsify run build:test && gjsify run test:gjs",
+        "test:gjs": "gjsify run test.gjs.mjs"
+    },
+    "keywords": [
+        "gjs",
+        "gstreamer",
+        "capture",
+        "microphone",
+        "camera",
+        "pipewire",
+        "pcm"
+    ],
+    "dependencies": {
+        "@gjsify/dom-exception": "^0.64.14",
+        "@gjsify/node-globals": "^0.64.14"
+    },
+    "devDependencies": {
+        "@girs/gjs": "5.4.0",
+        "@girs/glib-2.0": "5.4.0",
+        "@girs/gobject-2.0": "5.4.0",
+        "@girs/gst-1.0": "5.4.0",
+        "@girs/gstapp-1.0": "5.4.0",
+        "@gjsify/cli": "^0.64.14",
+        "@gjsify/unit": "^0.64.14",
+        "@types/node": "^25.9.2",
+        "typescript": "^6.0.3"
+    },
+    "gjsify": {
+        "runtimes": {
+            "gjs": "polyfill",
+            "node": "none",
+            "browser": "none",
+            "nativescript": "none"
+        },
+        "tier": 1
+    },
+    "license": "MIT",
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/gjsify/gjsify.git",
+        "directory": "packages/web/media-capture"
+    },
+    "bugs": {
+        "url": "https://github.com/gjsify/gjsify/issues"
+    },
+    "homepage": "https://github.com/gjsify/gjsify/tree/main/packages/web/media-capture#readme"
+}