@solidrt/core 0.0.8 → 0.0.10

package/package.json

@@ -1,13 +1,16 @@
 {
   "name": "@solidrt/core",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "license": "MIT",
   "author": "Antoine van Wel",
   "type": "module",
   "main": "src/index.ts",
   "exports": {
     ".": "./src/index.ts",
+    "./camera": "./src/camera.ts",
     "./gpu": "./src/gpu.ts",
+    "./microphone": "./src/microphone.ts",
+    "./speech": "./src/speech.ts",
     "./jsx-runtime": "./jsx-runtime.d.ts",
     "./jsx-runtime-dev": "./jsx-runtime.d.ts"
   },
@@ -20,7 +23,7 @@
     "colord": "^2.9.3"
   },
   "devDependencies": {
-    "@solidrt/flux-types": "0.0.8"
+    "@solidrt/flux-types": "0.0.10"
   },
   "peerDependencies": {
     "@solidjs/signals": "2.0.0-beta.14",

package/src/camera-view.tsx

@@ -0,0 +1,55 @@
+import { createSignal, onCleanup } from "@solidjs/signals"
+import { openCamera, type BarcodeResult, type Camera } from "./camera"
+
+// Convenience viewfinder over openCamera: opens on mount, renders the stream
+// texture, closes on cleanup. Use openCamera directly for anything it does not
+// cover; this is just composition, no extra capability.
+
+export interface CameraViewProps {
+  /** Explicit device id from listCameras(); takes precedence over facing. */
+  camera?: number
+  facing?: "front" | "back"
+  /**
+   * Size hint for the stream and explicit size of the view. Omit height to
+   * follow the stream's aspect ratio, which can flip when a phone rotates.
+   */
+  width?: number
+  height?: number
+  scan?: "qr"[]
+  onReady?: (cam: Camera) => void
+  onError?: (error: Error) => void
+  onBarcode?: (result: BarcodeResult) => void
+}
+
+export function CameraView(props: CameraViewProps) {
+  let [texture, setTexture] = createSignal<number | undefined>(undefined)
+  let cam: Camera | undefined
+  let disposed = false
+
+  openCamera({
+    camera: props.camera,
+    facing: props.facing,
+    width: props.width,
+    height: props.height,
+    scan: props.scan,
+  })
+    .then((opened) => {
+      if (disposed) {
+        opened.close()
+        return
+      }
+      cam = opened
+      if (props.onBarcode) opened.onBarcode(props.onBarcode)
+      setTexture(opened.texture)
+      props.onReady?.(opened)
+    })
+    .catch((e) => props.onError?.(e instanceof Error ? e : new Error(String(e))))
+
+  onCleanup(() => {
+    disposed = true
+    cam?.close()
+    cam = undefined
+  })
+
+  return <texture src={texture()} width={props.width} height={props.height} />
+}

package/src/camera.ts

@@ -0,0 +1,78 @@
+// Camera capture. A camera streams into a texture id, so a viewfinder is just
+// <texture src={cam.texture} />. Opening the camera IS the permission request
+// (SDL semantics): the promise resolves once the stream is configured and
+// rejects if the user denies access.
+
+import { on } from "srt:events"
+
+export type CameraFacing = "front" | "back" | "unknown"
+
+export type CameraInfo = {
+  id: number
+  name: string
+  facing: CameraFacing
+}
+
+export type BarcodeFormat = "qr"
+
+export type BarcodeResult = {
+  data: string
+  format: BarcodeFormat
+}
+
+export type CameraOptions = {
+  /** Explicit device id from listCameras(); takes precedence over facing. */
+  camera?: number
+  /** Pick the first camera with this facing (falls back to the first camera). */
+  facing?: "front" | "back"
+  /** Size hint; the device picks the closest supported mode. */
+  width?: number
+  height?: number
+  /** Decode these barcode formats from the stream (delivered via onBarcode). */
+  scan?: BarcodeFormat[]
+}
+
+export type Camera = {
+  /** Texture id updated every frame while open; render with <texture src={...}>. */
+  texture: number
+  /** Actual stream size (may differ from the requested hint). */
+  width: number
+  height: number
+  /** Receive decoded barcodes (requires the scan option; replaces any previous callback). */
+  onBarcode(callback: (result: BarcodeResult) => void): void
+  /** Release the device. The texture keeps showing the last frame. */
+  close(): void
+}
+
+export function listCameras(): CameraInfo[] {
+  return camera.listCameras()
+}
+
+// Camera hotplug. Re-enumerate with listCameras() to see the new device set.
+// Events only flow once the camera subsystem is up, i.e. after the first
+// listCameras() or openCamera() call. Returns an unsubscribe function.
+// Coverage caveat (SDL 3.4.8): only Android delivers both add and remove. On
+// Linux you get added=true but not added=false (removal is broken upstream);
+// on macOS/Windows there is no camera hotplug at all, so nothing fires.
+export function onDeviceChange(callback: (event: { added: boolean }) => void): () => void {
+  return on("cameraDeviceChange", callback)
+}
+
+// One-shot scan of an RGBA8 pixel buffer for QR codes; composes with
+// decodeImage: scanBarcodes(img.data, img.width, img.height).
+export function scanBarcodes(data: Uint8Array, width: number, height: number): BarcodeResult[] {
+  return camera.scanImage(data, width, height)
+}
+
+export async function openCamera(options: CameraOptions = {}): Promise<Camera> {
+  let opened = await camera.open(options)
+  return {
+    texture: opened.texture,
+    width: opened.width,
+    height: opened.height,
+    onBarcode: (callback: (result: BarcodeResult) => void) => camera.setBarcodeCallback(opened.handle, callback),
+    close: () => camera.close(opened.handle),
+  }
+}
+
+export { CameraView, type CameraViewProps } from "./camera-view"

package/src/gpu.ts

@@ -10,4 +10,14 @@ export function decodeImage(bytes: Uint8Array): DecodedImage {
 
 export function createTexture(data: Uint8Array, width: number, height: number): number {
   return gpu.createTexture(data, width, height)
+}
+
+// The texture keeps reading from `data` (which may hold multiple frames):
+// mutate it in place, then call uploadTexture to push the pixels to the GPU.
+export function createMutableTexture(data: Uint8Array, width: number, height: number): number {
+  return gpu.createMutableTexture(data, width, height)
+}
+
+export function uploadTexture(textureId: number, offset: number = 0): void {
+  gpu.uploadTexture(textureId, offset)
 }

package/src/microphone.ts

@@ -0,0 +1,40 @@
+// Microphone capture. A session delivers raw mono float32 samples at the
+// requested sample rate (the device format is converted by SDL); read()
+// drains whatever was captured since the last call. Captured audio buffers
+// until read, so poll read() regularly (e.g. once per frame) while open.
+
+export type MicrophoneInfo = {
+  id: number
+  name: string
+}
+
+export type MicrophoneOptions = {
+  /** Explicit device id from listMicrophones(); default is the system default recording device. */
+  microphone?: number
+  /** Sample rate of the delivered samples (the device rate is converted). Default 16000. */
+  sampleRate?: number
+}
+
+export type Microphone = {
+  /** Sample rate of read() samples. */
+  sampleRate: number
+  /** Drain the mono float32 samples captured since the last read. */
+  read(): Float32Array
+  /** Release the device. */
+  close(): void
+}
+
+export function listMicrophones(): MicrophoneInfo[] {
+  return microphone.listMicrophones()
+}
+
+// Async to leave room for an OS permission prompt on platforms that need one
+// (the desktop backends open synchronously).
+export async function openMicrophone(options: MicrophoneOptions = {}): Promise<Microphone> {
+  let opened = microphone.open(options)
+  return {
+    sampleRate: opened.sampleRate,
+    read: () => microphone.read(opened.handle),
+    close: () => microphone.close(opened.handle),
+  }
+}

package/src/speech.ts

@@ -0,0 +1,67 @@
+// Speech recognition. A session captures the microphone, segments utterances
+// by silence (Silero VAD) and transcribes each one with Whisper, delivering
+// final transcripts through onResult. With wakeWord the session starts
+// asleep behind an efficient wake word detector (livekit-wakeword) and only
+// transcribes after the wake word. startRecognition resolves once the models
+// are loaded and listening has begun; it rejects when loading fails.
+// Models are passed as bytes so any source composes: flux:fs file(), fetch
+// (incl. the dev-server file proxy), or a download cache layered on top.
+// Requires a runtime built with speech support.
+
+export type SpeechOptions = {
+  /** A ggml Whisper model (file contents, e.g. ggml-tiny.en.bin). */
+  model: Uint8Array
+  /** A ggml Silero VAD model (file contents). */
+  vadModel: Uint8Array
+  /** Whisper language code; "auto" detects (multilingual models only). Default "en". */
+  language?: string
+  /** Explicit microphone device id from listMicrophones(). */
+  microphone?: number
+  /** Stop automatically after the first final result (with wakeWord: re-arm instead, one result per wake). */
+  singleUtterance?: boolean
+  /** Also deliver snapshot transcripts (final: false) while an utterance is still being spoken. */
+  interimResults?: boolean
+  /**
+   * Wake word: start asleep, fire onWake when it is heard, then transcribe
+   * the speech that follows. How the wake word is specified depends on the
+   * engine. The current engine detects with a trained classifier and takes
+   * the model's bytes (livekit-wakeword ONNX, e.g. the pretrained "hey
+   * livekit"; custom phrases are trained offline with its toolkit). Phrase
+   * strings are reserved for engines that match text; passing them to this
+   * engine rejects with an error.
+   */
+  wakeWord?: Uint8Array | string | string[]
+  /** Detector confidence (0..1) that counts as a wake. Default 0.5. */
+  wakeThreshold?: number
+}
+
+export type SpeechResult = {
+  /** Transcript of the utterance (a snapshot of it when final is false). */
+  text: string
+  /** True for the completed utterance, false for interim snapshots. */
+  final: boolean
+}
+
+export type SpeechSession = {
+  /** Receive transcripts (replaces any previous callback). */
+  onResult(callback: (result: SpeechResult) => void): void
+  /** The user started speaking (replaces any previous callback). */
+  onSpeechStart(callback: () => void): void
+  /** The utterance ended; its final result follows once transcribed. */
+  onSpeechEnd(callback: () => void): void
+  /** The wake word was heard (wakeWordModel sessions only). */
+  onWake(callback: () => void): void
+  /** Release the microphone and discard any utterance in progress. */
+  stop(): void
+}
+
+export async function startRecognition(options: SpeechOptions): Promise<SpeechSession> {
+  let started = await speech.start(options)
+  return {
+    onResult: (callback: (result: SpeechResult) => void) => speech.setResultCallback(started.handle, callback),
+    onSpeechStart: (callback: () => void) => speech.setSpeechStartCallback(started.handle, callback),
+    onSpeechEnd: (callback: () => void) => speech.setSpeechEndCallback(started.handle, callback),
+    onWake: (callback: () => void) => speech.setWakeCallback(started.handle, callback),
+    stop: () => speech.stop(started.handle),
+  }
+}

package/src/types.d.ts

@@ -2,16 +2,28 @@
 
 import type { JSX as SolidJSX } from "@solidjs/signals"
 
+// UI event bus (lattice), provided by the runtime as a builtin module.
+// on/once return an unsubscribe function.
+declare module "srt:events" {
+  export function on(event: string, callback: (data: any) => void): () => void
+  export function once(event: string, callback: (data: any) => void): () => void
+}
+
+// Dev-server control surface (lattice). Present only in dev/go builds; in other
+// builds `available` is false and the functions are no-ops.
+declare module "srt:dev" {
+  export const available: boolean
+  export const canDiscover: boolean
+  export const recents: string[]
+  export function connect(address: string): void
+  export function discover(): void
+  export function stop(): void
+}
+
 declare global {
   function requestAnimationFrame(callback: (time: number) => void): number
   function cancelAnimationFrame(id: number): void
 
-  // UI event bus (lattice). on/once return an unsubscribe function.
-  let srt: {
-    on(event: string, callback: (data: any) => void): () => void
-    once(event: string, callback: (data: any) => void): () => void
-  }
-
   let ffi: {
     createRoot(id: number): void
     createNode(id: number, kind: string): void
@@ -19,14 +31,45 @@ declare global {
     deleteNode(parentId: number, nodeId: number): void
     setProperty(nodeId: number, name: string, value: unknown): void
     setTextInputActive(active: boolean): void
+    requestFrame(): void
     measureText(text: string, options?: MeasureTextOptions): { width: number, height: number }
     getBoundingBox(id: number): { x: number, y: number, width: number, height: number } | null
   }
 
   let gpu: {
     createTexture(data: Uint8Array, width: number, height: number): number
+    createMutableTexture(data: Uint8Array, width: number, height: number): number
+    uploadTexture(textureId: number, offset?: number): void
     decodeImage(bytes: Uint8Array): { data: Uint8Array, width: number, height: number }
   }
+
+  let camera: {
+    listCameras(): { id: number, name: string, facing: "front" | "back" | "unknown" }[]
+    open(options: { camera?: number, facing?: "front" | "back", width?: number, height?: number, scan?: string[] }):
+      Promise<{ handle: number, texture: number, width: number, height: number }>
+    setBarcodeCallback(handle: number, callback: (result: { data: string, format: "qr" }) => void): void
+    scanImage(data: Uint8Array, width: number, height: number): { data: string, format: "qr" }[]
+    close(handle: number): void
+  }
+
+  let microphone: {
+    listMicrophones(): { id: number, name: string }[]
+    open(options: { microphone?: number, sampleRate?: number }): { handle: number, sampleRate: number }
+    read(handle: number): Float32Array
+    close(handle: number): void
+  }
+
+  let speech: {
+    start(options: {
+      model: Uint8Array, vadModel: Uint8Array, language?: string, microphone?: number,
+      singleUtterance?: boolean, interimResults?: boolean, wakeWord?: Uint8Array | string | string[], wakeThreshold?: number,
+    }): Promise<{ handle: number }>
+    setResultCallback(handle: number, callback: (result: { text: string, final: boolean }) => void): void
+    setSpeechStartCallback(handle: number, callback: () => void): void
+    setSpeechEndCallback(handle: number, callback: () => void): void
+    setWakeCallback(handle: number, callback: () => void): void
+    stop(handle: number): void
+  }
 }
 
 export interface MeasureTextOptions {
@@ -119,6 +162,19 @@ export interface PaintProps {
 export interface TransformProps {
   rotate?: number
   scale?: number
+  // Per-axis scale; overrides `scale` on that axis (e.g. scaleX for a flip).
+  scaleX?: number
+  scaleY?: number
+  // 3D rotation about the horizontal axis, in radians (a top/bottom tilt). Like
+  // rotateY, reads as 3D only with `perspective` set.
+  rotateX?: number
+  // 3D rotation about the vertical axis, in radians, for a card-flip. Reads as a
+  // real flip only with `perspective` set; on its own it is an orthographic
+  // squash (like scaleX).
+  rotateY?: number
+  // Perspective viewing distance in pixels (CSS `perspective`). Enables the 3D
+  // depth for rotateY; larger values give a shallower effect.
+  perspective?: number
   x?: number
   y?: number
   cx?: number
@@ -180,6 +236,25 @@ export interface WindowProps extends LayoutProps {
 export interface ViewProps extends LayoutProps, TransformProps, PointerProps {
   children?: Children
   trace?: boolean
+  /**
+   * Corner radii for the clip applied when overflow is non-visible (hidden,
+   * clip, scroll on both axes). A single number rounds all four corners; an
+   * array is [top-left, top-right, bottom-right, bottom-left]. Without overflow
+   * clipping this has no effect.
+   */
+  clipRadius?: number | [number, number, number, number]
+  /**
+   * Marks a repaint boundary: the subtree is recorded into its own retained
+   * display list and reused until something inside it changes. Place around
+   * heavy static content that sits next to frequently changing content.
+   *
+   * "snapshot" additionally retains the rasterized pixels as a GPU texture,
+   * skipping rasterization entirely. Costs texture memory and re-rasterizes
+   * on layout-size or display-scale changes. Content painted outside the
+   * element's layout box is cropped, and ancestor scale animations smear the
+   * bitmap; best for screen-aligned, static, raster-expensive content.
+   */
+  repaintBoundary?: boolean | "snapshot"
 }
 
 export interface AudioProps {

package/src/window.ts

@@ -1,4 +1,5 @@
 import { onCleanup, onSettled, flush } from "@solidjs/signals"
+import { on, once } from "srt:events"
 import { getEventHandler, getFocusedNodeId, setFocus } from "./core"
 
 // ------ Animation frames ----------------
@@ -26,10 +27,13 @@ export function onFrame(fn: (tick: number, frame: number, rate: number) => void)
     fn(tick, frame, rate)
     frameId = nextFrameId++
     animationFrames.set(frameId, extendedFn)
+    // A pending onFrame callback is a standing request for the next frame.
+    ffi.requestFrame()
   }
 
   frameId = nextFrameId++
   animationFrames.set(frameId, extendedFn)
+  ffi.requestFrame()
 
   let cleanup = () => animationFrames.delete(frameId)
   onCleanup(cleanup)
@@ -53,7 +57,7 @@ interface ResizeEvent {
 }
 
 export function onResize(fn: (data: ResizeEvent) => void) {
-  let unsubscribe = srt.on("resize", fn)
+  let unsubscribe = on("resize", fn)
   onCleanup(unsubscribe)
   return unsubscribe
 }
@@ -63,19 +67,19 @@ export function onResize(fn: (data: ResizeEvent) => void) {
 // by a re-layout pass before painting (one extra pass; cascades beyond that
 // paint stale).
 export function onLayout(fn: () => void) {
-  let unsubscribe = srt.on("postLayout", fn)
+  let unsubscribe = on("postLayout", fn)
   onCleanup(unsubscribe)
   return unsubscribe
 }
 
 export function onWindowFocus(fn: () => void) {
-  let unsubscribe = srt.on("windowFocus", fn)
+  let unsubscribe = on("windowFocus", fn)
   onCleanup(unsubscribe)
   return unsubscribe
 }
 
 export function onWindowBlur(fn: () => void) {
-  let unsubscribe = srt.on("windowBlur", fn)
+  let unsubscribe = on("windowBlur", fn)
   onCleanup(unsubscribe)
   return unsubscribe
 }
@@ -109,15 +113,15 @@ export function attachWindow(_nodeId: number) {
 
   onSettled(() => {
     // Sticky event: a late subscriber still receives the current rate.
-    unsubRefreshRate = srt.on("displayRefreshRate", ({ hz }: { hz: number }) => {
+    unsubRefreshRate = on("displayRefreshRate", ({ hz }: { hz: number }) => {
       if (hz > 0) refreshRate = hz
     })
 
-    unsubscribe = srt.on("render", ({ time, frame }: { time: number; frame: number }) => {
+    unsubscribe = on("render", ({ time, frame }: { time: number; frame: number }) => {
       runFrame(time * 1000, frame)
     })
 
-    unsubDown = srt.on(
+    unsubDown = on(
       "pointerDown",
       ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
         for (let nodeId of targets) {
@@ -132,13 +136,13 @@ export function attachWindow(_nodeId: number) {
       },
     )
 
-    unsubUp = srt.on("pointerUp", ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
+    unsubUp = on("pointerUp", ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
       for (let nodeId of targets) {
         getEventHandler(nodeId, "onPointerUp")?.(e)
       }
     })
 
-    unsubMove = srt.on(
+    unsubMove = on(
       "pointerMove",
       ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
         for (let nodeId of targets) {
@@ -147,7 +151,7 @@ export function attachWindow(_nodeId: number) {
       },
     )
 
-    unsubEnter = srt.on(
+    unsubEnter = on(
       "pointerEnter",
       ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
         for (let nodeId of targets) {
@@ -156,7 +160,7 @@ export function attachWindow(_nodeId: number) {
       },
     )
 
-    unsubLeave = srt.on(
+    unsubLeave = on(
       "pointerLeave",
       ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
         for (let nodeId of targets) {
@@ -165,27 +169,27 @@ export function attachWindow(_nodeId: number) {
       },
     )
 
-    unsubWheel = srt.on("wheel", ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
+    unsubWheel = on("wheel", ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
       for (let nodeId of targets) {
         getEventHandler(nodeId, "onWheel")?.(e)
       }
     })
 
-    unsubKeyDown = srt.on("keydown", (e: any) => {
+    unsubKeyDown = on("keydown", (e: any) => {
       let id = getFocusedNodeId()
       if (id != null) {
         getEventHandler(id, "onKeyDown")?.(e)
       }
     })
 
-    unsubKeyUp = srt.on("keyup", (e: any) => {
+    unsubKeyUp = on("keyup", (e: any) => {
       let id = getFocusedNodeId()
       if (id != null) {
         getEventHandler(id, "onKeyUp")?.(e)
       }
     })
 
-    unsubTextInput = srt.on("textInput", (e: any) => {
+    unsubTextInput = on("textInput", (e: any) => {
       let id = getFocusedNodeId()
       if (id != null) {
         getEventHandler(id, "onTextInput")?.(e)
@@ -194,7 +198,7 @@ export function attachWindow(_nodeId: number) {
 
     // When the user dismisses the on-screen keyboard (swipe down, "Done",
     // back button), blur the focused node so the app's UI state catches up.
-    unsubKeyboardVisibility = srt.on("keyboardVisibility", ({ shown }: { shown: boolean }) => {
+    unsubKeyboardVisibility = on("keyboardVisibility", ({ shown }: { shown: boolean }) => {
       if (!shown) setFocus(null)
     })
 
@@ -205,7 +209,7 @@ export function attachWindow(_nodeId: number) {
     // synchronously here, while we are still inside this onSettled callback
     // where flush() is illegal (not reentrant). Defer runFrame to a microtask
     // so the first frame always runs after this callback returns.
-    unsubFirstResize = srt.once("resize", () => {
+    unsubFirstResize = once("resize", () => {
       queueMicrotask(() => runFrame(0, 0))
     })
   })