@solidrt/core 0.0.9 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +108 -0
- package/README.md +2 -0
- package/package.json +8 -4
- package/src/camera-view.tsx +55 -0
- package/src/camera.ts +78 -0
- package/src/gpu.ts +33 -0
- package/src/index.ts +1 -1
- package/src/microphone.ts +40 -0
- package/src/speech.ts +67 -0
- package/src/types.d.ts +89 -6
- package/src/window.ts +21 -17
package/AGENTS.md
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# @solidrt/core - agent notes
|
|
2
|
+
|
|
3
|
+
Dense, self-contained facts for writing a SolidRT app.
|
|
4
|
+
Full docs live in docs/ (and the website). When this conflicts with prose docs,
|
|
5
|
+
trust this file and the types in src/types.d.ts and jsx-runtime.d.ts.
|
|
6
|
+
|
|
7
|
+
SolidRT is a custom SolidJS renderer: it paints through a Rust runtime, not the
|
|
8
|
+
DOM. There is no HTML, no CSS cascade, no `className`.
|
|
9
|
+
|
|
10
|
+
## Setup
|
|
11
|
+
|
|
12
|
+
```sh
|
|
13
|
+
bun add @solidrt/core # the renderer
|
|
14
|
+
bun add -d @solidrt/cli # the `srt` tool (see its AGENTS.md)
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
`@solidrt/components` is a separate, optional package of higher-level components
|
|
18
|
+
(see its own AGENTS.md); core primitives alone are enough to build a full app.
|
|
19
|
+
|
|
20
|
+
tsconfig.json - the two load-bearing lines are jsx + jsxImportSource:
|
|
21
|
+
|
|
22
|
+
```json
|
|
23
|
+
{ "compilerOptions": {
|
|
24
|
+
"jsx": "preserve",
|
|
25
|
+
"jsxImportSource": "@solidrt/core",
|
|
26
|
+
"moduleResolution": "bundler",
|
|
27
|
+
"strict": true
|
|
28
|
+
} }
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Peer deps @solidjs/signals and @solidjs/universal must match (currently
|
|
32
|
+
2.0.0-beta.14); bun resolves them from peerDependencies.
|
|
33
|
+
|
|
34
|
+
## Element model (the parts that are easy to get wrong)
|
|
35
|
+
|
|
36
|
+
- `render(() => <App />)`. The returned root MUST be a `<window>` or it throws.
|
|
37
|
+
Call render once, at the top level.
|
|
38
|
+
|
|
39
|
+
- Two kinds of element:
|
|
40
|
+
- Containers - `<window>`, `<view>`. Do layout + transform + pointer events.
|
|
41
|
+
THEY DO NOT PAINT. A `<view>` has no background/fill prop.
|
|
42
|
+
- Draw primitives - `<rect>`, `<oval>`, `<path>`, `<texture>`, `<text>`.
|
|
43
|
+
These paint. To give a view a background, render a draw primitive (e.g.
|
|
44
|
+
`<d-rect>`) as a child, behind the content.
|
|
45
|
+
|
|
46
|
+
- Paint color is the `color` prop (a CSS color string). There is NO `fill`,
|
|
47
|
+
`stroke`, or `background` prop (some older doc examples are wrong about this).
|
|
48
|
+
Outlines: `drawStyle="stroke"` (or "stroke-and-fill") plus `strokeWidth`.
|
|
49
|
+
Corner radius on draw primitives: `radius` (number or [tl, tr, br, bl]).
|
|
50
|
+
|
|
51
|
+
- Registered JSX intrinsics: `window`, `view`, `text`, `rect`, `oval`, `path`,
|
|
52
|
+
`texture`, `audio`, plus the `d-` variants `d-view`, `d-rect`, `d-oval`,
|
|
53
|
+
`d-path`, `d-texture`, `d-text`. NOTE: `<line>` has a LineProps type but is
|
|
54
|
+
NOT a registered intrinsic - it will not typecheck.
|
|
55
|
+
|
|
56
|
+
- Plain vs `d-` variant (the `d-` prefix means "detached" - detached from the
|
|
57
|
+
layout engine, Taffy): a plain element (e.g. `rect`) is `RectProps &
|
|
58
|
+
LayoutProps`, so it draws AND is laid out by Taffy. The detached variant
|
|
59
|
+
(`d-rect`) is `RectProps` only - it draws but is NOT in the layout pass; you
|
|
60
|
+
place it yourself with `x`/`y` (omit them and it fills the parent, which is
|
|
61
|
+
how backgrounds work). Reach for `d-` whenever you want explicit coordinate
|
|
62
|
+
positioning instead of layout. It is also a performance lever: for many
|
|
63
|
+
directly-positioned, often-animating elements (e.g. hundreds of balls), `d-`
|
|
64
|
+
skips the per-element layout that plain elements would incur.
|
|
65
|
+
|
|
66
|
+
- Events: there is NO `onClick`/`onPress`. A "button" is a `<view>`/`<rect>`
|
|
67
|
+
with `onPointerDown`. Handlers: onPointerDown/Up/Move/Enter/Leave, onWheel,
|
|
68
|
+
onKeyDown/Up, onTextInput, onFocus/onBlur. Text entry: focus a node with an
|
|
69
|
+
`onTextInput` handler (setFocus activates the on-screen keyboard).
|
|
70
|
+
|
|
71
|
+
- Reactivity is plain SolidJS (`createSignal`, `createEffect`, ... from
|
|
72
|
+
@solidjs/signals). Per-frame work: `onFrame((tick, frame) => {})` (returns a
|
|
73
|
+
cleanup; auto-cleaned inside a reactive scope). Also onResize, onLayout,
|
|
74
|
+
onWindowFocus, onWindowBlur.
|
|
75
|
+
|
|
76
|
+
- Device/GPU access via subpath imports: @solidrt/core/camera, /microphone,
|
|
77
|
+
/speech, /gpu. Image flow: `decodeImage(bytes)` -> `createTexture(data,w,h)`
|
|
78
|
+
-> `<texture src={id} />`.
|
|
79
|
+
|
|
80
|
+
## Minimal app, core primitives only (verified to render)
|
|
81
|
+
|
|
82
|
+
```tsx
|
|
83
|
+
import { render } from "@solidrt/core"
|
|
84
|
+
import { createSignal } from "@solidjs/signals"
|
|
85
|
+
|
|
86
|
+
function App() {
|
|
87
|
+
let [count, setCount] = createSignal(0)
|
|
88
|
+
return (
|
|
89
|
+
<window flexDirection="column" alignItems="center" justifyContent="center" gap={24}>
|
|
90
|
+
<d-rect color="#0b0f17" /> {/* window background */}
|
|
91
|
+
<text color="#1f6feb" fontSize={48} fontWeight={800}>{count()}</text>
|
|
92
|
+
<view onPointerDown={() => setCount((c) => c + 1)}
|
|
93
|
+
padding={16} alignItems="center" justifyContent="center">
|
|
94
|
+
<d-rect color="#1f6feb" radius={12} /> {/* button background, underlays the label */}
|
|
95
|
+
<text color="#ffffff" fontSize={20}>increment</text>
|
|
96
|
+
</view>
|
|
97
|
+
</window>
|
|
98
|
+
)
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
render(() => <App />)
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
Note the two `<d-rect>` underlays: a `<view>`/`<window>` does not paint, so a
|
|
105
|
+
background is a draw-primitive child placed behind the content.
|
|
106
|
+
|
|
107
|
+
To run and verify (incl. headless), see @solidrt/cli (its AGENTS.md). For
|
|
108
|
+
higher-level components, see @solidrt/components (its AGENTS.md).
|
package/README.md
CHANGED
|
@@ -4,6 +4,8 @@ A low-level toolkit for creating cross-platform applications.
|
|
|
4
4
|
|
|
5
5
|
_SolidRT is in pre-alpha stage. Anything can and will be changed._
|
|
6
6
|
|
|
7
|
+
> LLM agents: see [AGENTS.md](./AGENTS.md) for a dense, self-contained quickstart.
|
|
8
|
+
|
|
7
9
|
## Getting started
|
|
8
10
|
|
|
9
11
|
Prerequisites: [bun](https://bun.sh) (only required for development; not needed to run built apps).
|
package/package.json
CHANGED
|
@@ -1,29 +1,33 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@solidrt/core",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.11",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"author": "Antoine van Wel",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"main": "src/index.ts",
|
|
8
8
|
"exports": {
|
|
9
9
|
".": "./src/index.ts",
|
|
10
|
+
"./camera": "./src/camera.ts",
|
|
10
11
|
"./gpu": "./src/gpu.ts",
|
|
12
|
+
"./microphone": "./src/microphone.ts",
|
|
13
|
+
"./speech": "./src/speech.ts",
|
|
11
14
|
"./jsx-runtime": "./jsx-runtime.d.ts",
|
|
12
15
|
"./jsx-runtime-dev": "./jsx-runtime.d.ts"
|
|
13
16
|
},
|
|
14
17
|
"files": [
|
|
15
18
|
"src/",
|
|
16
19
|
"jsx-runtime.d.ts",
|
|
17
|
-
"
|
|
20
|
+
"AGENTS.md"
|
|
18
21
|
],
|
|
19
22
|
"dependencies": {
|
|
20
23
|
"colord": "^2.9.3"
|
|
21
24
|
},
|
|
22
25
|
"devDependencies": {
|
|
23
|
-
"@solidrt/flux-types": "0.0.
|
|
26
|
+
"@solidrt/flux-types": "0.0.11"
|
|
24
27
|
},
|
|
25
28
|
"peerDependencies": {
|
|
26
29
|
"@solidjs/signals": "2.0.0-beta.14",
|
|
27
|
-
"@solidjs/universal": "2.0.0-beta.14"
|
|
30
|
+
"@solidjs/universal": "2.0.0-beta.14",
|
|
31
|
+
"solid-js": "2.0.0-beta.14"
|
|
28
32
|
}
|
|
29
33
|
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
import { createSignal, onCleanup } from "@solidjs/signals"
|
|
2
|
+
import { openCamera, type BarcodeResult, type Camera } from "./camera"
|
|
3
|
+
|
|
4
|
+
// Convenience viewfinder over openCamera: opens on mount, renders the stream
|
|
5
|
+
// texture, closes on cleanup. Use openCamera directly for anything it does not
|
|
6
|
+
// cover; this is just composition, no extra capability.
|
|
7
|
+
|
|
8
|
+
export interface CameraViewProps {
|
|
9
|
+
/** Explicit device id from listCameras(); takes precedence over facing. */
|
|
10
|
+
camera?: number
|
|
11
|
+
facing?: "front" | "back"
|
|
12
|
+
/**
|
|
13
|
+
* Size hint for the stream and explicit size of the view. Omit height to
|
|
14
|
+
* follow the stream's aspect ratio, which can flip when a phone rotates.
|
|
15
|
+
*/
|
|
16
|
+
width?: number
|
|
17
|
+
height?: number
|
|
18
|
+
scan?: "qr"[]
|
|
19
|
+
onReady?: (cam: Camera) => void
|
|
20
|
+
onError?: (error: Error) => void
|
|
21
|
+
onBarcode?: (result: BarcodeResult) => void
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export function CameraView(props: CameraViewProps) {
|
|
25
|
+
let [texture, setTexture] = createSignal<number | undefined>(undefined)
|
|
26
|
+
let cam: Camera | undefined
|
|
27
|
+
let disposed = false
|
|
28
|
+
|
|
29
|
+
openCamera({
|
|
30
|
+
camera: props.camera,
|
|
31
|
+
facing: props.facing,
|
|
32
|
+
width: props.width,
|
|
33
|
+
height: props.height,
|
|
34
|
+
scan: props.scan,
|
|
35
|
+
})
|
|
36
|
+
.then((opened) => {
|
|
37
|
+
if (disposed) {
|
|
38
|
+
opened.close()
|
|
39
|
+
return
|
|
40
|
+
}
|
|
41
|
+
cam = opened
|
|
42
|
+
if (props.onBarcode) opened.onBarcode(props.onBarcode)
|
|
43
|
+
setTexture(opened.texture)
|
|
44
|
+
props.onReady?.(opened)
|
|
45
|
+
})
|
|
46
|
+
.catch((e) => props.onError?.(e instanceof Error ? e : new Error(String(e))))
|
|
47
|
+
|
|
48
|
+
onCleanup(() => {
|
|
49
|
+
disposed = true
|
|
50
|
+
cam?.close()
|
|
51
|
+
cam = undefined
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
return <texture src={texture()} width={props.width} height={props.height} />
|
|
55
|
+
}
|
package/src/camera.ts
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// Camera capture. A camera streams into a texture id, so a viewfinder is just
|
|
2
|
+
// <texture src={cam.texture} />. Opening the camera IS the permission request
|
|
3
|
+
// (SDL semantics): the promise resolves once the stream is configured and
|
|
4
|
+
// rejects if the user denies access.
|
|
5
|
+
|
|
6
|
+
import { on } from "srt:events"
|
|
7
|
+
|
|
8
|
+
export type CameraFacing = "front" | "back" | "unknown"
|
|
9
|
+
|
|
10
|
+
export type CameraInfo = {
|
|
11
|
+
id: number
|
|
12
|
+
name: string
|
|
13
|
+
facing: CameraFacing
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export type BarcodeFormat = "qr"
|
|
17
|
+
|
|
18
|
+
export type BarcodeResult = {
|
|
19
|
+
data: string
|
|
20
|
+
format: BarcodeFormat
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export type CameraOptions = {
|
|
24
|
+
/** Explicit device id from listCameras(); takes precedence over facing. */
|
|
25
|
+
camera?: number
|
|
26
|
+
/** Pick the first camera with this facing (falls back to the first camera). */
|
|
27
|
+
facing?: "front" | "back"
|
|
28
|
+
/** Size hint; the device picks the closest supported mode. */
|
|
29
|
+
width?: number
|
|
30
|
+
height?: number
|
|
31
|
+
/** Decode these barcode formats from the stream (delivered via onBarcode). */
|
|
32
|
+
scan?: BarcodeFormat[]
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export type Camera = {
|
|
36
|
+
/** Texture id updated every frame while open; render with <texture src={...}>. */
|
|
37
|
+
texture: number
|
|
38
|
+
/** Actual stream size (may differ from the requested hint). */
|
|
39
|
+
width: number
|
|
40
|
+
height: number
|
|
41
|
+
/** Receive decoded barcodes (requires the scan option; replaces any previous callback). */
|
|
42
|
+
onBarcode(callback: (result: BarcodeResult) => void): void
|
|
43
|
+
/** Release the device. The texture keeps showing the last frame. */
|
|
44
|
+
close(): void
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export function listCameras(): CameraInfo[] {
|
|
48
|
+
return camera.listCameras()
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
// Camera hotplug. Re-enumerate with listCameras() to see the new device set.
|
|
52
|
+
// Events only flow once the camera subsystem is up, i.e. after the first
|
|
53
|
+
// listCameras() or openCamera() call. Returns an unsubscribe function.
|
|
54
|
+
// Coverage caveat (SDL 3.4.8): only Android delivers both add and remove. On
|
|
55
|
+
// Linux you get added=true but not added=false (removal is broken upstream);
|
|
56
|
+
// on macOS/Windows there is no camera hotplug at all, so nothing fires.
|
|
57
|
+
export function onDeviceChange(callback: (event: { added: boolean }) => void): () => void {
|
|
58
|
+
return on("cameraDeviceChange", callback)
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// One-shot scan of an RGBA8 pixel buffer for QR codes; composes with
|
|
62
|
+
// decodeImage: scanBarcodes(img.data, img.width, img.height).
|
|
63
|
+
export function scanBarcodes(data: Uint8Array, width: number, height: number): BarcodeResult[] {
|
|
64
|
+
return camera.scanImage(data, width, height)
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export async function openCamera(options: CameraOptions = {}): Promise<Camera> {
|
|
68
|
+
let opened = await camera.open(options)
|
|
69
|
+
return {
|
|
70
|
+
texture: opened.texture,
|
|
71
|
+
width: opened.width,
|
|
72
|
+
height: opened.height,
|
|
73
|
+
onBarcode: (callback: (result: BarcodeResult) => void) => camera.setBarcodeCallback(opened.handle, callback),
|
|
74
|
+
close: () => camera.close(opened.handle),
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export { CameraView, type CameraViewProps } from "./camera-view"
|
package/src/gpu.ts
CHANGED
|
@@ -10,4 +10,37 @@ export function decodeImage(bytes: Uint8Array): DecodedImage {
|
|
|
10
10
|
|
|
11
11
|
export function createTexture(data: Uint8Array, width: number, height: number): number {
|
|
12
12
|
return gpu.createTexture(data, width, height)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// The texture keeps reading from `data` (which may hold multiple frames):
|
|
16
|
+
// mutate it in place, then call uploadTexture to push the pixels to the GPU.
|
|
17
|
+
export function createMutableTexture(data: Uint8Array, width: number, height: number): number {
|
|
18
|
+
return gpu.createMutableTexture(data, width, height)
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function uploadTexture(textureId: number, offset: number = 0): void {
|
|
22
|
+
gpu.uploadTexture(textureId, offset)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Compile a GLSL ES 3.00 fragment shader and render it into a texture, returning
|
|
26
|
+
// the texture id (usable anywhere a normal texture id is, e.g. <texture src>).
|
|
27
|
+
// The fragment body may reference vUV (0..1, top-left origin), iResolution,
|
|
28
|
+
// iTime, and any `uniform float` it declares; pass their values via `params`.
|
|
29
|
+
// `textures` binds each declared `uniform sampler2D` to an existing texture id
|
|
30
|
+
// (e.g. a camera or decoded image) so the shader can read it; those inputs are
|
|
31
|
+
// re-sampled on every setShaderParams call, so live sources stay current.
|
|
32
|
+
export function createShader(
|
|
33
|
+
fragmentSrc: string,
|
|
34
|
+
width: number,
|
|
35
|
+
height: number,
|
|
36
|
+
params?: Record<string, number>,
|
|
37
|
+
textures?: Record<string, number>,
|
|
38
|
+
): number {
|
|
39
|
+
return gpu.createShader(fragmentSrc, width, height, params, textures)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Re-render an existing shader texture with new param values and request a
|
|
43
|
+
// frame. Use this to animate (e.g. update iTime each frame).
|
|
44
|
+
export function setShaderParams(textureId: number, params: Record<string, number>): void {
|
|
45
|
+
gpu.setShaderParams(textureId, params)
|
|
13
46
|
}
|
package/src/index.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export * from "./renderer"
|
|
2
|
-
export { setFocus, getFocusedNodeId, measureText, getBoundingBox } from "./core"
|
|
2
|
+
export { setFocus, getFocusedNodeId, measureText, getBoundingBox, parseColorToU32 } from "./core"
|
|
3
3
|
export type { BoundingBox } from "./core"
|
|
4
4
|
export { onFrame, onLayout, onResize, onWindowFocus, onWindowBlur } from "./window"
|
|
5
5
|
export { createTexture, decodeImage } from "./gpu"
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
// Microphone capture. A session delivers raw mono float32 samples at the
|
|
2
|
+
// requested sample rate (the device format is converted by SDL); read()
|
|
3
|
+
// drains whatever was captured since the last call. Captured audio buffers
|
|
4
|
+
// until read, so poll read() regularly (e.g. once per frame) while open.
|
|
5
|
+
|
|
6
|
+
export type MicrophoneInfo = {
|
|
7
|
+
id: number
|
|
8
|
+
name: string
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export type MicrophoneOptions = {
|
|
12
|
+
/** Explicit device id from listMicrophones(); default is the system default recording device. */
|
|
13
|
+
microphone?: number
|
|
14
|
+
/** Sample rate of the delivered samples (the device rate is converted). Default 16000. */
|
|
15
|
+
sampleRate?: number
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export type Microphone = {
|
|
19
|
+
/** Sample rate of read() samples. */
|
|
20
|
+
sampleRate: number
|
|
21
|
+
/** Drain the mono float32 samples captured since the last read. */
|
|
22
|
+
read(): Float32Array
|
|
23
|
+
/** Release the device. */
|
|
24
|
+
close(): void
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function listMicrophones(): MicrophoneInfo[] {
|
|
28
|
+
return microphone.listMicrophones()
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Async to leave room for an OS permission prompt on platforms that need one
|
|
32
|
+
// (the desktop backends open synchronously).
|
|
33
|
+
export async function openMicrophone(options: MicrophoneOptions = {}): Promise<Microphone> {
|
|
34
|
+
let opened = microphone.open(options)
|
|
35
|
+
return {
|
|
36
|
+
sampleRate: opened.sampleRate,
|
|
37
|
+
read: () => microphone.read(opened.handle),
|
|
38
|
+
close: () => microphone.close(opened.handle),
|
|
39
|
+
}
|
|
40
|
+
}
|
package/src/speech.ts
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// Speech recognition. A session captures the microphone, segments utterances
|
|
2
|
+
// by silence (Silero VAD) and transcribes each one with Whisper, delivering
|
|
3
|
+
// final transcripts through onResult. With wakeWord the session starts
|
|
4
|
+
// asleep behind an efficient wake word detector (livekit-wakeword) and only
|
|
5
|
+
// transcribes after the wake word. startRecognition resolves once the models
|
|
6
|
+
// are loaded and listening has begun; it rejects when loading fails.
|
|
7
|
+
// Models are passed as bytes so any source composes: flux:fs file(), fetch
|
|
8
|
+
// (incl. the dev-server file proxy), or a download cache layered on top.
|
|
9
|
+
// Requires a runtime built with speech support.
|
|
10
|
+
|
|
11
|
+
export type SpeechOptions = {
|
|
12
|
+
/** A ggml Whisper model (file contents, e.g. ggml-tiny.en.bin). */
|
|
13
|
+
model: Uint8Array
|
|
14
|
+
/** A ggml Silero VAD model (file contents). */
|
|
15
|
+
vadModel: Uint8Array
|
|
16
|
+
/** Whisper language code; "auto" detects (multilingual models only). Default "en". */
|
|
17
|
+
language?: string
|
|
18
|
+
/** Explicit microphone device id from listMicrophones(). */
|
|
19
|
+
microphone?: number
|
|
20
|
+
/** Stop automatically after the first final result (with wakeWord: re-arm instead, one result per wake). */
|
|
21
|
+
singleUtterance?: boolean
|
|
22
|
+
/** Also deliver snapshot transcripts (final: false) while an utterance is still being spoken. */
|
|
23
|
+
interimResults?: boolean
|
|
24
|
+
/**
|
|
25
|
+
* Wake word: start asleep, fire onWake when it is heard, then transcribe
|
|
26
|
+
* the speech that follows. How the wake word is specified depends on the
|
|
27
|
+
* engine. The current engine detects with a trained classifier and takes
|
|
28
|
+
* the model's bytes (livekit-wakeword ONNX, e.g. the pretrained "hey
|
|
29
|
+
* livekit"; custom phrases are trained offline with its toolkit). Phrase
|
|
30
|
+
* strings are reserved for engines that match text; passing them to this
|
|
31
|
+
* engine rejects with an error.
|
|
32
|
+
*/
|
|
33
|
+
wakeWord?: Uint8Array | string | string[]
|
|
34
|
+
/** Detector confidence (0..1) that counts as a wake. Default 0.5. */
|
|
35
|
+
wakeThreshold?: number
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export type SpeechResult = {
|
|
39
|
+
/** Transcript of the utterance (a snapshot of it when final is false). */
|
|
40
|
+
text: string
|
|
41
|
+
/** True for the completed utterance, false for interim snapshots. */
|
|
42
|
+
final: boolean
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
export type SpeechSession = {
|
|
46
|
+
/** Receive transcripts (replaces any previous callback). */
|
|
47
|
+
onResult(callback: (result: SpeechResult) => void): void
|
|
48
|
+
/** The user started speaking (replaces any previous callback). */
|
|
49
|
+
onSpeechStart(callback: () => void): void
|
|
50
|
+
/** The utterance ended; its final result follows once transcribed. */
|
|
51
|
+
onSpeechEnd(callback: () => void): void
|
|
52
|
+
/** The wake word was heard (wakeWordModel sessions only). */
|
|
53
|
+
onWake(callback: () => void): void
|
|
54
|
+
/** Release the microphone and discard any utterance in progress. */
|
|
55
|
+
stop(): void
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function startRecognition(options: SpeechOptions): Promise<SpeechSession> {
|
|
59
|
+
let started = await speech.start(options)
|
|
60
|
+
return {
|
|
61
|
+
onResult: (callback: (result: SpeechResult) => void) => speech.setResultCallback(started.handle, callback),
|
|
62
|
+
onSpeechStart: (callback: () => void) => speech.setSpeechStartCallback(started.handle, callback),
|
|
63
|
+
onSpeechEnd: (callback: () => void) => speech.setSpeechEndCallback(started.handle, callback),
|
|
64
|
+
onWake: (callback: () => void) => speech.setWakeCallback(started.handle, callback),
|
|
65
|
+
stop: () => speech.stop(started.handle),
|
|
66
|
+
}
|
|
67
|
+
}
|
package/src/types.d.ts
CHANGED
|
@@ -2,16 +2,28 @@
|
|
|
2
2
|
|
|
3
3
|
import type { JSX as SolidJSX } from "@solidjs/signals"
|
|
4
4
|
|
|
5
|
+
// UI event bus (lattice), provided by the runtime as a builtin module.
|
|
6
|
+
// on/once return an unsubscribe function.
|
|
7
|
+
declare module "srt:events" {
|
|
8
|
+
export function on(event: string, callback: (data: any) => void): () => void
|
|
9
|
+
export function once(event: string, callback: (data: any) => void): () => void
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// Dev-server control surface (lattice). Present only in dev/go builds; in other
|
|
13
|
+
// builds `available` is false and the functions are no-ops.
|
|
14
|
+
declare module "srt:dev" {
|
|
15
|
+
export const available: boolean
|
|
16
|
+
export const canDiscover: boolean
|
|
17
|
+
export const recents: string[]
|
|
18
|
+
export function connect(address: string): void
|
|
19
|
+
export function discover(): void
|
|
20
|
+
export function stop(): void
|
|
21
|
+
}
|
|
22
|
+
|
|
5
23
|
declare global {
|
|
6
24
|
function requestAnimationFrame(callback: (time: number) => void): number
|
|
7
25
|
function cancelAnimationFrame(id: number): void
|
|
8
26
|
|
|
9
|
-
// UI event bus (lattice). on/once return an unsubscribe function.
|
|
10
|
-
let srt: {
|
|
11
|
-
on(event: string, callback: (data: any) => void): () => void
|
|
12
|
-
once(event: string, callback: (data: any) => void): () => void
|
|
13
|
-
}
|
|
14
|
-
|
|
15
27
|
let ffi: {
|
|
16
28
|
createRoot(id: number): void
|
|
17
29
|
createNode(id: number, kind: string): void
|
|
@@ -19,14 +31,53 @@ declare global {
|
|
|
19
31
|
deleteNode(parentId: number, nodeId: number): void
|
|
20
32
|
setProperty(nodeId: number, name: string, value: unknown): void
|
|
21
33
|
setTextInputActive(active: boolean): void
|
|
34
|
+
requestFrame(): void
|
|
22
35
|
measureText(text: string, options?: MeasureTextOptions): { width: number, height: number }
|
|
23
36
|
getBoundingBox(id: number): { x: number, y: number, width: number, height: number } | null
|
|
24
37
|
}
|
|
25
38
|
|
|
26
39
|
let gpu: {
|
|
27
40
|
createTexture(data: Uint8Array, width: number, height: number): number
|
|
41
|
+
createMutableTexture(data: Uint8Array, width: number, height: number): number
|
|
42
|
+
uploadTexture(textureId: number, offset?: number): void
|
|
43
|
+
createShader(
|
|
44
|
+
fragmentSrc: string,
|
|
45
|
+
width: number,
|
|
46
|
+
height: number,
|
|
47
|
+
params?: Record<string, number>,
|
|
48
|
+
textures?: Record<string, number>,
|
|
49
|
+
): number
|
|
50
|
+
setShaderParams(textureId: number, params: Record<string, number>): void
|
|
28
51
|
decodeImage(bytes: Uint8Array): { data: Uint8Array, width: number, height: number }
|
|
29
52
|
}
|
|
53
|
+
|
|
54
|
+
let camera: {
|
|
55
|
+
listCameras(): { id: number, name: string, facing: "front" | "back" | "unknown" }[]
|
|
56
|
+
open(options: { camera?: number, facing?: "front" | "back", width?: number, height?: number, scan?: string[] }):
|
|
57
|
+
Promise<{ handle: number, texture: number, width: number, height: number }>
|
|
58
|
+
setBarcodeCallback(handle: number, callback: (result: { data: string, format: "qr" }) => void): void
|
|
59
|
+
scanImage(data: Uint8Array, width: number, height: number): { data: string, format: "qr" }[]
|
|
60
|
+
close(handle: number): void
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
let microphone: {
|
|
64
|
+
listMicrophones(): { id: number, name: string }[]
|
|
65
|
+
open(options: { microphone?: number, sampleRate?: number }): { handle: number, sampleRate: number }
|
|
66
|
+
read(handle: number): Float32Array
|
|
67
|
+
close(handle: number): void
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
let speech: {
|
|
71
|
+
start(options: {
|
|
72
|
+
model: Uint8Array, vadModel: Uint8Array, language?: string, microphone?: number,
|
|
73
|
+
singleUtterance?: boolean, interimResults?: boolean, wakeWord?: Uint8Array | string | string[], wakeThreshold?: number,
|
|
74
|
+
}): Promise<{ handle: number }>
|
|
75
|
+
setResultCallback(handle: number, callback: (result: { text: string, final: boolean }) => void): void
|
|
76
|
+
setSpeechStartCallback(handle: number, callback: () => void): void
|
|
77
|
+
setSpeechEndCallback(handle: number, callback: () => void): void
|
|
78
|
+
setWakeCallback(handle: number, callback: () => void): void
|
|
79
|
+
stop(handle: number): void
|
|
80
|
+
}
|
|
30
81
|
}
|
|
31
82
|
|
|
32
83
|
export interface MeasureTextOptions {
|
|
@@ -119,6 +170,19 @@ export interface PaintProps {
|
|
|
119
170
|
export interface TransformProps {
|
|
120
171
|
rotate?: number
|
|
121
172
|
scale?: number
|
|
173
|
+
// Per-axis scale; overrides `scale` on that axis (e.g. scaleX for a flip).
|
|
174
|
+
scaleX?: number
|
|
175
|
+
scaleY?: number
|
|
176
|
+
// 3D rotation about the horizontal axis, in radians (a top/bottom tilt). Like
|
|
177
|
+
// rotateY, reads as 3D only with `perspective` set.
|
|
178
|
+
rotateX?: number
|
|
179
|
+
// 3D rotation about the vertical axis, in radians, for a card-flip. Reads as a
|
|
180
|
+
// real flip only with `perspective` set; on its own it is an orthographic
|
|
181
|
+
// squash (like scaleX).
|
|
182
|
+
rotateY?: number
|
|
183
|
+
// Perspective viewing distance in pixels (CSS `perspective`). Enables the 3D
|
|
184
|
+
// depth for rotateY; larger values give a shallower effect.
|
|
185
|
+
perspective?: number
|
|
122
186
|
x?: number
|
|
123
187
|
y?: number
|
|
124
188
|
cx?: number
|
|
@@ -180,6 +244,25 @@ export interface WindowProps extends LayoutProps {
|
|
|
180
244
|
export interface ViewProps extends LayoutProps, TransformProps, PointerProps {
|
|
181
245
|
children?: Children
|
|
182
246
|
trace?: boolean
|
|
247
|
+
/**
|
|
248
|
+
* Corner radii for the clip applied when overflow is non-visible (hidden,
|
|
249
|
+
* clip, scroll on both axes). A single number rounds all four corners; an
|
|
250
|
+
* array is [top-left, top-right, bottom-right, bottom-left]. Without overflow
|
|
251
|
+
* clipping this has no effect.
|
|
252
|
+
*/
|
|
253
|
+
clipRadius?: number | [number, number, number, number]
|
|
254
|
+
/**
|
|
255
|
+
* Marks a repaint boundary: the subtree is recorded into its own retained
|
|
256
|
+
* display list and reused until something inside it changes. Place around
|
|
257
|
+
* heavy static content that sits next to frequently changing content.
|
|
258
|
+
*
|
|
259
|
+
* "snapshot" additionally retains the rasterized pixels as a GPU texture,
|
|
260
|
+
* skipping rasterization entirely. Costs texture memory and re-rasterizes
|
|
261
|
+
* on layout-size or display-scale changes. Content painted outside the
|
|
262
|
+
* element's layout box is cropped, and ancestor scale animations smear the
|
|
263
|
+
* bitmap; best for screen-aligned, static, raster-expensive content.
|
|
264
|
+
*/
|
|
265
|
+
repaintBoundary?: boolean | "snapshot"
|
|
183
266
|
}
|
|
184
267
|
|
|
185
268
|
export interface AudioProps {
|
package/src/window.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { onCleanup, onSettled, flush } from "@solidjs/signals"
|
|
2
|
+
import { on, once } from "srt:events"
|
|
2
3
|
import { getEventHandler, getFocusedNodeId, setFocus } from "./core"
|
|
3
4
|
|
|
4
5
|
// ------ Animation frames ----------------
|
|
@@ -26,10 +27,13 @@ export function onFrame(fn: (tick: number, frame: number, rate: number) => void)
|
|
|
26
27
|
fn(tick, frame, rate)
|
|
27
28
|
frameId = nextFrameId++
|
|
28
29
|
animationFrames.set(frameId, extendedFn)
|
|
30
|
+
// A pending onFrame callback is a standing request for the next frame.
|
|
31
|
+
ffi.requestFrame()
|
|
29
32
|
}
|
|
30
33
|
|
|
31
34
|
frameId = nextFrameId++
|
|
32
35
|
animationFrames.set(frameId, extendedFn)
|
|
36
|
+
ffi.requestFrame()
|
|
33
37
|
|
|
34
38
|
let cleanup = () => animationFrames.delete(frameId)
|
|
35
39
|
onCleanup(cleanup)
|
|
@@ -53,7 +57,7 @@ interface ResizeEvent {
|
|
|
53
57
|
}
|
|
54
58
|
|
|
55
59
|
export function onResize(fn: (data: ResizeEvent) => void) {
|
|
56
|
-
let unsubscribe =
|
|
60
|
+
let unsubscribe = on("resize", fn)
|
|
57
61
|
onCleanup(unsubscribe)
|
|
58
62
|
return unsubscribe
|
|
59
63
|
}
|
|
@@ -63,19 +67,19 @@ export function onResize(fn: (data: ResizeEvent) => void) {
|
|
|
63
67
|
// by a re-layout pass before painting (one extra pass; cascades beyond that
|
|
64
68
|
// paint stale).
|
|
65
69
|
export function onLayout(fn: () => void) {
|
|
66
|
-
let unsubscribe =
|
|
70
|
+
let unsubscribe = on("postLayout", fn)
|
|
67
71
|
onCleanup(unsubscribe)
|
|
68
72
|
return unsubscribe
|
|
69
73
|
}
|
|
70
74
|
|
|
71
75
|
export function onWindowFocus(fn: () => void) {
|
|
72
|
-
let unsubscribe =
|
|
76
|
+
let unsubscribe = on("windowFocus", fn)
|
|
73
77
|
onCleanup(unsubscribe)
|
|
74
78
|
return unsubscribe
|
|
75
79
|
}
|
|
76
80
|
|
|
77
81
|
export function onWindowBlur(fn: () => void) {
|
|
78
|
-
let unsubscribe =
|
|
82
|
+
let unsubscribe = on("windowBlur", fn)
|
|
79
83
|
onCleanup(unsubscribe)
|
|
80
84
|
return unsubscribe
|
|
81
85
|
}
|
|
@@ -109,15 +113,15 @@ export function attachWindow(_nodeId: number) {
|
|
|
109
113
|
|
|
110
114
|
onSettled(() => {
|
|
111
115
|
// Sticky event: a late subscriber still receives the current rate.
|
|
112
|
-
unsubRefreshRate =
|
|
116
|
+
unsubRefreshRate = on("displayRefreshRate", ({ hz }: { hz: number }) => {
|
|
113
117
|
if (hz > 0) refreshRate = hz
|
|
114
118
|
})
|
|
115
119
|
|
|
116
|
-
unsubscribe =
|
|
120
|
+
unsubscribe = on("render", ({ time, frame }: { time: number; frame: number }) => {
|
|
117
121
|
runFrame(time * 1000, frame)
|
|
118
122
|
})
|
|
119
123
|
|
|
120
|
-
unsubDown =
|
|
124
|
+
unsubDown = on(
|
|
121
125
|
"pointerDown",
|
|
122
126
|
({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
|
|
123
127
|
for (let nodeId of targets) {
|
|
@@ -132,13 +136,13 @@ export function attachWindow(_nodeId: number) {
|
|
|
132
136
|
},
|
|
133
137
|
)
|
|
134
138
|
|
|
135
|
-
unsubUp =
|
|
139
|
+
unsubUp = on("pointerUp", ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
|
|
136
140
|
for (let nodeId of targets) {
|
|
137
141
|
getEventHandler(nodeId, "onPointerUp")?.(e)
|
|
138
142
|
}
|
|
139
143
|
})
|
|
140
144
|
|
|
141
|
-
unsubMove =
|
|
145
|
+
unsubMove = on(
|
|
142
146
|
"pointerMove",
|
|
143
147
|
({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
|
|
144
148
|
for (let nodeId of targets) {
|
|
@@ -147,7 +151,7 @@ export function attachWindow(_nodeId: number) {
|
|
|
147
151
|
},
|
|
148
152
|
)
|
|
149
153
|
|
|
150
|
-
unsubEnter =
|
|
154
|
+
unsubEnter = on(
|
|
151
155
|
"pointerEnter",
|
|
152
156
|
({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
|
|
153
157
|
for (let nodeId of targets) {
|
|
@@ -156,7 +160,7 @@ export function attachWindow(_nodeId: number) {
|
|
|
156
160
|
},
|
|
157
161
|
)
|
|
158
162
|
|
|
159
|
-
unsubLeave =
|
|
163
|
+
unsubLeave = on(
|
|
160
164
|
"pointerLeave",
|
|
161
165
|
({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
|
|
162
166
|
for (let nodeId of targets) {
|
|
@@ -165,27 +169,27 @@ export function attachWindow(_nodeId: number) {
|
|
|
165
169
|
},
|
|
166
170
|
)
|
|
167
171
|
|
|
168
|
-
unsubWheel =
|
|
172
|
+
unsubWheel = on("wheel", ({ targets, ...e }: { targets: number[]; [k: string]: any }) => {
|
|
169
173
|
for (let nodeId of targets) {
|
|
170
174
|
getEventHandler(nodeId, "onWheel")?.(e)
|
|
171
175
|
}
|
|
172
176
|
})
|
|
173
177
|
|
|
174
|
-
unsubKeyDown =
|
|
178
|
+
unsubKeyDown = on("keydown", (e: any) => {
|
|
175
179
|
let id = getFocusedNodeId()
|
|
176
180
|
if (id != null) {
|
|
177
181
|
getEventHandler(id, "onKeyDown")?.(e)
|
|
178
182
|
}
|
|
179
183
|
})
|
|
180
184
|
|
|
181
|
-
unsubKeyUp =
|
|
185
|
+
unsubKeyUp = on("keyup", (e: any) => {
|
|
182
186
|
let id = getFocusedNodeId()
|
|
183
187
|
if (id != null) {
|
|
184
188
|
getEventHandler(id, "onKeyUp")?.(e)
|
|
185
189
|
}
|
|
186
190
|
})
|
|
187
191
|
|
|
188
|
-
unsubTextInput =
|
|
192
|
+
unsubTextInput = on("textInput", (e: any) => {
|
|
189
193
|
let id = getFocusedNodeId()
|
|
190
194
|
if (id != null) {
|
|
191
195
|
getEventHandler(id, "onTextInput")?.(e)
|
|
@@ -194,7 +198,7 @@ export function attachWindow(_nodeId: number) {
|
|
|
194
198
|
|
|
195
199
|
// When the user dismisses the on-screen keyboard (swipe down, "Done",
|
|
196
200
|
// back button), blur the focused node so the app's UI state catches up.
|
|
197
|
-
unsubKeyboardVisibility =
|
|
201
|
+
unsubKeyboardVisibility = on("keyboardVisibility", ({ shown }: { shown: boolean }) => {
|
|
198
202
|
if (!shown) setFocus(null)
|
|
199
203
|
})
|
|
200
204
|
|
|
@@ -205,7 +209,7 @@ export function attachWindow(_nodeId: number) {
|
|
|
205
209
|
// synchronously here, while we are still inside this onSettled callback
|
|
206
210
|
// where flush() is illegal (not reentrant). Defer runFrame to a microtask
|
|
207
211
|
// so the first frame always runs after this callback returns.
|
|
208
|
-
unsubFirstResize =
|
|
212
|
+
unsubFirstResize = once("resize", () => {
|
|
209
213
|
queueMicrotask(() => runFrame(0, 0))
|
|
210
214
|
})
|
|
211
215
|
})
|