lecodes-sdk 2.2.9 → 2.2.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/global.d.ts +42 -1
- package/dist/types/canvas/Canvas.d.ts +6 -6
- package/dist/types/gl/Camera.d.ts +5 -2
- package/dist/types/gl/Node.d.ts +0 -7
- package/dist/types/gl/Scene.d.ts +6 -24
- package/dist/types/inject.d.ts +5 -1
- package/dist/types/plugin.d.ts +27 -0
- package/dist/types/plugins/gen/ar/sdk/ar.d.ts +228 -0
- package/dist/types/plugins/gen/ar/sdk/ar.gen.d.ts +73 -0
- package/dist/types/plugins/gen/ar/sdk/controls.d.ts +28 -0
- package/dist/types/plugins/gen/audio-recorder/sdk/audio-recorder.d.ts +43 -0
- package/dist/types/plugins/gen/audio-recorder/sdk/audio-recorder.gen.d.ts +47 -0
- package/dist/types/plugins/gen/geolocation/sdk/geolocation.gen.d.ts +6 -2
- package/dist/types/plugins/gen/speech/sdk/speech.d.ts +44 -0
- package/dist/types/plugins/gen/speech/sdk/speech.gen.d.ts +39 -0
- package/dist/types/runtime/media.d.ts +3 -2
- package/dist/types/runtime/permissions.d.ts +16 -0
- package/dist/types/version.d.ts +1 -1
- package/dist/types.json +1 -1
- package/package.json +1 -1
- package/prompts/ar.md +81 -48
- package/prompts/dist/2d-game.md +1567 -0
- package/prompts/dist/3d-app.md +1712 -0
- package/prompts/dist/ar-app.md +1585 -0
- package/prompts/dist/concept.md +49 -0
- package/prompts/dist/design.md +521 -0
- package/prompts/dist/ui-app.md +1175 -0
- package/prompts/index.md +3 -1
- package/src/animate/tween/Timeline.ts +175 -175
- package/src/bridges/device.d.ts +20 -6
- package/src/bridges/gl.d.ts +33 -19
- package/src/bridges/media.d.ts +3 -1
- package/src/canvas/Canvas.ts +6 -6
- package/src/compile/header.ts +11 -4
- package/src/gl/Camera.ts +5 -2
- package/src/gl/CameraPlace.ts +52 -52
- package/src/gl/Light.ts +257 -257
- package/src/gl/Mesh.ts +120 -120
- package/src/gl/Node.ts +0 -8
- package/src/gl/Scene.ts +14 -81
- package/src/gl/animation/DynamicBone.ts +482 -482
- package/src/gl/scenarios.ts +291 -291
- package/src/inject.ts +13 -1
- package/src/plugin.ts +45 -2
- package/src/plugins/gen/ar/contract.d.ts +118 -0
- package/src/plugins/gen/ar/sdk/ar.gen.ts +112 -0
- package/src/plugins/gen/ar/sdk/ar.ts +699 -0
- package/src/plugins/gen/ar/sdk/controls.ts +148 -0
- package/src/plugins/gen/audio-recorder/contract.d.ts +64 -0
- package/src/plugins/gen/audio-recorder/sdk/audio-recorder.gen.ts +76 -0
- package/src/plugins/gen/audio-recorder/sdk/audio-recorder.ts +113 -0
- package/src/plugins/gen/geolocation/contract.d.ts +7 -5
- package/src/plugins/gen/geolocation/sdk/geolocation.gen.ts +6 -2
- package/src/plugins/gen/speech/contract.d.ts +67 -0
- package/src/plugins/gen/speech/sdk/speech.gen.ts +64 -0
- package/src/plugins/gen/speech/sdk/speech.ts +104 -0
- package/src/runtime/media.ts +11 -4
- package/src/runtime/permissions.ts +40 -0
- package/src/server/auth/host.ts +1 -1
- package/src/ui/presentable.ts +5 -0
- package/src/ui/router.ts +4 -3
- package/src/version.ts +1 -1
- package/dist/types/gl/controls.d.ts +0 -19
- package/dist/types/plugins/permission.d.ts +0 -1
- package/src/gl/controls.ts +0 -95
- package/src/plugins/permission.ts +0 -8
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
// GENERATED by scripts/vendor-plugins.ts from lecodes-plugins/plugins/ar/sdk/controls.ts — do not edit here.
|
|
2
|
+
// Change it in the plugin repo and run `bun run vendor:plugins`; `bun run check:plugins` fails when this copy is stale.
|
|
3
|
+
// AR object-placement controls — the classic "drop a model in the room and manipulate it" gesture
|
|
4
|
+
// recipe: ONE finger drags the target over what the session finds under the finger (`hitTest`: a
|
|
5
|
+
// plane), and where it finds nothing — the web, which finds no surfaces, or before one is known —
|
|
6
|
+
// across the ground plane, relative to where the camera looks; TWO fingers pinch to scale and twist
|
|
7
|
+
// to rotate (yaw). Lives on ARScene (`scene.addControls`) because it orchestrates the scene's camera,
|
|
8
|
+
// its hit test and touch input against a target node; it's built entirely on the public API, so
|
|
9
|
+
// it's a convenience, not a primitive.
|
|
10
|
+
|
|
11
|
+
import { Quat, Vec2, Vec3, type Node, type Scene, type TouchStartEvent } from "../../../../plugin"
|
|
12
|
+
import type { HitTarget } from "../contract"
|
|
13
|
+
|
|
14
|
+
export type ControlsOptions = {
|
|
15
|
+
/** Smallest uniform scale the pinch gesture allows (default 0.08). */
|
|
16
|
+
minScale?: number
|
|
17
|
+
/** Largest uniform scale the pinch gesture allows (default 2). */
|
|
18
|
+
maxScale?: number
|
|
19
|
+
/** One-finger drag moves the target over the surfaces under the finger (default true). */
|
|
20
|
+
pan?: boolean
|
|
21
|
+
/** Two-finger pinch scales the target (default true). */
|
|
22
|
+
pinch?: boolean
|
|
23
|
+
/** Two-finger twist rotates the target around its Y axis (default true). */
|
|
24
|
+
twist?: boolean
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/** Returned by `addControls` — call `remove()` to detach the gesture listeners. */
|
|
28
|
+
export type ControlsHandle = { remove(): void }
|
|
29
|
+
|
|
30
|
+
/** The scene the controls work on: ARScene's hit test, a Scene's camera and touches. */
|
|
31
|
+
type PlacingScene = Scene & {
|
|
32
|
+
hitTest(x: number, y: number, target?: HitTarget): Promise<{ pose: { position: Vec3 } }[]>
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
type ActiveTouch = { pointerId: number, x: number, y: number }
|
|
36
|
+
|
|
37
|
+
const clamp = (v: number, lo: number, hi: number): number => (v < lo ? lo : v > hi ? hi : v)
|
|
38
|
+
|
|
39
|
+
export const attachControls = (scene: PlacingScene, target: Node, options: ControlsOptions = {}): ControlsHandle => {
|
|
40
|
+
const minScale = options.minScale ?? 0.08
|
|
41
|
+
const maxScale = options.maxScale ?? 2
|
|
42
|
+
const pan = options.pan ?? true
|
|
43
|
+
const pinch = options.pinch ?? true
|
|
44
|
+
const twist = options.twist ?? true
|
|
45
|
+
|
|
46
|
+
const active: ActiveTouch[] = []
|
|
47
|
+
let removed = false
|
|
48
|
+
|
|
49
|
+
// --- one finger: the hit test is a call to the session, so ONE is in flight at a time; the moves
|
|
50
|
+
// that come meanwhile add up and go with the next ---
|
|
51
|
+
let pending: { x: number, y: number, dx: number, dy: number } | null = null
|
|
52
|
+
let asking = false
|
|
53
|
+
|
|
54
|
+
/** The target onto a point of the world, its height and turn kept by the surface's. */
|
|
55
|
+
const placeAt = (world: Vec3): void => {
|
|
56
|
+
const w = target.worldMatrix.toArray()
|
|
57
|
+
w[12] = world.x; w[13] = world.y; w[14] = world.z
|
|
58
|
+
target.worldMatrix = w
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/** Nothing under the finger: across the ground plane, relative to where the camera looks. */
|
|
62
|
+
const slide = (dx: number, dy: number): void => {
|
|
63
|
+
let dir = scene.camera.forward
|
|
64
|
+
const parent = target.parent
|
|
65
|
+
if (parent) dir = dir.rotate(parent.quaternion.invert()) // into the parent's local frame
|
|
66
|
+
dir = dir.withY(0)
|
|
67
|
+
const k = Math.hypot(dir.x, dir.z)
|
|
68
|
+
// the move's delta over the view's size: both logical units (a scene touch's, displaySize's), so
|
|
69
|
+
// a drag across the view slides the target by its distance from the camera on every host
|
|
70
|
+
const screen = Math.max(...scene.camera.displaySize)
|
|
71
|
+
if (k > 1e-4 && screen > 0) {
|
|
72
|
+
const dist = target.worldPosition.distanceTo(scene.camera.worldPosition)
|
|
73
|
+
const right = new Vec3(-dir.z, 0, dir.x).scale(dx / k / screen * dist)
|
|
74
|
+
const fwd = dir.scale(-dy / k / screen * dist)
|
|
75
|
+
const move = right.add(fwd)
|
|
76
|
+
const p = target.position
|
|
77
|
+
target.position = new Vec3(p.x + move.x, p.y, p.z + move.z) // keep height fixed
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const ask = (): void => {
|
|
82
|
+
const drag = pending
|
|
83
|
+
if (!drag || asking) return
|
|
84
|
+
pending = null
|
|
85
|
+
asking = true
|
|
86
|
+
scene.hitTest(drag.x, drag.y, "planes").catch(() => []).then((hits) => {
|
|
87
|
+
asking = false
|
|
88
|
+
if (removed) return
|
|
89
|
+
const hit = hits[0]
|
|
90
|
+
if (hit) placeAt(hit.pose.position)
|
|
91
|
+
else slide(drag.dx, drag.dy)
|
|
92
|
+
ask()
|
|
93
|
+
})
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const drag = (x: number, y: number, dx: number, dy: number): void => {
|
|
97
|
+
if (pending) { pending.x = x; pending.y = y; pending.dx += dx; pending.dy += dy }
|
|
98
|
+
else pending = { x, y, dx, dy }
|
|
99
|
+
ask()
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const onTouchStart = (e: TouchStartEvent<Node | null>): void => {
|
|
103
|
+
const touch: ActiveTouch = { pointerId: e.pointerId, x: e.clientX, y: e.clientY }
|
|
104
|
+
active.push(touch)
|
|
105
|
+
e.track({
|
|
106
|
+
onMove(m) {
|
|
107
|
+
if (active.length === 2) {
|
|
108
|
+
// --- two-finger: pinch-scale + twist-rotate about the gesture axis ---
|
|
109
|
+
const other = active.find((t) => t !== touch)
|
|
110
|
+
if (other) {
|
|
111
|
+
const axis = new Vec2(other.x - touch.x, other.y - touch.y)
|
|
112
|
+
const span = axis.length()
|
|
113
|
+
if (span > 1e-3) {
|
|
114
|
+
const dir = axis.scale(1 / span)
|
|
115
|
+
const delta = new Vec2(m.deltaX, m.deltaY)
|
|
116
|
+
if (pinch) {
|
|
117
|
+
const along = -dir.dot(delta) // movement along the pinch axis
|
|
118
|
+
target.scale = clamp(target.scale.x * (span + along) / span, minScale * 0.4, maxScale * 1.5)
|
|
119
|
+
}
|
|
120
|
+
if (twist) {
|
|
121
|
+
const tangent = dir.perp().dot(delta) // movement across the axis
|
|
122
|
+
target.quaternion = target.quaternion.mul(Quat.fromAxisAngle(Vec3.up, Math.atan2(tangent, span)))
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
} else if (active.length === 1 && pan) {
|
|
127
|
+
drag(m.clientX, m.clientY, m.deltaX, m.deltaY)
|
|
128
|
+
}
|
|
129
|
+
touch.x = m.clientX
|
|
130
|
+
touch.y = m.clientY
|
|
131
|
+
},
|
|
132
|
+
onEnd() {
|
|
133
|
+
const i = active.indexOf(touch)
|
|
134
|
+
if (i >= 0) active.splice(i, 1)
|
|
135
|
+
},
|
|
136
|
+
})
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
scene.addEventListener("touchstart", onTouchStart)
|
|
140
|
+
return {
|
|
141
|
+
remove() {
|
|
142
|
+
removed = true
|
|
143
|
+
pending = null
|
|
144
|
+
scene.removeEventListener("touchstart", onTouchStart)
|
|
145
|
+
active.length = 0
|
|
146
|
+
},
|
|
147
|
+
}
|
|
148
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// GENERATED by scripts/vendor-plugins.ts from lecodes-plugins/plugins/audio-recorder/contract.d.ts — do not edit here.
|
|
2
|
+
// Change it in the plugin repo and run `bun run vendor:plugins`; `bun run check:plugins` fails when this copy is stale.
|
|
3
|
+
// audio-recorder — audio recording to a file: voice messages, voice notes.
|
|
4
|
+
// A session is ONE recording (the app-facing `AudioRecorder` opens one per `start()`): start, pause
|
|
5
|
+
// / resume, then `stop()` for the file or `cancel()` to discard it. The microphone permission is the
|
|
6
|
+
// contract's (`@permission microphone` on start): the host asks, the halves never do.
|
|
7
|
+
//
|
|
8
|
+
// The format is the platform's encoder: AAC in an .m4a on iOS and Android, what the browser's
|
|
9
|
+
// MediaRecorder gives on the web (webm/opus in Chrome and Firefox, mp4 in Safari) — `mimeType`
|
|
10
|
+
// says which, the app's server decides whether to transcode.
|
|
11
|
+
|
|
12
|
+
import type { File, Service, f64 } from "../../../plugin"
|
|
13
|
+
|
|
14
|
+
export interface RecordOptions {
|
|
15
|
+
/** Kilobits per second of the encoded audio. Default 64 — speech; 128 for music. */
|
|
16
|
+
bitrate?: f64
|
|
17
|
+
/** Samples per second. Default 44100; the browser keeps its own. */
|
|
18
|
+
sampleRate?: f64
|
|
19
|
+
/** Stop on its own after this many ms: `ended` fires with "maxDuration", `stop()` then returns
|
|
20
|
+
* the recording. Default: no limit. */
|
|
21
|
+
maxDuration?: f64
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
export type EndReason = "maxDuration" | "interrupted" | "failed"
|
|
25
|
+
|
|
26
|
+
export interface Recording {
|
|
27
|
+
/** The audio file — `recording.m4a` / `.webm` / `.mp4` by the encoder. */
|
|
28
|
+
file: File
|
|
29
|
+
/** "audio/mp4", "audio/webm", … — the container, without codec parameters. */
|
|
30
|
+
mimeType: string
|
|
31
|
+
/** Seconds of audio (the pauses left out). */
|
|
32
|
+
duration: f64
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export interface AudioRecorderParams {}
|
|
36
|
+
|
|
37
|
+
export interface AudioRecorderEvents {
|
|
38
|
+
/** The input level, 0..1, about ten times a second while recording (not while paused): a meter,
|
|
39
|
+
* a waveform. */
|
|
40
|
+
level: { value: f64 }
|
|
41
|
+
/** The recording stopped on its own: the limit (`maxDuration`), an interruption (a call came in),
|
|
42
|
+
* a failure of the encoder. What was recorded is kept until `stop()` (which returns it) or
|
|
43
|
+
* `cancel()`. */
|
|
44
|
+
ended: { reason: EndReason }
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
export interface AudioRecorder extends Service<"audioRecorder", AudioRecorderParams, AudioRecorderEvents> {
|
|
48
|
+
/** Start recording. One recording per session: a second start rejects "busy".
|
|
49
|
+
* @permission microphone
|
|
50
|
+
* @rejects busy failed */
|
|
51
|
+
start(options?: RecordOptions): Promise<void>
|
|
52
|
+
/** Pause: the level stops, the duration does not grow. Rejects "notRecording" before a start
|
|
53
|
+
* or after the recording ended.
|
|
54
|
+
* @rejects notRecording failed */
|
|
55
|
+
pause(): Promise<void>
|
|
56
|
+
/** @rejects notRecording failed */
|
|
57
|
+
resume(): Promise<void>
|
|
58
|
+
/** Stop and get the recording (also after `ended`: the file was kept). Rejects "failed" when
|
|
59
|
+
* nothing could be encoded.
|
|
60
|
+
* @rejects notRecording failed */
|
|
61
|
+
stop(): Promise<Recording>
|
|
62
|
+
/** Stop and discard. Nothing to discard is not an error. */
|
|
63
|
+
cancel(): Promise<void>
|
|
64
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
// GENERATED by scripts/vendor-plugins.ts from lecodes-plugins/plugins/audio-recorder/sdk/audio-recorder.gen.ts — do not edit here.
|
|
2
|
+
// Change it in the plugin repo and run `bun run vendor:plugins`; `bun run check:plugins` fails when this copy is stale.
|
|
3
|
+
// GENERATED by `lecodes plugin gen` from contract.d.ts — do not edit.
|
|
4
|
+
// audio-recorder: the service "audioRecorder", contract version 1.
|
|
5
|
+
|
|
6
|
+
import { ServiceChannel, _toFile } from "../../../../plugin"
|
|
7
|
+
import type { AudioRecorderEvents, AudioRecorderParams, RecordOptions, Recording } from "../contract"
|
|
8
|
+
|
|
9
|
+
/** What the halves register under. */
|
|
10
|
+
export const NAME = "audioRecorder"
|
|
11
|
+
/** The contract's version: the highest `@since` of its members. */
|
|
12
|
+
export const VERSION = 1
|
|
13
|
+
|
|
14
|
+
const _toRecording = (w: any): Recording => ({ ...w, file: _toFile(w.file) })
|
|
15
|
+
|
|
16
|
+
// ---- the wire, call by call ----------------------------------------------------------------------
|
|
17
|
+
|
|
18
|
+
/** Start recording. One recording per session: a second start rejects "busy".
|
|
19
|
+
* Needs the "microphone" permission: the host asks before the call, "denied" when refused.
|
|
20
|
+
* Rejects with "denied" / "busy" / "failed". */
|
|
21
|
+
export const start = (service: ServiceChannel<AudioRecorderEvents>, options?: RecordOptions): Promise<void> =>
|
|
22
|
+
service._call("start", [options])
|
|
23
|
+
|
|
24
|
+
/** Pause: the level stops, the duration does not grow. Rejects "notRecording" before a start
|
|
25
|
+
* or after the recording ended.
|
|
26
|
+
* Rejects with "notRecording" / "failed". */
|
|
27
|
+
export const pause = (service: ServiceChannel<AudioRecorderEvents>): Promise<void> =>
|
|
28
|
+
service._call("pause", [])
|
|
29
|
+
|
|
30
|
+
/** Rejects with "notRecording" / "failed". */
|
|
31
|
+
export const resume = (service: ServiceChannel<AudioRecorderEvents>): Promise<void> =>
|
|
32
|
+
service._call("resume", [])
|
|
33
|
+
|
|
34
|
+
/** Stop and get the recording (also after `ended`: the file was kept). Rejects "failed" when
|
|
35
|
+
* nothing could be encoded.
|
|
36
|
+
* Rejects with "notRecording" / "failed". */
|
|
37
|
+
export const stop = (service: ServiceChannel<AudioRecorderEvents>): Promise<Recording> =>
|
|
38
|
+
service._call("stop", []).then((w: any) => _toRecording(w))
|
|
39
|
+
|
|
40
|
+
/** Stop and discard. Nothing to discard is not an error. */
|
|
41
|
+
export const cancel = (service: ServiceChannel<AudioRecorderEvents>): Promise<void> =>
|
|
42
|
+
service._call("cancel", [])
|
|
43
|
+
|
|
44
|
+
// ---- what the host's half has --------------------------------------------------------------------
|
|
45
|
+
|
|
46
|
+
/** A method of the contract. */
|
|
47
|
+
export type Method = "start" | "pause" | "resume" | "stop" | "cancel"
|
|
48
|
+
|
|
49
|
+
/** Whether this host has the service "audioRecorder" — and, given a method, a half new enough for it
|
|
50
|
+
* (a method added after the first version of the contract is tagged `@since`). */
|
|
51
|
+
export const supports = (method?: Method): boolean =>
|
|
52
|
+
ServiceChannel.supports(NAME) // every method is of version 1
|
|
53
|
+
|
|
54
|
+
// ---- the client -----------------------------------------------------------------------------------
|
|
55
|
+
|
|
56
|
+
/** The service "audioRecorder" with the wire's methods as its own. */
|
|
57
|
+
export class AudioRecorderBase extends ServiceChannel<AudioRecorderEvents> {
|
|
58
|
+
constructor(params: AudioRecorderParams = {}) { super("AudioRecorder", NAME, params) }
|
|
59
|
+
|
|
60
|
+
/** Start recording. One recording per session: a second start rejects "busy".
|
|
61
|
+
* Needs the "microphone" permission: the host asks before the call, "denied" when refused.
|
|
62
|
+
* Rejects with "denied" / "busy" / "failed". */
|
|
63
|
+
start(options?: RecordOptions): Promise<void> { return start(this, options) }
|
|
64
|
+
/** Pause: the level stops, the duration does not grow. Rejects "notRecording" before a start
|
|
65
|
+
* or after the recording ended.
|
|
66
|
+
* Rejects with "notRecording" / "failed". */
|
|
67
|
+
pause(): Promise<void> { return pause(this) }
|
|
68
|
+
/** Rejects with "notRecording" / "failed". */
|
|
69
|
+
resume(): Promise<void> { return resume(this) }
|
|
70
|
+
/** Stop and get the recording (also after `ended`: the file was kept). Rejects "failed" when
|
|
71
|
+
* nothing could be encoded.
|
|
72
|
+
* Rejects with "notRecording" / "failed". */
|
|
73
|
+
stop(): Promise<Recording> { return stop(this) }
|
|
74
|
+
/** Stop and discard. Nothing to discard is not an error. */
|
|
75
|
+
cancel(): Promise<void> { return cancel(this) }
|
|
76
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
// GENERATED by scripts/vendor-plugins.ts from lecodes-plugins/plugins/audio-recorder/sdk/audio-recorder.ts — do not edit here.
|
|
2
|
+
// Change it in the plugin repo and run `bun run vendor:plugins`; `bun run check:plugins` fails when this copy is stale.
|
|
3
|
+
// AudioRecorder — audio recording to a file: voice messages, voice notes.
|
|
4
|
+
//
|
|
5
|
+
// Hand-written over the generated wire because the app's object is not the wire's session: an
|
|
6
|
+
// `AudioRecorder` is reusable — every `start()` opens a fresh host session (one recording), `stop()`
|
|
7
|
+
// / `cancel()` close it — and its events are the SDK's `addEventListener` pattern.
|
|
8
|
+
//
|
|
9
|
+
// The microphone permission is asked by the host on `start()` (the contract's `@permission`); to
|
|
10
|
+
// ask earlier, or to act on a denial, `permissions.request("microphone")` / `openSettings()`.
|
|
11
|
+
|
|
12
|
+
import { Emitter } from "../../../../plugin"
|
|
13
|
+
import { AudioRecorderBase, NAME } from "./audio-recorder.gen"
|
|
14
|
+
import type { EndReason, RecordOptions, Recording } from "../contract"
|
|
15
|
+
|
|
16
|
+
export type { EndReason, RecordOptions, Recording }
|
|
17
|
+
|
|
18
|
+
/** `"stopped"` = the recording ended on its own (`ended` fired): `stop()` still returns it. */
|
|
19
|
+
export type RecorderState = "inactive" | "recording" | "paused" | "stopped"
|
|
20
|
+
|
|
21
|
+
export type AudioRecorderEvents = {
|
|
22
|
+
/** The input level, 0..1, about ten times a second while recording — a meter, a waveform. */
|
|
23
|
+
level: (value: number) => void
|
|
24
|
+
/** The recording stopped on its own: the limit, an interruption (a call came in), a failure.
|
|
25
|
+
* `stop()` returns what was recorded. */
|
|
26
|
+
ended: (reason: EndReason) => void
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Records the microphone to a file. `AudioRecorder.isSupported` reports whether this host
|
|
31
|
+
* registered the service — check it before showing a record button.
|
|
32
|
+
*
|
|
33
|
+
* ```ts
|
|
34
|
+
* const recorder = new AudioRecorder()
|
|
35
|
+
* recorder.addEventListener('level', v => meter.style({ width: v * 100 }))
|
|
36
|
+
* await recorder.start({ maxDuration: 60_000 })
|
|
37
|
+
* const { file, duration, mimeType } = await recorder.stop()
|
|
38
|
+
* ```
|
|
39
|
+
*/
|
|
40
|
+
export class AudioRecorder extends Emitter<AudioRecorderEvents> {
|
|
41
|
+
/** Whether this host registered an "audioRecorder" service. */
|
|
42
|
+
static get isSupported(): boolean { return AudioRecorderBase.supports(NAME) }
|
|
43
|
+
|
|
44
|
+
private _session: AudioRecorderBase | null = null
|
|
45
|
+
private _state: RecorderState = "inactive"
|
|
46
|
+
|
|
47
|
+
get state(): RecorderState { return this._state }
|
|
48
|
+
|
|
49
|
+
/** Start a recording. The host asks for the microphone first; rejects `"denied"` when refused,
|
|
50
|
+
* `"failed"` when the recorder could not start, and before touching the host when it is not
|
|
51
|
+
* supported or a recording is under way. */
|
|
52
|
+
async start(options?: RecordOptions): Promise<void> {
|
|
53
|
+
if (!AudioRecorder.isSupported) {
|
|
54
|
+
throw new Error('AudioRecorder isn\'t available on this host — it needs a registered "audioRecorder" service.')
|
|
55
|
+
}
|
|
56
|
+
if (this._session) throw new Error("AudioRecorder: a recording is under way — stop() or cancel() it first")
|
|
57
|
+
const session = new AudioRecorderBase()
|
|
58
|
+
session.on("level", payload => { if (payload) this.dispatch("level", payload.value) })
|
|
59
|
+
session.on("ended", payload => {
|
|
60
|
+
if (!payload || this._session !== session) return
|
|
61
|
+
this._state = "stopped"
|
|
62
|
+
this.dispatch("ended", payload.reason)
|
|
63
|
+
})
|
|
64
|
+
this._session = session
|
|
65
|
+
try {
|
|
66
|
+
await session.start(options ?? {})
|
|
67
|
+
} catch (e) {
|
|
68
|
+
this._drop(session)
|
|
69
|
+
throw e
|
|
70
|
+
}
|
|
71
|
+
if (this._session === session && this._state === "inactive") this._state = "recording"
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** Pause: the level stops, the duration does not grow. */
|
|
75
|
+
async pause(): Promise<void> {
|
|
76
|
+
if (!this._session || this._state !== "recording") throw new Error("AudioRecorder: not recording")
|
|
77
|
+
await this._session.pause()
|
|
78
|
+
if (this._state === "recording") this._state = "paused"
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async resume(): Promise<void> {
|
|
82
|
+
if (!this._session || this._state !== "paused") throw new Error("AudioRecorder: not paused")
|
|
83
|
+
await this._session.resume()
|
|
84
|
+
if (this._state === "paused") this._state = "recording"
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/** Stop and get the recording — also after `ended`. Rejects `"failed"` when nothing could be
|
|
88
|
+
* encoded. The recorder is ready for another `start()` afterwards. */
|
|
89
|
+
async stop(): Promise<Recording> {
|
|
90
|
+
const session = this._session
|
|
91
|
+
if (!session) throw new Error("AudioRecorder: nothing to stop")
|
|
92
|
+
try {
|
|
93
|
+
return await session.stop()
|
|
94
|
+
} finally {
|
|
95
|
+
this._drop(session)
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** Stop and discard. Nothing under way is not an error. */
|
|
100
|
+
cancel(): void {
|
|
101
|
+
const session = this._session
|
|
102
|
+
if (!session) return
|
|
103
|
+
session.cancel().catch(() => {})
|
|
104
|
+
this._drop(session)
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
private _drop(session: AudioRecorderBase): void {
|
|
108
|
+
if (this._session !== session) return
|
|
109
|
+
session.close()
|
|
110
|
+
this._session = null
|
|
111
|
+
this._state = "inactive"
|
|
112
|
+
}
|
|
113
|
+
}
|
|
@@ -4,8 +4,8 @@
|
|
|
4
4
|
// The wire has ONE watch per session (startWatch / stopWatch + the `position` event); the
|
|
5
5
|
// app-facing `Geolocation.watch(callback)` fans it out to its subscribers in sdk/.
|
|
6
6
|
//
|
|
7
|
-
// A session opens without a prompt. The OS permission
|
|
8
|
-
// the position.
|
|
7
|
+
// A session opens without a prompt. The OS permission is the calls' (`@permission location`): the
|
|
8
|
+
// host asks before the first call that needs the position, the halves never do.
|
|
9
9
|
|
|
10
10
|
import type { Service, f64 } from "../../../plugin"
|
|
11
11
|
|
|
@@ -40,11 +40,13 @@ export interface GeolocationEvents {
|
|
|
40
40
|
|
|
41
41
|
export interface Geolocation extends Service<"geolocation", GeolocationParams, GeolocationEvents> {
|
|
42
42
|
/** One fix.
|
|
43
|
-
* @
|
|
43
|
+
* @permission location
|
|
44
|
+
* @rejects unavailable timeout */
|
|
44
45
|
getCurrent(options?: GeoOptions): Promise<GeoPosition>
|
|
45
|
-
/** Start the session's watch; resolves once it is live (
|
|
46
|
+
/** Start the session's watch; resolves once it is live (the provider started).
|
|
46
47
|
* A second call while watching resolves at once and keeps the first call's options.
|
|
47
|
-
* @
|
|
48
|
+
* @permission location
|
|
49
|
+
* @rejects unavailable */
|
|
48
50
|
startWatch(options?: GeoOptions): Promise<void>
|
|
49
51
|
stopWatch(): Promise<void>
|
|
50
52
|
}
|
|
@@ -14,12 +14,14 @@ export const VERSION = 1
|
|
|
14
14
|
// ---- the wire, call by call ----------------------------------------------------------------------
|
|
15
15
|
|
|
16
16
|
/** One fix.
|
|
17
|
+
* Needs the "location" permission: the host asks before the call, "denied" when refused.
|
|
17
18
|
* Rejects with "denied" / "unavailable" / "timeout". */
|
|
18
19
|
export const getCurrent = (service: ServiceChannel<GeolocationEvents>, options?: GeoOptions): Promise<GeoPosition> =>
|
|
19
20
|
service._call("getCurrent", [options])
|
|
20
21
|
|
|
21
|
-
/** Start the session's watch; resolves once it is live (
|
|
22
|
+
/** Start the session's watch; resolves once it is live (the provider started).
|
|
22
23
|
* A second call while watching resolves at once and keeps the first call's options.
|
|
24
|
+
* Needs the "location" permission: the host asks before the call, "denied" when refused.
|
|
23
25
|
* Rejects with "denied" / "unavailable". */
|
|
24
26
|
export const startWatch = (service: ServiceChannel<GeolocationEvents>, options?: GeoOptions): Promise<void> =>
|
|
25
27
|
service._call("startWatch", [options])
|
|
@@ -44,10 +46,12 @@ export class GeolocationBase extends ServiceChannel<GeolocationEvents> {
|
|
|
44
46
|
constructor(params: GeolocationParams = {}) { super("Geolocation", NAME, params) }
|
|
45
47
|
|
|
46
48
|
/** One fix.
|
|
49
|
+
* Needs the "location" permission: the host asks before the call, "denied" when refused.
|
|
47
50
|
* Rejects with "denied" / "unavailable" / "timeout". */
|
|
48
51
|
getCurrent(options?: GeoOptions): Promise<GeoPosition> { return getCurrent(this, options) }
|
|
49
|
-
/** Start the session's watch; resolves once it is live (
|
|
52
|
+
/** Start the session's watch; resolves once it is live (the provider started).
|
|
50
53
|
* A second call while watching resolves at once and keeps the first call's options.
|
|
54
|
+
* Needs the "location" permission: the host asks before the call, "denied" when refused.
|
|
51
55
|
* Rejects with "denied" / "unavailable". */
|
|
52
56
|
startWatch(options?: GeoOptions): Promise<void> { return startWatch(this, options) }
|
|
53
57
|
stopWatch(): Promise<void> { return stopWatch(this) }
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
// GENERATED by scripts/vendor-plugins.ts from lecodes-plugins/plugins/speech/contract.d.ts — do not edit here.
|
|
2
|
+
// Change it in the plugin repo and run `bun run vendor:plugins`; `bun run check:plugins` fails when this copy is stale.
|
|
3
|
+
// speech — speech recognition: the microphone to text, live (dictation, voice commands).
|
|
4
|
+
// A session is ONE listening (the app-facing `SpeechRecognizer` opens one per `start()`): results
|
|
5
|
+
// arrive as events, partial ones while the user speaks and a final one after, then `end`. The
|
|
6
|
+
// permissions are the contract's (`@permission speech microphone` on start): the host asks, the
|
|
7
|
+
// halves never do.
|
|
8
|
+
//
|
|
9
|
+
// The recognizer is the platform's: Apple's (SFSpeechRecognizer: on-device where the language
|
|
10
|
+
// allows, else Apple's servers), Google's on Android (the device's RecognitionService — absent on a
|
|
11
|
+
// phone without Google's app: `start` rejects "unavailable"), the browser's on the web (Chrome sends
|
|
12
|
+
// the audio to Google; Safari to Apple). Languages and quality differ with them.
|
|
13
|
+
|
|
14
|
+
import type { Service, f64 } from "../../../plugin"
|
|
15
|
+
|
|
16
|
+
export interface SpeechOptions {
|
|
17
|
+
/** BCP-47 — "en-US", "ru-RU". Default: the device's language. */
|
|
18
|
+
language?: string
|
|
19
|
+
/** Keep listening and delivering results until `stop()`; default false: one utterance — the
|
|
20
|
+
* listening ends after a pause in the speech. */
|
|
21
|
+
continuous?: boolean
|
|
22
|
+
/** Deliver partial results while the user speaks (`isFinal` false). Default true. */
|
|
23
|
+
interimResults?: boolean
|
|
24
|
+
/** Recognize on the device only — no audio leaves it; fewer languages, and "unavailable" where
|
|
25
|
+
* the device cannot. Default false. The web ignores it. */
|
|
26
|
+
onDevice?: boolean
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
export interface SpeechResult {
|
|
30
|
+
/** The text so far (a partial), or of the utterance (final). */
|
|
31
|
+
transcript: string
|
|
32
|
+
isFinal: boolean
|
|
33
|
+
/** 0..1 when the recognizer says; null when it does not (partials, Android). */
|
|
34
|
+
confidence: f64 | null
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export type SpeechErrorCode = "noSpeech" | "audio" | "network" | "denied" | "aborted" | "unavailable" | "failed"
|
|
38
|
+
|
|
39
|
+
export interface SpeechParams {}
|
|
40
|
+
|
|
41
|
+
export interface SpeechEvents {
|
|
42
|
+
/** A result: partials while speaking (when `interimResults`), then the final one. */
|
|
43
|
+
result: SpeechResult
|
|
44
|
+
/** The user started speaking. */
|
|
45
|
+
speechStart: void
|
|
46
|
+
/** The user stopped speaking (a pause): the final result follows. */
|
|
47
|
+
speechEnd: void
|
|
48
|
+
/** The listening is over — after the final result, after `stop()` / `abort()`, after an error. */
|
|
49
|
+
end: void
|
|
50
|
+
/** The listening failed; `end` follows. "noSpeech": nothing was heard (not continuous). */
|
|
51
|
+
error: { code: SpeechErrorCode, message: string }
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export interface Speech extends Service<"speech", SpeechParams, SpeechEvents> {
|
|
55
|
+
/** Start listening. One listening per session: a second start rejects "busy". "unavailable":
|
|
56
|
+
* no recognizer on this device, or none for the language / on the device.
|
|
57
|
+
* @permission speech microphone
|
|
58
|
+
* @rejects busy unavailable failed */
|
|
59
|
+
start(options?: SpeechOptions): Promise<void>
|
|
60
|
+
/** Stop listening and finish: the final result of what was heard, then `end`. Not listening is
|
|
61
|
+
* not an error. */
|
|
62
|
+
stop(): Promise<void>
|
|
63
|
+
/** Stop and drop what was heard: `end`, no result. */
|
|
64
|
+
abort(): Promise<void>
|
|
65
|
+
/** The languages this host recognizes, BCP-47; empty when the host cannot say (the web). */
|
|
66
|
+
supportedLanguages(): Promise<string[]>
|
|
67
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// GENERATED by scripts/vendor-plugins.ts from lecodes-plugins/plugins/speech/sdk/speech.gen.ts — do not edit here.
|
|
2
|
+
// Change it in the plugin repo and run `bun run vendor:plugins`; `bun run check:plugins` fails when this copy is stale.
|
|
3
|
+
// GENERATED by `lecodes plugin gen` from contract.d.ts — do not edit.
|
|
4
|
+
// speech: the service "speech", contract version 1.
|
|
5
|
+
|
|
6
|
+
import { ServiceChannel } from "../../../../plugin"
|
|
7
|
+
import type { SpeechEvents, SpeechOptions, SpeechParams } from "../contract"
|
|
8
|
+
|
|
9
|
+
/** What the halves register under. */
|
|
10
|
+
export const NAME = "speech"
|
|
11
|
+
/** The contract's version: the highest `@since` of its members. */
|
|
12
|
+
export const VERSION = 1
|
|
13
|
+
|
|
14
|
+
// ---- the wire, call by call ----------------------------------------------------------------------
|
|
15
|
+
|
|
16
|
+
/** Start listening. One listening per session: a second start rejects "busy". "unavailable":
|
|
17
|
+
* no recognizer on this device, or none for the language / on the device.
|
|
18
|
+
* Needs the "speech" and "microphone" permissions: the host asks before the call, "denied" when refused.
|
|
19
|
+
* Rejects with "denied" / "busy" / "unavailable" / "failed". */
|
|
20
|
+
export const start = (service: ServiceChannel<SpeechEvents>, options?: SpeechOptions): Promise<void> =>
|
|
21
|
+
service._call("start", [options])
|
|
22
|
+
|
|
23
|
+
/** Stop listening and finish: the final result of what was heard, then `end`. Not listening is
|
|
24
|
+
* not an error. */
|
|
25
|
+
export const stop = (service: ServiceChannel<SpeechEvents>): Promise<void> =>
|
|
26
|
+
service._call("stop", [])
|
|
27
|
+
|
|
28
|
+
/** Stop and drop what was heard: `end`, no result. */
|
|
29
|
+
export const abort = (service: ServiceChannel<SpeechEvents>): Promise<void> =>
|
|
30
|
+
service._call("abort", [])
|
|
31
|
+
|
|
32
|
+
/** The languages this host recognizes, BCP-47; empty when the host cannot say (the web). */
|
|
33
|
+
export const supportedLanguages = (service: ServiceChannel<SpeechEvents>): Promise<string[]> =>
|
|
34
|
+
service._call("supportedLanguages", [])
|
|
35
|
+
|
|
36
|
+
// ---- what the host's half has --------------------------------------------------------------------
|
|
37
|
+
|
|
38
|
+
/** A method of the contract. */
|
|
39
|
+
export type Method = "start" | "stop" | "abort" | "supportedLanguages"
|
|
40
|
+
|
|
41
|
+
/** Whether this host has the service "speech" — and, given a method, a half new enough for it
|
|
42
|
+
* (a method added after the first version of the contract is tagged `@since`). */
|
|
43
|
+
export const supports = (method?: Method): boolean =>
|
|
44
|
+
ServiceChannel.supports(NAME) // every method is of version 1
|
|
45
|
+
|
|
46
|
+
// ---- the client -----------------------------------------------------------------------------------
|
|
47
|
+
|
|
48
|
+
/** The service "speech" with the wire's methods as its own. */
|
|
49
|
+
export class SpeechBase extends ServiceChannel<SpeechEvents> {
|
|
50
|
+
constructor(params: SpeechParams = {}) { super("Speech", NAME, params) }
|
|
51
|
+
|
|
52
|
+
/** Start listening. One listening per session: a second start rejects "busy". "unavailable":
|
|
53
|
+
* no recognizer on this device, or none for the language / on the device.
|
|
54
|
+
* Needs the "speech" and "microphone" permissions: the host asks before the call, "denied" when refused.
|
|
55
|
+
* Rejects with "denied" / "busy" / "unavailable" / "failed". */
|
|
56
|
+
start(options?: SpeechOptions): Promise<void> { return start(this, options) }
|
|
57
|
+
/** Stop listening and finish: the final result of what was heard, then `end`. Not listening is
|
|
58
|
+
* not an error. */
|
|
59
|
+
stop(): Promise<void> { return stop(this) }
|
|
60
|
+
/** Stop and drop what was heard: `end`, no result. */
|
|
61
|
+
abort(): Promise<void> { return abort(this) }
|
|
62
|
+
/** The languages this host recognizes, BCP-47; empty when the host cannot say (the web). */
|
|
63
|
+
supportedLanguages(): Promise<string[]> { return supportedLanguages(this) }
|
|
64
|
+
}
|