@marver-design/marver 0.25.1 → 0.26.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. package/CHANGELOG.md +116 -0
  2. package/README.md +5 -3
  3. package/dist/{anchor-DVV732zR.mjs → anchor-5H1vnoLu.mjs} +6 -1
  4. package/dist/{anchor-check-DGZnYviN.mjs → anchor-check-l-ny3UzH.mjs} +1 -1
  5. package/dist/audio-Dw1lfMdb.mjs +443 -0
  6. package/dist/{auth-DdUKeVKt.mjs → auth-Dix8wrXV.mjs} +1 -1
  7. package/dist/{bake-BpxMZfij.mjs → bake-C-awKgLA.mjs} +1 -1
  8. package/dist/{boards-Csxj2WJn.mjs → boards-DYG22tOe.mjs} +2 -2
  9. package/dist/{build-DKaGhoVN.mjs → build-BDRiO_mP.mjs} +16 -9
  10. package/dist/capture-vadKjuCo.mjs +730 -0
  11. package/dist/cli.mjs +24 -18
  12. package/dist/{collab-gqNdScUE.mjs → collab-Dxg2trOK.mjs} +4 -3
  13. package/dist/{comments-Zoch254k.mjs → comments-0BSbb04K.mjs} +14 -3
  14. package/dist/{comments-pTLlhjh4.mjs → comments-BqXua-XL.mjs} +7 -7
  15. package/dist/{config-cKgZBRj0.mjs → config-Cf7EjAMf.mjs} +12 -1
  16. package/dist/{context-D2ugHqwO.mjs → context-HQ4Giw58.mjs} +1 -1
  17. package/dist/daemon-DL6Fwq6D.mjs +2569 -0
  18. package/dist/{dev-Diqx1Rt8.mjs → dev-Bw_qYWWq.mjs} +202 -18
  19. package/dist/dev-lock-DWSpa6nj.mjs +179 -0
  20. package/dist/{events-DnK_AnrW.mjs → events-CLH4ijKP.mjs} +14 -4
  21. package/dist/{init-DO4DhUt_.mjs → init-D8DtrK_F.mjs} +2 -2
  22. package/dist/{ledger-BNXW14Xw.mjs → ledger-D54iROST.mjs} +17 -1
  23. package/dist/{link-C-qUIuKU.mjs → link-BNzLvdhI.mjs} +2 -2
  24. package/dist/live-Cgzsicwi.mjs +4 -0
  25. package/dist/{manifest-DnAte_fB.mjs → manifest-Nl3KSVOk.mjs} +1 -1
  26. package/dist/{marks-koRsbHfe.mjs → marks-CKgzucHZ.mjs} +2 -2
  27. package/dist/{marks-DFuintmJ.mjs → marks-DyWMv_Z1.mjs} +23 -5
  28. package/dist/{marver-id-gate-DGANtVyC.mjs → marver-id-gate-BjF6v4bT.mjs} +4 -4
  29. package/dist/perf-C75MZ73h.mjs +167 -0
  30. package/dist/{perf-CCisg-ny.mjs → perf-DPbLGhhO.mjs} +1 -1
  31. package/dist/perf-jam-BU3TVrZu.mjs +239 -0
  32. package/dist/{plugin-dTF5reck.mjs → plugin-DkGmZs_r.mjs} +298 -38
  33. package/dist/{poster-D-zjYpgk.mjs → poster-BdtSKMDm.mjs} +2 -1
  34. package/dist/{profile-Do0nGkXO.mjs → profile-WU9qcbIo.mjs} +1 -1
  35. package/dist/{publish-bakes-C_9VZ-Mq.mjs → publish-bakes-9RzRToXV.mjs} +3 -3
  36. package/dist/queue-29ok5_eZ.mjs +1319 -0
  37. package/dist/reserve-BURNQ2Y3.mjs +53 -0
  38. package/dist/{rolldown-runtime-D7D4PA-g.mjs → rolldown-runtime-8H4AJuhK.mjs} +1 -0
  39. package/dist/{serve-wv2QWdxB.mjs → serve-C8rVheYk.mjs} +12 -12
  40. package/dist/session-BAaJsK5c.mjs +191 -0
  41. package/dist/{share-ScamJcks.mjs → share--oFJ1bCI.mjs} +2 -2
  42. package/dist/{shot-CUlmkqFX.mjs → shot-DQeR_Pfn.mjs} +339 -20
  43. package/dist/shot-XQV7HC9A.mjs +139 -0
  44. package/dist/speech-models-DomiLQcY.mjs +299 -0
  45. package/dist/speech-worker.mjs +354 -0
  46. package/dist/{summary-CrW88e3q.mjs → summary-Y8MY1Wuc.mjs} +2 -2
  47. package/dist/{sync-C3035OnJ.mjs → sync-DKk-yT1V.mjs} +9 -8
  48. package/dist/{work-GE88wb_3.mjs → work-F5u-twpJ.mjs} +28 -5
  49. package/dist/{work-CpJn1gwY.mjs → work-VvX_JIgl.mjs} +2 -2
  50. package/docs/jam-session.md +194 -0
  51. package/docs/live-jam.md +29 -3
  52. package/package.json +7 -1
  53. package/src/client/frame-host/bridge.js +20 -2
  54. package/src/client/frame-host/inspect.d.ts +4 -0
  55. package/src/client/frame-host/inspect.js +96 -6
  56. package/src/client/frame-host/keys.d.ts +6 -0
  57. package/src/client/frame-host/keys.js +59 -0
  58. package/src/client/frame-host/trail.d.ts +7 -0
  59. package/src/client/frame-host/trail.js +204 -0
  60. package/src/client/shell/App.tsx +52 -8
  61. package/src/client/shell/Comments.tsx +20 -6
  62. package/src/client/shell/JamSurface.tsx +346 -0
  63. package/src/client/shell/Play.tsx +32 -9
  64. package/src/client/shell/Tip.tsx +8 -3
  65. package/src/client/shell/Toolbar.tsx +5 -4
  66. package/src/client/shell/canvas/Canvas.tsx +12 -2
  67. package/src/client/shell/canvas/FrameNode.tsx +14 -4
  68. package/src/client/shell/comments-store.ts +23 -16
  69. package/src/client/shell/icons.tsx +17 -0
  70. package/src/client/shell/jam-capture.ts +1005 -0
  71. package/src/client/shell/jam-keys.ts +358 -0
  72. package/src/client/shell/jam-lane.tsx +249 -0
  73. package/src/client/shell/jam-lines.ts +115 -0
  74. package/src/client/shell/jam-panel.tsx +556 -0
  75. package/src/client/shell/jam-queue.ts +126 -0
  76. package/src/client/shell/jam-session.tsx +389 -0
  77. package/src/client/shell/jam-state.ts +103 -0
  78. package/src/client/shell/jam-surface-parts.tsx +106 -0
  79. package/src/client/shell/jam-ui.tsx +104 -0
  80. package/src/client/shell/jam-voice.ts +479 -0
  81. package/src/client/shell/sound.ts +22 -0
  82. package/src/client/shell/stage-size.ts +11 -0
  83. package/src/client/shell/store.ts +47 -17
  84. package/src/client/shell/styles.css +564 -2
  85. package/src/client/stage/main.tsx +18 -2
  86. package/src/shared/capture-frames.ts +41 -0
  87. package/src/shared/events.ts +19 -4
  88. package/src/shared/speech.ts +16 -0
  89. package/templates/AGENTS-embedded.md +10 -2
  90. package/templates/AGENTS-studio.md +10 -2
  91. package/templates/instructions/iterate.md +8 -6
  92. package/templates/instructions/jam.md +131 -59
  93. package/dist/daemon-Cl8KXGW4.mjs +0 -1523
  94. package/dist/perf-QJIWtTw1.mjs +0 -68
  95. package/dist/shot-jdO_B6Qw.mjs +0 -86
@@ -0,0 +1,479 @@
1
+ /**
2
+ * Words on this Mac, in the page (spec 27, Parts 4 and 5): the microphone, the audio socket, and the
3
+ * words that come back.
4
+ *
5
+ * The first `S` (or `J`) on a canvas asks once whether to use the microphone (the answer is kept for this
6
+ * canvas); if the model is missing, it offers the download. Outside a session the microphone is open only
7
+ * while a capture records; in a session it stays open, and what is said between captures is the session's
8
+ * talk (kept on this Mac in its transcript).
9
+ *
10
+ * The audio is cut into SEGMENTS, each with its own id on the socket (`<owner>~<n>`): a capture's audio
11
+ * (maybe several segments - the microphone closed and opened again), or a stretch of session talk. An
12
+ * AudioWorklet makes 16 kHz mono PCM in 100 ms chunks, each sent with its sample offset from the OWNER's
13
+ * start, so words get the owner's time from the audio itself. A cut is a mark in the worklet: the chunk up
14
+ * to it goes to the old segment, then `end` with that last sample, on the same socket (it cannot overtake
15
+ * the audio) - and the next segment, if any, starts exactly at the mark. A capture's words are complete
16
+ * only if every one of its segments ended cleanly and came back final.
17
+ */
18
+ import { create } from 'zustand'
19
+ import { ROUTE } from '../const.ts'
20
+ import { CONFIG, useStore } from './store.ts'
21
+
22
+ export const RATE = 16_000
23
+ const FINAL_MS = 3_000
24
+ const CHUNK = 1_600 // 100 ms
25
+
26
+ const csrf = () => /(?:^|;\s*)mv_c=([\w-]+)/.exec(document.cookie)?.[1] ?? ''
27
+ const CHOICE = `mv-jam-voice:${CONFIG.workspace ?? 'unknown'}`
28
+ const readChoice = (): 'yes' | 'no' | null => { try { const v = localStorage.getItem(CHOICE); return v === 'yes' || v === 'no' ? v : null } catch { return null } }
29
+ const now = () => performance.timeOrigin + performance.now()
30
+
31
+ export type ServerVoice = 'unknown' | 'unavailable' | 'needs-model' | 'loading' | 'ready'
32
+ export interface Heard { w: string; at: number; lost?: true } // `at`: the owner's sample
33
+
34
+ interface VoiceStore {
35
+ /** your answer for this canvas: use the microphone, or not (null: not asked yet) */
36
+ choice: 'yes' | 'no' | null
37
+ server: ServerVoice
38
+ reason?: string
39
+ download: { state: 'idle' | 'downloading' | 'checking' | 'done' | 'failed'; received: number; total: number; error?: string } | null
40
+ mb: number
41
+ /** the question is on screen (first S or J, or the model missing) */
42
+ asking: boolean
43
+ /** the microphone is live for this capture (null: not a capture's - off, or a session's talk) */
44
+ hearing: string | null
45
+ /** the microphone is live at all (a capture or a session's talk) */
46
+ live: boolean
47
+ /** the browser refused the microphone */
48
+ denied: boolean
49
+ /** words heard so far, by capture - the live caption */
50
+ heard: Record<string, Heard[]>
51
+ setChoice(c: 'yes' | 'no'): void
52
+ refresh(): Promise<void>
53
+ /** try the engine again (it failed to start before) and refresh */
54
+ retry(): Promise<void>
55
+ startDownload(): Promise<void>
56
+ dismiss(): void
57
+ }
58
+
59
+ export const useJamVoice = create<VoiceStore>((set, get) => ({
60
+ choice: readChoice(), server: 'unknown', download: null, mb: 490, asking: false, hearing: null, live: false, denied: false, heard: {},
61
+ setChoice(c) {
62
+ try { localStorage.setItem(CHOICE, c) } catch { /* this page only */ }
63
+ set({ choice: c, ...(c === 'no' ? { asking: false } : {}) })
64
+ // no: an opening is cancelled, an open microphone is cut now - the words said so far are kept
65
+ if (c === 'no') void reconcile()
66
+ // yes: an engine that failed to start is tried again, then the microphone opens for what runs now
67
+ if (c === 'yes') void get().retry().then(() => { if (get().server === 'ready' || get().server === 'loading') set({ asking: false }); void reconcile() })
68
+ },
69
+ async refresh() {
70
+ try {
71
+ const r = await fetch(`${ROUTE}/api/jam/voice`, { headers: { 'x-mv-c': csrf() } })
72
+ if (!r.ok) { set({ server: 'unavailable', reason: 'jam is not running here' }); return }
73
+ const b = await r.json()
74
+ set({ server: b.state, reason: b.reason, download: b.download ?? null, mb: b.mb ?? 490 })
75
+ } catch { set({ server: 'unavailable', reason: 'marver dev did not answer' }) }
76
+ },
77
+ async startDownload() {
78
+ try { await fetch(`${ROUTE}/api/jam/voice`, { method: 'POST', headers: { 'x-mv-c': csrf(), 'content-type': 'application/json' }, body: JSON.stringify({ download: true }) }) } catch { /* the poll says */ }
79
+ poll()
80
+ },
81
+ async retry() {
82
+ toldUnavailable = false
83
+ try {
84
+ const r = await fetch(`${ROUTE}/api/jam/voice`, { method: 'POST', headers: { 'x-mv-c': csrf(), 'content-type': 'application/json' }, body: JSON.stringify({ retry: true }) })
85
+ if (r.ok) { const b = await r.json(); set({ server: b.state, reason: b.reason, download: b.download ?? null, mb: b.mb ?? 490 }); return }
86
+ } catch { /* below */ }
87
+ await get().refresh()
88
+ },
89
+ dismiss() { set({ asking: false }) },
90
+ }))
91
+
92
+ let polling: ReturnType<typeof setInterval> | null = null
93
+ function poll() {
94
+ if (polling) return
95
+ polling = setInterval(async () => {
96
+ await useJamVoice.getState().refresh()
97
+ const s = useJamVoice.getState()
98
+ const d = s.download?.state
99
+ if (d !== 'downloading' && d !== 'checking') {
100
+ clearInterval(polling!); polling = null
101
+ if (s.server === 'ready' || s.server === 'loading') { useJamVoice.setState({ asking: false }); void reconcile() }
102
+ }
103
+ }, 1_000)
104
+ }
105
+
106
+ // ---- who the microphone is for --------------------------------------------------------------------------
107
+
108
+ /** What the microphone hears for: a capture recording, or - in a session, between captures - its talk. */
109
+ interface Owner { kind: 'capture' | 'talk'; id: string; t0: number }
110
+ /** The capture recording now (its start on the page clock). */
111
+ let capture: { id: string; t0: number } | null = null
112
+ /** The session running now: its talk is heard between captures. */
113
+ let session: { id: string; t0: number } | null = null
114
+ const target = (): Owner | null => capture ? { kind: 'capture', ...capture } : session ? { kind: 'talk', ...session } : null
115
+
116
+ // ---- the audio --------------------------------------------------------------------------------------
117
+
118
+ /** The worklet: 16 kHz mono floats to Int16 chunks of 100 ms, each with the sample number it starts at.
119
+ * `mark` answers with the samples since the last chunk and the sample count at that instant. */
120
+ const WORKLET = `
121
+ class MvPcm extends AudioWorkletProcessor {
122
+ constructor() { super(); this.n = 0; this.buf = new Int16Array(${CHUNK}); this.k = 0; this.port.onmessage = (e) => { if (e.data === 'mark') { this.port.postMessage({ mark: this.n, start: this.n - this.k, pcm: this.buf.slice(0, this.k) }); this.k = 0 } } }
123
+ process(inputs) {
124
+ const ch = inputs[0] && inputs[0][0]
125
+ if (!ch) return true
126
+ for (let i = 0; i < ch.length; i++) {
127
+ const v = Math.max(-1, Math.min(1, ch[i]))
128
+ this.buf[this.k++] = v < 0 ? v * 32768 : v * 32767
129
+ this.n++
130
+ if (this.k === ${CHUNK}) { this.port.postMessage({ start: this.n - this.k, pcm: this.buf.slice(0) }); this.k = 0 }
131
+ }
132
+ return true
133
+ }
134
+ }
135
+ registerProcessor('mv-pcm', MvPcm)`
136
+
137
+ /** One segment on the socket: whose it is, and where its audio starts. */
138
+ interface Segment {
139
+ seg: string // `<owner>~<n>` - its own id on the socket
140
+ owner: Owner
141
+ base: number // the owner's sample at the segment's first sample (the time before it, unheard)
142
+ from: number // the worklet sample the segment starts at (-1 until the first mark)
143
+ sent: number // the owner's sample just after the last one sent
144
+ }
145
+ interface Mic { ctx: AudioContext; stream: MediaStream; node: AudioWorkletNode; seg: Segment | null; closed: boolean }
146
+ let mic: Mic | null = null
147
+ /** A microphone being opened (its permission, its worklet): a newer change cancels it. */
148
+ let opening: { cancelled: boolean } | null = null
149
+ let segN = 0
150
+ /** segment -> its owner (the words it brings back belong to it) */
151
+ const owners = new Map<string, Owner>()
152
+ /** per owner: every cut segment's promise - true when its words came back whole */
153
+ const cuts = new Map<string, Promise<boolean>[]>()
154
+ /** owners whose words may be incomplete: audio lost on the way */
155
+ const broken = new Set<string>()
156
+ /** captures whose words are still taken (recording or ending) - a dropped or sealed one takes none */
157
+ const accepting = new Set<string>()
158
+ let ws: WebSocket | null = null
159
+ let wsOpen: Promise<WebSocket | null> | null = null
160
+ const finals = new Map<string, (complete: boolean) => void>() // segment -> its final
161
+ let toldUnavailable = false
162
+ let takenOver = false
163
+
164
+ const setLive = () => useJamVoice.setState({ live: !!mic && !mic.closed, hearing: mic && !mic.closed && mic.seg?.owner.kind === 'capture' ? mic.seg.owner.id : null })
165
+
166
+ function socket(reclaim = false): Promise<WebSocket | null> {
167
+ if (ws && ws.readyState === WebSocket.OPEN) return Promise.resolve(ws)
168
+ if (wsOpen) return wsOpen
169
+ wsOpen = (async () => {
170
+ try {
171
+ const r = await fetch(`${ROUTE}/api/jam/audio-ticket`, { method: 'POST', headers: { 'x-mv-c': csrf() } })
172
+ if (!r.ok) return null
173
+ const { ticket } = await r.json()
174
+ const s = new WebSocket(`${location.protocol === 'https:' ? 'wss' : 'ws'}://${location.host}${ROUTE}/api/jam/audio?ticket=${encodeURIComponent(ticket)}${reclaim ? '&reclaim=1' : ''}`)
175
+ s.binaryType = 'arraybuffer'
176
+ await new Promise<void>((res, rej) => { s.onopen = () => res(); s.onerror = () => rej(new Error('audio socket refused')) })
177
+ s.onmessage = (e) => {
178
+ let m: any
179
+ try { m = JSON.parse(String(e.data)) } catch { return }
180
+ if (typeof m?.capture !== 'string') return
181
+ const owner = owners.get(m.capture)
182
+ if (m.type === 'words' && Array.isArray(m.words)) {
183
+ if (!owner || owner.kind !== 'capture' || !accepting.has(owner.id)) return // talk goes to the transcript on the server
184
+ const words = (m.words as Heard[]).filter((x) => typeof x?.w === 'string' && Number.isFinite(x?.at))
185
+ useJamVoice.setState((st) => ({ heard: { ...st.heard, [owner.id]: [...(st.heard[owner.id] ?? []), ...words] } }))
186
+ onWords?.(owner.id, words)
187
+ } else if (m.type === 'final') {
188
+ const f = finals.get(m.capture)
189
+ finals.delete(m.capture)
190
+ if (owner && (owner.kind === 'talk' || !accepting.has(owner.id))) owners.delete(m.capture)
191
+ if (f) f(m.incomplete !== true)
192
+ else if (m.incomplete === true && owner) lost(owner, m.capture, 'the words stopped (the speech engine went)')
193
+ // the server ended the segment we are still sending (the session's talk, finished by another tab):
194
+ // the microphone goes on in a fresh segment
195
+ else if (mic && !mic.closed && mic.seg?.seg === m.capture) { mic.seg = null; void reconcile() }
196
+ } else if (m.type === 'unavailable') {
197
+ // no words for it: the microphone closes, and the reason is said once
198
+ useJamVoice.setState({ server: 'unavailable', reason: typeof m.reason === 'string' ? m.reason : undefined })
199
+ if (owner) broken.add(owner.id)
200
+ const cur = mic
201
+ if (cur && cur.seg?.seg === m.capture) { closeNow(cur); mic = null; setLive() }
202
+ void reconcile()
203
+ if (!toldUnavailable) { toldUnavailable = true; useStore.getState().toast(typeof m.reason === 'string' ? m.reason : 'words are not available - typed notes only') }
204
+ }
205
+ }
206
+ s.onclose = (ev) => {
207
+ if (ws === s) ws = null
208
+ takenOver = ev.code === 4000
209
+ // a closed connection is not a final: what was waiting may be incomplete
210
+ for (const f of finals.values()) f(false)
211
+ finals.clear()
212
+ if (mic && !mic.closed && mic.seg) lost(mic.seg.owner, mic.seg.seg, takenOver ? 'another tab took the microphone - it comes back here when that tab is done' : 'the microphone connection closed')
213
+ if (takenOver) takeBackLater()
214
+ }
215
+ ws = s
216
+ return s
217
+ } catch { return null } finally { wsOpen = null }
218
+ })()
219
+ return wsOpen
220
+ }
221
+
222
+ /** The microphone stops (its connection or the engine went): the words heard so far are kept, the owner's
223
+ * words are marked as maybe incomplete. */
224
+ function lost(owner: Owner, seg: string, why: string) {
225
+ broken.add(owner.id)
226
+ // only the very segment that failed closes the microphone (a late word about an older one never does)
227
+ const m = mic
228
+ if (m && !m.closed && m.seg?.seg === seg) { closeNow(m); mic = null; setLive() }
229
+ useStore.getState().toast(`${why} - the words so far are kept`)
230
+ }
231
+ const closeNow = (m: Mic) => { m.closed = true; m.node.port.onmessage = null; m.stream.getTracks().forEach((t) => t.stop()); void m.ctx.close().catch(() => {}) }
232
+
233
+ /** One audio connection per canvas: a capture in another tab takes it over. The session's tab (the one
234
+ * hearing its talk) then asks the server every two seconds whether the connection is still busy - a
235
+ * segment open on it, or last words not back yet - and takes the microphone back once it is not (that
236
+ * tab done, closed, or its microphone never started). */
237
+ let waitingBack: ReturnType<typeof setInterval> | null = null
238
+ /** the next opening takes the connection back (refused by the server while it is busy) */
239
+ let reclaiming = false
240
+ function takeBackLater() {
241
+ if (waitingBack || !session) return
242
+ waitingBack = setInterval(async () => {
243
+ if (!session || (mic && !mic.closed)) { clearInterval(waitingBack!); waitingBack = null; return }
244
+ try {
245
+ const r = await fetch(`${ROUTE}/api/jam/voice`, { headers: { 'x-mv-c': csrf() } })
246
+ const b = r.ok ? await r.json() : null
247
+ if (b && b.busy === false) { clearInterval(waitingBack!); waitingBack = null; reclaiming = true; void reconcile() }
248
+ } catch { /* ask again */ }
249
+ }, 2_000)
250
+ }
251
+
252
+ /** Words for a capture, as they come (jam-capture.ts records them in its timeline). */
253
+ let onWords: ((capture: string, words: Heard[]) => void) | null = null
254
+ export const setWordsListener = (f: typeof onWords) => { onWords = f }
255
+
256
+ const enc = new TextEncoder()
257
+ /** One chunk, naming its segment: [u8 id length][id][u32 offset][Int16 samples]. */
258
+ function sendChunk(s: Segment, start: number, pcm: Int16Array) {
259
+ if (s.from < 0 || !pcm.length) return
260
+ const offset = s.base + start - s.from
261
+ s.sent = Math.max(s.sent, offset + pcm.length)
262
+ if (!ws || ws.readyState !== WebSocket.OPEN) return
263
+ const id = enc.encode(s.seg)
264
+ const out = new Uint8Array(1 + id.length + 4 + pcm.byteLength)
265
+ out[0] = id.length
266
+ out.set(id, 1)
267
+ new DataView(out.buffer).setUint32(1 + id.length, Math.max(0, offset), true)
268
+ out.set(new Uint8Array(pcm.buffer, pcm.byteOffset, pcm.byteLength), 1 + id.length + 4)
269
+ ws.send(out)
270
+ }
271
+
272
+ /** A new segment for `owner`, starting at worklet sample `from` (-1: at the next mark). */
273
+ /** This page's own mark in its segment ids: a reload never reuses the id of a segment still finishing. */
274
+ const PAGE = Math.random().toString(36).slice(2, 8).padEnd(6, '0')
275
+ function segmentFor(owner: Owner, from: number): Segment {
276
+ const seg = `${owner.id}~${PAGE}${++segN}`
277
+ owners.set(seg, owner)
278
+ // the owner's clock: the time before this segment counts too (unheard)
279
+ const base = Math.max(0, Math.round((now() - owner.t0) / 1000 * RATE))
280
+ const s: Segment = { seg, owner, base, from, sent: base }
281
+ ws?.send(JSON.stringify({ type: 'begin', capture: seg, ...(owner.kind === 'talk' ? { t0: owner.t0 } : {}) }))
282
+ return s
283
+ }
284
+
285
+ /** Mark the worklet: resolves with the partial chunk and the sample count at this instant (null if it
286
+ * does not answer in 500 ms). Chunks that arrive first go to `s`. */
287
+ function mark(m: Mic, s: Segment | null): Promise<{ start: number; pcm: Int16Array; mark: number } | null> {
288
+ return new Promise((res) => {
289
+ const t = setTimeout(() => res(null), 500)
290
+ m.node.port.onmessage = (e) => {
291
+ if ((e.data as any).mark !== undefined) { clearTimeout(t); res(e.data) }
292
+ else if (s) sendChunk(s, e.data.start, e.data.pcm)
293
+ }
294
+ m.node.port.postMessage('mark')
295
+ })
296
+ }
297
+ const listenTo = (m: Mic) => { m.node.port.onmessage = (e) => { if (!m.closed && m.seg && (e.data as any).mark === undefined) sendChunk(m.seg, e.data.start, e.data.pcm) } }
298
+
299
+ /** End a segment at a mark: what it holds up to the mark, then `end`; its final is awaited by its owner. */
300
+ function endSegment(s: Segment, at: { start: number; pcm: Int16Array; mark: number } | null) {
301
+ const clean = !!at && s.from >= 0
302
+ if (clean) sendChunk(s, at!.start, at!.pcm)
303
+ const lastSample = clean ? s.base + at!.mark - s.from : s.sent
304
+ const done = !ws || ws.readyState !== WebSocket.OPEN ? Promise.resolve(false) : new Promise<boolean>((res) => {
305
+ const t = setTimeout(() => { finals.delete(s.seg); res(false) }, FINAL_MS)
306
+ finals.set(s.seg, (complete) => { clearTimeout(t); res(complete && clean) })
307
+ ws!.send(JSON.stringify({ type: 'end', capture: s.seg, lastSample }))
308
+ })
309
+ const list = cuts.get(s.owner.id) ?? []
310
+ list.push(done)
311
+ cuts.set(s.owner.id, list)
312
+ }
313
+
314
+ /** Who the microphone should hear now: what runs (a capture, else the session's talk) - if you said yes
315
+ * and the engine can run. */
316
+ const wanted = (): Owner | null => {
317
+ const v = useJamVoice.getState()
318
+ return v.choice === 'yes' && (v.server === 'ready' || v.server === 'loading') ? target() : null
319
+ }
320
+
321
+ /** Bring the microphone to what should be heard NOW. Every change - open, switch, close - is one step in
322
+ * one chain, so no change overtakes another: a switch ends the current segment and starts the next at
323
+ * the same worklet mark (nothing lost, nothing heard twice); an opening still waiting gives way to a
324
+ * newer state (it checks after every wait) and the step after it looks again. */
325
+ let chain: Promise<void> = Promise.resolve()
326
+ function reconcile(): Promise<void> {
327
+ cancelOpening()
328
+ chain = chain.then(step, step)
329
+ return chain
330
+ }
331
+ async function step() {
332
+ const want = wanted()
333
+ const m = mic && !mic.closed ? mic : null
334
+ if (m && want && m.seg?.owner.id === want.id) return
335
+ if (m) {
336
+ const cur = m.seg
337
+ const at = await mark(m, cur)
338
+ if (cur) endSegment(cur, at)
339
+ const next = wanted() // looked at again: it may have changed during the mark
340
+ if (next && at && !m.closed) { m.seg = segmentFor(next, at.mark); listenTo(m); setLive(); return }
341
+ closeNow(m)
342
+ if (mic === m) mic = null
343
+ setLive()
344
+ }
345
+ const now_ = wanted()
346
+ if (now_) await open(now_)
347
+ }
348
+
349
+ /** Open the microphone for `owner` - one step of the chain; a newer state cancels it at its next wait. */
350
+ async function open(owner: Owner) {
351
+ const op = { cancelled: false }
352
+ opening = op
353
+ const stale = () => op.cancelled || wanted()?.id !== owner.id
354
+ let stream: MediaStream | null = null
355
+ let ctx: AudioContext | null = null
356
+ const undo = () => { stream?.getTracks().forEach((t) => t.stop()); void ctx?.close().catch(() => {}) }
357
+ try {
358
+ // the microphone first (its permission, its worklet), the connection last: a page never holds the one
359
+ // audio connection while it is still starting - and a reclaim takes it only if it is free
360
+ try { stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } }) }
361
+ catch { useJamVoice.setState({ denied: true }); useStore.getState().toast('the microphone is blocked for this page - typed notes only'); return }
362
+ if (stale()) { undo(); return }
363
+ ctx = new AudioContext({ sampleRate: RATE })
364
+ await ctx.audioWorklet.addModule(URL.createObjectURL(new Blob([WORKLET], { type: 'application/javascript' })))
365
+ if (stale()) { undo(); return }
366
+ // a capture is something you just did here: it takes the connection; a session's talk only takes it if it
367
+ // is free (after a reload too) - refused, it asks again until it is
368
+ const reclaim = owner.kind === 'talk' || reclaiming
369
+ reclaiming = false
370
+ const s = await socket(reclaim)
371
+ if (!s) { undo(); if (reclaim && owner.kind === 'talk') takeBackLater(); return } // still busy: ask again later
372
+ if (stale() || s.readyState !== WebSocket.OPEN) { undo(); return }
373
+ const node = new AudioWorkletNode(ctx, 'mv-pcm')
374
+ ctx.createMediaStreamSource(stream).connect(node)
375
+ const m: Mic = { ctx, stream, node, seg: null, closed: false }
376
+ mic = m
377
+ const seg = segmentFor(owner, -1)
378
+ m.seg = seg
379
+ // the first mark sets where the segment starts; chunks before it are not its
380
+ node.port.onmessage = (e) => { if ((e.data as any).mark !== undefined && seg.from < 0) { seg.from = e.data.mark; listenTo(m) } }
381
+ node.port.postMessage('mark')
382
+ useJamVoice.setState({ denied: false })
383
+ setLive()
384
+ } catch {
385
+ undo()
386
+ useStore.getState().toast('the microphone could not start - typed notes only')
387
+ } finally { if (opening === op) opening = null }
388
+ }
389
+ const cancelOpening = () => { if (opening) opening.cancelled = true }
390
+ /** The chain settled - but never longer than `ms` (an opening may wait on the browser's permission
391
+ * question; what the caller needs - the cuts of its segments - is done before it). */
392
+ const settle = (ms = 2_000) => Promise.race([reconcile(), new Promise<void>((r) => setTimeout(r, ms))])
393
+
394
+ /** What the question and the server say before the microphone opens for what runs now. */
395
+ async function begin() {
396
+ const v = useJamVoice.getState()
397
+ if (v.choice === 'no') return
398
+ await v.refresh()
399
+ const s = useJamVoice.getState()
400
+ if (s.server === 'unavailable') {
401
+ // the one line why, once - and a first S or J takes it as the answer
402
+ if (s.choice === null) useJamVoice.setState({ choice: 'no' })
403
+ if (!toldUnavailable) { toldUnavailable = true; useStore.getState().toast(s.reason ?? 'words are not available here - typed notes only') }
404
+ return
405
+ }
406
+ // first time, or yes but the model is missing: the question (the capture or session runs meanwhile)
407
+ if (s.choice === null || s.server === 'needs-model') { useJamVoice.setState({ asking: true }); return }
408
+ await reconcile()
409
+ }
410
+
411
+ /** Every segment of an owner ended: true when all its words came back whole. */
412
+ async function settled(owner: string): Promise<boolean> {
413
+ const list = cuts.get(owner) ?? []
414
+ cuts.delete(owner)
415
+ const all = await Promise.all(list)
416
+ const ok = all.every(Boolean) && !broken.has(owner)
417
+ broken.delete(owner)
418
+ return ok
419
+ }
420
+
421
+ // ---- captures and sessions --------------------------------------------------------------------------------
422
+
423
+ /** A capture starts: the microphone hears it (from the session's talk at the same instant, or opened now). */
424
+ export async function voiceBegin(id: string, t0: number) {
425
+ capture = { id, t0 }
426
+ accepting.add(id)
427
+ if (mic && !mic.closed) { await reconcile(); return }
428
+ await begin()
429
+ }
430
+
431
+ /** A capture ends: its audio cut at this instant (in a session the talk goes on from it), its words awaited
432
+ * (3 s at most). Resolves true when the words are complete, false when they may not be. */
433
+ export async function voiceEnd(id: string): Promise<boolean> {
434
+ if (capture?.id === id) capture = null
435
+ useJamVoice.setState((st) => ({ asking: st.asking && st.choice === null && !session ? false : st.asking }))
436
+ await settle()
437
+ return settled(id)
438
+ }
439
+
440
+ /** A capture dropped: what was heard for it is discarded - nothing of it is transcribed. */
441
+ export async function voiceDrop(id: string) {
442
+ if (capture?.id === id) capture = null
443
+ accepting.delete(id)
444
+ useJamVoice.setState((st) => { const heard = { ...st.heard }; delete heard[id]; return { heard } })
445
+ await settle()
446
+ // every segment it had: dropped on the server (discarded, untranscribed)
447
+ for (const [seg, o] of owners) if (o.id === id) { ws?.send(JSON.stringify({ type: 'drop', capture: seg })); finals.get(seg)?.(false); finals.delete(seg); owners.delete(seg) }
448
+ cuts.delete(id)
449
+ broken.delete(id)
450
+ }
451
+
452
+ /** A capture sealed: its words are in its record - nothing more of it is taken. */
453
+ export const forgetHeard = (id: string) => {
454
+ accepting.delete(id)
455
+ for (const [seg, o] of owners) if (o.id === id && !finals.has(seg)) owners.delete(seg)
456
+ useJamVoice.setState((st) => { if (!st.heard[id]) return st; const heard = { ...st.heard }; delete heard[id]; return { heard } })
457
+ }
458
+
459
+ /** A session starts: the microphone (if you said yes) hears its talk from now on, between captures. */
460
+ export async function voiceSession(id: string, t0: number) {
461
+ session = { id, t0 }
462
+ if (capture) return // the capture keeps the microphone; the talk starts when it ends
463
+ if (mic && !mic.closed) { await reconcile(); return }
464
+ await begin()
465
+ }
466
+
467
+ /** The session the microphone hears now (the page's own state - the server holds the session). */
468
+ export const voiceSessionId = (): string | null => session?.id ?? null
469
+
470
+ /** A session ends: its talk's last segment is cut; the microphone closes (unless a capture records). */
471
+ export async function voiceSessionEnd(): Promise<void> {
472
+ const s = session
473
+ session = null
474
+ if (!s) return
475
+ await settle()
476
+ await settled(s.id)
477
+ }
478
+
479
+ if (typeof window !== 'undefined' && (import.meta as any).env?.DEV) { (window as any).__mvJamVoice = useJamVoice; (window as any).__mvJamVoiceSession = voiceSessionId }
@@ -30,3 +30,25 @@ export function playPing(): void {
30
30
  }
31
31
  } catch { /* no audio here - fine */ }
32
32
  }
33
+
34
+ /** Something is ready to review (spec 27): a soft, lower two-note chime - once per batch that leaves a
35
+ * capture to review, only while the page has focus (`jam.chime`, on by default). */
36
+ export function playChime(): void {
37
+ try {
38
+ ctx ??= new AudioContext()
39
+ if (ctx.state === 'suspended') { void ctx.resume().catch(() => {}); if (ctx.state === 'suspended') return }
40
+ const t0 = ctx.currentTime
41
+ for (const [freq, at, dur] of [[659.3, 0, .14], [987.8, .12, .26]] as const) {
42
+ const osc = ctx.createOscillator()
43
+ const gain = ctx.createGain()
44
+ osc.type = 'sine'
45
+ osc.frequency.value = freq
46
+ gain.gain.setValueAtTime(0, t0 + at)
47
+ gain.gain.linearRampToValueAtTime(0.05, t0 + at + 0.02)
48
+ gain.gain.exponentialRampToValueAtTime(0.0001, t0 + at + dur)
49
+ osc.connect(gain).connect(ctx.destination)
50
+ osc.start(t0 + at)
51
+ osc.stop(t0 + at + dur + 0.05)
52
+ }
53
+ } catch { /* no audio here */ }
54
+ }
@@ -0,0 +1,11 @@
1
+ /** The play stage's size as it is drawn - a device, the window when it fills, a slide's own stage, the reading
2
+ * width of a document - in the frame's CSS pixels. Play publishes it after every layout (a resize, a device, the
3
+ * screen it shows); a capture reads it, and hears when it changes (what a frame on the stage is pictured at). */
4
+ export const stageSize: { at: string; w: number; h: number } = { at: '', w: 0, h: 0 }
5
+ const listeners = new Set<() => void>()
6
+ export function setStageSize(at: string, w: number, h: number) {
7
+ if (stageSize.at === at && stageSize.w === w && stageSize.h === h) return
8
+ stageSize.at = at; stageSize.w = w; stageSize.h = h
9
+ for (const fn of listeners) { try { fn() } catch { /* a listener's own trouble */ } }
10
+ }
11
+ export function onStageSize(fn: () => void): () => void { listeners.add(fn); return () => listeners.delete(fn) }