@dxos/react-ui-transcription 0.10.0 → 0.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. package/dist/lib/index.mjs +963 -0
  2. package/dist/lib/index.mjs.map +1 -0
  3. package/dist/types/src/capture/create-transcriber.d.ts +1 -1
  4. package/dist/types/src/capture/media-stream-recorder.d.ts +1 -1
  5. package/dist/types/src/capture/media-stream-recorder.d.ts.map +1 -1
  6. package/dist/types/src/components/PipelineStatus/PipelineStatus.d.ts +1 -1
  7. package/dist/types/src/components/Transcription/Transcription.d.ts +1 -1
  8. package/dist/types/src/components/Transcription/Transcription.d.ts.map +1 -1
  9. package/dist/types/src/hooks/useRecordingPipeline.d.ts +1 -1
  10. package/dist/types/src/hooks/useTranscriber.d.ts +1 -1
  11. package/dist/types/src/index.d.ts +0 -1
  12. package/dist/types/src/index.d.ts.map +1 -1
  13. package/dist/types/tsconfig.tsbuildinfo +1 -1
  14. package/package.json +23 -24
  15. package/src/capture/create-transcriber.ts +1 -1
  16. package/src/capture/media-stream-recorder.ts +1 -3
  17. package/src/components/PipelineStatus/PipelineStatus.tsx +1 -1
  18. package/src/hooks/useRecordingPipeline.ts +1 -1
  19. package/src/hooks/useTranscriber.ts +1 -1
  20. package/src/index.ts +0 -1
  21. package/dist/lib/browser/index.mjs +0 -1002
  22. package/dist/lib/browser/index.mjs.map +0 -7
  23. package/dist/lib/browser/meta.json +0 -1
  24. package/dist/lib/node-esm/index.mjs +0 -1004
  25. package/dist/lib/node-esm/index.mjs.map +0 -7
  26. package/dist/lib/node-esm/meta.json +0 -1
  27. package/dist/types/src/util/index.d.ts +0 -2
  28. package/dist/types/src/util/index.d.ts.map +0 -1
  29. package/dist/types/src/util/render-byline.d.ts +0 -4
  30. package/dist/types/src/util/render-byline.d.ts.map +0 -1
  31. package/src/util/index.ts +0 -5
  32. package/src/util/render-byline.ts +0 -32
@@ -0,0 +1,963 @@
1
+ import { Transcriber, runAsrPipeline } from "@dxos/pipeline-transcription";
2
+ import { WaveFile } from "wavefile";
3
+ import { Event, addEventListener, combine, scheduleTask, synchronized } from "@dxos/async";
4
+ import { invariant } from "@dxos/invariant";
5
+ import { log } from "@dxos/log";
6
+ import { Fragment, useEffect, useMemo, useRef, useState } from "react";
7
+ import { DropdownMenu, Icon, IconButton, composable, composableProps, useThemeContext, useTranslation } from "@dxos/react-ui";
8
+ import { jsx, jsxs } from "react/jsx-runtime";
9
+ import { useTextEditor } from "@dxos/react-ui-editor";
10
+ import { AnchorWidget, createBasicExtensions, createMarkdownExtensions, createThemeExtensions, decorateMarkdown, documentSlots, scroller, xmlTags } from "@dxos/ui-editor";
11
+ import { RangeSetBuilder, Text } from "@codemirror/state";
12
+ import { EditorView, GutterMarker, ViewPlugin, gutter } from "@codemirror/view";
13
+ import { format } from "date-fns/format";
14
+ import { intervalToDuration } from "date-fns/intervalToDuration";
15
+ import { Context } from "@dxos/context";
16
+ import { SpeakingMonitor } from "@dxos/av";
17
+ //#region \0@oxc-project+runtime@0.139.0/helpers/esm/decorate.js
18
+ function __decorate(decorators, target, key, desc) {
19
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
20
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
21
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
22
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
23
+ }
24
+ //#endregion
25
+ //#region src/capture/media-stream-recorder.ts
26
+ var __dxlog_file$5 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/capture/media-stream-recorder.ts";
27
+ var MediaRecorderClass;
28
+ var initializingPromise;
29
+ var initializeExtendableMediaRecorder = async () => {
30
+ const { MediaRecorder, register } = await import("extendable-media-recorder");
31
+ const { connect } = await import("extendable-media-recorder-wav-encoder");
32
+ await register(await connect());
33
+ MediaRecorderClass = MediaRecorder;
34
+ };
35
+ /**
36
+ * Recorder that uses the MediaContext API and AudioNode API to record audio.
37
+ *
38
+ * It records MediaStream using https://www.npmjs.com/package/extendable-media-recorder.
39
+ */
40
+ var MediaStreamRecorder = class {
41
+ _mediaStreamTrack;
42
+ /**
43
+ * Default MediaRecorder implementation do not support wav encoding.
44
+ */
45
+ _config;
46
+ _mediaRecorder = void 0;
47
+ _header = void 0;
48
+ _onChunk;
49
+ constructor({ mediaStreamTrack, config }) {
50
+ this._mediaStreamTrack = mediaStreamTrack;
51
+ this._config = config;
52
+ }
53
+ get wavConfig() {
54
+ const settings = this._mediaStreamTrack.getSettings();
55
+ return {
56
+ channels: settings.channelCount ?? 1,
57
+ sampleRate: settings.sampleRate ?? 16e3,
58
+ bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
59
+ };
60
+ }
61
+ get mediaStreamTrack() {
62
+ return this._mediaStreamTrack;
63
+ }
64
+ setOnChunk(onChunk) {
65
+ this._onChunk = onChunk;
66
+ }
67
+ async start() {
68
+ await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log.info("initializeExtendableMediaRecorder", { err }, {
69
+ "~LogMeta": "~LogMeta",
70
+ F: __dxlog_file$5,
71
+ L: 76,
72
+ S: this
73
+ }));
74
+ if (this._mediaRecorder?.state === "recording") return;
75
+ invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", {
76
+ "~LogMeta": "~LogMeta",
77
+ F: __dxlog_file$5,
78
+ L: 82,
79
+ S: this,
80
+ A: ["this._onChunk", "'MediaStreamRecorder: onChunk is not set'"]
81
+ });
82
+ invariant(MediaRecorderClass, "MediaStreamRecorder: extendable-media-recorder not initialized", {
83
+ "~LogMeta": "~LogMeta",
84
+ F: __dxlog_file$5,
85
+ L: 83,
86
+ S: this,
87
+ A: ["MediaRecorderClass", "'MediaStreamRecorder: extendable-media-recorder not initialized'"]
88
+ });
89
+ const stream = new MediaStream([this._mediaStreamTrack]);
90
+ const recorder = new MediaRecorderClass(stream, { mimeType: "audio/wav" });
91
+ recorder.ondataavailable = (event) => this._ondataavailable(event);
92
+ this._mediaRecorder = recorder;
93
+ recorder.start(this._config.interval);
94
+ }
95
+ async stop() {
96
+ const recorder = this._mediaRecorder;
97
+ if (!recorder || recorder.state !== "recording") return;
98
+ await new Promise((resolve) => {
99
+ recorder.addEventListener("stop", () => resolve(), { once: true });
100
+ recorder.stop();
101
+ });
102
+ this._mediaRecorder = void 0;
103
+ }
104
+ async _ondataavailable(event) {
105
+ const blob = event.data;
106
+ const uint8Array = new Uint8Array(await blob.arrayBuffer());
107
+ const isHeader = uint8Array[0] === 82 && uint8Array[1] === 73 && uint8Array[2] === 70 && uint8Array[3] === 70;
108
+ let wav;
109
+ if (isHeader) {
110
+ wav = new WaveFile(uint8Array);
111
+ this._header = uint8Array.slice(0, 44);
112
+ } else if (this._header) wav = new WaveFile(new Uint8Array([...this._header, ...uint8Array]));
113
+ else {
114
+ log.warn("MediaStreamRecorder: no header", void 0, {
115
+ "~LogMeta": "~LogMeta",
116
+ F: __dxlog_file$5,
117
+ L: 126,
118
+ S: this
119
+ });
120
+ return;
121
+ }
122
+ this._onChunk({
123
+ timestamp: Date.now(),
124
+ data: wav.getSamples()
125
+ });
126
+ }
127
+ };
128
+ __decorate([synchronized], MediaStreamRecorder.prototype, "start", null);
129
+ __decorate([synchronized], MediaStreamRecorder.prototype, "stop", null);
130
+ __decorate([synchronized], MediaStreamRecorder.prototype, "_ondataavailable", null);
131
+ //#endregion
132
+ //#region src/capture/create-transcriber.ts
133
+ var RECORD_INTERVAL = 200;
134
+ var PREFIXED_CHUNKS_AMOUNT = 10;
135
+ var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
136
+ /**
137
+ * Builds a {@link Transcriber} backed by a browser {@link MediaStreamRecorder} for the given track —
138
+ * the low-level audio→text construction, co-located with the recorder so it can be tested outside the
139
+ * plugin. Consumed directly via the {@link useTranscriber} hook.
140
+ */
141
+ var createTranscriber = ({ audioStreamTrack, onSegments, transcriberConfig, recorderConfig, transcribe }) => new Transcriber({
142
+ config: {
143
+ transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
144
+ prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
145
+ ...transcriberConfig
146
+ },
147
+ recorder: new MediaStreamRecorder({
148
+ mediaStreamTrack: audioStreamTrack,
149
+ config: {
150
+ interval: RECORD_INTERVAL,
151
+ ...recorderConfig
152
+ }
153
+ }),
154
+ onSegments,
155
+ transcribe
156
+ });
157
+ //#endregion
158
+ //#region src/components/MicSettings/MicSettings.tsx
159
+ /**
160
+ * Dropdown of recording options for the mic: record mode (toggle/push-to-talk), the input device,
161
+ * and whether entity extraction is applied. Presentational — state lives in the consumer.
162
+ */
163
+ var MicSettings = ({ translationNs, recordMode, entityExtraction, devices, selectedDeviceId, onRecordModeChange, onEntityExtractionChange, onSelectDevice }) => {
164
+ const { t } = useTranslation(translationNs);
165
+ return /* @__PURE__ */ jsxs(DropdownMenu.Root, { children: [/* @__PURE__ */ jsx(DropdownMenu.Trigger, {
166
+ asChild: true,
167
+ children: /* @__PURE__ */ jsx(IconButton, {
168
+ icon: "ph--caret-down--regular",
169
+ iconOnly: true,
170
+ label: t("recording-options.label"),
171
+ variant: "ghost",
172
+ classNames: "w-4",
173
+ "data-testid": "transcription.record.options"
174
+ })
175
+ }), /* @__PURE__ */ jsx(DropdownMenu.Portal, { children: /* @__PURE__ */ jsxs(DropdownMenu.Content, { children: [/* @__PURE__ */ jsxs(DropdownMenu.Viewport, { children: [
176
+ /* @__PURE__ */ jsx(DropdownMenu.GroupLabel, { children: t("record-mode.label") }),
177
+ /* @__PURE__ */ jsx(SelectableItem, {
178
+ label: t("record-mode.toggle.label"),
179
+ selected: recordMode === "toggle",
180
+ onSelect: () => onRecordModeChange("toggle")
181
+ }),
182
+ /* @__PURE__ */ jsx(SelectableItem, {
183
+ label: t("record-mode.hold.label"),
184
+ selected: recordMode === "hold",
185
+ onSelect: () => onRecordModeChange("hold")
186
+ }),
187
+ /* @__PURE__ */ jsx(DropdownMenu.Separator, {}),
188
+ /* @__PURE__ */ jsx(DropdownMenu.GroupLabel, { children: t("audio-device.label") }),
189
+ /* @__PURE__ */ jsx(SelectableItem, {
190
+ label: t("audio-device.default.label"),
191
+ selected: selectedDeviceId === "",
192
+ onSelect: () => onSelectDevice("")
193
+ }),
194
+ devices.map((device) => /* @__PURE__ */ jsx(SelectableItem, {
195
+ label: device.label,
196
+ selected: selectedDeviceId === device.deviceId,
197
+ onSelect: () => onSelectDevice(device.deviceId)
198
+ }, device.deviceId)),
199
+ /* @__PURE__ */ jsx(DropdownMenu.Separator, {}),
200
+ /* @__PURE__ */ jsxs(DropdownMenu.CheckboxItem, {
201
+ checked: entityExtraction,
202
+ onCheckedChange: onEntityExtractionChange,
203
+ classNames: "gap-2",
204
+ "data-testid": "transcription.entity-extraction",
205
+ children: [/* @__PURE__ */ jsx("span", {
206
+ className: "grow truncate",
207
+ children: t("settings.entity-extraction.label")
208
+ }), /* @__PURE__ */ jsx(DropdownMenu.ItemIndicator, {
209
+ asChild: true,
210
+ children: /* @__PURE__ */ jsx(Icon, {
211
+ icon: "ph--check--regular",
212
+ size: 4
213
+ })
214
+ })]
215
+ })
216
+ ] }), /* @__PURE__ */ jsx(DropdownMenu.Arrow, {})] }) })] });
217
+ };
218
+ var SelectableItem = ({ label, selected, onSelect }) => /* @__PURE__ */ jsxs(DropdownMenu.Item, {
219
+ classNames: "gap-2",
220
+ onSelect,
221
+ children: [/* @__PURE__ */ jsx("span", {
222
+ className: "grow truncate",
223
+ children: label
224
+ }), selected && /* @__PURE__ */ jsx(Icon, {
225
+ icon: "ph--check--regular",
226
+ size: 4
227
+ })]
228
+ });
229
+ //#endregion
230
+ //#region src/components/PipelineStatus/PipelineStatus.tsx
231
+ /**
232
+ * Read-only status panel reflecting the transcription/pipeline state: mic + pipeline phase, the
233
+ * active stage list, per-stage telemetry, and the running summary. Driven by the `PipelineStatus`
234
+ * capability (phase) and `PipelineRuntime` telemetry.
235
+ */
236
+ var PipelineStatus = ({ phase = "idle", stages, telemetry, summary }) => /* @__PURE__ */ jsxs("div", {
237
+ className: "flex flex-col gap-3 overflow-y-auto p-2 text-sm",
238
+ children: [
239
+ /* @__PURE__ */ jsx(Section, {
240
+ title: "Status",
241
+ children: /* @__PURE__ */ jsxs("div", {
242
+ className: "grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-description",
243
+ children: [
244
+ /* @__PURE__ */ jsx("span", { children: "Mic" }),
245
+ /* @__PURE__ */ jsx("span", { children: phase === "recording" ? "● on" : "○ off" }),
246
+ /* @__PURE__ */ jsx("span", { children: "Pipeline" }),
247
+ /* @__PURE__ */ jsx("span", { children: phase })
248
+ ]
249
+ })
250
+ }),
251
+ stages && /* @__PURE__ */ jsx(Section, {
252
+ title: "Stages",
253
+ children: /* @__PURE__ */ jsx("div", {
254
+ className: "text-description",
255
+ children: stages.join(" → ") || "(none)"
256
+ })
257
+ }),
258
+ telemetry && telemetry.length > 0 && /* @__PURE__ */ jsx(Section, {
259
+ title: "Telemetry",
260
+ children: /* @__PURE__ */ jsx("div", {
261
+ className: "grid w-fit grid-cols-[1fr_auto_auto] gap-x-4 gap-y-1",
262
+ children: telemetry.map((event, index) => /* @__PURE__ */ jsxs(Fragment, { children: [
263
+ /* @__PURE__ */ jsx("span", { children: event.stageId }),
264
+ /* @__PURE__ */ jsx("span", {
265
+ className: "text-description",
266
+ children: event.outcome
267
+ }),
268
+ /* @__PURE__ */ jsxs("span", {
269
+ className: "text-right tabular-nums text-description",
270
+ children: [event.durationMs, "ms"]
271
+ })
272
+ ] }, index))
273
+ })
274
+ }),
275
+ summary && /* @__PURE__ */ jsx(Section, {
276
+ title: "Summary",
277
+ children: /* @__PURE__ */ jsx("p", {
278
+ className: "text-description",
279
+ children: summary
280
+ })
281
+ })
282
+ ]
283
+ });
284
+ var Section = ({ title, children }) => /* @__PURE__ */ jsxs("div", { children: [/* @__PURE__ */ jsx("h3", {
285
+ className: "mb-1 font-medium",
286
+ children: title
287
+ }), children] });
288
+ //#endregion
289
+ //#region src/model/transcript-model.ts
290
+ var __dxlog_file$4 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/model/transcript-model.ts";
291
+ /**
292
+ * CodeMirror-backed {@link ChunkDocument}.
293
+ */
294
+ var EditorChunkDocument = class {
295
+ _view;
296
+ constructor(_view) {
297
+ this._view = _view;
298
+ }
299
+ /** The document must always have at least one line. */
300
+ lineCount() {
301
+ return this._view.state.doc.lines - 1;
302
+ }
303
+ replaceLines(from, remove, insert) {
304
+ log("replaceLines", {
305
+ count: this._view.state.doc.lines,
306
+ from,
307
+ remove,
308
+ insert
309
+ }, {
310
+ "~LogMeta": "~LogMeta",
311
+ F: __dxlog_file$4,
312
+ L: 39,
313
+ S: this
314
+ });
315
+ const numLines = this._view.state.doc.lines;
316
+ const posFrom = this._view.state.doc.line(from).from;
317
+ const posTo = remove === 0 ? posFrom : this._view.state.doc.line(Math.min(numLines, from + remove)).from;
318
+ let text = insert.join("\n");
319
+ if (posTo === this._view.state.doc.length) text += "\n";
320
+ this._view.dispatch({ changes: {
321
+ from: posFrom,
322
+ to: posTo,
323
+ insert: text
324
+ } });
325
+ }
326
+ };
327
+ /**
328
+ * Each Chunk can be converted into a set of lines.
329
+ * Chunks can be added, updated or deleted from the Queue.
330
+ * A Document is an immutable projection that represents a flat list of lines corresponding to the Queue.
331
+ *
332
+ * Ideally we would implement a custom virtual Text model for the View, but this currently isn't possible in Codemirror.
333
+ * Instead this model tracks changes and syncs them with the ChunkDocument.
334
+ */
335
+ var TranscriptModel = class {
336
+ _renderer;
337
+ /** Emits when the document is updated. */
338
+ update = new Event();
339
+ /** Ordered set of chunks. */
340
+ _chunks = [];
341
+ /** Map of chunk IDs to their current line counts. */
342
+ _chunkLineCounts = /* @__PURE__ */ new Map();
343
+ /** Map of line numbers to chunk IDs. */
344
+ _lineToChunk = /* @__PURE__ */ new Map();
345
+ /** Track chunk changes since last sync. */
346
+ _changes = [];
347
+ constructor(_renderer, initialChunks = []) {
348
+ this._renderer = _renderer;
349
+ initialChunks.forEach((chunk) => {
350
+ this.appendChunk(chunk);
351
+ });
352
+ }
353
+ toJSON() {
354
+ return {
355
+ chunks: this._chunks.length,
356
+ changes: this._changes.length
357
+ };
358
+ }
359
+ get doc() {
360
+ return Text.of([...this._chunks.flatMap((chunk, index) => this._renderer(chunk, index)), ""]);
361
+ }
362
+ get chunks() {
363
+ return this._chunks;
364
+ }
365
+ /**
366
+ * Get the block ID at the given line number.
367
+ */
368
+ getChunkAtLine(line) {
369
+ return this._lineToChunk.get(line);
370
+ }
371
+ reset() {
372
+ this._chunks.length = 0;
373
+ this._chunkLineCounts.clear();
374
+ this._lineToChunk.clear();
375
+ this.update.emit();
376
+ return this;
377
+ }
378
+ appendChunk(chunk) {
379
+ const index = this._chunks.length;
380
+ this._chunks.push(chunk);
381
+ this._changes.push({
382
+ type: "append",
383
+ index,
384
+ chunk
385
+ });
386
+ this.update.emit();
387
+ return this;
388
+ }
389
+ updateChunk(chunk) {
390
+ const index = this._chunks.findIndex((c) => c.id === chunk.id);
391
+ invariant(index !== -1, void 0, {
392
+ "~LogMeta": "~LogMeta",
393
+ F: __dxlog_file$4,
394
+ L: 147,
395
+ S: this,
396
+ A: ["index !== -1", ""]
397
+ });
398
+ this._chunks[index] = chunk;
399
+ this._changes.push({
400
+ type: "update",
401
+ index,
402
+ chunk
403
+ });
404
+ this.update.emit();
405
+ return this;
406
+ }
407
+ deleteBlock(id) {
408
+ const index = this._chunks.findIndex((c) => c.id === id);
409
+ invariant(index !== -1, void 0, {
410
+ "~LogMeta": "~LogMeta",
411
+ F: __dxlog_file$4,
412
+ L: 156,
413
+ S: this,
414
+ A: ["index !== -1", ""]
415
+ });
416
+ this._chunks.splice(index, 1);
417
+ this._changes.push({
418
+ type: "delete",
419
+ index,
420
+ id
421
+ });
422
+ this.update.emit();
423
+ return this;
424
+ }
425
+ /**
426
+ * Syncs the tracked changes with the ChunkDocument.
427
+ */
428
+ sync(document) {
429
+ log("sync", { changes: this._changes }, {
430
+ "~LogMeta": "~LogMeta",
431
+ F: __dxlog_file$4,
432
+ L: 167,
433
+ S: this
434
+ });
435
+ for (const change of this._changes) switch (change.type) {
436
+ case "append": {
437
+ const lines = this._renderer(change.chunk, change.index);
438
+ const lastLine = document.lineCount();
439
+ document.replaceLines(lastLine + 1, 0, lines);
440
+ break;
441
+ }
442
+ case "update": {
443
+ let lineStart = 1;
444
+ for (let i = 0; i < change.index; i++) lineStart += this._chunkLineCounts.get(this._chunks[i].id);
445
+ const previousLineCount = this._chunkLineCounts.get(change.chunk.id);
446
+ const newLines = this._renderer(change.chunk, change.index);
447
+ document.replaceLines(lineStart, previousLineCount, newLines);
448
+ break;
449
+ }
450
+ case "delete": {
451
+ let lineStart = 1;
452
+ for (let i = 0; i < change.index; i++) lineStart += this._chunkLineCounts.get(this._chunks[i].id);
453
+ const previousLineCount = this._chunkLineCounts.get(change.id);
454
+ document.replaceLines(lineStart, previousLineCount, []);
455
+ break;
456
+ }
457
+ }
458
+ this._lineToChunk.clear();
459
+ let currentLine = 1;
460
+ for (const [index, chunk] of this._chunks.entries()) {
461
+ const lineCount = this._renderer(chunk, index).length;
462
+ this._chunkLineCounts.set(chunk.id, lineCount);
463
+ this._lineToChunk.set(currentLine, chunk);
464
+ currentLine += lineCount;
465
+ }
466
+ const chunkIds = new Set(this._chunks.map((chunk) => chunk.id));
467
+ for (const id of this._chunkLineCounts.keys()) if (!chunkIds.has(id)) this._chunkLineCounts.delete(id);
468
+ this._changes.length = 0;
469
+ return this;
470
+ }
471
+ };
472
+ //#endregion
473
+ //#region src/components/Transcription/transcription-extension.ts
474
+ /**
475
+ * Scrolling transcript with timestamps.
476
+ */
477
+ var transcription = ({ model, started }) => {
478
+ return [
479
+ gutter({
480
+ class: "cm-timestamp-gutter",
481
+ lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
482
+ markers: (view) => {
483
+ const start = getStartTime(started, model.chunks[0]);
484
+ const builder = new RangeSetBuilder();
485
+ for (const { from, to } of view.visibleRanges) {
486
+ let line = view.state.doc.lineAt(from);
487
+ while (line.from <= to) {
488
+ const block = model.getChunkAtLine(line.number)?.blocks[0];
489
+ const timestamp = block?._tag === "transcript" && block.started;
490
+ if (timestamp) builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
491
+ if (line.to + 1 > view.state.doc.length) break;
492
+ line = view.state.doc.lineAt(line.to + 1);
493
+ }
494
+ }
495
+ return builder.finish();
496
+ }
497
+ }),
498
+ ViewPlugin.fromClass(class {
499
+ _controls;
500
+ _cleanup;
501
+ _adapter;
502
+ _initialized = false;
503
+ constructor(view) {
504
+ this._adapter = new EditorChunkDocument(view);
505
+ const scroller = view.scrollDOM;
506
+ let isAutoScrolling = false;
507
+ let hasScrolled = false;
508
+ let timeout;
509
+ const scrollToBottom = (smooth = false) => {
510
+ scroller.style.scrollBehavior = smooth ? "smooth" : "";
511
+ scroller.classList.add("cm-hide-scrollbar");
512
+ isAutoScrolling = true;
513
+ clearTimeout(timeout);
514
+ timeout = setTimeout(() => {
515
+ this._controls?.classList.add("opacity-0");
516
+ scroller.classList.remove("cm-hide-scrollbar");
517
+ isAutoScrolling = false;
518
+ }, 500);
519
+ view.dispatch({ effects: EditorView.scrollIntoView(view.state.doc.length, { y: "end" }) });
520
+ };
521
+ this._cleanup = combine(addEventListener(view.scrollDOM, "scroll", () => {
522
+ if (!isAutoScrolling) {
523
+ hasScrolled = true;
524
+ this._controls?.classList.remove("opacity-0");
525
+ }
526
+ }), model.update.on(() => {
527
+ const autoScroll = scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
528
+ model.sync(this._adapter);
529
+ if (autoScroll) scrollToBottom(true);
530
+ }));
531
+ }
532
+ update(_update) {
533
+ if (!this._initialized) {
534
+ this._initialized = true;
535
+ setTimeout(() => {
536
+ model.sync(this._adapter);
537
+ });
538
+ }
539
+ }
540
+ destroy() {
541
+ this._controls?.remove();
542
+ this._cleanup();
543
+ }
544
+ }),
545
+ EditorView.theme({
546
+ ".cm-gutters": { backgroundColor: "var(--color-base-surface)" },
547
+ ".cm-timestamp-gutter": {
548
+ width: "6rem",
549
+ paddingLeft: "0.5rem",
550
+ paddingRight: "1rem"
551
+ },
552
+ ".cm-timestamp-gutter > .cm-gutterElement > div": {
553
+ display: "inline-flex",
554
+ textAlign: "right",
555
+ padding: "3px"
556
+ }
557
+ })
558
+ ];
559
+ };
560
+ /**
561
+ * Gutter marker that displays a timestamp.
562
+ */
563
+ var TimestampMarker = class extends GutterMarker {
564
+ _line;
565
+ _timestamp;
566
+ _started;
567
+ constructor(_line, _timestamp, _started) {
568
+ super();
569
+ this._line = _line;
570
+ this._timestamp = _timestamp;
571
+ this._started = _started;
572
+ }
573
+ eq(other) {
574
+ return this._timestamp === other._timestamp;
575
+ }
576
+ toDOM(view) {
577
+ const el = document.createElement("div");
578
+ el.className = "text-sm text-subdued hover:bg-hover-surface cursor-pointer";
579
+ el.textContent = formatTimestamp(this._timestamp, this._started);
580
+ el.onclick = () => {
581
+ const pos = view.state.doc.line(this._line).from;
582
+ view.dispatch({ effects: EditorView.scrollIntoView(pos, { y: "start" }) });
583
+ };
584
+ return el;
585
+ }
586
+ };
587
+ var getStartTime = (started, message) => {
588
+ if (started) return started;
589
+ if (message?.blocks[0]?._tag === "transcript" && message.blocks[0].started) return new Date(message.blocks[0].started);
590
+ };
591
+ var formatTimestamp = (timestamp, relative) => {
592
+ if (relative) {
593
+ const pad = (n = 0) => String(n).padStart(2, "0");
594
+ const { hours, minutes, seconds } = intervalToDuration({
595
+ start: relative,
596
+ end: timestamp
597
+ });
598
+ return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
599
+ } else return format(timestamp, "HH:mm:ss");
600
+ };
601
+ //#endregion
602
+ //#region src/components/Transcription/Transcription.tsx
603
+ var inlinePreviewRegistry = { "link-preview": {
604
+ block: false,
605
+ urlSchemes: ["dxn:", "echo:"],
606
+ factory: ({ label, dxn }) => typeof label === "string" && typeof dxn === "string" ? new AnchorWidget(label, dxn) : null
607
+ } };
608
+ var Transcription = composable(({ transcript: object, model, children, ...props }, forwardedRef) => {
609
+ const { themeMode } = useThemeContext();
610
+ const { parentRef } = useTextEditor(() => {
611
+ return { extensions: [
612
+ createBasicExtensions({
613
+ readOnly: true,
614
+ lineWrapping: true,
615
+ search: true
616
+ }),
617
+ createThemeExtensions({
618
+ themeMode,
619
+ slots: documentSlots
620
+ }),
621
+ createMarkdownExtensions(),
622
+ decorateMarkdown({ skip: ({ url }) => url.startsWith("dxn:") || url.startsWith("echo:") }),
623
+ xmlTags({ registry: inlinePreviewRegistry }),
624
+ transcription({
625
+ model,
626
+ started: object?.started ? new Date(object.started) : void 0
627
+ }),
628
+ scroller()
629
+ ] };
630
+ }, [model, themeMode]);
631
+ return /* @__PURE__ */ jsx("div", {
632
+ ...composableProps(props, { classNames: "dx-container" }),
633
+ "data-popover-collision-boundary": true,
634
+ ref: parentRef
635
+ });
636
+ });
637
+ //#endregion
638
+ //#region src/hooks/useAudioFile.ts
639
+ var __dxlog_file$3 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/hooks/useAudioFile.ts";
640
+ var useAudioFile = (audioUrl, constraints) => {
641
+ const [{ audio, stream, track }, setStream] = useState({});
642
+ useEffect(() => {
643
+ const ctx = new Context(void 0, {
644
+ "~LogMeta": "~LogMeta",
645
+ F: __dxlog_file$3,
646
+ L: 20
647
+ });
648
+ scheduleTask(ctx, async () => {
649
+ try {
650
+ const blob = await (await fetch(audioUrl)).blob();
651
+ const objectUrl = URL.createObjectURL(blob);
652
+ ctx.onDispose(() => {
653
+ URL.revokeObjectURL(objectUrl);
654
+ });
655
+ const audio = new Audio();
656
+ audio.src = objectUrl;
657
+ await new Promise((resolve, reject) => {
658
+ audio.addEventListener("error", (err) => {
659
+ log.error("error", { err }, {
660
+ "~LogMeta": "~LogMeta",
661
+ F: __dxlog_file$3,
662
+ L: 39,
663
+ S: void 0
664
+ });
665
+ reject(err);
666
+ }, { once: true });
667
+ audio.addEventListener("canplay", async () => {
668
+ resolve();
669
+ }, { once: true });
670
+ audio.load();
671
+ });
672
+ const audioCtx = new AudioContext();
673
+ if (audioCtx.state === "suspended") await audioCtx.resume();
674
+ const destination = audioCtx.createMediaStreamDestination();
675
+ destination.channelCount = 1;
676
+ audioCtx.createMediaElementSource(audio).connect(destination);
677
+ const track = destination.stream.getAudioTracks()[0];
678
+ if (constraints) await track.applyConstraints(constraints).catch((err) => log.catch(err, void 0, {
679
+ "~LogMeta": "~LogMeta",
680
+ F: __dxlog_file$3,
681
+ L: 68,
682
+ S: void 0
683
+ }));
684
+ setStream({
685
+ audio,
686
+ stream: destination.stream,
687
+ track: destination.stream.getAudioTracks()[0]
688
+ });
689
+ } catch (err) {
690
+ log.catch(err, void 0, {
691
+ "~LogMeta": "~LogMeta",
692
+ F: __dxlog_file$3,
693
+ L: 78,
694
+ S: void 0
695
+ });
696
+ }
697
+ });
698
+ return () => {
699
+ ctx.dispose();
700
+ };
701
+ }, [audioUrl]);
702
+ return {
703
+ audio,
704
+ stream,
705
+ track
706
+ };
707
+ };
708
+ //#endregion
709
+ //#region src/hooks/useAudioTrack.ts
710
+ /**
711
+ * Acquires a microphone audio track while `active`. Re-acquires when the `constraints` change (e.g.
712
+ * a different input device is selected) so the new device takes effect without a remount.
713
+ */
714
+ var useAudioTrack = (active, constraints) => {
715
+ const [track, setTrack] = useState();
716
+ useEffect(() => {
717
+ if (!active) {
718
+ track?.stop();
719
+ setTrack(void 0);
720
+ return;
721
+ }
722
+ let cancelled = false;
723
+ let acquired;
724
+ setTrack(void 0);
725
+ const initAudio = async () => {
726
+ try {
727
+ const [next] = (await navigator.mediaDevices.getUserMedia({ audio: constraints ?? true })).getAudioTracks();
728
+ if (cancelled || !next) {
729
+ next?.stop();
730
+ return;
731
+ }
732
+ acquired = next;
733
+ setTrack(next);
734
+ } catch {
735
+ if (!cancelled) setTrack(void 0);
736
+ }
737
+ };
738
+ initAudio();
739
+ return () => {
740
+ cancelled = true;
741
+ acquired?.stop();
742
+ };
743
+ }, [active, constraints ? JSON.stringify(constraints) : ""]);
744
+ return track;
745
+ };
746
+ //#endregion
747
+ //#region src/hooks/useFeedModelAdapter.ts
748
+ /**
749
+ * Feeds an array of objects (typically from `useQuery(db, Query.from(feed))`) into a
750
+ * `TranscriptModel`, appending each new tail item as the array grows.
751
+ *
752
+ * @param renderer - chunk renderer.
753
+ * @param objects - current snapshot of feed items.
754
+ * @param initialChunks - chunks to seed the model with on first render.
755
+ */
756
+ var useFeedModelAdapter = (renderer, objects, initialChunks = []) => {
757
+ const model = useMemo(() => new TranscriptModel(renderer, initialChunks), [renderer]);
758
+ useEffect(() => {
759
+ for (const chunk of objects) model.appendChunk(chunk);
760
+ }, [model, objects]);
761
+ useEffect(() => {
762
+ if (model.chunks.length === objects.length) return;
763
+ const chunk = objects.at(-1);
764
+ if (chunk) model.appendChunk(chunk);
765
+ }, [model, objects.length]);
766
+ return model;
767
+ };
768
+ //#endregion
769
+ //#region src/hooks/useIsSpeaking.ts
770
+ var __dxlog_file$2 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/hooks/useIsSpeaking.ts";
771
+ /**
772
+ * Tracks voice activity on an audio track via {@link SpeakingMonitor}.
773
+ */
774
+ var useIsSpeaking = (track) => {
775
+ const [isSpeaking, setIsSpeaking] = useState(false);
776
+ const speakingMonitor = useMemo(() => {
777
+ if (!track) return;
778
+ return new SpeakingMonitor(track);
779
+ }, [track]);
780
+ useEffect(() => {
781
+ if (!speakingMonitor) return;
782
+ const ctx = new Context(void 0, {
783
+ "~LogMeta": "~LogMeta",
784
+ F: __dxlog_file$2,
785
+ L: 29
786
+ });
787
+ scheduleTask(ctx, async () => {
788
+ speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
789
+ await speakingMonitor.open();
790
+ ctx.onDispose(() => speakingMonitor.close());
791
+ });
792
+ return () => {
793
+ ctx.dispose();
794
+ };
795
+ }, [speakingMonitor]);
796
+ return isSpeaking;
797
+ };
798
+ //#endregion
799
+ //#region src/hooks/useRecordingPipeline.ts
800
+ var __dxlog_file$1 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/hooks/useRecordingPipeline.ts";
801
+ var RECORDER_INTERVAL_MS = 200;
802
+ var DRAIN_SILENCE_MS = 5e3;
803
+ var DEFAULT_TRANSCRIBE_CONFIG = {
804
+ transcribeAfterChunksAmount: 50,
805
+ prefixBufferChunksAmount: 10
806
+ };
807
+ /**
808
+ * Single ASR-capture engine binding for React consumers: drives an {@link AsrPipeline}
809
+ * ({@link runAsrPipeline}) off a capture flag, sourcing audio from the microphone (owned here) or an
810
+ * external track. The editor driver, message-list recording, and stories all build on this — varying
811
+ * only by source, stages, and commit sink.
812
+ */
813
+ var useRecordingPipeline = ({ active, microphone, track: externalTrack, audioConstraints, stages, commit, lookup, onSegment, config, segmentSentences }) => {
814
+ const [phase, setPhase] = useState("idle");
815
+ const inUse = phase !== "idle";
816
+ useEffect(() => {
817
+ setPhase((current) => active ? "recording" : current === "recording" ? "draining" : current);
818
+ }, [active]);
819
+ const micTrack = useAudioTrack(!!microphone && inUse, audioConstraints);
820
+ const track = microphone ? micTrack : externalTrack;
821
+ const recorder = useMemo(() => track ? new MediaStreamRecorder({
822
+ mediaStreamTrack: track,
823
+ config: { interval: RECORDER_INTERVAL_MS }
824
+ }) : void 0, [track]);
825
+ const pipelineRef = useRef(null);
826
+ useEffect(() => {
827
+ if (phase !== "recording" || !recorder) return;
828
+ const asr = runAsrPipeline({
829
+ recorder,
830
+ config: {
831
+ ...DEFAULT_TRANSCRIBE_CONFIG,
832
+ ...config
833
+ },
834
+ segmentSentences,
835
+ stages,
836
+ commit,
837
+ lookup,
838
+ onSegment,
839
+ drainSilenceMs: DRAIN_SILENCE_MS
840
+ });
841
+ pipelineRef.current = asr;
842
+ let cancelled = false;
843
+ asr.open().then(() => {
844
+ if (!cancelled) asr.start();
845
+ }, (err) => {
846
+ if (!cancelled) {
847
+ log.catch(err, void 0, {
848
+ "~LogMeta": "~LogMeta",
849
+ F: __dxlog_file$1,
850
+ L: 131,
851
+ S: void 0
852
+ });
853
+ setPhase("idle");
854
+ }
855
+ });
856
+ return () => {
857
+ cancelled = true;
858
+ };
859
+ }, [phase, recorder]);
860
+ useEffect(() => {
861
+ if (phase !== "draining") return;
862
+ let cancelled = false;
863
+ (async () => {
864
+ try {
865
+ await pipelineRef.current?.end();
866
+ } catch (err) {
867
+ log.catch(err, void 0, {
868
+ "~LogMeta": "~LogMeta",
869
+ F: __dxlog_file$1,
870
+ L: 152,
871
+ S: void 0
872
+ });
873
+ } finally {
874
+ pipelineRef.current = null;
875
+ if (!cancelled) setPhase((current) => current === "draining" ? "idle" : current);
876
+ }
877
+ })();
878
+ return () => {
879
+ cancelled = true;
880
+ };
881
+ }, [phase]);
882
+ return { phase };
883
+ };
884
+ //#endregion
885
+ //#region src/hooks/useSpeechRecognition.ts
886
+ var __dxlog_file = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/hooks/useSpeechRecognition.ts";
887
+ var cleanText = (text) => text.toLowerCase().replace(/[.,!?]/g, "").replace(/\s+/g, " ").trim();
888
+ /**
889
+ * Use browser speech recognition API to transcribe speech to text.
890
+ */
891
+ var useSpeechRecognition = ({ active, onTranscript }) => {
892
+ const recognition = useMemo(() => {
893
+ const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
894
+ const SpeechGrammarList = window.SpeechGrammarList || window.webkitSpeechGrammarList;
895
+ if (!SpeechRecognition || !SpeechGrammarList) {
896
+ log.error("Speech recognition not supported", void 0, {
897
+ "~LogMeta": "~LogMeta",
898
+ F: __dxlog_file,
899
+ L: 29,
900
+ S: void 0
901
+ });
902
+ return;
903
+ }
904
+ return Object.assign(new SpeechRecognition(), {
905
+ grammars: new SpeechGrammarList(),
906
+ continuous: true,
907
+ interimResults: true,
908
+ lang: "en-US",
909
+ maxAlternatives: 1,
910
+ onresult: (event) => {
911
+ onTranscript(cleanText(event.results[event.resultIndex][0].transcript));
912
+ },
913
+ onerror: (event) => {
914
+ log.error("Speech recognition error", { error: event.error }, {
915
+ "~LogMeta": "~LogMeta",
916
+ F: __dxlog_file,
917
+ L: 44,
918
+ S: void 0
919
+ });
920
+ }
921
+ });
922
+ }, [onTranscript]);
923
+ useEffect(() => {
924
+ if (active) recognition?.start();
925
+ else recognition?.stop();
926
+ }, [active, recognition]);
927
+ };
928
+ //#endregion
929
+ //#region src/hooks/useTranscriber.ts
930
+ /**
931
+ * Records audio while the user is speaking and transcribes it after they pause. Builds a
932
+ * {@link Transcriber} directly from {@link createTranscriber} (no app-framework capability needed,
933
+ * since the browser construction lives in this package); returns `undefined` until a track and
934
+ * `onSegments` handler are supplied.
935
+ */
936
+ var useTranscriber = ({ audioStreamTrack, recorderConfig, transcriberConfig, transcribe, onSegments }) => {
937
+ const transcriber = useMemo(() => {
938
+ if (!audioStreamTrack || !onSegments) return;
939
+ return createTranscriber({
940
+ audioStreamTrack,
941
+ recorderConfig,
942
+ transcriberConfig,
943
+ transcribe,
944
+ onSegments
945
+ });
946
+ }, [
947
+ audioStreamTrack,
948
+ recorderConfig,
949
+ transcriberConfig,
950
+ transcribe,
951
+ onSegments
952
+ ]);
953
+ useEffect(() => {
954
+ return () => {
955
+ transcriber?.close();
956
+ };
957
+ }, [transcriber]);
958
+ return transcriber;
959
+ };
960
+ //#endregion
961
+ export { EditorChunkDocument, MediaStreamRecorder, MicSettings, PipelineStatus, TranscriptModel, Transcription, createTranscriber, transcription, useAudioFile, useAudioTrack, useFeedModelAdapter, useIsSpeaking, useRecordingPipeline, useSpeechRecognition, useTranscriber };
962
+
963
+ //# sourceMappingURL=index.mjs.map