@dxos/react-ui-transcription 0.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (84) hide show
  1. package/LICENSE +105 -0
  2. package/dist/lib/browser/index.mjs +1002 -0
  3. package/dist/lib/browser/index.mjs.map +7 -0
  4. package/dist/lib/browser/meta.json +1 -0
  5. package/dist/lib/node-esm/index.mjs +1004 -0
  6. package/dist/lib/node-esm/index.mjs.map +7 -0
  7. package/dist/lib/node-esm/meta.json +1 -0
  8. package/dist/types/src/capture/create-transcriber.d.ts +18 -0
  9. package/dist/types/src/capture/create-transcriber.d.ts.map +1 -0
  10. package/dist/types/src/capture/index.d.ts +3 -0
  11. package/dist/types/src/capture/index.d.ts.map +1 -0
  12. package/dist/types/src/capture/media-stream-recorder.d.ts +30 -0
  13. package/dist/types/src/capture/media-stream-recorder.d.ts.map +1 -0
  14. package/dist/types/src/components/MicSettings/MicSettings.d.ts +25 -0
  15. package/dist/types/src/components/MicSettings/MicSettings.d.ts.map +1 -0
  16. package/dist/types/src/components/MicSettings/index.d.ts +2 -0
  17. package/dist/types/src/components/MicSettings/index.d.ts.map +1 -0
  18. package/dist/types/src/components/PipelineStatus/PipelineStatus.d.ts +20 -0
  19. package/dist/types/src/components/PipelineStatus/PipelineStatus.d.ts.map +1 -0
  20. package/dist/types/src/components/PipelineStatus/index.d.ts +2 -0
  21. package/dist/types/src/components/PipelineStatus/index.d.ts.map +1 -0
  22. package/dist/types/src/components/Transcription/Transcription.d.ts +11 -0
  23. package/dist/types/src/components/Transcription/Transcription.d.ts.map +1 -0
  24. package/dist/types/src/components/Transcription/index.d.ts +3 -0
  25. package/dist/types/src/components/Transcription/index.d.ts.map +1 -0
  26. package/dist/types/src/components/Transcription/transcription-extension.d.ts +15 -0
  27. package/dist/types/src/components/Transcription/transcription-extension.d.ts.map +1 -0
  28. package/dist/types/src/components/index.d.ts +4 -0
  29. package/dist/types/src/components/index.d.ts.map +1 -0
  30. package/dist/types/src/hooks/index.d.ts +8 -0
  31. package/dist/types/src/hooks/index.d.ts.map +1 -0
  32. package/dist/types/src/hooks/useAudioFile.d.ts +7 -0
  33. package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -0
  34. package/dist/types/src/hooks/useAudioTrack.d.ts +6 -0
  35. package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -0
  36. package/dist/types/src/hooks/useFeedModelAdapter.d.ts +12 -0
  37. package/dist/types/src/hooks/useFeedModelAdapter.d.ts.map +1 -0
  38. package/dist/types/src/hooks/useIsSpeaking.d.ts +5 -0
  39. package/dist/types/src/hooks/useIsSpeaking.d.ts.map +1 -0
  40. package/dist/types/src/hooks/useRecordingPipeline.d.ts +40 -0
  41. package/dist/types/src/hooks/useRecordingPipeline.d.ts.map +1 -0
  42. package/dist/types/src/hooks/useSpeechRecognition.d.ts +9 -0
  43. package/dist/types/src/hooks/useSpeechRecognition.d.ts.map +1 -0
  44. package/dist/types/src/hooks/useTranscriber.d.ts +10 -0
  45. package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -0
  46. package/dist/types/src/index.d.ts +7 -0
  47. package/dist/types/src/index.d.ts.map +1 -0
  48. package/dist/types/src/model/index.d.ts +2 -0
  49. package/dist/types/src/model/index.d.ts.map +1 -0
  50. package/dist/types/src/model/transcript-model.d.ts +68 -0
  51. package/dist/types/src/model/transcript-model.d.ts.map +1 -0
  52. package/dist/types/src/types.d.ts +3 -0
  53. package/dist/types/src/types.d.ts.map +1 -0
  54. package/dist/types/src/util/index.d.ts +2 -0
  55. package/dist/types/src/util/index.d.ts.map +1 -0
  56. package/dist/types/src/util/render-byline.d.ts +4 -0
  57. package/dist/types/src/util/render-byline.d.ts.map +1 -0
  58. package/dist/types/tsconfig.tsbuildinfo +1 -0
  59. package/package.json +69 -0
  60. package/src/capture/create-transcriber.ts +49 -0
  61. package/src/capture/index.ts +6 -0
  62. package/src/capture/media-stream-recorder.ts +137 -0
  63. package/src/components/MicSettings/MicSettings.tsx +126 -0
  64. package/src/components/MicSettings/index.ts +5 -0
  65. package/src/components/PipelineStatus/PipelineStatus.tsx +71 -0
  66. package/src/components/PipelineStatus/index.ts +5 -0
  67. package/src/components/Transcription/Transcription.tsx +67 -0
  68. package/src/components/Transcription/index.ts +6 -0
  69. package/src/components/Transcription/transcription-extension.ts +203 -0
  70. package/src/components/index.ts +7 -0
  71. package/src/hooks/index.ts +11 -0
  72. package/src/hooks/useAudioFile.ts +88 -0
  73. package/src/hooks/useAudioTrack.ts +56 -0
  74. package/src/hooks/useFeedModelAdapter.ts +46 -0
  75. package/src/hooks/useIsSpeaking.ts +42 -0
  76. package/src/hooks/useRecordingPipeline.ts +166 -0
  77. package/src/hooks/useSpeechRecognition.ts +56 -0
  78. package/src/hooks/useTranscriber.ts +45 -0
  79. package/src/index.ts +10 -0
  80. package/src/model/index.ts +5 -0
  81. package/src/model/transcript-model.ts +229 -0
  82. package/src/types.ts +6 -0
  83. package/src/util/index.ts +5 -0
  84. package/src/util/render-byline.ts +32 -0
@@ -0,0 +1,1002 @@
1
+ // src/capture/create-transcriber.ts
2
+ import { Transcriber } from "@dxos/transcription-pipeline";
3
+
4
+ // src/capture/media-stream-recorder.ts
5
+ import { WaveFile } from "wavefile";
6
+ import { synchronized } from "@dxos/async";
7
+ import { invariant } from "@dxos/invariant";
8
+ import { log } from "@dxos/log";
9
+ import { trace } from "@dxos/tracing";
10
+ var __dxlog_file = "/Users/jdw/Code/dxos/packages/ui/react-ui-transcription/src/capture/media-stream-recorder.ts";
11
+ function _ts_decorate(decorators, target, key, desc) {
12
+ var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
13
+ if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
14
+ else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
15
+ return c > 3 && r && Object.defineProperty(target, key, r), r;
16
+ }
17
+ var MediaRecorderClass;
18
+ var initializingPromise;
19
+ var initializeExtendableMediaRecorder = async () => {
20
+ const { MediaRecorder, register } = await import("extendable-media-recorder");
21
+ const { connect } = await import("extendable-media-recorder-wav-encoder");
22
+ await register(await connect());
23
+ MediaRecorderClass = MediaRecorder;
24
+ };
25
+ var MediaStreamRecorder = class {
26
+ _mediaStreamTrack;
27
+ /**
28
+ * Default MediaRecorder implementation do not support wav encoding.
29
+ */
30
+ _config;
31
+ _mediaRecorder = void 0;
32
+ _header = void 0;
33
+ _onChunk;
34
+ constructor({ mediaStreamTrack, config }) {
35
+ this._mediaStreamTrack = mediaStreamTrack;
36
+ this._config = config;
37
+ }
38
+ get wavConfig() {
39
+ const settings = this._mediaStreamTrack.getSettings();
40
+ return {
41
+ channels: settings.channelCount ?? 1,
42
+ sampleRate: settings.sampleRate ?? 16e3,
43
+ bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
44
+ };
45
+ }
46
+ get mediaStreamTrack() {
47
+ return this._mediaStreamTrack;
48
+ }
49
+ setOnChunk(onChunk) {
50
+ this._onChunk = onChunk;
51
+ }
52
+ // `@synchronized` serializes start/stop (they share the instance lock) so a rapid stop→start cycle
53
+ // (e.g. React StrictMode double-mount) cannot initiate a new WAV encoding before the previous one
54
+ // has flushed — which throws "Another request was made to initiate an encoding" and kills capture.
55
+ async start() {
56
+ await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log.info("initializeExtendableMediaRecorder", {
57
+ err
58
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 56, S: this }));
59
+ if (this._mediaRecorder?.state === "recording") {
60
+ return;
61
+ }
62
+ invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 62, S: this, A: ["this._onChunk", "'MediaStreamRecorder: onChunk is not set'"] });
63
+ invariant(MediaRecorderClass, "MediaStreamRecorder: extendable-media-recorder not initialized", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 63, S: this, A: ["MediaRecorderClass", "'MediaStreamRecorder: extendable-media-recorder not initialized'"] });
64
+ const stream = new MediaStream([
65
+ this._mediaStreamTrack
66
+ ]);
67
+ const recorder = new MediaRecorderClass(stream, {
68
+ mimeType: "audio/wav"
69
+ });
70
+ recorder.ondataavailable = (event) => this._ondataavailable(event);
71
+ this._mediaRecorder = recorder;
72
+ recorder.start(this._config.interval);
73
+ }
74
+ async stop() {
75
+ const recorder = this._mediaRecorder;
76
+ if (!recorder || recorder.state !== "recording") {
77
+ return;
78
+ }
79
+ await new Promise((resolve) => {
80
+ recorder.addEventListener("stop", () => resolve(), {
81
+ once: true
82
+ });
83
+ recorder.stop();
84
+ });
85
+ this._mediaRecorder = void 0;
86
+ }
87
+ async _ondataavailable(event) {
88
+ const blob = event.data;
89
+ const uint8Array = new Uint8Array(await blob.arrayBuffer());
90
+ const isHeader = uint8Array[0] === 82 && // R
91
+ uint8Array[1] === 73 && // I
92
+ uint8Array[2] === 70 && // F
93
+ uint8Array[3] === 70;
94
+ let wav;
95
+ if (isHeader) {
96
+ wav = new WaveFile(uint8Array);
97
+ this._header = uint8Array.slice(0, 44);
98
+ } else if (this._header) {
99
+ wav = new WaveFile(new Uint8Array([
100
+ ...this._header,
101
+ ...uint8Array
102
+ ]));
103
+ } else {
104
+ log.warn("MediaStreamRecorder: no header", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 108, S: this });
105
+ return;
106
+ }
107
+ this._onChunk({
108
+ timestamp: Date.now(),
109
+ data: wav.getSamples()
110
+ });
111
+ }
112
+ };
113
+ _ts_decorate([
114
+ synchronized
115
+ ], MediaStreamRecorder.prototype, "start", null);
116
+ _ts_decorate([
117
+ synchronized
118
+ ], MediaStreamRecorder.prototype, "stop", null);
119
+ _ts_decorate([
120
+ synchronized
121
+ ], MediaStreamRecorder.prototype, "_ondataavailable", null);
122
+ MediaStreamRecorder = _ts_decorate([
123
+ trace.resource()
124
+ ], MediaStreamRecorder);
125
+
126
+ // src/capture/create-transcriber.ts
127
+ var RECORD_INTERVAL = 200;
128
+ var PREFIXED_CHUNKS_AMOUNT = 10;
129
+ var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
130
+ var createTranscriber = ({ audioStreamTrack, onSegments, transcriberConfig, recorderConfig, transcribe }) => new Transcriber({
131
+ config: {
132
+ transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
133
+ prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
134
+ ...transcriberConfig
135
+ },
136
+ recorder: new MediaStreamRecorder({
137
+ mediaStreamTrack: audioStreamTrack,
138
+ config: {
139
+ interval: RECORD_INTERVAL,
140
+ ...recorderConfig
141
+ }
142
+ }),
143
+ onSegments,
144
+ transcribe
145
+ });
146
+
147
+ // src/components/MicSettings/MicSettings.tsx
148
+ import React from "react";
149
+ import { DropdownMenu, Icon, IconButton, useTranslation } from "@dxos/react-ui";
150
+ var MicSettings = ({ translationNs, recordMode, entityExtraction, devices, selectedDeviceId, onRecordModeChange, onEntityExtractionChange, onSelectDevice }) => {
151
+ const { t } = useTranslation(translationNs);
152
+ return /* @__PURE__ */ React.createElement(DropdownMenu.Root, null, /* @__PURE__ */ React.createElement(DropdownMenu.Trigger, {
153
+ asChild: true
154
+ }, /* @__PURE__ */ React.createElement(IconButton, {
155
+ icon: "ph--caret-down--regular",
156
+ iconOnly: true,
157
+ label: t("recording-options.label"),
158
+ variant: "ghost",
159
+ // Half-width slim caret abutting the mic (split-control affordance).
160
+ classNames: "w-4",
161
+ "data-testid": "transcription.record.options"
162
+ })), /* @__PURE__ */ React.createElement(DropdownMenu.Portal, null, /* @__PURE__ */ React.createElement(DropdownMenu.Content, null, /* @__PURE__ */ React.createElement(DropdownMenu.Viewport, null, /* @__PURE__ */ React.createElement(DropdownMenu.GroupLabel, null, t("record-mode.label")), /* @__PURE__ */ React.createElement(SelectableItem, {
163
+ label: t("record-mode.toggle.label"),
164
+ selected: recordMode === "toggle",
165
+ onSelect: () => onRecordModeChange("toggle")
166
+ }), /* @__PURE__ */ React.createElement(SelectableItem, {
167
+ label: t("record-mode.hold.label"),
168
+ selected: recordMode === "hold",
169
+ onSelect: () => onRecordModeChange("hold")
170
+ }), /* @__PURE__ */ React.createElement(DropdownMenu.Separator, null), /* @__PURE__ */ React.createElement(DropdownMenu.GroupLabel, null, t("audio-device.label")), /* @__PURE__ */ React.createElement(SelectableItem, {
171
+ label: t("audio-device.default.label"),
172
+ selected: selectedDeviceId === "",
173
+ onSelect: () => onSelectDevice("")
174
+ }), devices.map((device) => /* @__PURE__ */ React.createElement(SelectableItem, {
175
+ key: device.deviceId,
176
+ label: device.label,
177
+ selected: selectedDeviceId === device.deviceId,
178
+ onSelect: () => onSelectDevice(device.deviceId)
179
+ })), /* @__PURE__ */ React.createElement(DropdownMenu.Separator, null), /* @__PURE__ */ React.createElement(DropdownMenu.CheckboxItem, {
180
+ checked: entityExtraction,
181
+ onCheckedChange: onEntityExtractionChange,
182
+ classNames: "gap-2",
183
+ "data-testid": "transcription.entity-extraction"
184
+ }, /* @__PURE__ */ React.createElement("span", {
185
+ className: "grow truncate"
186
+ }, t("settings.entity-extraction.label")), /* @__PURE__ */ React.createElement(DropdownMenu.ItemIndicator, {
187
+ asChild: true
188
+ }, /* @__PURE__ */ React.createElement(Icon, {
189
+ icon: "ph--check--regular",
190
+ size: 4
191
+ })))), /* @__PURE__ */ React.createElement(DropdownMenu.Arrow, null))));
192
+ };
193
+ var SelectableItem = ({ label, selected, onSelect }) => /* @__PURE__ */ React.createElement(DropdownMenu.Item, {
194
+ classNames: "gap-2",
195
+ onSelect
196
+ }, /* @__PURE__ */ React.createElement("span", {
197
+ className: "grow truncate"
198
+ }, label), selected && /* @__PURE__ */ React.createElement(Icon, {
199
+ icon: "ph--check--regular",
200
+ size: 4
201
+ }));
202
+
203
+ // src/components/PipelineStatus/PipelineStatus.tsx
204
+ import React2, { Fragment } from "react";
205
+ var PipelineStatus = ({ phase = "idle", stages, telemetry, summary }) => /* @__PURE__ */ React2.createElement("div", {
206
+ className: "flex flex-col gap-3 overflow-y-auto p-2 text-sm"
207
+ }, /* @__PURE__ */ React2.createElement(Section, {
208
+ title: "Status"
209
+ }, /* @__PURE__ */ React2.createElement("div", {
210
+ className: "grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-description"
211
+ }, /* @__PURE__ */ React2.createElement("span", null, "Mic"), /* @__PURE__ */ React2.createElement("span", null, phase === "recording" ? "\u25CF on" : "\u25CB off"), /* @__PURE__ */ React2.createElement("span", null, "Pipeline"), /* @__PURE__ */ React2.createElement("span", null, phase))), stages && /* @__PURE__ */ React2.createElement(Section, {
212
+ title: "Stages"
213
+ }, /* @__PURE__ */ React2.createElement("div", {
214
+ className: "text-description"
215
+ }, stages.join(" \u2192 ") || "(none)")), telemetry && telemetry.length > 0 && /* @__PURE__ */ React2.createElement(Section, {
216
+ title: "Telemetry"
217
+ }, /* @__PURE__ */ React2.createElement("div", {
218
+ className: "grid w-fit grid-cols-[1fr_auto_auto] gap-x-4 gap-y-1"
219
+ }, telemetry.map((event, index) => /* @__PURE__ */ React2.createElement(Fragment, {
220
+ key: index
221
+ }, /* @__PURE__ */ React2.createElement("span", null, event.stageId), /* @__PURE__ */ React2.createElement("span", {
222
+ className: "text-description"
223
+ }, event.outcome), /* @__PURE__ */ React2.createElement("span", {
224
+ className: "text-right tabular-nums text-description"
225
+ }, event.durationMs, "ms"))))), summary && /* @__PURE__ */ React2.createElement(Section, {
226
+ title: "Summary"
227
+ }, /* @__PURE__ */ React2.createElement("p", {
228
+ className: "text-description"
229
+ }, summary)));
230
+ var Section = ({ title, children }) => /* @__PURE__ */ React2.createElement("div", null, /* @__PURE__ */ React2.createElement("h3", {
231
+ className: "mb-1 font-medium"
232
+ }, title), children);
233
+
234
+ // src/components/Transcription/Transcription.tsx
235
+ import React3 from "react";
236
+ import { composable, composableProps, useThemeContext } from "@dxos/react-ui";
237
+ import { useTextEditor } from "@dxos/react-ui-editor";
238
+ import { AnchorWidget, createBasicExtensions, createMarkdownExtensions, createThemeExtensions, decorateMarkdown, documentSlots, scroller, xmlTags } from "@dxos/ui-editor";
239
+
240
+ // src/components/Transcription/transcription-extension.ts
241
+ import { RangeSetBuilder } from "@codemirror/state";
242
+ import { EditorView, GutterMarker, ViewPlugin, gutter } from "@codemirror/view";
243
+ import { format } from "date-fns/format";
244
+ import { intervalToDuration } from "date-fns/intervalToDuration";
245
+ import { addEventListener, combine } from "@dxos/async";
246
+
247
+ // src/model/transcript-model.ts
248
+ import { Text } from "@codemirror/state";
249
+ import { Event } from "@dxos/async";
250
+ import { invariant as invariant2 } from "@dxos/invariant";
251
+ import { log as log2 } from "@dxos/log";
252
+ var __dxlog_file2 = "/Users/jdw/Code/dxos/packages/ui/react-ui-transcription/src/model/transcript-model.ts";
253
+ var EditorChunkDocument = class {
254
+ _view;
255
+ constructor(_view) {
256
+ this._view = _view;
257
+ }
258
+ /** The document must always have at least one line. */
259
+ lineCount() {
260
+ return this._view.state.doc.lines - 1;
261
+ }
262
+ replaceLines(from, remove, insert) {
263
+ log2("replaceLines", {
264
+ count: this._view.state.doc.lines,
265
+ from,
266
+ remove,
267
+ insert
268
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 19, S: this });
269
+ const numLines = this._view.state.doc.lines;
270
+ const posFrom = this._view.state.doc.line(from).from;
271
+ const posTo = remove === 0 ? posFrom : this._view.state.doc.line(Math.min(numLines, from + remove)).from;
272
+ let text = insert.join("\n");
273
+ if (posTo === this._view.state.doc.length) {
274
+ text += "\n";
275
+ }
276
+ this._view.dispatch({
277
+ changes: {
278
+ from: posFrom,
279
+ to: posTo,
280
+ insert: text
281
+ }
282
+ });
283
+ }
284
+ };
285
+ var TranscriptModel = class {
286
+ _renderer;
287
+ /** Emits when the document is updated. */
288
+ update = new Event();
289
+ /** Ordered set of chunks. */
290
+ _chunks = [];
291
+ /** Map of chunk IDs to their current line counts. */
292
+ _chunkLineCounts = /* @__PURE__ */ new Map();
293
+ /** Map of line numbers to chunk IDs. */
294
+ _lineToChunk = /* @__PURE__ */ new Map();
295
+ /** Track chunk changes since last sync. */
296
+ _changes = [];
297
+ constructor(_renderer, initialChunks = []) {
298
+ this._renderer = _renderer;
299
+ initialChunks.forEach((chunk) => {
300
+ this.appendChunk(chunk);
301
+ });
302
+ }
303
+ toJSON() {
304
+ return {
305
+ chunks: this._chunks.length,
306
+ changes: this._changes.length
307
+ };
308
+ }
309
+ get doc() {
310
+ return Text.of([
311
+ ...this._chunks.flatMap((chunk, index) => this._renderer(chunk, index)),
312
+ ""
313
+ ]);
314
+ }
315
+ get chunks() {
316
+ return this._chunks;
317
+ }
318
+ /**
319
+ * Get the block ID at the given line number.
320
+ */
321
+ getChunkAtLine(line) {
322
+ return this._lineToChunk.get(line);
323
+ }
324
+ reset() {
325
+ this._chunks.length = 0;
326
+ this._chunkLineCounts.clear();
327
+ this._lineToChunk.clear();
328
+ this.update.emit();
329
+ return this;
330
+ }
331
+ appendChunk(chunk) {
332
+ const index = this._chunks.length;
333
+ this._chunks.push(chunk);
334
+ this._changes.push({
335
+ type: "append",
336
+ index,
337
+ chunk
338
+ });
339
+ this.update.emit();
340
+ return this;
341
+ }
342
+ updateChunk(chunk) {
343
+ const index = this._chunks.findIndex((c) => c.id === chunk.id);
344
+ invariant2(index !== -1, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 102, S: this, A: ["index !== -1", ""] });
345
+ this._chunks[index] = chunk;
346
+ this._changes.push({
347
+ type: "update",
348
+ index,
349
+ chunk
350
+ });
351
+ this.update.emit();
352
+ return this;
353
+ }
354
+ deleteBlock(id) {
355
+ const index = this._chunks.findIndex((c) => c.id === id);
356
+ invariant2(index !== -1, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 114, S: this, A: ["index !== -1", ""] });
357
+ this._chunks.splice(index, 1);
358
+ this._changes.push({
359
+ type: "delete",
360
+ index,
361
+ id
362
+ });
363
+ this.update.emit();
364
+ return this;
365
+ }
366
+ /**
367
+ * Syncs the tracked changes with the ChunkDocument.
368
+ */
369
+ sync(document2) {
370
+ log2("sync", {
371
+ changes: this._changes
372
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 127, S: this });
373
+ for (const change of this._changes) {
374
+ switch (change.type) {
375
+ case "append": {
376
+ const lines = this._renderer(change.chunk, change.index);
377
+ const lastLine = document2.lineCount();
378
+ document2.replaceLines(lastLine + 1, 0, lines);
379
+ break;
380
+ }
381
+ case "update": {
382
+ let lineStart = 1;
383
+ for (let i = 0; i < change.index; i++) {
384
+ lineStart += this._chunkLineCounts.get(this._chunks[i].id);
385
+ }
386
+ const previousLineCount = this._chunkLineCounts.get(change.chunk.id);
387
+ const newLines = this._renderer(change.chunk, change.index);
388
+ document2.replaceLines(lineStart, previousLineCount, newLines);
389
+ break;
390
+ }
391
+ case "delete": {
392
+ let lineStart = 1;
393
+ for (let i = 0; i < change.index; i++) {
394
+ lineStart += this._chunkLineCounts.get(this._chunks[i].id);
395
+ }
396
+ const previousLineCount = this._chunkLineCounts.get(change.id);
397
+ document2.replaceLines(lineStart, previousLineCount, []);
398
+ break;
399
+ }
400
+ }
401
+ }
402
+ this._lineToChunk.clear();
403
+ let currentLine = 1;
404
+ for (const [index, chunk] of this._chunks.entries()) {
405
+ const lineCount = this._renderer(chunk, index).length;
406
+ this._chunkLineCounts.set(chunk.id, lineCount);
407
+ this._lineToChunk.set(currentLine, chunk);
408
+ currentLine += lineCount;
409
+ }
410
+ const chunkIds = new Set(this._chunks.map((chunk) => chunk.id));
411
+ for (const id of this._chunkLineCounts.keys()) {
412
+ if (!chunkIds.has(id)) {
413
+ this._chunkLineCounts.delete(id);
414
+ }
415
+ }
416
+ this._changes.length = 0;
417
+ return this;
418
+ }
419
+ };
420
+
421
+ // src/components/Transcription/transcription-extension.ts
422
+ var transcription = ({ model, started }) => {
423
+ return [
424
+ // Show timestamps in the gutter.
425
+ gutter({
426
+ class: "cm-timestamp-gutter",
427
+ lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
428
+ markers: (view) => {
429
+ const start = getStartTime(started, model.chunks[0]);
430
+ const builder = new RangeSetBuilder();
431
+ for (const { from, to } of view.visibleRanges) {
432
+ let line = view.state.doc.lineAt(from);
433
+ while (line.from <= to) {
434
+ const block = model.getChunkAtLine(line.number)?.blocks[0];
435
+ const timestamp = block?._tag === "transcript" && block.started;
436
+ if (timestamp) {
437
+ builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
438
+ }
439
+ if (line.to + 1 > view.state.doc.length) {
440
+ break;
441
+ }
442
+ line = view.state.doc.lineAt(line.to + 1);
443
+ }
444
+ }
445
+ return builder.finish();
446
+ }
447
+ }),
448
+ // Listen for model updates.
449
+ ViewPlugin.fromClass(class {
450
+ _controls;
451
+ _cleanup;
452
+ _adapter;
453
+ _initialized = false;
454
+ constructor(view) {
455
+ this._adapter = new EditorChunkDocument(view);
456
+ const scroller2 = view.scrollDOM;
457
+ let isAutoScrolling = false;
458
+ let hasScrolled = false;
459
+ let timeout;
460
+ const scrollToBottom = (smooth = false) => {
461
+ scroller2.style.scrollBehavior = smooth ? "smooth" : "";
462
+ scroller2.classList.add("cm-hide-scrollbar");
463
+ isAutoScrolling = true;
464
+ clearTimeout(timeout);
465
+ timeout = setTimeout(() => {
466
+ this._controls?.classList.add("opacity-0");
467
+ scroller2.classList.remove("cm-hide-scrollbar");
468
+ isAutoScrolling = false;
469
+ }, 500);
470
+ view.dispatch({
471
+ effects: EditorView.scrollIntoView(view.state.doc.length, {
472
+ y: "end"
473
+ })
474
+ });
475
+ };
476
+ this._cleanup = combine(addEventListener(view.scrollDOM, "scroll", () => {
477
+ if (!isAutoScrolling) {
478
+ hasScrolled = true;
479
+ this._controls?.classList.remove("opacity-0");
480
+ }
481
+ }), model.update.on(() => {
482
+ const autoScroll = scroller2.scrollHeight - scroller2.scrollTop - scroller2.clientHeight === 0 || !hasScrolled;
483
+ model.sync(this._adapter);
484
+ if (autoScroll) {
485
+ scrollToBottom(true);
486
+ }
487
+ }));
488
+ }
489
+ update(_update) {
490
+ if (!this._initialized) {
491
+ this._initialized = true;
492
+ setTimeout(() => {
493
+ model.sync(this._adapter);
494
+ });
495
+ }
496
+ }
497
+ destroy() {
498
+ this._controls?.remove();
499
+ this._cleanup();
500
+ }
501
+ }),
502
+ EditorView.theme({
503
+ ".cm-gutters": {
504
+ backgroundColor: "var(--color-base-surface)"
505
+ },
506
+ ".cm-timestamp-gutter": {
507
+ width: "6rem",
508
+ paddingLeft: "0.5rem",
509
+ paddingRight: "1rem"
510
+ },
511
+ ".cm-timestamp-gutter > .cm-gutterElement > div": {
512
+ display: "inline-flex",
513
+ textAlign: "right",
514
+ padding: "3px"
515
+ }
516
+ })
517
+ ];
518
+ };
519
+ var TimestampMarker = class extends GutterMarker {
520
+ _line;
521
+ _timestamp;
522
+ _started;
523
+ constructor(_line, _timestamp, _started) {
524
+ super(), this._line = _line, this._timestamp = _timestamp, this._started = _started;
525
+ }
526
+ eq(other) {
527
+ return this._timestamp === other._timestamp;
528
+ }
529
+ toDOM(view) {
530
+ const el = document.createElement("div");
531
+ el.className = "text-sm text-subdued hover:bg-hover-surface cursor-pointer";
532
+ el.textContent = formatTimestamp(this._timestamp, this._started);
533
+ el.onclick = () => {
534
+ const pos = view.state.doc.line(this._line).from;
535
+ view.dispatch({
536
+ effects: EditorView.scrollIntoView(pos, {
537
+ y: "start"
538
+ })
539
+ });
540
+ };
541
+ return el;
542
+ }
543
+ };
544
+ var getStartTime = (started, message) => {
545
+ if (started) {
546
+ return started;
547
+ }
548
+ if (message?.blocks[0]?._tag === "transcript" && message.blocks[0].started) {
549
+ return new Date(message.blocks[0].started);
550
+ }
551
+ return void 0;
552
+ };
553
+ var formatTimestamp = (timestamp, relative) => {
554
+ if (relative) {
555
+ const pad = (n = 0) => String(n).padStart(2, "0");
556
+ const { hours, minutes, seconds } = intervalToDuration({
557
+ start: relative,
558
+ end: timestamp
559
+ });
560
+ return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
561
+ } else {
562
+ return format(timestamp, "HH:mm:ss");
563
+ }
564
+ };
565
+
566
+ // src/components/Transcription/Transcription.tsx
567
+ var inlinePreviewRegistry = {
568
+ "link-preview": {
569
+ block: false,
570
+ urlSchemes: [
571
+ "dxn:",
572
+ "echo:"
573
+ ],
574
+ factory: ({ label, dxn }) => typeof label === "string" && typeof dxn === "string" ? new AnchorWidget(label, dxn) : null
575
+ }
576
+ };
577
+ var Transcription = composable(({ transcript: object, model, children, ...props }, forwardedRef) => {
578
+ const { themeMode } = useThemeContext();
579
+ const { parentRef } = useTextEditor(() => {
580
+ return {
581
+ extensions: [
582
+ createBasicExtensions({
583
+ readOnly: true,
584
+ lineWrapping: true,
585
+ search: true
586
+ }),
587
+ createThemeExtensions({
588
+ themeMode,
589
+ slots: documentSlots
590
+ }),
591
+ createMarkdownExtensions(),
592
+ // xmlTags() handles dxn:/echo: links via url-scheme widgets; skip here to avoid double-processing.
593
+ decorateMarkdown({
594
+ skip: ({ url }) => url.startsWith("dxn:") || url.startsWith("echo:")
595
+ }),
596
+ xmlTags({
597
+ registry: inlinePreviewRegistry
598
+ }),
599
+ transcription({
600
+ model,
601
+ started: object?.started ? new Date(object.started) : void 0
602
+ }),
603
+ scroller()
604
+ ]
605
+ };
606
+ }, [
607
+ model,
608
+ themeMode
609
+ ]);
610
+ return /* @__PURE__ */ React3.createElement("div", {
611
+ ...composableProps(props, {
612
+ classNames: "dx-container"
613
+ }),
614
+ "data-popover-collision-boundary": true,
615
+ ref: parentRef
616
+ });
617
+ });
618
+
619
+ // src/hooks/useAudioFile.ts
620
+ import { useEffect, useState } from "react";
621
+ import { scheduleTask } from "@dxos/async";
622
+ import { Context } from "@dxos/context";
623
+ import { log as log3 } from "@dxos/log";
624
+ var __dxlog_file3 = "/Users/jdw/Code/dxos/packages/ui/react-ui-transcription/src/hooks/useAudioFile.ts";
625
+ var useAudioFile = (audioUrl, constraints) => {
626
+ const [{ audio, stream, track }, setStream] = useState({});
627
+ useEffect(() => {
628
+ const ctx = new Context(void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file3, L: 11 });
629
+ scheduleTask(ctx, async () => {
630
+ try {
631
+ const response = await fetch(audioUrl);
632
+ const blob = await response.blob();
633
+ const objectUrl = URL.createObjectURL(blob);
634
+ ctx.onDispose(() => {
635
+ URL.revokeObjectURL(objectUrl);
636
+ });
637
+ const audio2 = new Audio();
638
+ audio2.src = objectUrl;
639
+ await new Promise((resolve, reject) => {
640
+ audio2.addEventListener("error", (err) => {
641
+ log3.error("error", {
642
+ err
643
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file3, L: 25, S: void 0 });
644
+ reject(err);
645
+ }, {
646
+ once: true
647
+ });
648
+ audio2.addEventListener("canplay", async () => {
649
+ resolve();
650
+ }, {
651
+ once: true
652
+ });
653
+ audio2.load();
654
+ });
655
+ const audioCtx = new AudioContext();
656
+ if (audioCtx.state === "suspended") {
657
+ await audioCtx.resume();
658
+ }
659
+ const destination = audioCtx.createMediaStreamDestination();
660
+ destination.channelCount = 1;
661
+ const source = audioCtx.createMediaElementSource(audio2);
662
+ source.connect(destination);
663
+ const track2 = destination.stream.getAudioTracks()[0];
664
+ if (constraints) {
665
+ await track2.applyConstraints(constraints).catch((err) => log3.catch(err, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file3, L: 50, S: void 0 }));
666
+ }
667
+ setStream({
668
+ audio: audio2,
669
+ stream: destination.stream,
670
+ track: destination.stream.getAudioTracks()[0]
671
+ });
672
+ } catch (err) {
673
+ log3.catch(err, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file3, L: 59, S: void 0 });
674
+ }
675
+ });
676
+ return () => {
677
+ void ctx.dispose();
678
+ };
679
+ }, [
680
+ audioUrl
681
+ ]);
682
+ return {
683
+ audio,
684
+ stream,
685
+ track
686
+ };
687
+ };
688
+
689
+ // src/hooks/useAudioTrack.ts
690
+ import { useEffect as useEffect2, useState as useState2 } from "react";
691
+ var useAudioTrack = (active, constraints) => {
692
+ const [track, setTrack] = useState2();
693
+ const constraintsKey = constraints ? JSON.stringify(constraints) : "";
694
+ useEffect2(() => {
695
+ if (!active) {
696
+ track?.stop();
697
+ setTrack(void 0);
698
+ return;
699
+ }
700
+ let cancelled = false;
701
+ let acquired;
702
+ setTrack(void 0);
703
+ const initAudio = async () => {
704
+ try {
705
+ const stream = await navigator.mediaDevices.getUserMedia({
706
+ audio: constraints ?? true
707
+ });
708
+ const [next] = stream.getAudioTracks();
709
+ if (cancelled || !next) {
710
+ next?.stop();
711
+ return;
712
+ }
713
+ acquired = next;
714
+ setTrack(next);
715
+ } catch {
716
+ if (!cancelled) {
717
+ setTrack(void 0);
718
+ }
719
+ }
720
+ };
721
+ void initAudio();
722
+ return () => {
723
+ cancelled = true;
724
+ acquired?.stop();
725
+ };
726
+ }, [
727
+ active,
728
+ constraintsKey
729
+ ]);
730
+ return track;
731
+ };
732
+
733
+ // src/hooks/useFeedModelAdapter.ts
734
+ import { useEffect as useEffect3, useMemo } from "react";
735
+ var useFeedModelAdapter = (renderer, objects, initialChunks = []) => {
736
+ const model = useMemo(() => new TranscriptModel(renderer, initialChunks), [
737
+ renderer
738
+ ]);
739
+ useEffect3(() => {
740
+ for (const chunk of objects) {
741
+ model.appendChunk(chunk);
742
+ }
743
+ }, [
744
+ model,
745
+ objects
746
+ ]);
747
+ useEffect3(() => {
748
+ if (model.chunks.length === objects.length) {
749
+ return;
750
+ }
751
+ const chunk = objects.at(-1);
752
+ if (chunk) {
753
+ model.appendChunk(chunk);
754
+ }
755
+ }, [
756
+ model,
757
+ objects.length
758
+ ]);
759
+ return model;
760
+ };
761
+
762
+ // src/hooks/useIsSpeaking.ts
763
+ import { useEffect as useEffect4, useMemo as useMemo2, useState as useState3 } from "react";
764
+ import { scheduleTask as scheduleTask2 } from "@dxos/async";
765
+ import { SpeakingMonitor } from "@dxos/av";
766
+ import { Context as Context2 } from "@dxos/context";
767
+ var __dxlog_file4 = "/Users/jdw/Code/dxos/packages/ui/react-ui-transcription/src/hooks/useIsSpeaking.ts";
768
+ var useIsSpeaking = (track) => {
769
+ const [isSpeaking, setIsSpeaking] = useState3(false);
770
+ const speakingMonitor = useMemo2(() => {
771
+ if (!track) {
772
+ return;
773
+ }
774
+ return new SpeakingMonitor(track);
775
+ }, [
776
+ track
777
+ ]);
778
+ useEffect4(() => {
779
+ if (!speakingMonitor) {
780
+ return;
781
+ }
782
+ const ctx = new Context2(void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file4, L: 24 });
783
+ scheduleTask2(ctx, async () => {
784
+ speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
785
+ await speakingMonitor.open();
786
+ ctx.onDispose(() => speakingMonitor.close());
787
+ });
788
+ return () => {
789
+ void ctx.dispose();
790
+ };
791
+ }, [
792
+ speakingMonitor
793
+ ]);
794
+ return isSpeaking;
795
+ };
796
+
797
+ // src/hooks/useRecordingPipeline.ts
798
+ import { useEffect as useEffect5, useMemo as useMemo3, useRef, useState as useState4 } from "react";
799
+ import { log as log4 } from "@dxos/log";
800
+ import { runAsrPipeline } from "@dxos/transcription-pipeline";
801
+ var __dxlog_file5 = "/Users/jdw/Code/dxos/packages/ui/react-ui-transcription/src/hooks/useRecordingPipeline.ts";
802
+ var RECORDER_INTERVAL_MS = 200;
803
+ var DRAIN_SILENCE_MS = 5e3;
804
+ var DEFAULT_TRANSCRIBE_CONFIG = {
805
+ transcribeAfterChunksAmount: 50,
806
+ prefixBufferChunksAmount: 10
807
+ };
808
+ var useRecordingPipeline = ({ active, microphone, track: externalTrack, audioConstraints, stages, commit, lookup, onSegment, config, segmentSentences }) => {
809
+ const [phase, setPhase] = useState4("idle");
810
+ const inUse = phase !== "idle";
811
+ useEffect5(() => {
812
+ setPhase((current) => active ? "recording" : current === "recording" ? "draining" : current);
813
+ }, [
814
+ active
815
+ ]);
816
+ const micTrack = useAudioTrack(!!microphone && inUse, audioConstraints);
817
+ const track = microphone ? micTrack : externalTrack;
818
+ const recorder = useMemo3(() => track ? new MediaStreamRecorder({
819
+ mediaStreamTrack: track,
820
+ config: {
821
+ interval: RECORDER_INTERVAL_MS
822
+ }
823
+ }) : void 0, [
824
+ track
825
+ ]);
826
+ const pipelineRef = useRef(null);
827
+ useEffect5(() => {
828
+ if (phase !== "recording" || !recorder) {
829
+ return;
830
+ }
831
+ const asr = runAsrPipeline({
832
+ recorder,
833
+ config: {
834
+ ...DEFAULT_TRANSCRIBE_CONFIG,
835
+ ...config
836
+ },
837
+ segmentSentences,
838
+ stages,
839
+ commit,
840
+ lookup,
841
+ onSegment,
842
+ drainSilenceMs: DRAIN_SILENCE_MS
843
+ });
844
+ pipelineRef.current = asr;
845
+ let cancelled = false;
846
+ void asr.open().then(() => {
847
+ if (!cancelled) {
848
+ asr.start();
849
+ }
850
+ }, (err) => {
851
+ if (!cancelled) {
852
+ log4.catch(err, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file5, L: 72, S: void 0 });
853
+ setPhase("idle");
854
+ }
855
+ });
856
+ return () => {
857
+ cancelled = true;
858
+ };
859
+ }, [
860
+ phase,
861
+ recorder
862
+ ]);
863
+ useEffect5(() => {
864
+ if (phase !== "draining") {
865
+ return;
866
+ }
867
+ let cancelled = false;
868
+ void (async () => {
869
+ try {
870
+ await pipelineRef.current?.end();
871
+ } catch (err) {
872
+ log4.catch(err, void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file5, L: 94, S: void 0 });
873
+ } finally {
874
+ pipelineRef.current = null;
875
+ if (!cancelled) {
876
+ setPhase((current) => current === "draining" ? "idle" : current);
877
+ }
878
+ }
879
+ })();
880
+ return () => {
881
+ cancelled = true;
882
+ };
883
+ }, [
884
+ phase
885
+ ]);
886
+ return {
887
+ phase
888
+ };
889
+ };
890
+
891
+ // src/hooks/useSpeechRecognition.ts
892
+ import { useEffect as useEffect6, useMemo as useMemo4 } from "react";
893
+ import { log as log5 } from "@dxos/log";
894
+ var __dxlog_file6 = "/Users/jdw/Code/dxos/packages/ui/react-ui-transcription/src/hooks/useSpeechRecognition.ts";
895
+ var cleanText = (text) => text.toLowerCase().replace(/[.,!?]/g, "").replace(/\s+/g, " ").trim();
896
+ var useSpeechRecognition = ({ active, onTranscript }) => {
897
+ const recognition = useMemo4(() => {
898
+ const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
899
+ const SpeechGrammarList = window.SpeechGrammarList || window.webkitSpeechGrammarList;
900
+ if (!SpeechRecognition || !SpeechGrammarList) {
901
+ log5.error("Speech recognition not supported", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file6, L: 14, S: void 0 });
902
+ return;
903
+ }
904
+ return Object.assign(new SpeechRecognition(), {
905
+ grammars: new SpeechGrammarList(),
906
+ continuous: true,
907
+ interimResults: true,
908
+ lang: "en-US",
909
+ maxAlternatives: 1,
910
+ onresult: (event) => {
911
+ const transcript = cleanText(event.results[event.resultIndex][0].transcript);
912
+ onTranscript(transcript);
913
+ },
914
+ onerror: (event) => {
915
+ log5.error("Speech recognition error", {
916
+ error: event.error
917
+ }, { "~LogMeta": "~LogMeta", F: __dxlog_file6, L: 28, S: void 0 });
918
+ }
919
+ });
920
+ }, [
921
+ onTranscript
922
+ ]);
923
+ useEffect6(() => {
924
+ if (active) {
925
+ recognition?.start();
926
+ } else {
927
+ recognition?.stop();
928
+ }
929
+ }, [
930
+ active,
931
+ recognition
932
+ ]);
933
+ };
934
+
935
+ // src/hooks/useTranscriber.ts
936
+ import { useEffect as useEffect7, useMemo as useMemo5 } from "react";
937
+ var useTranscriber = ({ audioStreamTrack, recorderConfig, transcriberConfig, transcribe, onSegments }) => {
938
+ const transcriber = useMemo5(() => {
939
+ if (!audioStreamTrack || !onSegments) {
940
+ return void 0;
941
+ }
942
+ return createTranscriber({
943
+ audioStreamTrack,
944
+ recorderConfig,
945
+ transcriberConfig,
946
+ transcribe,
947
+ onSegments
948
+ });
949
+ }, [
950
+ audioStreamTrack,
951
+ recorderConfig,
952
+ transcriberConfig,
953
+ transcribe,
954
+ onSegments
955
+ ]);
956
+ useEffect7(() => {
957
+ return () => {
958
+ void transcriber?.close();
959
+ };
960
+ }, [
961
+ transcriber
962
+ ]);
963
+ return transcriber;
964
+ };
965
+
966
+ // src/util/render-byline.ts
967
+ var renderByline = (identities) => (message, index, debug = false) => {
968
+ if (message.sender.role === "assistant") {
969
+ return [
970
+ message.blocks.find((block) => block._tag === "transcript")?.text ?? "",
971
+ ""
972
+ ];
973
+ }
974
+ const identity = identities.find((identity2) => identity2.did === message.sender.identityDid);
975
+ const name = identity?.profile?.displayName ?? message.sender.contact?.target?.fullName ?? message.sender.name ?? message.sender.email ?? message.sender.identityDid;
976
+ const blocks = message.blocks.filter((block) => block._tag === "transcript");
977
+ return [
978
+ // TODO(thure): Use an XML tag with the bits needed here.
979
+ `###### ${name}` + (debug ? ` [${index}]:${message.id}` : ""),
980
+ blocks.map((block) => block.text.trim()).join(" "),
981
+ ""
982
+ ];
983
+ };
984
+ export {
985
+ EditorChunkDocument,
986
+ MediaStreamRecorder,
987
+ MicSettings,
988
+ PipelineStatus,
989
+ TranscriptModel,
990
+ Transcription,
991
+ createTranscriber,
992
+ renderByline,
993
+ transcription,
994
+ useAudioFile,
995
+ useAudioTrack,
996
+ useFeedModelAdapter,
997
+ useIsSpeaking,
998
+ useRecordingPipeline,
999
+ useSpeechRecognition,
1000
+ useTranscriber
1001
+ };
1002
+ //# sourceMappingURL=index.mjs.map