@dxos/react-ui-transcription 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/index.mjs +963 -0
- package/dist/lib/index.mjs.map +1 -0
- package/dist/types/src/capture/create-transcriber.d.ts +1 -1
- package/dist/types/src/capture/media-stream-recorder.d.ts +1 -1
- package/dist/types/src/capture/media-stream-recorder.d.ts.map +1 -1
- package/dist/types/src/components/PipelineStatus/PipelineStatus.d.ts +1 -1
- package/dist/types/src/components/Transcription/Transcription.d.ts +1 -1
- package/dist/types/src/components/Transcription/Transcription.d.ts.map +1 -1
- package/dist/types/src/hooks/useRecordingPipeline.d.ts +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts +1 -1
- package/dist/types/src/index.d.ts +0 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +23 -24
- package/src/capture/create-transcriber.ts +1 -1
- package/src/capture/media-stream-recorder.ts +1 -3
- package/src/components/PipelineStatus/PipelineStatus.tsx +1 -1
- package/src/hooks/useRecordingPipeline.ts +1 -1
- package/src/hooks/useTranscriber.ts +1 -1
- package/src/index.ts +0 -1
- package/dist/lib/browser/index.mjs +0 -1002
- package/dist/lib/browser/index.mjs.map +0 -7
- package/dist/lib/browser/meta.json +0 -1
- package/dist/lib/node-esm/index.mjs +0 -1004
- package/dist/lib/node-esm/index.mjs.map +0 -7
- package/dist/lib/node-esm/meta.json +0 -1
- package/dist/types/src/util/index.d.ts +0 -2
- package/dist/types/src/util/index.d.ts.map +0 -1
- package/dist/types/src/util/render-byline.d.ts +0 -4
- package/dist/types/src/util/render-byline.d.ts.map +0 -1
- package/src/util/index.ts +0 -5
- package/src/util/render-byline.ts +0 -32
|
@@ -0,0 +1,963 @@
|
|
|
1
|
+
import { Transcriber, runAsrPipeline } from "@dxos/pipeline-transcription";
|
|
2
|
+
import { WaveFile } from "wavefile";
|
|
3
|
+
import { Event, addEventListener, combine, scheduleTask, synchronized } from "@dxos/async";
|
|
4
|
+
import { invariant } from "@dxos/invariant";
|
|
5
|
+
import { log } from "@dxos/log";
|
|
6
|
+
import { Fragment, useEffect, useMemo, useRef, useState } from "react";
|
|
7
|
+
import { DropdownMenu, Icon, IconButton, composable, composableProps, useThemeContext, useTranslation } from "@dxos/react-ui";
|
|
8
|
+
import { jsx, jsxs } from "react/jsx-runtime";
|
|
9
|
+
import { useTextEditor } from "@dxos/react-ui-editor";
|
|
10
|
+
import { AnchorWidget, createBasicExtensions, createMarkdownExtensions, createThemeExtensions, decorateMarkdown, documentSlots, scroller, xmlTags } from "@dxos/ui-editor";
|
|
11
|
+
import { RangeSetBuilder, Text } from "@codemirror/state";
|
|
12
|
+
import { EditorView, GutterMarker, ViewPlugin, gutter } from "@codemirror/view";
|
|
13
|
+
import { format } from "date-fns/format";
|
|
14
|
+
import { intervalToDuration } from "date-fns/intervalToDuration";
|
|
15
|
+
import { Context } from "@dxos/context";
|
|
16
|
+
import { SpeakingMonitor } from "@dxos/av";
|
|
17
|
+
//#region \0@oxc-project+runtime@0.139.0/helpers/esm/decorate.js
|
|
18
|
+
function __decorate(decorators, target, key, desc) {
|
|
19
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
20
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
21
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
22
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
23
|
+
}
|
|
24
|
+
//#endregion
|
|
25
|
+
//#region src/capture/media-stream-recorder.ts
|
|
26
|
+
var __dxlog_file$5 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/capture/media-stream-recorder.ts";
|
|
27
|
+
var MediaRecorderClass;
|
|
28
|
+
var initializingPromise;
|
|
29
|
+
var initializeExtendableMediaRecorder = async () => {
|
|
30
|
+
const { MediaRecorder, register } = await import("extendable-media-recorder");
|
|
31
|
+
const { connect } = await import("extendable-media-recorder-wav-encoder");
|
|
32
|
+
await register(await connect());
|
|
33
|
+
MediaRecorderClass = MediaRecorder;
|
|
34
|
+
};
|
|
35
|
+
/**
|
|
36
|
+
* Recorder that uses the MediaContext API and AudioNode API to record audio.
|
|
37
|
+
*
|
|
38
|
+
* It records MediaStream using https://www.npmjs.com/package/extendable-media-recorder.
|
|
39
|
+
*/
|
|
40
|
+
var MediaStreamRecorder = class {
|
|
41
|
+
_mediaStreamTrack;
|
|
42
|
+
/**
|
|
43
|
+
* Default MediaRecorder implementation do not support wav encoding.
|
|
44
|
+
*/
|
|
45
|
+
_config;
|
|
46
|
+
_mediaRecorder = void 0;
|
|
47
|
+
_header = void 0;
|
|
48
|
+
_onChunk;
|
|
49
|
+
constructor({ mediaStreamTrack, config }) {
|
|
50
|
+
this._mediaStreamTrack = mediaStreamTrack;
|
|
51
|
+
this._config = config;
|
|
52
|
+
}
|
|
53
|
+
get wavConfig() {
|
|
54
|
+
const settings = this._mediaStreamTrack.getSettings();
|
|
55
|
+
return {
|
|
56
|
+
channels: settings.channelCount ?? 1,
|
|
57
|
+
sampleRate: settings.sampleRate ?? 16e3,
|
|
58
|
+
bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : "16"
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
get mediaStreamTrack() {
|
|
62
|
+
return this._mediaStreamTrack;
|
|
63
|
+
}
|
|
64
|
+
setOnChunk(onChunk) {
|
|
65
|
+
this._onChunk = onChunk;
|
|
66
|
+
}
|
|
67
|
+
async start() {
|
|
68
|
+
await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log.info("initializeExtendableMediaRecorder", { err }, {
|
|
69
|
+
"~LogMeta": "~LogMeta",
|
|
70
|
+
F: __dxlog_file$5,
|
|
71
|
+
L: 76,
|
|
72
|
+
S: this
|
|
73
|
+
}));
|
|
74
|
+
if (this._mediaRecorder?.state === "recording") return;
|
|
75
|
+
invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", {
|
|
76
|
+
"~LogMeta": "~LogMeta",
|
|
77
|
+
F: __dxlog_file$5,
|
|
78
|
+
L: 82,
|
|
79
|
+
S: this,
|
|
80
|
+
A: ["this._onChunk", "'MediaStreamRecorder: onChunk is not set'"]
|
|
81
|
+
});
|
|
82
|
+
invariant(MediaRecorderClass, "MediaStreamRecorder: extendable-media-recorder not initialized", {
|
|
83
|
+
"~LogMeta": "~LogMeta",
|
|
84
|
+
F: __dxlog_file$5,
|
|
85
|
+
L: 83,
|
|
86
|
+
S: this,
|
|
87
|
+
A: ["MediaRecorderClass", "'MediaStreamRecorder: extendable-media-recorder not initialized'"]
|
|
88
|
+
});
|
|
89
|
+
const stream = new MediaStream([this._mediaStreamTrack]);
|
|
90
|
+
const recorder = new MediaRecorderClass(stream, { mimeType: "audio/wav" });
|
|
91
|
+
recorder.ondataavailable = (event) => this._ondataavailable(event);
|
|
92
|
+
this._mediaRecorder = recorder;
|
|
93
|
+
recorder.start(this._config.interval);
|
|
94
|
+
}
|
|
95
|
+
async stop() {
|
|
96
|
+
const recorder = this._mediaRecorder;
|
|
97
|
+
if (!recorder || recorder.state !== "recording") return;
|
|
98
|
+
await new Promise((resolve) => {
|
|
99
|
+
recorder.addEventListener("stop", () => resolve(), { once: true });
|
|
100
|
+
recorder.stop();
|
|
101
|
+
});
|
|
102
|
+
this._mediaRecorder = void 0;
|
|
103
|
+
}
|
|
104
|
+
async _ondataavailable(event) {
|
|
105
|
+
const blob = event.data;
|
|
106
|
+
const uint8Array = new Uint8Array(await blob.arrayBuffer());
|
|
107
|
+
const isHeader = uint8Array[0] === 82 && uint8Array[1] === 73 && uint8Array[2] === 70 && uint8Array[3] === 70;
|
|
108
|
+
let wav;
|
|
109
|
+
if (isHeader) {
|
|
110
|
+
wav = new WaveFile(uint8Array);
|
|
111
|
+
this._header = uint8Array.slice(0, 44);
|
|
112
|
+
} else if (this._header) wav = new WaveFile(new Uint8Array([...this._header, ...uint8Array]));
|
|
113
|
+
else {
|
|
114
|
+
log.warn("MediaStreamRecorder: no header", void 0, {
|
|
115
|
+
"~LogMeta": "~LogMeta",
|
|
116
|
+
F: __dxlog_file$5,
|
|
117
|
+
L: 126,
|
|
118
|
+
S: this
|
|
119
|
+
});
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
this._onChunk({
|
|
123
|
+
timestamp: Date.now(),
|
|
124
|
+
data: wav.getSamples()
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
};
|
|
128
|
+
__decorate([synchronized], MediaStreamRecorder.prototype, "start", null);
|
|
129
|
+
__decorate([synchronized], MediaStreamRecorder.prototype, "stop", null);
|
|
130
|
+
__decorate([synchronized], MediaStreamRecorder.prototype, "_ondataavailable", null);
|
|
131
|
+
//#endregion
|
|
132
|
+
//#region src/capture/create-transcriber.ts
|
|
133
|
+
var RECORD_INTERVAL = 200;
|
|
134
|
+
var PREFIXED_CHUNKS_AMOUNT = 10;
|
|
135
|
+
var TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
|
|
136
|
+
/**
|
|
137
|
+
* Builds a {@link Transcriber} backed by a browser {@link MediaStreamRecorder} for the given track —
|
|
138
|
+
* the low-level audio→text construction, co-located with the recorder so it can be tested outside the
|
|
139
|
+
* plugin. Consumed directly via the {@link useTranscriber} hook.
|
|
140
|
+
*/
|
|
141
|
+
var createTranscriber = ({ audioStreamTrack, onSegments, transcriberConfig, recorderConfig, transcribe }) => new Transcriber({
|
|
142
|
+
config: {
|
|
143
|
+
transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
|
|
144
|
+
prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
|
|
145
|
+
...transcriberConfig
|
|
146
|
+
},
|
|
147
|
+
recorder: new MediaStreamRecorder({
|
|
148
|
+
mediaStreamTrack: audioStreamTrack,
|
|
149
|
+
config: {
|
|
150
|
+
interval: RECORD_INTERVAL,
|
|
151
|
+
...recorderConfig
|
|
152
|
+
}
|
|
153
|
+
}),
|
|
154
|
+
onSegments,
|
|
155
|
+
transcribe
|
|
156
|
+
});
|
|
157
|
+
//#endregion
|
|
158
|
+
//#region src/components/MicSettings/MicSettings.tsx
|
|
159
|
+
/**
|
|
160
|
+
* Dropdown of recording options for the mic: record mode (toggle/push-to-talk), the input device,
|
|
161
|
+
* and whether entity extraction is applied. Presentational — state lives in the consumer.
|
|
162
|
+
*/
|
|
163
|
+
var MicSettings = ({ translationNs, recordMode, entityExtraction, devices, selectedDeviceId, onRecordModeChange, onEntityExtractionChange, onSelectDevice }) => {
|
|
164
|
+
const { t } = useTranslation(translationNs);
|
|
165
|
+
return /* @__PURE__ */ jsxs(DropdownMenu.Root, { children: [/* @__PURE__ */ jsx(DropdownMenu.Trigger, {
|
|
166
|
+
asChild: true,
|
|
167
|
+
children: /* @__PURE__ */ jsx(IconButton, {
|
|
168
|
+
icon: "ph--caret-down--regular",
|
|
169
|
+
iconOnly: true,
|
|
170
|
+
label: t("recording-options.label"),
|
|
171
|
+
variant: "ghost",
|
|
172
|
+
classNames: "w-4",
|
|
173
|
+
"data-testid": "transcription.record.options"
|
|
174
|
+
})
|
|
175
|
+
}), /* @__PURE__ */ jsx(DropdownMenu.Portal, { children: /* @__PURE__ */ jsxs(DropdownMenu.Content, { children: [/* @__PURE__ */ jsxs(DropdownMenu.Viewport, { children: [
|
|
176
|
+
/* @__PURE__ */ jsx(DropdownMenu.GroupLabel, { children: t("record-mode.label") }),
|
|
177
|
+
/* @__PURE__ */ jsx(SelectableItem, {
|
|
178
|
+
label: t("record-mode.toggle.label"),
|
|
179
|
+
selected: recordMode === "toggle",
|
|
180
|
+
onSelect: () => onRecordModeChange("toggle")
|
|
181
|
+
}),
|
|
182
|
+
/* @__PURE__ */ jsx(SelectableItem, {
|
|
183
|
+
label: t("record-mode.hold.label"),
|
|
184
|
+
selected: recordMode === "hold",
|
|
185
|
+
onSelect: () => onRecordModeChange("hold")
|
|
186
|
+
}),
|
|
187
|
+
/* @__PURE__ */ jsx(DropdownMenu.Separator, {}),
|
|
188
|
+
/* @__PURE__ */ jsx(DropdownMenu.GroupLabel, { children: t("audio-device.label") }),
|
|
189
|
+
/* @__PURE__ */ jsx(SelectableItem, {
|
|
190
|
+
label: t("audio-device.default.label"),
|
|
191
|
+
selected: selectedDeviceId === "",
|
|
192
|
+
onSelect: () => onSelectDevice("")
|
|
193
|
+
}),
|
|
194
|
+
devices.map((device) => /* @__PURE__ */ jsx(SelectableItem, {
|
|
195
|
+
label: device.label,
|
|
196
|
+
selected: selectedDeviceId === device.deviceId,
|
|
197
|
+
onSelect: () => onSelectDevice(device.deviceId)
|
|
198
|
+
}, device.deviceId)),
|
|
199
|
+
/* @__PURE__ */ jsx(DropdownMenu.Separator, {}),
|
|
200
|
+
/* @__PURE__ */ jsxs(DropdownMenu.CheckboxItem, {
|
|
201
|
+
checked: entityExtraction,
|
|
202
|
+
onCheckedChange: onEntityExtractionChange,
|
|
203
|
+
classNames: "gap-2",
|
|
204
|
+
"data-testid": "transcription.entity-extraction",
|
|
205
|
+
children: [/* @__PURE__ */ jsx("span", {
|
|
206
|
+
className: "grow truncate",
|
|
207
|
+
children: t("settings.entity-extraction.label")
|
|
208
|
+
}), /* @__PURE__ */ jsx(DropdownMenu.ItemIndicator, {
|
|
209
|
+
asChild: true,
|
|
210
|
+
children: /* @__PURE__ */ jsx(Icon, {
|
|
211
|
+
icon: "ph--check--regular",
|
|
212
|
+
size: 4
|
|
213
|
+
})
|
|
214
|
+
})]
|
|
215
|
+
})
|
|
216
|
+
] }), /* @__PURE__ */ jsx(DropdownMenu.Arrow, {})] }) })] });
|
|
217
|
+
};
|
|
218
|
+
var SelectableItem = ({ label, selected, onSelect }) => /* @__PURE__ */ jsxs(DropdownMenu.Item, {
|
|
219
|
+
classNames: "gap-2",
|
|
220
|
+
onSelect,
|
|
221
|
+
children: [/* @__PURE__ */ jsx("span", {
|
|
222
|
+
className: "grow truncate",
|
|
223
|
+
children: label
|
|
224
|
+
}), selected && /* @__PURE__ */ jsx(Icon, {
|
|
225
|
+
icon: "ph--check--regular",
|
|
226
|
+
size: 4
|
|
227
|
+
})]
|
|
228
|
+
});
|
|
229
|
+
//#endregion
|
|
230
|
+
//#region src/components/PipelineStatus/PipelineStatus.tsx
|
|
231
|
+
/**
|
|
232
|
+
* Read-only status panel reflecting the transcription/pipeline state: mic + pipeline phase, the
|
|
233
|
+
* active stage list, per-stage telemetry, and the running summary. Driven by the `PipelineStatus`
|
|
234
|
+
* capability (phase) and `PipelineRuntime` telemetry.
|
|
235
|
+
*/
|
|
236
|
+
var PipelineStatus = ({ phase = "idle", stages, telemetry, summary }) => /* @__PURE__ */ jsxs("div", {
|
|
237
|
+
className: "flex flex-col gap-3 overflow-y-auto p-2 text-sm",
|
|
238
|
+
children: [
|
|
239
|
+
/* @__PURE__ */ jsx(Section, {
|
|
240
|
+
title: "Status",
|
|
241
|
+
children: /* @__PURE__ */ jsxs("div", {
|
|
242
|
+
className: "grid grid-cols-[auto_1fr] gap-x-4 gap-y-1 text-description",
|
|
243
|
+
children: [
|
|
244
|
+
/* @__PURE__ */ jsx("span", { children: "Mic" }),
|
|
245
|
+
/* @__PURE__ */ jsx("span", { children: phase === "recording" ? "● on" : "○ off" }),
|
|
246
|
+
/* @__PURE__ */ jsx("span", { children: "Pipeline" }),
|
|
247
|
+
/* @__PURE__ */ jsx("span", { children: phase })
|
|
248
|
+
]
|
|
249
|
+
})
|
|
250
|
+
}),
|
|
251
|
+
stages && /* @__PURE__ */ jsx(Section, {
|
|
252
|
+
title: "Stages",
|
|
253
|
+
children: /* @__PURE__ */ jsx("div", {
|
|
254
|
+
className: "text-description",
|
|
255
|
+
children: stages.join(" → ") || "(none)"
|
|
256
|
+
})
|
|
257
|
+
}),
|
|
258
|
+
telemetry && telemetry.length > 0 && /* @__PURE__ */ jsx(Section, {
|
|
259
|
+
title: "Telemetry",
|
|
260
|
+
children: /* @__PURE__ */ jsx("div", {
|
|
261
|
+
className: "grid w-fit grid-cols-[1fr_auto_auto] gap-x-4 gap-y-1",
|
|
262
|
+
children: telemetry.map((event, index) => /* @__PURE__ */ jsxs(Fragment, { children: [
|
|
263
|
+
/* @__PURE__ */ jsx("span", { children: event.stageId }),
|
|
264
|
+
/* @__PURE__ */ jsx("span", {
|
|
265
|
+
className: "text-description",
|
|
266
|
+
children: event.outcome
|
|
267
|
+
}),
|
|
268
|
+
/* @__PURE__ */ jsxs("span", {
|
|
269
|
+
className: "text-right tabular-nums text-description",
|
|
270
|
+
children: [event.durationMs, "ms"]
|
|
271
|
+
})
|
|
272
|
+
] }, index))
|
|
273
|
+
})
|
|
274
|
+
}),
|
|
275
|
+
summary && /* @__PURE__ */ jsx(Section, {
|
|
276
|
+
title: "Summary",
|
|
277
|
+
children: /* @__PURE__ */ jsx("p", {
|
|
278
|
+
className: "text-description",
|
|
279
|
+
children: summary
|
|
280
|
+
})
|
|
281
|
+
})
|
|
282
|
+
]
|
|
283
|
+
});
|
|
284
|
+
var Section = ({ title, children }) => /* @__PURE__ */ jsxs("div", { children: [/* @__PURE__ */ jsx("h3", {
|
|
285
|
+
className: "mb-1 font-medium",
|
|
286
|
+
children: title
|
|
287
|
+
}), children] });
|
|
288
|
+
//#endregion
|
|
289
|
+
//#region src/model/transcript-model.ts
|
|
290
|
+
var __dxlog_file$4 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/model/transcript-model.ts";
|
|
291
|
+
/**
|
|
292
|
+
* CodeMirror-backed {@link ChunkDocument}.
|
|
293
|
+
*/
|
|
294
|
+
var EditorChunkDocument = class {
|
|
295
|
+
_view;
|
|
296
|
+
constructor(_view) {
|
|
297
|
+
this._view = _view;
|
|
298
|
+
}
|
|
299
|
+
/** The document must always have at least one line. */
|
|
300
|
+
lineCount() {
|
|
301
|
+
return this._view.state.doc.lines - 1;
|
|
302
|
+
}
|
|
303
|
+
replaceLines(from, remove, insert) {
|
|
304
|
+
log("replaceLines", {
|
|
305
|
+
count: this._view.state.doc.lines,
|
|
306
|
+
from,
|
|
307
|
+
remove,
|
|
308
|
+
insert
|
|
309
|
+
}, {
|
|
310
|
+
"~LogMeta": "~LogMeta",
|
|
311
|
+
F: __dxlog_file$4,
|
|
312
|
+
L: 39,
|
|
313
|
+
S: this
|
|
314
|
+
});
|
|
315
|
+
const numLines = this._view.state.doc.lines;
|
|
316
|
+
const posFrom = this._view.state.doc.line(from).from;
|
|
317
|
+
const posTo = remove === 0 ? posFrom : this._view.state.doc.line(Math.min(numLines, from + remove)).from;
|
|
318
|
+
let text = insert.join("\n");
|
|
319
|
+
if (posTo === this._view.state.doc.length) text += "\n";
|
|
320
|
+
this._view.dispatch({ changes: {
|
|
321
|
+
from: posFrom,
|
|
322
|
+
to: posTo,
|
|
323
|
+
insert: text
|
|
324
|
+
} });
|
|
325
|
+
}
|
|
326
|
+
};
|
|
327
|
+
/**
|
|
328
|
+
* Each Chunk can be converted into a set of lines.
|
|
329
|
+
* Chunks can be added, updated or deleted from the Queue.
|
|
330
|
+
* A Document is an immutable projection that represents a flat list of lines corresponding to the Queue.
|
|
331
|
+
*
|
|
332
|
+
* Ideally we would implement a custom virtual Text model for the View, but this currently isn't possible in Codemirror.
|
|
333
|
+
* Instead this model tracks changes and syncs them with the ChunkDocument.
|
|
334
|
+
*/
|
|
335
|
+
var TranscriptModel = class {
|
|
336
|
+
_renderer;
|
|
337
|
+
/** Emits when the document is updated. */
|
|
338
|
+
update = new Event();
|
|
339
|
+
/** Ordered set of chunks. */
|
|
340
|
+
_chunks = [];
|
|
341
|
+
/** Map of chunk IDs to their current line counts. */
|
|
342
|
+
_chunkLineCounts = /* @__PURE__ */ new Map();
|
|
343
|
+
/** Map of line numbers to chunk IDs. */
|
|
344
|
+
_lineToChunk = /* @__PURE__ */ new Map();
|
|
345
|
+
/** Track chunk changes since last sync. */
|
|
346
|
+
_changes = [];
|
|
347
|
+
constructor(_renderer, initialChunks = []) {
|
|
348
|
+
this._renderer = _renderer;
|
|
349
|
+
initialChunks.forEach((chunk) => {
|
|
350
|
+
this.appendChunk(chunk);
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
toJSON() {
|
|
354
|
+
return {
|
|
355
|
+
chunks: this._chunks.length,
|
|
356
|
+
changes: this._changes.length
|
|
357
|
+
};
|
|
358
|
+
}
|
|
359
|
+
get doc() {
|
|
360
|
+
return Text.of([...this._chunks.flatMap((chunk, index) => this._renderer(chunk, index)), ""]);
|
|
361
|
+
}
|
|
362
|
+
get chunks() {
|
|
363
|
+
return this._chunks;
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Get the block ID at the given line number.
|
|
367
|
+
*/
|
|
368
|
+
getChunkAtLine(line) {
|
|
369
|
+
return this._lineToChunk.get(line);
|
|
370
|
+
}
|
|
371
|
+
reset() {
|
|
372
|
+
this._chunks.length = 0;
|
|
373
|
+
this._chunkLineCounts.clear();
|
|
374
|
+
this._lineToChunk.clear();
|
|
375
|
+
this.update.emit();
|
|
376
|
+
return this;
|
|
377
|
+
}
|
|
378
|
+
appendChunk(chunk) {
|
|
379
|
+
const index = this._chunks.length;
|
|
380
|
+
this._chunks.push(chunk);
|
|
381
|
+
this._changes.push({
|
|
382
|
+
type: "append",
|
|
383
|
+
index,
|
|
384
|
+
chunk
|
|
385
|
+
});
|
|
386
|
+
this.update.emit();
|
|
387
|
+
return this;
|
|
388
|
+
}
|
|
389
|
+
updateChunk(chunk) {
|
|
390
|
+
const index = this._chunks.findIndex((c) => c.id === chunk.id);
|
|
391
|
+
invariant(index !== -1, void 0, {
|
|
392
|
+
"~LogMeta": "~LogMeta",
|
|
393
|
+
F: __dxlog_file$4,
|
|
394
|
+
L: 147,
|
|
395
|
+
S: this,
|
|
396
|
+
A: ["index !== -1", ""]
|
|
397
|
+
});
|
|
398
|
+
this._chunks[index] = chunk;
|
|
399
|
+
this._changes.push({
|
|
400
|
+
type: "update",
|
|
401
|
+
index,
|
|
402
|
+
chunk
|
|
403
|
+
});
|
|
404
|
+
this.update.emit();
|
|
405
|
+
return this;
|
|
406
|
+
}
|
|
407
|
+
deleteBlock(id) {
|
|
408
|
+
const index = this._chunks.findIndex((c) => c.id === id);
|
|
409
|
+
invariant(index !== -1, void 0, {
|
|
410
|
+
"~LogMeta": "~LogMeta",
|
|
411
|
+
F: __dxlog_file$4,
|
|
412
|
+
L: 156,
|
|
413
|
+
S: this,
|
|
414
|
+
A: ["index !== -1", ""]
|
|
415
|
+
});
|
|
416
|
+
this._chunks.splice(index, 1);
|
|
417
|
+
this._changes.push({
|
|
418
|
+
type: "delete",
|
|
419
|
+
index,
|
|
420
|
+
id
|
|
421
|
+
});
|
|
422
|
+
this.update.emit();
|
|
423
|
+
return this;
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* Syncs the tracked changes with the ChunkDocument.
|
|
427
|
+
*/
|
|
428
|
+
sync(document) {
|
|
429
|
+
log("sync", { changes: this._changes }, {
|
|
430
|
+
"~LogMeta": "~LogMeta",
|
|
431
|
+
F: __dxlog_file$4,
|
|
432
|
+
L: 167,
|
|
433
|
+
S: this
|
|
434
|
+
});
|
|
435
|
+
for (const change of this._changes) switch (change.type) {
|
|
436
|
+
case "append": {
|
|
437
|
+
const lines = this._renderer(change.chunk, change.index);
|
|
438
|
+
const lastLine = document.lineCount();
|
|
439
|
+
document.replaceLines(lastLine + 1, 0, lines);
|
|
440
|
+
break;
|
|
441
|
+
}
|
|
442
|
+
case "update": {
|
|
443
|
+
let lineStart = 1;
|
|
444
|
+
for (let i = 0; i < change.index; i++) lineStart += this._chunkLineCounts.get(this._chunks[i].id);
|
|
445
|
+
const previousLineCount = this._chunkLineCounts.get(change.chunk.id);
|
|
446
|
+
const newLines = this._renderer(change.chunk, change.index);
|
|
447
|
+
document.replaceLines(lineStart, previousLineCount, newLines);
|
|
448
|
+
break;
|
|
449
|
+
}
|
|
450
|
+
case "delete": {
|
|
451
|
+
let lineStart = 1;
|
|
452
|
+
for (let i = 0; i < change.index; i++) lineStart += this._chunkLineCounts.get(this._chunks[i].id);
|
|
453
|
+
const previousLineCount = this._chunkLineCounts.get(change.id);
|
|
454
|
+
document.replaceLines(lineStart, previousLineCount, []);
|
|
455
|
+
break;
|
|
456
|
+
}
|
|
457
|
+
}
|
|
458
|
+
this._lineToChunk.clear();
|
|
459
|
+
let currentLine = 1;
|
|
460
|
+
for (const [index, chunk] of this._chunks.entries()) {
|
|
461
|
+
const lineCount = this._renderer(chunk, index).length;
|
|
462
|
+
this._chunkLineCounts.set(chunk.id, lineCount);
|
|
463
|
+
this._lineToChunk.set(currentLine, chunk);
|
|
464
|
+
currentLine += lineCount;
|
|
465
|
+
}
|
|
466
|
+
const chunkIds = new Set(this._chunks.map((chunk) => chunk.id));
|
|
467
|
+
for (const id of this._chunkLineCounts.keys()) if (!chunkIds.has(id)) this._chunkLineCounts.delete(id);
|
|
468
|
+
this._changes.length = 0;
|
|
469
|
+
return this;
|
|
470
|
+
}
|
|
471
|
+
};
|
|
472
|
+
//#endregion
|
|
473
|
+
//#region src/components/Transcription/transcription-extension.ts
|
|
474
|
+
/**
|
|
475
|
+
* Scrolling transcript with timestamps.
|
|
476
|
+
*/
|
|
477
|
+
var transcription = ({ model, started }) => {
|
|
478
|
+
return [
|
|
479
|
+
gutter({
|
|
480
|
+
class: "cm-timestamp-gutter",
|
|
481
|
+
lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
|
|
482
|
+
markers: (view) => {
|
|
483
|
+
const start = getStartTime(started, model.chunks[0]);
|
|
484
|
+
const builder = new RangeSetBuilder();
|
|
485
|
+
for (const { from, to } of view.visibleRanges) {
|
|
486
|
+
let line = view.state.doc.lineAt(from);
|
|
487
|
+
while (line.from <= to) {
|
|
488
|
+
const block = model.getChunkAtLine(line.number)?.blocks[0];
|
|
489
|
+
const timestamp = block?._tag === "transcript" && block.started;
|
|
490
|
+
if (timestamp) builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
|
|
491
|
+
if (line.to + 1 > view.state.doc.length) break;
|
|
492
|
+
line = view.state.doc.lineAt(line.to + 1);
|
|
493
|
+
}
|
|
494
|
+
}
|
|
495
|
+
return builder.finish();
|
|
496
|
+
}
|
|
497
|
+
}),
|
|
498
|
+
ViewPlugin.fromClass(class {
|
|
499
|
+
_controls;
|
|
500
|
+
_cleanup;
|
|
501
|
+
_adapter;
|
|
502
|
+
_initialized = false;
|
|
503
|
+
constructor(view) {
|
|
504
|
+
this._adapter = new EditorChunkDocument(view);
|
|
505
|
+
const scroller = view.scrollDOM;
|
|
506
|
+
let isAutoScrolling = false;
|
|
507
|
+
let hasScrolled = false;
|
|
508
|
+
let timeout;
|
|
509
|
+
const scrollToBottom = (smooth = false) => {
|
|
510
|
+
scroller.style.scrollBehavior = smooth ? "smooth" : "";
|
|
511
|
+
scroller.classList.add("cm-hide-scrollbar");
|
|
512
|
+
isAutoScrolling = true;
|
|
513
|
+
clearTimeout(timeout);
|
|
514
|
+
timeout = setTimeout(() => {
|
|
515
|
+
this._controls?.classList.add("opacity-0");
|
|
516
|
+
scroller.classList.remove("cm-hide-scrollbar");
|
|
517
|
+
isAutoScrolling = false;
|
|
518
|
+
}, 500);
|
|
519
|
+
view.dispatch({ effects: EditorView.scrollIntoView(view.state.doc.length, { y: "end" }) });
|
|
520
|
+
};
|
|
521
|
+
this._cleanup = combine(addEventListener(view.scrollDOM, "scroll", () => {
|
|
522
|
+
if (!isAutoScrolling) {
|
|
523
|
+
hasScrolled = true;
|
|
524
|
+
this._controls?.classList.remove("opacity-0");
|
|
525
|
+
}
|
|
526
|
+
}), model.update.on(() => {
|
|
527
|
+
const autoScroll = scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
|
|
528
|
+
model.sync(this._adapter);
|
|
529
|
+
if (autoScroll) scrollToBottom(true);
|
|
530
|
+
}));
|
|
531
|
+
}
|
|
532
|
+
update(_update) {
|
|
533
|
+
if (!this._initialized) {
|
|
534
|
+
this._initialized = true;
|
|
535
|
+
setTimeout(() => {
|
|
536
|
+
model.sync(this._adapter);
|
|
537
|
+
});
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
destroy() {
|
|
541
|
+
this._controls?.remove();
|
|
542
|
+
this._cleanup();
|
|
543
|
+
}
|
|
544
|
+
}),
|
|
545
|
+
EditorView.theme({
|
|
546
|
+
".cm-gutters": { backgroundColor: "var(--color-base-surface)" },
|
|
547
|
+
".cm-timestamp-gutter": {
|
|
548
|
+
width: "6rem",
|
|
549
|
+
paddingLeft: "0.5rem",
|
|
550
|
+
paddingRight: "1rem"
|
|
551
|
+
},
|
|
552
|
+
".cm-timestamp-gutter > .cm-gutterElement > div": {
|
|
553
|
+
display: "inline-flex",
|
|
554
|
+
textAlign: "right",
|
|
555
|
+
padding: "3px"
|
|
556
|
+
}
|
|
557
|
+
})
|
|
558
|
+
];
|
|
559
|
+
};
|
|
560
|
+
/**
|
|
561
|
+
* Gutter marker that displays a timestamp.
|
|
562
|
+
*/
|
|
563
|
+
var TimestampMarker = class extends GutterMarker {
|
|
564
|
+
_line;
|
|
565
|
+
_timestamp;
|
|
566
|
+
_started;
|
|
567
|
+
constructor(_line, _timestamp, _started) {
|
|
568
|
+
super();
|
|
569
|
+
this._line = _line;
|
|
570
|
+
this._timestamp = _timestamp;
|
|
571
|
+
this._started = _started;
|
|
572
|
+
}
|
|
573
|
+
eq(other) {
|
|
574
|
+
return this._timestamp === other._timestamp;
|
|
575
|
+
}
|
|
576
|
+
toDOM(view) {
|
|
577
|
+
const el = document.createElement("div");
|
|
578
|
+
el.className = "text-sm text-subdued hover:bg-hover-surface cursor-pointer";
|
|
579
|
+
el.textContent = formatTimestamp(this._timestamp, this._started);
|
|
580
|
+
el.onclick = () => {
|
|
581
|
+
const pos = view.state.doc.line(this._line).from;
|
|
582
|
+
view.dispatch({ effects: EditorView.scrollIntoView(pos, { y: "start" }) });
|
|
583
|
+
};
|
|
584
|
+
return el;
|
|
585
|
+
}
|
|
586
|
+
};
|
|
587
|
+
var getStartTime = (started, message) => {
|
|
588
|
+
if (started) return started;
|
|
589
|
+
if (message?.blocks[0]?._tag === "transcript" && message.blocks[0].started) return new Date(message.blocks[0].started);
|
|
590
|
+
};
|
|
591
|
+
var formatTimestamp = (timestamp, relative) => {
|
|
592
|
+
if (relative) {
|
|
593
|
+
const pad = (n = 0) => String(n).padStart(2, "0");
|
|
594
|
+
const { hours, minutes, seconds } = intervalToDuration({
|
|
595
|
+
start: relative,
|
|
596
|
+
end: timestamp
|
|
597
|
+
});
|
|
598
|
+
return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
|
|
599
|
+
} else return format(timestamp, "HH:mm:ss");
|
|
600
|
+
};
|
|
601
|
+
//#endregion
|
|
602
|
+
//#region src/components/Transcription/Transcription.tsx
|
|
603
|
+
var inlinePreviewRegistry = { "link-preview": {
|
|
604
|
+
block: false,
|
|
605
|
+
urlSchemes: ["dxn:", "echo:"],
|
|
606
|
+
factory: ({ label, dxn }) => typeof label === "string" && typeof dxn === "string" ? new AnchorWidget(label, dxn) : null
|
|
607
|
+
} };
|
|
608
|
+
var Transcription = composable(({ transcript: object, model, children, ...props }, forwardedRef) => {
|
|
609
|
+
const { themeMode } = useThemeContext();
|
|
610
|
+
const { parentRef } = useTextEditor(() => {
|
|
611
|
+
return { extensions: [
|
|
612
|
+
createBasicExtensions({
|
|
613
|
+
readOnly: true,
|
|
614
|
+
lineWrapping: true,
|
|
615
|
+
search: true
|
|
616
|
+
}),
|
|
617
|
+
createThemeExtensions({
|
|
618
|
+
themeMode,
|
|
619
|
+
slots: documentSlots
|
|
620
|
+
}),
|
|
621
|
+
createMarkdownExtensions(),
|
|
622
|
+
decorateMarkdown({ skip: ({ url }) => url.startsWith("dxn:") || url.startsWith("echo:") }),
|
|
623
|
+
xmlTags({ registry: inlinePreviewRegistry }),
|
|
624
|
+
transcription({
|
|
625
|
+
model,
|
|
626
|
+
started: object?.started ? new Date(object.started) : void 0
|
|
627
|
+
}),
|
|
628
|
+
scroller()
|
|
629
|
+
] };
|
|
630
|
+
}, [model, themeMode]);
|
|
631
|
+
return /* @__PURE__ */ jsx("div", {
|
|
632
|
+
...composableProps(props, { classNames: "dx-container" }),
|
|
633
|
+
"data-popover-collision-boundary": true,
|
|
634
|
+
ref: parentRef
|
|
635
|
+
});
|
|
636
|
+
});
|
|
637
|
+
//#endregion
|
|
638
|
+
//#region src/hooks/useAudioFile.ts
|
|
639
|
+
var __dxlog_file$3 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/hooks/useAudioFile.ts";
|
|
640
|
+
var useAudioFile = (audioUrl, constraints) => {
|
|
641
|
+
const [{ audio, stream, track }, setStream] = useState({});
|
|
642
|
+
useEffect(() => {
|
|
643
|
+
const ctx = new Context(void 0, {
|
|
644
|
+
"~LogMeta": "~LogMeta",
|
|
645
|
+
F: __dxlog_file$3,
|
|
646
|
+
L: 20
|
|
647
|
+
});
|
|
648
|
+
scheduleTask(ctx, async () => {
|
|
649
|
+
try {
|
|
650
|
+
const blob = await (await fetch(audioUrl)).blob();
|
|
651
|
+
const objectUrl = URL.createObjectURL(blob);
|
|
652
|
+
ctx.onDispose(() => {
|
|
653
|
+
URL.revokeObjectURL(objectUrl);
|
|
654
|
+
});
|
|
655
|
+
const audio = new Audio();
|
|
656
|
+
audio.src = objectUrl;
|
|
657
|
+
await new Promise((resolve, reject) => {
|
|
658
|
+
audio.addEventListener("error", (err) => {
|
|
659
|
+
log.error("error", { err }, {
|
|
660
|
+
"~LogMeta": "~LogMeta",
|
|
661
|
+
F: __dxlog_file$3,
|
|
662
|
+
L: 39,
|
|
663
|
+
S: void 0
|
|
664
|
+
});
|
|
665
|
+
reject(err);
|
|
666
|
+
}, { once: true });
|
|
667
|
+
audio.addEventListener("canplay", async () => {
|
|
668
|
+
resolve();
|
|
669
|
+
}, { once: true });
|
|
670
|
+
audio.load();
|
|
671
|
+
});
|
|
672
|
+
const audioCtx = new AudioContext();
|
|
673
|
+
if (audioCtx.state === "suspended") await audioCtx.resume();
|
|
674
|
+
const destination = audioCtx.createMediaStreamDestination();
|
|
675
|
+
destination.channelCount = 1;
|
|
676
|
+
audioCtx.createMediaElementSource(audio).connect(destination);
|
|
677
|
+
const track = destination.stream.getAudioTracks()[0];
|
|
678
|
+
if (constraints) await track.applyConstraints(constraints).catch((err) => log.catch(err, void 0, {
|
|
679
|
+
"~LogMeta": "~LogMeta",
|
|
680
|
+
F: __dxlog_file$3,
|
|
681
|
+
L: 68,
|
|
682
|
+
S: void 0
|
|
683
|
+
}));
|
|
684
|
+
setStream({
|
|
685
|
+
audio,
|
|
686
|
+
stream: destination.stream,
|
|
687
|
+
track: destination.stream.getAudioTracks()[0]
|
|
688
|
+
});
|
|
689
|
+
} catch (err) {
|
|
690
|
+
log.catch(err, void 0, {
|
|
691
|
+
"~LogMeta": "~LogMeta",
|
|
692
|
+
F: __dxlog_file$3,
|
|
693
|
+
L: 78,
|
|
694
|
+
S: void 0
|
|
695
|
+
});
|
|
696
|
+
}
|
|
697
|
+
});
|
|
698
|
+
return () => {
|
|
699
|
+
ctx.dispose();
|
|
700
|
+
};
|
|
701
|
+
}, [audioUrl]);
|
|
702
|
+
return {
|
|
703
|
+
audio,
|
|
704
|
+
stream,
|
|
705
|
+
track
|
|
706
|
+
};
|
|
707
|
+
};
|
|
708
|
+
//#endregion
|
|
709
|
+
//#region src/hooks/useAudioTrack.ts
|
|
710
|
+
/**
|
|
711
|
+
* Acquires a microphone audio track while `active`. Re-acquires when the `constraints` change (e.g.
|
|
712
|
+
* a different input device is selected) so the new device takes effect without a remount.
|
|
713
|
+
*/
|
|
714
|
+
var useAudioTrack = (active, constraints) => {
|
|
715
|
+
const [track, setTrack] = useState();
|
|
716
|
+
useEffect(() => {
|
|
717
|
+
if (!active) {
|
|
718
|
+
track?.stop();
|
|
719
|
+
setTrack(void 0);
|
|
720
|
+
return;
|
|
721
|
+
}
|
|
722
|
+
let cancelled = false;
|
|
723
|
+
let acquired;
|
|
724
|
+
setTrack(void 0);
|
|
725
|
+
const initAudio = async () => {
|
|
726
|
+
try {
|
|
727
|
+
const [next] = (await navigator.mediaDevices.getUserMedia({ audio: constraints ?? true })).getAudioTracks();
|
|
728
|
+
if (cancelled || !next) {
|
|
729
|
+
next?.stop();
|
|
730
|
+
return;
|
|
731
|
+
}
|
|
732
|
+
acquired = next;
|
|
733
|
+
setTrack(next);
|
|
734
|
+
} catch {
|
|
735
|
+
if (!cancelled) setTrack(void 0);
|
|
736
|
+
}
|
|
737
|
+
};
|
|
738
|
+
initAudio();
|
|
739
|
+
return () => {
|
|
740
|
+
cancelled = true;
|
|
741
|
+
acquired?.stop();
|
|
742
|
+
};
|
|
743
|
+
}, [active, constraints ? JSON.stringify(constraints) : ""]);
|
|
744
|
+
return track;
|
|
745
|
+
};
|
|
746
|
+
//#endregion
|
|
747
|
+
//#region src/hooks/useFeedModelAdapter.ts
|
|
748
|
+
/**
|
|
749
|
+
* Feeds an array of objects (typically from `useQuery(db, Query.from(feed))`) into a
|
|
750
|
+
* `TranscriptModel`, appending each new tail item as the array grows.
|
|
751
|
+
*
|
|
752
|
+
* @param renderer - chunk renderer.
|
|
753
|
+
* @param objects - current snapshot of feed items.
|
|
754
|
+
* @param initialChunks - chunks to seed the model with on first render.
|
|
755
|
+
*/
|
|
756
|
+
var useFeedModelAdapter = (renderer, objects, initialChunks = []) => {
|
|
757
|
+
const model = useMemo(() => new TranscriptModel(renderer, initialChunks), [renderer]);
|
|
758
|
+
useEffect(() => {
|
|
759
|
+
for (const chunk of objects) model.appendChunk(chunk);
|
|
760
|
+
}, [model, objects]);
|
|
761
|
+
useEffect(() => {
|
|
762
|
+
if (model.chunks.length === objects.length) return;
|
|
763
|
+
const chunk = objects.at(-1);
|
|
764
|
+
if (chunk) model.appendChunk(chunk);
|
|
765
|
+
}, [model, objects.length]);
|
|
766
|
+
return model;
|
|
767
|
+
};
|
|
768
|
+
//#endregion
|
|
769
|
+
//#region src/hooks/useIsSpeaking.ts
|
|
770
|
+
var __dxlog_file$2 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/hooks/useIsSpeaking.ts";
|
|
771
|
+
/**
|
|
772
|
+
* Tracks voice activity on an audio track via {@link SpeakingMonitor}.
|
|
773
|
+
*/
|
|
774
|
+
var useIsSpeaking = (track) => {
|
|
775
|
+
const [isSpeaking, setIsSpeaking] = useState(false);
|
|
776
|
+
const speakingMonitor = useMemo(() => {
|
|
777
|
+
if (!track) return;
|
|
778
|
+
return new SpeakingMonitor(track);
|
|
779
|
+
}, [track]);
|
|
780
|
+
useEffect(() => {
|
|
781
|
+
if (!speakingMonitor) return;
|
|
782
|
+
const ctx = new Context(void 0, {
|
|
783
|
+
"~LogMeta": "~LogMeta",
|
|
784
|
+
F: __dxlog_file$2,
|
|
785
|
+
L: 29
|
|
786
|
+
});
|
|
787
|
+
scheduleTask(ctx, async () => {
|
|
788
|
+
speakingMonitor.speakingChanged.on(ctx, () => setIsSpeaking(speakingMonitor.isSpeaking));
|
|
789
|
+
await speakingMonitor.open();
|
|
790
|
+
ctx.onDispose(() => speakingMonitor.close());
|
|
791
|
+
});
|
|
792
|
+
return () => {
|
|
793
|
+
ctx.dispose();
|
|
794
|
+
};
|
|
795
|
+
}, [speakingMonitor]);
|
|
796
|
+
return isSpeaking;
|
|
797
|
+
};
|
|
798
|
+
//#endregion
|
|
799
|
+
//#region src/hooks/useRecordingPipeline.ts
|
|
800
|
+
var __dxlog_file$1 = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/hooks/useRecordingPipeline.ts";
|
|
801
|
+
var RECORDER_INTERVAL_MS = 200;
|
|
802
|
+
var DRAIN_SILENCE_MS = 5e3;
|
|
803
|
+
var DEFAULT_TRANSCRIBE_CONFIG = {
|
|
804
|
+
transcribeAfterChunksAmount: 50,
|
|
805
|
+
prefixBufferChunksAmount: 10
|
|
806
|
+
};
|
|
807
|
+
/**
|
|
808
|
+
* Single ASR-capture engine binding for React consumers: drives an {@link AsrPipeline}
|
|
809
|
+
* ({@link runAsrPipeline}) off a capture flag, sourcing audio from the microphone (owned here) or an
|
|
810
|
+
* external track. The editor driver, message-list recording, and stories all build on this — varying
|
|
811
|
+
* only by source, stages, and commit sink.
|
|
812
|
+
*/
|
|
813
|
+
var useRecordingPipeline = ({ active, microphone, track: externalTrack, audioConstraints, stages, commit, lookup, onSegment, config, segmentSentences }) => {
|
|
814
|
+
const [phase, setPhase] = useState("idle");
|
|
815
|
+
const inUse = phase !== "idle";
|
|
816
|
+
useEffect(() => {
|
|
817
|
+
setPhase((current) => active ? "recording" : current === "recording" ? "draining" : current);
|
|
818
|
+
}, [active]);
|
|
819
|
+
const micTrack = useAudioTrack(!!microphone && inUse, audioConstraints);
|
|
820
|
+
const track = microphone ? micTrack : externalTrack;
|
|
821
|
+
const recorder = useMemo(() => track ? new MediaStreamRecorder({
|
|
822
|
+
mediaStreamTrack: track,
|
|
823
|
+
config: { interval: RECORDER_INTERVAL_MS }
|
|
824
|
+
}) : void 0, [track]);
|
|
825
|
+
const pipelineRef = useRef(null);
|
|
826
|
+
useEffect(() => {
|
|
827
|
+
if (phase !== "recording" || !recorder) return;
|
|
828
|
+
const asr = runAsrPipeline({
|
|
829
|
+
recorder,
|
|
830
|
+
config: {
|
|
831
|
+
...DEFAULT_TRANSCRIBE_CONFIG,
|
|
832
|
+
...config
|
|
833
|
+
},
|
|
834
|
+
segmentSentences,
|
|
835
|
+
stages,
|
|
836
|
+
commit,
|
|
837
|
+
lookup,
|
|
838
|
+
onSegment,
|
|
839
|
+
drainSilenceMs: DRAIN_SILENCE_MS
|
|
840
|
+
});
|
|
841
|
+
pipelineRef.current = asr;
|
|
842
|
+
let cancelled = false;
|
|
843
|
+
asr.open().then(() => {
|
|
844
|
+
if (!cancelled) asr.start();
|
|
845
|
+
}, (err) => {
|
|
846
|
+
if (!cancelled) {
|
|
847
|
+
log.catch(err, void 0, {
|
|
848
|
+
"~LogMeta": "~LogMeta",
|
|
849
|
+
F: __dxlog_file$1,
|
|
850
|
+
L: 131,
|
|
851
|
+
S: void 0
|
|
852
|
+
});
|
|
853
|
+
setPhase("idle");
|
|
854
|
+
}
|
|
855
|
+
});
|
|
856
|
+
return () => {
|
|
857
|
+
cancelled = true;
|
|
858
|
+
};
|
|
859
|
+
}, [phase, recorder]);
|
|
860
|
+
useEffect(() => {
|
|
861
|
+
if (phase !== "draining") return;
|
|
862
|
+
let cancelled = false;
|
|
863
|
+
(async () => {
|
|
864
|
+
try {
|
|
865
|
+
await pipelineRef.current?.end();
|
|
866
|
+
} catch (err) {
|
|
867
|
+
log.catch(err, void 0, {
|
|
868
|
+
"~LogMeta": "~LogMeta",
|
|
869
|
+
F: __dxlog_file$1,
|
|
870
|
+
L: 152,
|
|
871
|
+
S: void 0
|
|
872
|
+
});
|
|
873
|
+
} finally {
|
|
874
|
+
pipelineRef.current = null;
|
|
875
|
+
if (!cancelled) setPhase((current) => current === "draining" ? "idle" : current);
|
|
876
|
+
}
|
|
877
|
+
})();
|
|
878
|
+
return () => {
|
|
879
|
+
cancelled = true;
|
|
880
|
+
};
|
|
881
|
+
}, [phase]);
|
|
882
|
+
return { phase };
|
|
883
|
+
};
|
|
884
|
+
//#endregion
|
|
885
|
+
//#region src/hooks/useSpeechRecognition.ts
|
|
886
|
+
var __dxlog_file = "/__w/dxos/dxos/packages/ui/react-ui-transcription/src/hooks/useSpeechRecognition.ts";
|
|
887
|
+
var cleanText = (text) => text.toLowerCase().replace(/[.,!?]/g, "").replace(/\s+/g, " ").trim();
|
|
888
|
+
/**
|
|
889
|
+
* Use browser speech recognition API to transcribe speech to text.
|
|
890
|
+
*/
|
|
891
|
+
var useSpeechRecognition = ({ active, onTranscript }) => {
|
|
892
|
+
const recognition = useMemo(() => {
|
|
893
|
+
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
|
|
894
|
+
const SpeechGrammarList = window.SpeechGrammarList || window.webkitSpeechGrammarList;
|
|
895
|
+
if (!SpeechRecognition || !SpeechGrammarList) {
|
|
896
|
+
log.error("Speech recognition not supported", void 0, {
|
|
897
|
+
"~LogMeta": "~LogMeta",
|
|
898
|
+
F: __dxlog_file,
|
|
899
|
+
L: 29,
|
|
900
|
+
S: void 0
|
|
901
|
+
});
|
|
902
|
+
return;
|
|
903
|
+
}
|
|
904
|
+
return Object.assign(new SpeechRecognition(), {
|
|
905
|
+
grammars: new SpeechGrammarList(),
|
|
906
|
+
continuous: true,
|
|
907
|
+
interimResults: true,
|
|
908
|
+
lang: "en-US",
|
|
909
|
+
maxAlternatives: 1,
|
|
910
|
+
onresult: (event) => {
|
|
911
|
+
onTranscript(cleanText(event.results[event.resultIndex][0].transcript));
|
|
912
|
+
},
|
|
913
|
+
onerror: (event) => {
|
|
914
|
+
log.error("Speech recognition error", { error: event.error }, {
|
|
915
|
+
"~LogMeta": "~LogMeta",
|
|
916
|
+
F: __dxlog_file,
|
|
917
|
+
L: 44,
|
|
918
|
+
S: void 0
|
|
919
|
+
});
|
|
920
|
+
}
|
|
921
|
+
});
|
|
922
|
+
}, [onTranscript]);
|
|
923
|
+
useEffect(() => {
|
|
924
|
+
if (active) recognition?.start();
|
|
925
|
+
else recognition?.stop();
|
|
926
|
+
}, [active, recognition]);
|
|
927
|
+
};
|
|
928
|
+
//#endregion
|
|
929
|
+
//#region src/hooks/useTranscriber.ts
|
|
930
|
+
/**
|
|
931
|
+
* Records audio while the user is speaking and transcribes it after they pause. Builds a
|
|
932
|
+
* {@link Transcriber} directly from {@link createTranscriber} (no app-framework capability needed,
|
|
933
|
+
* since the browser construction lives in this package); returns `undefined` until a track and
|
|
934
|
+
* `onSegments` handler are supplied.
|
|
935
|
+
*/
|
|
936
|
+
var useTranscriber = ({ audioStreamTrack, recorderConfig, transcriberConfig, transcribe, onSegments }) => {
|
|
937
|
+
const transcriber = useMemo(() => {
|
|
938
|
+
if (!audioStreamTrack || !onSegments) return;
|
|
939
|
+
return createTranscriber({
|
|
940
|
+
audioStreamTrack,
|
|
941
|
+
recorderConfig,
|
|
942
|
+
transcriberConfig,
|
|
943
|
+
transcribe,
|
|
944
|
+
onSegments
|
|
945
|
+
});
|
|
946
|
+
}, [
|
|
947
|
+
audioStreamTrack,
|
|
948
|
+
recorderConfig,
|
|
949
|
+
transcriberConfig,
|
|
950
|
+
transcribe,
|
|
951
|
+
onSegments
|
|
952
|
+
]);
|
|
953
|
+
useEffect(() => {
|
|
954
|
+
return () => {
|
|
955
|
+
transcriber?.close();
|
|
956
|
+
};
|
|
957
|
+
}, [transcriber]);
|
|
958
|
+
return transcriber;
|
|
959
|
+
};
|
|
960
|
+
//#endregion
|
|
961
|
+
export { EditorChunkDocument, MediaStreamRecorder, MicSettings, PipelineStatus, TranscriptModel, Transcription, createTranscriber, transcription, useAudioFile, useAudioTrack, useFeedModelAdapter, useIsSpeaking, useRecordingPipeline, useSpeechRecognition, useTranscriber };
|
|
962
|
+
|
|
963
|
+
//# sourceMappingURL=index.mjs.map
|