@dxos/plugin-transcription 0.8.4-main.9be5663bfe → 0.8.4-main.abd8ff62ef
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/blueprints/index.mjs +1 -1
- package/dist/lib/browser/blueprints/index.mjs.map +3 -3
- package/dist/lib/browser/{chunk-5LWSJZTZ.mjs → chunk-3L623EVL.mjs} +2 -2
- package/dist/lib/browser/chunk-3L623EVL.mjs.map +7 -0
- package/dist/lib/browser/{chunk-HOOPZ7EM.mjs → chunk-KAEXRX3Y.mjs} +1 -1
- package/dist/lib/browser/{chunk-RIUSFL62.mjs → chunk-L4HW35L6.mjs} +4 -5
- package/dist/lib/browser/chunk-L4HW35L6.mjs.map +7 -0
- package/dist/lib/browser/chunk-MLRJWAQH.mjs +179 -0
- package/dist/lib/browser/chunk-MLRJWAQH.mjs.map +7 -0
- package/dist/lib/browser/chunk-NQQK7UZQ.mjs +244 -0
- package/dist/lib/browser/chunk-NQQK7UZQ.mjs.map +7 -0
- package/dist/lib/browser/components/index.mjs +207 -0
- package/dist/lib/browser/components/index.mjs.map +7 -0
- package/dist/lib/browser/{create-IZPB7VZW.mjs → create-ULR5DPJZ.mjs} +6 -4
- package/dist/lib/browser/create-ULR5DPJZ.mjs.map +7 -0
- package/dist/lib/browser/hooks/index.mjs +17 -0
- package/dist/lib/browser/hooks/index.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +93 -831
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/open-SFMPLZVW.mjs +27 -0
- package/dist/lib/browser/open-SFMPLZVW.mjs.map +7 -0
- package/dist/lib/browser/operations/index.mjs +3 -3
- package/dist/lib/browser/operations/index.mjs.map +3 -3
- package/dist/lib/browser/{summarize-5MZENAU5.mjs → summarize-BDFX4APN.mjs} +15 -8
- package/dist/lib/browser/summarize-BDFX4APN.mjs.map +7 -0
- package/dist/lib/browser/types/index.mjs +1 -1
- package/dist/lib/node-esm/blueprints/index.mjs +1 -1
- package/dist/lib/node-esm/blueprints/index.mjs.map +3 -3
- package/dist/lib/node-esm/{chunk-3GSFEN4L.mjs → chunk-7UIZWCD4.mjs} +4 -5
- package/dist/lib/node-esm/chunk-7UIZWCD4.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-C3AE2NYJ.mjs +181 -0
- package/dist/lib/node-esm/chunk-C3AE2NYJ.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-X74WYCCE.mjs → chunk-KO3IOD2O.mjs} +2 -2
- package/dist/lib/node-esm/chunk-KO3IOD2O.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-MQK4R6GT.mjs +245 -0
- package/dist/lib/node-esm/chunk-MQK4R6GT.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-NVM2QUMX.mjs → chunk-O533NBMK.mjs} +1 -1
- package/dist/lib/node-esm/components/index.mjs +208 -0
- package/dist/lib/node-esm/components/index.mjs.map +7 -0
- package/dist/lib/node-esm/{create-TQDGXQBL.mjs → create-A5QHNWRF.mjs} +6 -4
- package/dist/lib/node-esm/create-A5QHNWRF.mjs.map +7 -0
- package/dist/lib/node-esm/hooks/index.mjs +18 -0
- package/dist/lib/node-esm/hooks/index.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +93 -831
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/open-OI24QNTO.mjs +28 -0
- package/dist/lib/node-esm/open-OI24QNTO.mjs.map +7 -0
- package/dist/lib/node-esm/operations/index.mjs +3 -3
- package/dist/lib/node-esm/operations/index.mjs.map +3 -3
- package/dist/lib/node-esm/{summarize-RVOSRHEC.mjs → summarize-3UNN24I2.mjs} +15 -8
- package/dist/lib/node-esm/summarize-3UNN24I2.mjs.map +7 -0
- package/dist/lib/node-esm/types/index.mjs +1 -1
- package/dist/types/src/TranscriptionPlugin.d.ts +1 -0
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/TranscriptionPlugin.node.d.ts +4 -0
- package/dist/types/src/TranscriptionPlugin.node.d.ts.map +1 -0
- package/dist/types/src/TranscriptionPlugin.test.d.ts +2 -0
- package/dist/types/src/TranscriptionPlugin.test.d.ts.map +1 -0
- package/dist/types/src/blueprints/transcription-blueprint.d.ts +2 -2
- package/dist/types/src/blueprints/transcription-blueprint.d.ts.map +1 -1
- package/dist/types/src/capabilities/blueprint-definition.d.ts +2 -2
- package/dist/types/src/capabilities/blueprint-definition.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts +2 -2
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/operation-handler.d.ts +1 -1
- package/dist/types/src/capabilities/operation-handler.d.ts.map +1 -1
- package/dist/types/src/capabilities/react-surface.d.ts.map +1 -1
- package/dist/types/src/capabilities/transcriber.d.ts.map +1 -1
- package/dist/types/src/components/Transcription/Transcription.d.ts +1 -1
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts +7 -7
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts.map +1 -1
- package/dist/types/src/components/Transcription/transcription-extension.d.ts.map +1 -1
- package/dist/types/src/containers/TranscriptionContainer/TranscriptionContainer.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -1
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
- package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +3 -3
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/model/model.d.ts +2 -2
- package/dist/types/src/model/model.d.ts.map +1 -1
- package/dist/types/src/{segments-normalization → normalization}/index.d.ts +1 -1
- package/dist/types/src/normalization/index.d.ts.map +1 -0
- package/dist/types/src/normalization/message-normalizer.d.ts.map +1 -0
- package/dist/types/src/{segments-normalization → normalization}/normalization.d.ts +155 -153
- package/dist/types/src/normalization/normalization.d.ts.map +1 -0
- package/dist/types/src/normalization/normalization.test.d.ts.map +1 -0
- package/dist/types/src/normalization/utils.d.ts.map +1 -0
- package/dist/types/src/operations/create.d.ts +1 -1
- package/dist/types/src/operations/create.d.ts.map +1 -1
- package/dist/types/src/operations/definitions.d.ts +394 -382
- package/dist/types/src/operations/definitions.d.ts.map +1 -1
- package/dist/types/src/operations/index.d.ts +1 -1
- package/dist/types/src/operations/index.d.ts.map +1 -1
- package/dist/types/src/operations/open.d.ts +1 -1
- package/dist/types/src/operations/open.d.ts.map +1 -1
- package/dist/types/src/operations/summarize.d.ts +1 -1
- package/dist/types/src/operations/summarize.d.ts.map +1 -1
- package/dist/types/src/{components/stories → stories}/FileTranscription.stories.d.ts +3 -3
- package/dist/types/src/stories/FileTranscription.stories.d.ts.map +1 -0
- package/dist/types/src/stories/KeyWordRecognition.stories.d.ts.map +1 -0
- package/dist/types/src/{components/stories → stories}/MicrophoneTranscription.stories.d.ts +1 -1
- package/dist/types/src/stories/MicrophoneTranscription.stories.d.ts.map +1 -0
- package/dist/types/src/stories/TranscriptionStory.d.ts +23 -0
- package/dist/types/src/stories/TranscriptionStory.d.ts.map +1 -0
- package/dist/types/src/stories/useIsSpeaking.d.ts.map +1 -0
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/index.d.ts +1 -1
- package/dist/types/src/transcriber/index.d.ts.map +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +57 -2
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.test.d.ts +2 -0
- package/dist/types/src/transcriber/transcriber.test.d.ts.map +1 -0
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +7 -8
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/src/types/capabilities.d.ts +2 -1
- package/dist/types/src/types/capabilities.d.ts.map +1 -1
- package/dist/types/src/types/{types.d.ts → defs.d.ts} +1 -1
- package/dist/types/src/types/defs.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +1 -1
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/src/util/array.d.ts.map +1 -1
- package/dist/types/src/util/get-time-str.d.ts.map +1 -1
- package/dist/types/src/util/monitor-audio-level.d.ts.map +1 -1
- package/dist/types/src/util/render-byline.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +76 -67
- package/src/TranscriptionPlugin.node.ts +48 -0
- package/src/TranscriptionPlugin.test.ts +31 -0
- package/src/TranscriptionPlugin.tsx +7 -3
- package/src/blueprints/transcription-blueprint.ts +2 -3
- package/src/capabilities/blueprint-definition.ts +2 -0
- package/src/capabilities/index.ts +2 -1
- package/src/capabilities/operation-handler.ts +1 -1
- package/src/capabilities/react-surface.tsx +4 -2
- package/src/capabilities/transcriber.ts +2 -0
- package/src/components/Transcription/Transcription.stories.tsx +3 -5
- package/src/containers/TranscriptionContainer/TranscriptionContainer.tsx +4 -2
- package/src/hooks/useTranscriber.ts +3 -1
- package/src/index.ts +9 -4
- package/src/model/model.test.ts +3 -3
- package/src/model/model.ts +3 -3
- package/src/{segments-normalization → normalization}/message-normalizer.ts +1 -5
- package/src/{segments-normalization → normalization}/normalization.ts +5 -5
- package/src/operations/create.ts +4 -2
- package/src/operations/definitions.ts +3 -4
- package/src/operations/index.ts +1 -1
- package/src/operations/open.ts +5 -8
- package/src/operations/summarize.ts +11 -7
- package/src/{components/stories → stories}/FileTranscription.stories.tsx +24 -5
- package/src/{components/stories → stories}/KeyWordRecognition.stories.tsx +30 -28
- package/src/{components/stories → stories}/MicrophoneTranscription.stories.tsx +3 -3
- package/src/stories/TranscriptionStory.tsx +93 -0
- package/src/transcriber/index.ts +1 -1
- package/src/transcriber/transcriber.test.ts +104 -0
- package/src/transcriber/transcriber.ts +92 -51
- package/src/types/capabilities.ts +6 -1
- package/src/types/index.ts +1 -1
- package/dist/lib/browser/chunk-5LWSJZTZ.mjs.map +0 -7
- package/dist/lib/browser/chunk-RIUSFL62.mjs.map +0 -7
- package/dist/lib/browser/create-IZPB7VZW.mjs.map +0 -7
- package/dist/lib/browser/open-SN5O5I6X.mjs +0 -29
- package/dist/lib/browser/open-SN5O5I6X.mjs.map +0 -7
- package/dist/lib/browser/summarize-5MZENAU5.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-3GSFEN4L.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-X74WYCCE.mjs.map +0 -7
- package/dist/lib/node-esm/create-TQDGXQBL.mjs.map +0 -7
- package/dist/lib/node-esm/open-4GAAWWKJ.mjs +0 -30
- package/dist/lib/node-esm/open-4GAAWWKJ.mjs.map +0 -7
- package/dist/lib/node-esm/summarize-RVOSRHEC.mjs.map +0 -7
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +0 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +0 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +0 -1
- package/dist/types/src/components/stories/TranscriptionStory.d.ts +0 -12
- package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +0 -1
- package/dist/types/src/components/stories/useIsSpeaking.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/index.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/message-normalizer.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/normalization.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/normalization.test.d.ts.map +0 -1
- package/dist/types/src/segments-normalization/utils.d.ts.map +0 -1
- package/dist/types/src/types/types.d.ts.map +0 -1
- package/src/components/stories/TranscriptionStory.tsx +0 -43
- package/dist/lib/browser/{chunk-HOOPZ7EM.mjs.map → chunk-KAEXRX3Y.mjs.map} +0 -0
- package/dist/lib/node-esm/{chunk-NVM2QUMX.mjs.map → chunk-O533NBMK.mjs.map} +0 -0
- package/dist/types/src/{segments-normalization → normalization}/message-normalizer.d.ts +0 -0
- package/dist/types/src/{segments-normalization → normalization}/normalization.test.d.ts +0 -0
- package/dist/types/src/{segments-normalization → normalization}/utils.d.ts +0 -0
- package/dist/types/src/{components/stories → stories}/KeyWordRecognition.stories.d.ts +0 -0
- package/dist/types/src/{components/stories → stories}/useIsSpeaking.d.ts +0 -0
- package/src/{segments-normalization → normalization}/index.ts +1 -1
- /package/src/{segments-normalization → normalization}/normalization.test.ts +0 -0
- /package/src/{segments-normalization → normalization}/utils.ts +0 -0
- /package/src/{components/stories → stories}/useIsSpeaking.tsx +0 -0
- /package/src/types/{types.ts → defs.ts} +0 -0
|
@@ -1,670 +1,22 @@
|
|
|
1
1
|
import { createRequire } from 'node:module';const require = createRequire(import.meta.url);
|
|
2
|
+
import {
|
|
3
|
+
useAudioFile,
|
|
4
|
+
useAudioTrack,
|
|
5
|
+
useQueueModelAdapter,
|
|
6
|
+
useTranscriber,
|
|
7
|
+
useVoiceInput
|
|
8
|
+
} from "./chunk-MQK4R6GT.mjs";
|
|
9
|
+
import "./chunk-C3AE2NYJ.mjs";
|
|
2
10
|
import {
|
|
3
11
|
TranscriptionCapabilities
|
|
4
|
-
} from "./chunk-
|
|
12
|
+
} from "./chunk-KO3IOD2O.mjs";
|
|
5
13
|
import {
|
|
6
|
-
mergeFloat64Arrays
|
|
7
|
-
|
|
8
|
-
} from "./chunk-NVM2QUMX.mjs";
|
|
14
|
+
mergeFloat64Arrays
|
|
15
|
+
} from "./chunk-O533NBMK.mjs";
|
|
9
16
|
import "./chunk-HSLMI22Q.mjs";
|
|
10
17
|
|
|
11
|
-
// src/
|
|
12
|
-
import
|
|
13
|
-
import { useThemeContext } from "@dxos/react-ui";
|
|
14
|
-
import { useTextEditor } from "@dxos/react-ui-editor";
|
|
15
|
-
import { autoScroll, createBasicExtensions, createMarkdownExtensions, createThemeExtensions, decorateMarkdown, documentSlots, preview } from "@dxos/ui-editor";
|
|
16
|
-
import { composable, composableProps } from "@dxos/ui-theme";
|
|
17
|
-
import { isTruthy } from "@dxos/util";
|
|
18
|
-
|
|
19
|
-
// src/components/Transcription/transcription-extension.ts
|
|
20
|
-
import { RangeSetBuilder } from "@codemirror/state";
|
|
21
|
-
import { EditorView, GutterMarker, ViewPlugin, gutter } from "@codemirror/view";
|
|
22
|
-
import { format } from "date-fns/format";
|
|
23
|
-
import { intervalToDuration } from "date-fns/intervalToDuration";
|
|
24
|
-
import { addEventListener, combine } from "@dxos/async";
|
|
25
|
-
|
|
26
|
-
// src/model/model.ts
|
|
27
|
-
import { Text } from "@codemirror/state";
|
|
28
|
-
import { Event } from "@dxos/async";
|
|
29
|
-
import { invariant } from "@dxos/invariant";
|
|
30
|
-
import { log } from "@dxos/log";
|
|
31
|
-
var __dxlog_file = "/__w/dxos/dxos/packages/plugins/plugin-transcription/src/model/model.ts";
|
|
32
|
-
var DocumentAdapter = class {
|
|
33
|
-
_view;
|
|
34
|
-
constructor(_view) {
|
|
35
|
-
this._view = _view;
|
|
36
|
-
}
|
|
37
|
-
/** The document must always have at least one line. */
|
|
38
|
-
lineCount() {
|
|
39
|
-
return this._view.state.doc.lines - 1;
|
|
40
|
-
}
|
|
41
|
-
replaceLines(from, remove, insert) {
|
|
42
|
-
log("replaceLines", {
|
|
43
|
-
count: this._view.state.doc.lines,
|
|
44
|
-
from,
|
|
45
|
-
remove,
|
|
46
|
-
insert
|
|
47
|
-
}, {
|
|
48
|
-
F: __dxlog_file,
|
|
49
|
-
L: 39,
|
|
50
|
-
S: this,
|
|
51
|
-
C: (f, a) => f(...a)
|
|
52
|
-
});
|
|
53
|
-
const numLines = this._view.state.doc.lines;
|
|
54
|
-
const posFrom = this._view.state.doc.line(from).from;
|
|
55
|
-
const posTo = remove === 0 ? posFrom : this._view.state.doc.line(Math.min(numLines, from + remove)).from;
|
|
56
|
-
let text = insert.join("\n");
|
|
57
|
-
if (posTo === this._view.state.doc.length) {
|
|
58
|
-
text += "\n";
|
|
59
|
-
}
|
|
60
|
-
this._view.dispatch({
|
|
61
|
-
changes: {
|
|
62
|
-
from: posFrom,
|
|
63
|
-
to: posTo,
|
|
64
|
-
insert: text
|
|
65
|
-
}
|
|
66
|
-
});
|
|
67
|
-
}
|
|
68
|
-
};
|
|
69
|
-
var SerializationModel = class {
|
|
70
|
-
_renderer;
|
|
71
|
-
/** Ordered set of chunks. */
|
|
72
|
-
_chunks = [];
|
|
73
|
-
/** Map of chunk IDs to their current line counts. */
|
|
74
|
-
_chunkLineCounts = /* @__PURE__ */ new Map();
|
|
75
|
-
/** Map of line numbers to chunk IDs. */
|
|
76
|
-
_lineToChunk = /* @__PURE__ */ new Map();
|
|
77
|
-
/** Track chunk changes since last sync. */
|
|
78
|
-
_changes = [];
|
|
79
|
-
/** Emits when the document is updated. */
|
|
80
|
-
update = new Event();
|
|
81
|
-
constructor(_renderer, initialChunks = []) {
|
|
82
|
-
this._renderer = _renderer;
|
|
83
|
-
initialChunks.forEach((chunk) => {
|
|
84
|
-
this.appendChunk(chunk);
|
|
85
|
-
});
|
|
86
|
-
}
|
|
87
|
-
toJSON() {
|
|
88
|
-
return {
|
|
89
|
-
chunks: this._chunks.length,
|
|
90
|
-
changes: this._changes.length
|
|
91
|
-
};
|
|
92
|
-
}
|
|
93
|
-
get doc() {
|
|
94
|
-
return Text.of([
|
|
95
|
-
...this._chunks.flatMap((chunk, index) => this._renderer(chunk, index)),
|
|
96
|
-
""
|
|
97
|
-
]);
|
|
98
|
-
}
|
|
99
|
-
get chunks() {
|
|
100
|
-
return this._chunks;
|
|
101
|
-
}
|
|
102
|
-
/**
|
|
103
|
-
* Get the block ID at the given line number.
|
|
104
|
-
*/
|
|
105
|
-
getChunkAtLine(line) {
|
|
106
|
-
return this._lineToChunk.get(line);
|
|
107
|
-
}
|
|
108
|
-
reset() {
|
|
109
|
-
this._chunks.length = 0;
|
|
110
|
-
this._chunkLineCounts.clear();
|
|
111
|
-
this._lineToChunk.clear();
|
|
112
|
-
this.update.emit();
|
|
113
|
-
return this;
|
|
114
|
-
}
|
|
115
|
-
appendChunk(chunk) {
|
|
116
|
-
const index = this._chunks.length;
|
|
117
|
-
this._chunks.push(chunk);
|
|
118
|
-
this._changes.push({
|
|
119
|
-
type: "append",
|
|
120
|
-
index,
|
|
121
|
-
chunk
|
|
122
|
-
});
|
|
123
|
-
this.update.emit();
|
|
124
|
-
return this;
|
|
125
|
-
}
|
|
126
|
-
updateChunk(chunk) {
|
|
127
|
-
const index = this._chunks.findIndex((c) => c.id === chunk.id);
|
|
128
|
-
invariant(index !== -1, void 0, {
|
|
129
|
-
F: __dxlog_file,
|
|
130
|
-
L: 147,
|
|
131
|
-
S: this,
|
|
132
|
-
A: [
|
|
133
|
-
"index !== -1",
|
|
134
|
-
""
|
|
135
|
-
]
|
|
136
|
-
});
|
|
137
|
-
this._chunks[index] = chunk;
|
|
138
|
-
this._changes.push({
|
|
139
|
-
type: "update",
|
|
140
|
-
index,
|
|
141
|
-
chunk
|
|
142
|
-
});
|
|
143
|
-
this.update.emit();
|
|
144
|
-
return this;
|
|
145
|
-
}
|
|
146
|
-
deleteBlock(id) {
|
|
147
|
-
const index = this._chunks.findIndex((c) => c.id === id);
|
|
148
|
-
invariant(index !== -1, void 0, {
|
|
149
|
-
F: __dxlog_file,
|
|
150
|
-
L: 156,
|
|
151
|
-
S: this,
|
|
152
|
-
A: [
|
|
153
|
-
"index !== -1",
|
|
154
|
-
""
|
|
155
|
-
]
|
|
156
|
-
});
|
|
157
|
-
this._chunks.splice(index, 1);
|
|
158
|
-
this._changes.push({
|
|
159
|
-
type: "delete",
|
|
160
|
-
index,
|
|
161
|
-
id
|
|
162
|
-
});
|
|
163
|
-
this.update.emit();
|
|
164
|
-
return this;
|
|
165
|
-
}
|
|
166
|
-
/**
|
|
167
|
-
* Syncs the tracked changes with the ChunkDocument.
|
|
168
|
-
*/
|
|
169
|
-
sync(document2) {
|
|
170
|
-
log("sync", {
|
|
171
|
-
changes: this._changes
|
|
172
|
-
}, {
|
|
173
|
-
F: __dxlog_file,
|
|
174
|
-
L: 167,
|
|
175
|
-
S: this,
|
|
176
|
-
C: (f, a) => f(...a)
|
|
177
|
-
});
|
|
178
|
-
for (const change of this._changes) {
|
|
179
|
-
switch (change.type) {
|
|
180
|
-
case "append": {
|
|
181
|
-
const lines = this._renderer(change.chunk, change.index);
|
|
182
|
-
const lastLine = document2.lineCount();
|
|
183
|
-
document2.replaceLines(lastLine + 1, 0, lines);
|
|
184
|
-
break;
|
|
185
|
-
}
|
|
186
|
-
case "update": {
|
|
187
|
-
let lineStart = 1;
|
|
188
|
-
for (let i = 0; i < change.index; i++) {
|
|
189
|
-
lineStart += this._chunkLineCounts.get(this._chunks[i].id);
|
|
190
|
-
}
|
|
191
|
-
const previousLineCount = this._chunkLineCounts.get(change.chunk.id);
|
|
192
|
-
const newLines = this._renderer(change.chunk, change.index);
|
|
193
|
-
document2.replaceLines(lineStart, previousLineCount, newLines);
|
|
194
|
-
break;
|
|
195
|
-
}
|
|
196
|
-
case "delete": {
|
|
197
|
-
let lineStart = 1;
|
|
198
|
-
for (let i = 0; i < change.index; i++) {
|
|
199
|
-
lineStart += this._chunkLineCounts.get(this._chunks[i].id);
|
|
200
|
-
}
|
|
201
|
-
const previousLineCount = this._chunkLineCounts.get(change.id);
|
|
202
|
-
document2.replaceLines(lineStart, previousLineCount, []);
|
|
203
|
-
break;
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
}
|
|
207
|
-
this._lineToChunk.clear();
|
|
208
|
-
let currentLine = 1;
|
|
209
|
-
for (const [index, chunk] of this._chunks.entries()) {
|
|
210
|
-
const lineCount = this._renderer(chunk, index).length;
|
|
211
|
-
this._chunkLineCounts.set(chunk.id, lineCount);
|
|
212
|
-
this._lineToChunk.set(currentLine, chunk);
|
|
213
|
-
currentLine += lineCount;
|
|
214
|
-
}
|
|
215
|
-
const chunkIds = new Set(this._chunks.map((chunk) => chunk.id));
|
|
216
|
-
for (const id of this._chunkLineCounts.keys()) {
|
|
217
|
-
if (!chunkIds.has(id)) {
|
|
218
|
-
this._chunkLineCounts.delete(id);
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
this._changes.length = 0;
|
|
222
|
-
return this;
|
|
223
|
-
}
|
|
224
|
-
};
|
|
225
|
-
|
|
226
|
-
// src/components/Transcription/transcription-extension.ts
|
|
227
|
-
var transcription = ({ model, started }) => {
|
|
228
|
-
return [
|
|
229
|
-
// Show timestamps in the gutter.
|
|
230
|
-
gutter({
|
|
231
|
-
class: "cm-timestamp-gutter",
|
|
232
|
-
lineMarkerChange: (update) => update.docChanged || update.viewportChanged,
|
|
233
|
-
markers: (view) => {
|
|
234
|
-
const start = getStartTime(started, model.chunks[0]);
|
|
235
|
-
const builder = new RangeSetBuilder();
|
|
236
|
-
for (const { from, to } of view.visibleRanges) {
|
|
237
|
-
let line = view.state.doc.lineAt(from);
|
|
238
|
-
while (line.from <= to) {
|
|
239
|
-
const block = model.getChunkAtLine(line.number)?.blocks[0];
|
|
240
|
-
const timestamp = block?._tag === "transcript" && block.started;
|
|
241
|
-
if (timestamp) {
|
|
242
|
-
builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));
|
|
243
|
-
}
|
|
244
|
-
if (line.to + 1 > view.state.doc.length) {
|
|
245
|
-
break;
|
|
246
|
-
}
|
|
247
|
-
line = view.state.doc.lineAt(line.to + 1);
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
return builder.finish();
|
|
251
|
-
}
|
|
252
|
-
}),
|
|
253
|
-
// Listen for model updates.
|
|
254
|
-
ViewPlugin.fromClass(class {
|
|
255
|
-
_controls;
|
|
256
|
-
_cleanup;
|
|
257
|
-
_adapter;
|
|
258
|
-
_initialized = false;
|
|
259
|
-
constructor(view) {
|
|
260
|
-
this._adapter = new DocumentAdapter(view);
|
|
261
|
-
const scroller = view.scrollDOM;
|
|
262
|
-
let isAutoScrolling = false;
|
|
263
|
-
let hasScrolled = false;
|
|
264
|
-
let timeout;
|
|
265
|
-
const scrollToBottom = (smooth = false) => {
|
|
266
|
-
scroller.style.scrollBehavior = smooth ? "smooth" : "";
|
|
267
|
-
scroller.classList.add("cm-hide-scrollbar");
|
|
268
|
-
isAutoScrolling = true;
|
|
269
|
-
clearTimeout(timeout);
|
|
270
|
-
timeout = setTimeout(() => {
|
|
271
|
-
this._controls?.classList.add("opacity-0");
|
|
272
|
-
scroller.classList.remove("cm-hide-scrollbar");
|
|
273
|
-
isAutoScrolling = false;
|
|
274
|
-
}, 500);
|
|
275
|
-
view.dispatch({
|
|
276
|
-
effects: EditorView.scrollIntoView(view.state.doc.length, {
|
|
277
|
-
y: "end"
|
|
278
|
-
})
|
|
279
|
-
});
|
|
280
|
-
};
|
|
281
|
-
this._cleanup = combine(addEventListener(view.scrollDOM, "scroll", () => {
|
|
282
|
-
if (!isAutoScrolling) {
|
|
283
|
-
hasScrolled = true;
|
|
284
|
-
this._controls?.classList.remove("opacity-0");
|
|
285
|
-
}
|
|
286
|
-
}), model.update.on(() => {
|
|
287
|
-
const autoScroll2 = scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;
|
|
288
|
-
model.sync(this._adapter);
|
|
289
|
-
if (autoScroll2) {
|
|
290
|
-
scrollToBottom(true);
|
|
291
|
-
}
|
|
292
|
-
}));
|
|
293
|
-
}
|
|
294
|
-
update(_update) {
|
|
295
|
-
if (!this._initialized) {
|
|
296
|
-
this._initialized = true;
|
|
297
|
-
setTimeout(() => {
|
|
298
|
-
model.sync(this._adapter);
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
}
|
|
302
|
-
destroy() {
|
|
303
|
-
this._controls?.remove();
|
|
304
|
-
this._cleanup();
|
|
305
|
-
}
|
|
306
|
-
}),
|
|
307
|
-
EditorView.theme({
|
|
308
|
-
".cm-gutters": {
|
|
309
|
-
backgroundColor: "var(--color-base-surface)"
|
|
310
|
-
},
|
|
311
|
-
".cm-timestamp-gutter": {
|
|
312
|
-
width: "6rem",
|
|
313
|
-
paddingLeft: "0.5rem",
|
|
314
|
-
paddingRight: "1rem"
|
|
315
|
-
},
|
|
316
|
-
".cm-timestamp-gutter > .cm-gutterElement > div": {
|
|
317
|
-
display: "inline-flex",
|
|
318
|
-
textAlign: "right",
|
|
319
|
-
padding: "3px"
|
|
320
|
-
}
|
|
321
|
-
})
|
|
322
|
-
];
|
|
323
|
-
};
|
|
324
|
-
var TimestampMarker = class extends GutterMarker {
|
|
325
|
-
_line;
|
|
326
|
-
_timestamp;
|
|
327
|
-
_started;
|
|
328
|
-
constructor(_line, _timestamp, _started) {
|
|
329
|
-
super(), this._line = _line, this._timestamp = _timestamp, this._started = _started;
|
|
330
|
-
}
|
|
331
|
-
eq(other) {
|
|
332
|
-
return this._timestamp === other._timestamp;
|
|
333
|
-
}
|
|
334
|
-
toDOM(view) {
|
|
335
|
-
const el = document.createElement("div");
|
|
336
|
-
el.className = "text-sm text-subdued hover:bg-hover-surface cursor-pointer";
|
|
337
|
-
el.textContent = formatTimestamp(this._timestamp, this._started);
|
|
338
|
-
el.onclick = () => {
|
|
339
|
-
const pos = view.state.doc.line(this._line).from;
|
|
340
|
-
view.dispatch({
|
|
341
|
-
effects: EditorView.scrollIntoView(pos, {
|
|
342
|
-
y: "start"
|
|
343
|
-
})
|
|
344
|
-
});
|
|
345
|
-
};
|
|
346
|
-
return el;
|
|
347
|
-
}
|
|
348
|
-
};
|
|
349
|
-
var getStartTime = (started, message) => {
|
|
350
|
-
if (started) {
|
|
351
|
-
return started;
|
|
352
|
-
}
|
|
353
|
-
if (message?.blocks[0]?._tag === "transcript" && message.blocks[0].started) {
|
|
354
|
-
return new Date(message.blocks[0].started);
|
|
355
|
-
}
|
|
356
|
-
return void 0;
|
|
357
|
-
};
|
|
358
|
-
var formatTimestamp = (timestamp, relative) => {
|
|
359
|
-
if (relative) {
|
|
360
|
-
const pad = (n = 0) => String(n).padStart(2, "0");
|
|
361
|
-
const { hours, minutes, seconds } = intervalToDuration({
|
|
362
|
-
start: relative,
|
|
363
|
-
end: timestamp
|
|
364
|
-
});
|
|
365
|
-
return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;
|
|
366
|
-
} else {
|
|
367
|
-
return format(timestamp, "HH:mm:ss");
|
|
368
|
-
}
|
|
369
|
-
};
|
|
370
|
-
|
|
371
|
-
// src/components/Transcription/Transcription.tsx
|
|
372
|
-
var Transcription = composable(({ transcript: object, model, children, ...props }, forwardedRef) => {
|
|
373
|
-
const { themeMode } = useThemeContext();
|
|
374
|
-
const { parentRef } = useTextEditor(() => {
|
|
375
|
-
return {
|
|
376
|
-
extensions: [
|
|
377
|
-
createBasicExtensions({
|
|
378
|
-
readOnly: true,
|
|
379
|
-
lineWrapping: true,
|
|
380
|
-
search: true
|
|
381
|
-
}),
|
|
382
|
-
createMarkdownExtensions(),
|
|
383
|
-
createThemeExtensions({
|
|
384
|
-
themeMode,
|
|
385
|
-
slots: documentSlots
|
|
386
|
-
}),
|
|
387
|
-
decorateMarkdown(),
|
|
388
|
-
preview(),
|
|
389
|
-
transcription({
|
|
390
|
-
model,
|
|
391
|
-
started: object?.started ? new Date(object.started) : void 0
|
|
392
|
-
}),
|
|
393
|
-
autoScroll()
|
|
394
|
-
].filter(isTruthy)
|
|
395
|
-
};
|
|
396
|
-
}, [
|
|
397
|
-
model
|
|
398
|
-
]);
|
|
399
|
-
return /* @__PURE__ */ React.createElement("div", {
|
|
400
|
-
...composableProps(props, {
|
|
401
|
-
classNames: "dx-container"
|
|
402
|
-
}),
|
|
403
|
-
ref: parentRef,
|
|
404
|
-
"data-popover-collision-boundary": true
|
|
405
|
-
});
|
|
406
|
-
});
|
|
407
|
-
|
|
408
|
-
// src/hooks/useAudioFile.ts
|
|
409
|
-
import { useEffect, useState } from "react";
|
|
410
|
-
import { scheduleTask } from "@dxos/async";
|
|
411
|
-
import { Context } from "@dxos/context";
|
|
412
|
-
import { log as log2 } from "@dxos/log";
|
|
413
|
-
var __dxlog_file2 = "/__w/dxos/dxos/packages/plugins/plugin-transcription/src/hooks/useAudioFile.ts";
|
|
414
|
-
var useAudioFile = (audioUrl, constraints) => {
|
|
415
|
-
const [{ audio, stream, track }, setStream] = useState({});
|
|
416
|
-
useEffect(() => {
|
|
417
|
-
const ctx = new Context(void 0, {
|
|
418
|
-
F: __dxlog_file2,
|
|
419
|
-
L: 20
|
|
420
|
-
});
|
|
421
|
-
scheduleTask(ctx, async () => {
|
|
422
|
-
try {
|
|
423
|
-
const response = await fetch(audioUrl);
|
|
424
|
-
const blob = await response.blob();
|
|
425
|
-
const objectUrl = URL.createObjectURL(blob);
|
|
426
|
-
ctx.onDispose(() => {
|
|
427
|
-
URL.revokeObjectURL(objectUrl);
|
|
428
|
-
});
|
|
429
|
-
const audio2 = new Audio();
|
|
430
|
-
audio2.src = objectUrl;
|
|
431
|
-
await new Promise((resolve, reject) => {
|
|
432
|
-
audio2.addEventListener("error", (err) => {
|
|
433
|
-
log2.error("error", {
|
|
434
|
-
err
|
|
435
|
-
}, {
|
|
436
|
-
F: __dxlog_file2,
|
|
437
|
-
L: 39,
|
|
438
|
-
S: void 0,
|
|
439
|
-
C: (f, a) => f(...a)
|
|
440
|
-
});
|
|
441
|
-
reject(err);
|
|
442
|
-
}, {
|
|
443
|
-
once: true
|
|
444
|
-
});
|
|
445
|
-
audio2.addEventListener("canplay", async () => {
|
|
446
|
-
resolve();
|
|
447
|
-
}, {
|
|
448
|
-
once: true
|
|
449
|
-
});
|
|
450
|
-
audio2.load();
|
|
451
|
-
});
|
|
452
|
-
const audioCtx = new AudioContext();
|
|
453
|
-
if (audioCtx.state === "suspended") {
|
|
454
|
-
await audioCtx.resume();
|
|
455
|
-
}
|
|
456
|
-
const destination = audioCtx.createMediaStreamDestination();
|
|
457
|
-
destination.channelCount = 1;
|
|
458
|
-
const source = audioCtx.createMediaElementSource(audio2);
|
|
459
|
-
source.connect(destination);
|
|
460
|
-
const track2 = destination.stream.getAudioTracks()[0];
|
|
461
|
-
if (constraints) {
|
|
462
|
-
await track2.applyConstraints(constraints).catch((err) => log2.catch(err, void 0, {
|
|
463
|
-
F: __dxlog_file2,
|
|
464
|
-
L: 68,
|
|
465
|
-
S: void 0,
|
|
466
|
-
C: (f, a) => f(...a)
|
|
467
|
-
}));
|
|
468
|
-
}
|
|
469
|
-
setStream({
|
|
470
|
-
audio: audio2,
|
|
471
|
-
stream: destination.stream,
|
|
472
|
-
track: destination.stream.getAudioTracks()[0]
|
|
473
|
-
});
|
|
474
|
-
} catch (err) {
|
|
475
|
-
log2.catch(err, void 0, {
|
|
476
|
-
F: __dxlog_file2,
|
|
477
|
-
L: 78,
|
|
478
|
-
S: void 0,
|
|
479
|
-
C: (f, a) => f(...a)
|
|
480
|
-
});
|
|
481
|
-
}
|
|
482
|
-
});
|
|
483
|
-
return () => {
|
|
484
|
-
void ctx.dispose();
|
|
485
|
-
};
|
|
486
|
-
}, [
|
|
487
|
-
audioUrl
|
|
488
|
-
]);
|
|
489
|
-
return {
|
|
490
|
-
audio,
|
|
491
|
-
stream,
|
|
492
|
-
track
|
|
493
|
-
};
|
|
494
|
-
};
|
|
495
|
-
|
|
496
|
-
// src/hooks/useAudioTrack.ts
|
|
497
|
-
import { useEffect as useEffect2, useState as useState2 } from "react";
|
|
498
|
-
var useAudioTrack = (active, constraints) => {
|
|
499
|
-
const [track, setTrack] = useState2();
|
|
500
|
-
useEffect2(() => {
|
|
501
|
-
if (!active) {
|
|
502
|
-
track?.stop();
|
|
503
|
-
setTrack(void 0);
|
|
504
|
-
return;
|
|
505
|
-
}
|
|
506
|
-
const initAudio = async () => {
|
|
507
|
-
const audio = new Audio();
|
|
508
|
-
audio.srcObject = await navigator.mediaDevices.getUserMedia({
|
|
509
|
-
audio: constraints ?? true
|
|
510
|
-
});
|
|
511
|
-
const [track2] = audio.srcObject.getAudioTracks();
|
|
512
|
-
if (track2) {
|
|
513
|
-
setTrack(track2);
|
|
514
|
-
}
|
|
515
|
-
};
|
|
516
|
-
if (active) {
|
|
517
|
-
void initAudio();
|
|
518
|
-
}
|
|
519
|
-
}, [
|
|
520
|
-
active
|
|
521
|
-
]);
|
|
522
|
-
return track;
|
|
523
|
-
};
|
|
524
|
-
|
|
525
|
-
// src/hooks/useQueueModelAdapter.ts
|
|
526
|
-
import { useEffect as useEffect3, useMemo, useState as useState3 } from "react";
|
|
527
|
-
var useQueueModelAdapter = (renderer, queue, initialChunks = []) => {
|
|
528
|
-
const model = useMemo(() => new SerializationModel(renderer, initialChunks), [
|
|
529
|
-
queue
|
|
530
|
-
]);
|
|
531
|
-
const [loaded, setLoaded] = useState3(false);
|
|
532
|
-
useEffect3(() => {
|
|
533
|
-
if (!queue) {
|
|
534
|
-
return;
|
|
535
|
-
}
|
|
536
|
-
const update = () => {
|
|
537
|
-
for (const block of queue?.objects ?? []) {
|
|
538
|
-
model.appendChunk(block);
|
|
539
|
-
}
|
|
540
|
-
setLoaded(true);
|
|
541
|
-
};
|
|
542
|
-
let i;
|
|
543
|
-
if (queue.isLoading) {
|
|
544
|
-
i = setInterval(() => {
|
|
545
|
-
if (!queue?.isLoading) {
|
|
546
|
-
clearInterval(i);
|
|
547
|
-
update();
|
|
548
|
-
}
|
|
549
|
-
}, 1e3);
|
|
550
|
-
} else {
|
|
551
|
-
update();
|
|
552
|
-
}
|
|
553
|
-
return () => clearTimeout(i);
|
|
554
|
-
}, [
|
|
555
|
-
model,
|
|
556
|
-
queue
|
|
557
|
-
]);
|
|
558
|
-
useEffect3(() => {
|
|
559
|
-
if (!loaded || !queue || model.chunks.length === queue.objects.length) {
|
|
560
|
-
return;
|
|
561
|
-
}
|
|
562
|
-
const chunk = queue.objects.at(-1);
|
|
563
|
-
if (chunk) {
|
|
564
|
-
model.appendChunk(chunk);
|
|
565
|
-
}
|
|
566
|
-
}, [
|
|
567
|
-
model,
|
|
568
|
-
loaded,
|
|
569
|
-
queue?.objects.length
|
|
570
|
-
]);
|
|
571
|
-
return model;
|
|
572
|
-
};
|
|
573
|
-
|
|
574
|
-
// src/hooks/useTranscriber.ts
|
|
575
|
-
import { useEffect as useEffect4, useMemo as useMemo2 } from "react";
|
|
576
|
-
import { useCapabilities } from "@dxos/app-framework/ui";
|
|
577
|
-
import { TranscriptionCapabilities as TranscriptionCapabilities2 } from "#types";
|
|
578
|
-
var useTranscriber = ({ audioStreamTrack, recorderConfig, transcriberConfig, onSegments }) => {
|
|
579
|
-
const [getTranscriber] = useCapabilities(TranscriptionCapabilities2.Transcriber);
|
|
580
|
-
const transcriber = useMemo2(() => {
|
|
581
|
-
if (!getTranscriber || !audioStreamTrack || !onSegments) {
|
|
582
|
-
return void 0;
|
|
583
|
-
}
|
|
584
|
-
return getTranscriber({
|
|
585
|
-
audioStreamTrack,
|
|
586
|
-
recorderConfig,
|
|
587
|
-
transcriberConfig,
|
|
588
|
-
onSegments
|
|
589
|
-
});
|
|
590
|
-
}, [
|
|
591
|
-
getTranscriber,
|
|
592
|
-
audioStreamTrack,
|
|
593
|
-
recorderConfig,
|
|
594
|
-
transcriberConfig,
|
|
595
|
-
onSegments
|
|
596
|
-
]);
|
|
597
|
-
useEffect4(() => {
|
|
598
|
-
return () => {
|
|
599
|
-
void transcriber?.close();
|
|
600
|
-
};
|
|
601
|
-
}, [
|
|
602
|
-
transcriber
|
|
603
|
-
]);
|
|
604
|
-
return transcriber;
|
|
605
|
-
};
|
|
606
|
-
|
|
607
|
-
// src/hooks/useVoiceInput.ts
|
|
608
|
-
import { useCallback, useEffect as useEffect5, useState as useState4 } from "react";
|
|
609
|
-
import { scheduleMicroTask } from "@dxos/async";
|
|
610
|
-
import { Context as Context2 } from "@dxos/context";
|
|
611
|
-
import { log as log3 } from "@dxos/log";
|
|
612
|
-
import { useSoundEffect } from "@dxos/react-ui-sfx";
|
|
613
|
-
var __dxlog_file3 = "/__w/dxos/dxos/packages/plugins/plugin-transcription/src/hooks/useVoiceInput.ts";
|
|
614
|
-
var useVoiceInput = ({ active, onUpdate }) => {
|
|
615
|
-
const soundStart = useSoundEffect("StartRecording");
|
|
616
|
-
const soundStop = useSoundEffect("StopRecording");
|
|
617
|
-
const handleSegments = useCallback(async (segments) => {
|
|
618
|
-
const text = segments.map((str) => str.text.trim().replace(/[^\w\s]/g, "")).join(" ");
|
|
619
|
-
onUpdate(text);
|
|
620
|
-
}, []);
|
|
621
|
-
const [recording, setRecording] = useState4(false);
|
|
622
|
-
const track = useAudioTrack(active);
|
|
623
|
-
const transcriber = useTranscriber({
|
|
624
|
-
audioStreamTrack: track,
|
|
625
|
-
onSegments: handleSegments
|
|
626
|
-
});
|
|
627
|
-
useEffect5(() => {
|
|
628
|
-
const ctx = new Context2(void 0, {
|
|
629
|
-
F: __dxlog_file3,
|
|
630
|
-
L: 43
|
|
631
|
-
});
|
|
632
|
-
scheduleMicroTask(ctx, async () => {
|
|
633
|
-
if (active && transcriber) {
|
|
634
|
-
await transcriber.open();
|
|
635
|
-
log3.info("starting...", void 0, {
|
|
636
|
-
F: __dxlog_file3,
|
|
637
|
-
L: 47,
|
|
638
|
-
S: void 0,
|
|
639
|
-
C: (f, a) => f(...a)
|
|
640
|
-
});
|
|
641
|
-
setRecording(true);
|
|
642
|
-
void soundStart.play();
|
|
643
|
-
transcriber.startChunksRecording();
|
|
644
|
-
} else if (!active && transcriber?.isOpen) {
|
|
645
|
-
transcriber?.stopChunksRecording();
|
|
646
|
-
await transcriber?.close();
|
|
647
|
-
log3.info("stopped", void 0, {
|
|
648
|
-
F: __dxlog_file3,
|
|
649
|
-
L: 54,
|
|
650
|
-
S: void 0,
|
|
651
|
-
C: (f, a) => f(...a)
|
|
652
|
-
});
|
|
653
|
-
setRecording(false);
|
|
654
|
-
void soundStop.play();
|
|
655
|
-
}
|
|
656
|
-
});
|
|
657
|
-
return () => {
|
|
658
|
-
void ctx.dispose();
|
|
659
|
-
};
|
|
660
|
-
}, [
|
|
661
|
-
active,
|
|
662
|
-
transcriber
|
|
663
|
-
]);
|
|
664
|
-
return {
|
|
665
|
-
recording
|
|
666
|
-
};
|
|
667
|
-
};
|
|
18
|
+
// src/index.ts
|
|
19
|
+
import { Plugin } from "@dxos/app-framework";
|
|
668
20
|
|
|
669
21
|
// src/meta.ts
|
|
670
22
|
import { trim } from "@dxos/util";
|
|
@@ -688,16 +40,16 @@ var meta = {
|
|
|
688
40
|
import { MediaRecorder, register } from "extendable-media-recorder";
|
|
689
41
|
import { WaveFile } from "wavefile";
|
|
690
42
|
import { synchronized } from "@dxos/async";
|
|
691
|
-
import { invariant
|
|
692
|
-
import { log
|
|
43
|
+
import { invariant } from "@dxos/invariant";
|
|
44
|
+
import { log } from "@dxos/log";
|
|
693
45
|
import { trace } from "@dxos/tracing";
|
|
46
|
+
var __dxlog_file = "/__w/dxos/dxos/packages/plugins/plugin-transcription/src/transcriber/media-stream-recorder.ts";
|
|
694
47
|
function _ts_decorate(decorators, target, key, desc) {
|
|
695
48
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
696
49
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
697
50
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
698
51
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
699
52
|
}
|
|
700
|
-
var __dxlog_file4 = "/__w/dxos/dxos/packages/plugins/plugin-transcription/src/transcriber/media-stream-recorder.ts";
|
|
701
53
|
var initializingPromise;
|
|
702
54
|
var initializeExtendableMediaRecorder = async () => {
|
|
703
55
|
const { connect } = await import("extendable-media-recorder-wav-encoder");
|
|
@@ -731,14 +83,9 @@ var MediaStreamRecorder = class {
|
|
|
731
83
|
this._onChunk = onChunk;
|
|
732
84
|
}
|
|
733
85
|
async start() {
|
|
734
|
-
await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) =>
|
|
86
|
+
await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) => log.info("initializeExtendableMediaRecorder", {
|
|
735
87
|
err
|
|
736
|
-
}, {
|
|
737
|
-
F: __dxlog_file4,
|
|
738
|
-
L: 69,
|
|
739
|
-
S: this,
|
|
740
|
-
C: (f, a) => f(...a)
|
|
741
|
-
}));
|
|
88
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 48, S: this }));
|
|
742
89
|
if (!this._mediaRecorder) {
|
|
743
90
|
const stream = new MediaStream([
|
|
744
91
|
this._mediaStreamTrack
|
|
@@ -750,15 +97,7 @@ var MediaStreamRecorder = class {
|
|
|
750
97
|
if (this._mediaRecorder.state === "recording") {
|
|
751
98
|
return;
|
|
752
99
|
}
|
|
753
|
-
|
|
754
|
-
F: __dxlog_file4,
|
|
755
|
-
L: 80,
|
|
756
|
-
S: this,
|
|
757
|
-
A: [
|
|
758
|
-
"this._onChunk",
|
|
759
|
-
"'MediaStreamRecorder: onChunk is not set'"
|
|
760
|
-
]
|
|
761
|
-
});
|
|
100
|
+
invariant(this._onChunk, "MediaStreamRecorder: onChunk is not set", { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 62, S: this, A: ["this._onChunk", "'MediaStreamRecorder: onChunk is not set'"] });
|
|
762
101
|
this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);
|
|
763
102
|
this._mediaRecorder.start(this._config.interval);
|
|
764
103
|
}
|
|
@@ -785,12 +124,7 @@ var MediaStreamRecorder = class {
|
|
|
785
124
|
...uint8Array
|
|
786
125
|
]));
|
|
787
126
|
} else {
|
|
788
|
-
|
|
789
|
-
F: __dxlog_file4,
|
|
790
|
-
L: 111,
|
|
791
|
-
S: this,
|
|
792
|
-
C: (f, a) => f(...a)
|
|
793
|
-
});
|
|
127
|
+
log.warn("MediaStreamRecorder: no header", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file, L: 90, S: this });
|
|
794
128
|
return;
|
|
795
129
|
}
|
|
796
130
|
this._onChunk({
|
|
@@ -810,16 +144,16 @@ MediaStreamRecorder = _ts_decorate([
|
|
|
810
144
|
import { WaveFile as WaveFile2 } from "wavefile";
|
|
811
145
|
import { DeferredTask, Trigger } from "@dxos/async";
|
|
812
146
|
import { LifecycleState, Resource } from "@dxos/context";
|
|
813
|
-
import { log as
|
|
147
|
+
import { log as log2 } from "@dxos/log";
|
|
814
148
|
import { trace as trace2 } from "@dxos/tracing";
|
|
815
149
|
import { TRANSCRIPTION_URL } from "#types";
|
|
150
|
+
var __dxlog_file2 = "/__w/dxos/dxos/packages/plugins/plugin-transcription/src/transcriber/transcriber.ts";
|
|
816
151
|
function _ts_decorate2(decorators, target, key, desc) {
|
|
817
152
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
818
153
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
819
154
|
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
820
155
|
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
821
156
|
}
|
|
822
|
-
var __dxlog_file5 = "/__w/dxos/dxos/packages/plugins/plugin-transcription/src/transcriber/transcriber.ts";
|
|
823
157
|
var Transcriber = class extends Resource {
|
|
824
158
|
_audioChunks = [];
|
|
825
159
|
_lastUsedTimestamp = 0;
|
|
@@ -829,44 +163,31 @@ var Transcriber = class extends Resource {
|
|
|
829
163
|
_config;
|
|
830
164
|
_recorder;
|
|
831
165
|
_onSegments;
|
|
166
|
+
_transcribeFn;
|
|
832
167
|
_recording = false;
|
|
833
168
|
_transcribeTask = void 0;
|
|
834
|
-
constructor({ config, recorder, onSegments }) {
|
|
169
|
+
constructor({ config, recorder, onSegments, transcribe }) {
|
|
835
170
|
super();
|
|
836
171
|
this._config = config;
|
|
837
172
|
this._recorder = recorder;
|
|
838
173
|
this._onSegments = onSegments;
|
|
174
|
+
this._transcribeFn = transcribe;
|
|
839
175
|
}
|
|
840
176
|
async _open(ctx) {
|
|
841
|
-
|
|
842
|
-
F: __dxlog_file5,
|
|
843
|
-
L: 102,
|
|
844
|
-
S: this,
|
|
845
|
-
C: (f, a) => f(...a)
|
|
846
|
-
});
|
|
177
|
+
log2.info("opening", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 41, S: this });
|
|
847
178
|
this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));
|
|
848
179
|
await this._recorder.start();
|
|
849
180
|
this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());
|
|
850
181
|
this._openTrigger.wake();
|
|
851
182
|
}
|
|
852
183
|
async _close() {
|
|
853
|
-
|
|
854
|
-
F: __dxlog_file5,
|
|
855
|
-
L: 110,
|
|
856
|
-
S: this,
|
|
857
|
-
C: (f, a) => f(...a)
|
|
858
|
-
});
|
|
184
|
+
log2.info("closing", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 48, S: this });
|
|
859
185
|
this._recording = false;
|
|
860
186
|
this._transcribeTask = void 0;
|
|
861
187
|
await this._recorder.stop();
|
|
862
188
|
}
|
|
863
189
|
startChunksRecording() {
|
|
864
|
-
|
|
865
|
-
F: __dxlog_file5,
|
|
866
|
-
L: 117,
|
|
867
|
-
S: this,
|
|
868
|
-
C: (f, a) => f(...a)
|
|
869
|
-
});
|
|
190
|
+
log2.info("starting", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 54, S: this });
|
|
870
191
|
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
871
192
|
return;
|
|
872
193
|
}
|
|
@@ -878,22 +199,12 @@ var Transcriber = class extends Resource {
|
|
|
878
199
|
}
|
|
879
200
|
this._recording = false;
|
|
880
201
|
this._transcribeTask?.schedule();
|
|
881
|
-
|
|
882
|
-
F: __dxlog_file5,
|
|
883
|
-
L: 132,
|
|
884
|
-
S: this,
|
|
885
|
-
C: (f, a) => f(...a)
|
|
886
|
-
});
|
|
202
|
+
log2.info("stopped", void 0, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 66, S: this });
|
|
887
203
|
}
|
|
888
204
|
_saveAudioChunk(chunk) {
|
|
889
|
-
|
|
205
|
+
log2("saving audio chunk", {
|
|
890
206
|
chunk
|
|
891
|
-
}, {
|
|
892
|
-
F: __dxlog_file5,
|
|
893
|
-
L: 136,
|
|
894
|
-
S: this,
|
|
895
|
-
C: (f, a) => f(...a)
|
|
896
|
-
});
|
|
207
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 69, S: this });
|
|
897
208
|
this._audioChunks.push(chunk);
|
|
898
209
|
if (!this._recording && (!this._transcribeTask || !this._transcribeTask.scheduled) && this._audioChunks.length >= this._config.prefixBufferChunksAmount) {
|
|
899
210
|
this._audioChunks = this._dropOldChunks();
|
|
@@ -902,14 +213,9 @@ var Transcriber = class extends Resource {
|
|
|
902
213
|
}
|
|
903
214
|
}
|
|
904
215
|
async _transcribe() {
|
|
905
|
-
|
|
216
|
+
log2("transcribing", {
|
|
906
217
|
chunks: this._audioChunks.length
|
|
907
|
-
}, {
|
|
908
|
-
F: __dxlog_file5,
|
|
909
|
-
L: 152,
|
|
910
|
-
S: this,
|
|
911
|
-
C: (f, a) => f(...a)
|
|
912
|
-
});
|
|
218
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 81, S: this });
|
|
913
219
|
const chunks = this._audioChunks;
|
|
914
220
|
this._audioChunks = this._dropOldChunks();
|
|
915
221
|
if (chunks.length === 0) {
|
|
@@ -936,54 +242,41 @@ var Transcriber = class extends Resource {
|
|
|
936
242
|
}
|
|
937
243
|
async _fetchTranscription(audio) {
|
|
938
244
|
if (audio.length === 0) {
|
|
939
|
-
this._audioChunks = [];
|
|
940
245
|
throw new Error("No audio to send for transcribing");
|
|
941
246
|
}
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
247
|
+
let segments;
|
|
248
|
+
if (this._transcribeFn) {
|
|
249
|
+
segments = await this._transcribeFn(audio);
|
|
250
|
+
} else {
|
|
251
|
+
const endpoint = this._config.endpoint ?? TRANSCRIPTION_URL;
|
|
252
|
+
const response = await fetch(`${endpoint}/transcribe`, {
|
|
253
|
+
method: "POST",
|
|
254
|
+
body: JSON.stringify({
|
|
255
|
+
audio
|
|
256
|
+
}),
|
|
257
|
+
headers: {
|
|
258
|
+
"Content-Type": "application/json"
|
|
259
|
+
}
|
|
260
|
+
});
|
|
261
|
+
if (!response.ok) {
|
|
262
|
+
throw new Error(`Transcription failed: ${response.statusText}`);
|
|
949
263
|
}
|
|
950
|
-
|
|
951
|
-
if (!response.ok) {
|
|
952
|
-
this._audioChunks = [];
|
|
953
|
-
throw new Error(`Transcription failed: ${response.statusText}`);
|
|
264
|
+
({ segments } = await response.json());
|
|
954
265
|
}
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
L: 216,
|
|
962
|
-
S: this,
|
|
963
|
-
C: (f, a) => f(...a)
|
|
964
|
-
});
|
|
266
|
+
if (!Array.isArray(segments)) {
|
|
267
|
+
throw new Error("Transcription response payload is invalid");
|
|
268
|
+
}
|
|
269
|
+
log2.info("transcription response", {
|
|
270
|
+
segments: segments.length
|
|
271
|
+
}, { "~LogMeta": "~LogMeta", F: __dxlog_file2, L: 134, S: this });
|
|
965
272
|
return segments;
|
|
966
273
|
}
|
|
967
274
|
_alignSegments(segments, originalChunks) {
|
|
968
|
-
const
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
return {
|
|
972
|
-
...segment,
|
|
973
|
-
words,
|
|
974
|
-
text: words.map((word) => word.word).join(""),
|
|
975
|
-
start: words.at(0)?.start ?? segment.end
|
|
976
|
-
};
|
|
977
|
-
}).filter((segment) => segment.words.length > 0);
|
|
978
|
-
if (filteredSegments.length === 0) {
|
|
979
|
-
return [];
|
|
275
|
+
const result = alignWhisperSegments(segments, originalChunks, this._lastUsedTimestamp);
|
|
276
|
+
if (result.lastUsedTimestamp !== void 0) {
|
|
277
|
+
this._lastUsedTimestamp = result.lastUsedTimestamp;
|
|
980
278
|
}
|
|
981
|
-
|
|
982
|
-
return filteredSegments.map((segment) => ({
|
|
983
|
-
_tag: "transcript",
|
|
984
|
-
started: new Date(zeroTimestamp + segment.start * 1e3).toISOString(),
|
|
985
|
-
text: segment.text.trim()
|
|
986
|
-
}));
|
|
279
|
+
return result.transcripts;
|
|
987
280
|
}
|
|
988
281
|
};
|
|
989
282
|
_ts_decorate2([
|
|
@@ -996,6 +289,36 @@ _ts_decorate2([
|
|
|
996
289
|
showInBrowserTimeline: true
|
|
997
290
|
})
|
|
998
291
|
], Transcriber.prototype, "_fetchTranscription", null);
|
|
292
|
+
var alignWhisperSegments = (segments, originalChunks, lastUsedTimestamp) => {
|
|
293
|
+
if (originalChunks.length === 0) {
|
|
294
|
+
return {
|
|
295
|
+
transcripts: []
|
|
296
|
+
};
|
|
297
|
+
}
|
|
298
|
+
const zeroTimestamp = originalChunks[0].timestamp;
|
|
299
|
+
const filteredSegments = segments.filter((segment) => zeroTimestamp + segment.end * 1e3 >= lastUsedTimestamp).map((segment) => {
|
|
300
|
+
const words = segment.words.filter((word) => zeroTimestamp + word.start * 1e3 >= lastUsedTimestamp);
|
|
301
|
+
return {
|
|
302
|
+
...segment,
|
|
303
|
+
words,
|
|
304
|
+
text: words.map((word) => word.word).join(""),
|
|
305
|
+
start: words.at(0)?.start ?? segment.end
|
|
306
|
+
};
|
|
307
|
+
}).filter((segment) => segment.words.length > 0);
|
|
308
|
+
if (filteredSegments.length === 0) {
|
|
309
|
+
return {
|
|
310
|
+
transcripts: []
|
|
311
|
+
};
|
|
312
|
+
}
|
|
313
|
+
return {
|
|
314
|
+
transcripts: filteredSegments.map((segment) => ({
|
|
315
|
+
_tag: "transcript",
|
|
316
|
+
started: new Date(zeroTimestamp + segment.start * 1e3).toISOString(),
|
|
317
|
+
text: segment.text.trim()
|
|
318
|
+
})),
|
|
319
|
+
lastUsedTimestamp: zeroTimestamp + filteredSegments.at(-1).end * 1e3
|
|
320
|
+
};
|
|
321
|
+
};
|
|
999
322
|
|
|
1000
323
|
// src/transcriber/transcription-manager.ts
|
|
1001
324
|
import { Atom } from "@effect-atom/atom-react";
|
|
@@ -1132,77 +455,16 @@ _ts_decorate3([
|
|
|
1132
455
|
synchronized2
|
|
1133
456
|
], TranscriptionManager.prototype, "setAudioTrack", null);
|
|
1134
457
|
|
|
1135
|
-
// src/
|
|
1136
|
-
|
|
1137
|
-
import { Plugin } from "@dxos/app-framework";
|
|
1138
|
-
import { AppActivationEvents, AppPlugin } from "@dxos/app-toolkit";
|
|
1139
|
-
import { Annotation, Obj as Obj2 } from "@dxos/echo";
|
|
1140
|
-
import { getSpace } from "@dxos/react-client/echo";
|
|
1141
|
-
import { Message as Message2, Transcript } from "@dxos/types";
|
|
1142
|
-
import { BlueprintDefinition, OperationHandler, ReactSurface, Transcriber as Transcriber2 } from "#capabilities";
|
|
1143
|
-
import { meta as meta3 } from "#meta";
|
|
1144
|
-
|
|
1145
|
-
// src/translations.ts
|
|
1146
|
-
import { meta as meta2 } from "#meta";
|
|
1147
|
-
var translations = [
|
|
1148
|
-
{
|
|
1149
|
-
"en-US": {
|
|
1150
|
-
[meta2.id]: {
|
|
1151
|
-
"plugin.name": "Transcription",
|
|
1152
|
-
"transcript-companion.label": "Transcript",
|
|
1153
|
-
"delete.button": "Delete",
|
|
1154
|
-
"bookmark.button": "Bookmark",
|
|
1155
|
-
"scroll-to-end.label": "Scroll to latest",
|
|
1156
|
-
"settings.entity-extraction.label": "Entity extraction"
|
|
1157
|
-
}
|
|
1158
|
-
}
|
|
1159
|
-
}
|
|
1160
|
-
];
|
|
1161
|
-
|
|
1162
|
-
// src/TranscriptionPlugin.tsx
|
|
1163
|
-
var TranscriptionPlugin = Plugin.define(meta3).pipe(AppPlugin.addBlueprintDefinitionModule({
|
|
1164
|
-
activate: BlueprintDefinition
|
|
1165
|
-
}), AppPlugin.addMetadataModule({
|
|
1166
|
-
metadata: {
|
|
1167
|
-
id: Transcript.Transcript.typename,
|
|
1168
|
-
metadata: {
|
|
1169
|
-
icon: Annotation.IconAnnotation.get(Transcript.Transcript).pipe(Option.getOrThrow).icon,
|
|
1170
|
-
iconHue: Annotation.IconAnnotation.get(Transcript.Transcript).pipe(Option.getOrThrow).hue ?? "white",
|
|
1171
|
-
// TODO(wittjosiah): Factor out. Artifact? Separate capability?
|
|
1172
|
-
getTextContent: async (transcript) => {
|
|
1173
|
-
const space = getSpace(transcript);
|
|
1174
|
-
const members = space?.members.get().map((member) => member.identity) ?? [];
|
|
1175
|
-
const queue = space?.queues.get(transcript.queue.dxn);
|
|
1176
|
-
await queue?.refresh();
|
|
1177
|
-
const content = queue?.objects.filter((message) => Obj2.instanceOf(Message2.Message, message)).flatMap((message, index) => renderByline(members)(message, index)).join("\n\n");
|
|
1178
|
-
return content;
|
|
1179
|
-
}
|
|
1180
|
-
}
|
|
1181
|
-
}
|
|
1182
|
-
}), AppPlugin.addOperationHandlerModule({
|
|
1183
|
-
activate: OperationHandler
|
|
1184
|
-
}), AppPlugin.addSchemaModule({
|
|
1185
|
-
schema: [
|
|
1186
|
-
Transcript.Transcript
|
|
1187
|
-
]
|
|
1188
|
-
}), AppPlugin.addSurfaceModule({
|
|
1189
|
-
activate: ReactSurface
|
|
1190
|
-
}), AppPlugin.addTranslationsModule({
|
|
1191
|
-
translations
|
|
1192
|
-
}), Plugin.addModule({
|
|
1193
|
-
id: "transcription",
|
|
1194
|
-
activatesOn: AppActivationEvents.SetupAppGraph,
|
|
1195
|
-
activate: Transcriber2
|
|
1196
|
-
}), Plugin.make);
|
|
458
|
+
// src/index.ts
|
|
459
|
+
var TranscriptionPlugin = Plugin.lazy(meta, () => import("#plugin"));
|
|
1197
460
|
export {
|
|
1198
461
|
MediaStreamRecorder,
|
|
1199
462
|
Transcriber,
|
|
1200
|
-
Transcription,
|
|
1201
463
|
TranscriptionCapabilities,
|
|
1202
464
|
TranscriptionManager,
|
|
1203
465
|
TranscriptionPlugin,
|
|
466
|
+
alignWhisperSegments,
|
|
1204
467
|
meta,
|
|
1205
|
-
renderByline,
|
|
1206
468
|
useAudioFile,
|
|
1207
469
|
useAudioTrack,
|
|
1208
470
|
useQueueModelAdapter,
|