@dxos/plugin-transcription 0.8.0 → 0.8.1-main.303c73a
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{TranscriptContainer-SX6PNFEC.mjs → TranscriptContainer-N4KQHMY4.mjs} +2 -2
- package/dist/lib/browser/{chunk-AS364WZZ.mjs → chunk-BDVUIRBP.mjs} +2 -2
- package/dist/lib/browser/chunk-GHTRJTEB.mjs +35 -0
- package/dist/lib/browser/chunk-GHTRJTEB.mjs.map +7 -0
- package/dist/lib/browser/{chunk-R4HHS6FA.mjs → chunk-YX447BJ4.mjs} +16 -16
- package/dist/lib/browser/chunk-YX447BJ4.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +180 -72
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/{intent-resolver-EAR2TFK2.mjs → intent-resolver-RW6DEJC2.mjs} +2 -2
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{react-surface-FOEWLLTZ.mjs → react-surface-UPTZBHVB.mjs} +3 -3
- package/dist/lib/node/{TranscriptContainer-HU763SN3.cjs → TranscriptContainer-4BWXOVML.cjs} +7 -7
- package/dist/lib/node/{TranscriptContainer-HU763SN3.cjs.map → TranscriptContainer-4BWXOVML.cjs.map} +2 -2
- package/dist/lib/node/{chunk-YV74Z74I.cjs → chunk-ANSWGFMS.cjs} +18 -18
- package/dist/lib/node/chunk-ANSWGFMS.cjs.map +7 -0
- package/dist/lib/node/{chunk-JBN3P6ZF.cjs → chunk-HDRXDMK2.cjs} +4 -46
- package/dist/lib/node/chunk-HDRXDMK2.cjs.map +7 -0
- package/dist/lib/node/{chunk-QWXLQPP5.cjs → chunk-I35KHI2G.cjs} +5 -5
- package/dist/lib/node/index.cjs +182 -74
- package/dist/lib/node/index.cjs.map +4 -4
- package/dist/lib/node/{intent-resolver-3KJZGGUQ.cjs → intent-resolver-KLBZ25XJ.cjs} +7 -7
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/{react-surface-AGGGDOUC.cjs → react-surface-NCKXB7EQ.cjs} +7 -7
- package/dist/lib/node-esm/{TranscriptContainer-DVBXQIQ4.mjs → TranscriptContainer-6IKRASR6.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-WEZGRUG6.mjs → chunk-2CPKXRN6.mjs} +2 -2
- package/dist/lib/node-esm/chunk-LORUIVS4.mjs +37 -0
- package/dist/lib/node-esm/chunk-LORUIVS4.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-36EQXBWR.mjs → chunk-ZMK2QLW2.mjs} +16 -16
- package/dist/lib/node-esm/chunk-ZMK2QLW2.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +180 -72
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/{intent-resolver-UGLYRD4X.mjs → intent-resolver-4FMISUPW.mjs} +2 -2
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{react-surface-6AYPLJ2B.mjs → react-surface-2WJQ67LB.mjs} +3 -3
- package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/components/transcription.stories.d.ts +2 -1
- package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
- package/dist/types/src/hooks/index.d.ts +0 -1
- package/dist/types/src/hooks/index.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +1 -0
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts +1 -0
- package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
- package/dist/types/src/transcription-manager.d.ts +36 -0
- package/dist/types/src/transcription-manager.d.ts.map +1 -0
- package/package.json +37 -37
- package/src/components/TranscriptContainer.tsx +11 -15
- package/src/hooks/index.ts +0 -1
- package/src/index.ts +1 -0
- package/src/transcriber/media-stream-recorder.ts +4 -0
- package/src/transcription-manager.ts +173 -0
- package/dist/lib/browser/chunk-R4HHS6FA.mjs.map +0 -7
- package/dist/lib/browser/chunk-ZMES3U2D.mjs +0 -78
- package/dist/lib/browser/chunk-ZMES3U2D.mjs.map +0 -7
- package/dist/lib/node/chunk-JBN3P6ZF.cjs.map +0 -7
- package/dist/lib/node/chunk-YV74Z74I.cjs.map +0 -7
- package/dist/lib/node-esm/chunk-36EQXBWR.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-XYU6B73V.mjs +0 -80
- package/dist/lib/node-esm/chunk-XYU6B73V.mjs.map +0 -7
- package/dist/types/src/hooks/useIsSpeaking.d.ts +0 -2
- package/dist/types/src/hooks/useIsSpeaking.d.ts.map +0 -1
- package/src/hooks/useIsSpeaking.ts +0 -77
- /package/dist/lib/browser/{TranscriptContainer-SX6PNFEC.mjs.map → TranscriptContainer-N4KQHMY4.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-AS364WZZ.mjs.map → chunk-BDVUIRBP.mjs.map} +0 -0
- /package/dist/lib/browser/{intent-resolver-EAR2TFK2.mjs.map → intent-resolver-RW6DEJC2.mjs.map} +0 -0
- /package/dist/lib/browser/{react-surface-FOEWLLTZ.mjs.map → react-surface-UPTZBHVB.mjs.map} +0 -0
- /package/dist/lib/node/{chunk-QWXLQPP5.cjs.map → chunk-I35KHI2G.cjs.map} +0 -0
- /package/dist/lib/node/{intent-resolver-3KJZGGUQ.cjs.map → intent-resolver-KLBZ25XJ.cjs.map} +0 -0
- /package/dist/lib/node/{react-surface-AGGGDOUC.cjs.map → react-surface-NCKXB7EQ.cjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-DVBXQIQ4.mjs.map → TranscriptContainer-6IKRASR6.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-WEZGRUG6.mjs.map → chunk-2CPKXRN6.mjs.map} +0 -0
- /package/dist/lib/node-esm/{intent-resolver-UGLYRD4X.mjs.map → intent-resolver-4FMISUPW.mjs.map} +0 -0
- /package/dist/lib/node-esm/{react-surface-6AYPLJ2B.mjs.map → react-surface-2WJQ67LB.mjs.map} +0 -0
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Resource } from '@dxos/context';
|
|
2
|
+
import { type Queue } from '@dxos/echo-db';
|
|
3
|
+
import { type EdgeHttpClient } from '@dxos/react-edge-client';
|
|
4
|
+
import { TranscriptBlock } from './types';
|
|
5
|
+
/**
|
|
6
|
+
* Manages transcription state.
|
|
7
|
+
*/
|
|
8
|
+
export declare class TranscriptionManager extends Resource {
|
|
9
|
+
private readonly _edgeClient;
|
|
10
|
+
private _audioStreamTrack?;
|
|
11
|
+
private _name?;
|
|
12
|
+
private _mediaRecorder?;
|
|
13
|
+
private _transcriber?;
|
|
14
|
+
private _enabled;
|
|
15
|
+
private _queue?;
|
|
16
|
+
constructor(edgeClient: EdgeHttpClient);
|
|
17
|
+
protected _open(): Promise<void>;
|
|
18
|
+
protected _close(): Promise<void>;
|
|
19
|
+
setRecording(recording?: boolean): void;
|
|
20
|
+
/**
|
|
21
|
+
* Enable or disable transcription.
|
|
22
|
+
* @param enabled - Whether to enable transcription.
|
|
23
|
+
*/
|
|
24
|
+
setEnabled(enabled?: boolean): Promise<void>;
|
|
25
|
+
setAudioTrack(track?: MediaStreamTrack): Promise<void>;
|
|
26
|
+
/**
|
|
27
|
+
* Set the queue to save the transcription to.
|
|
28
|
+
* @param queue - The queue to save the transcription to or the DXN of the queue.
|
|
29
|
+
*/
|
|
30
|
+
setQueue(queue?: Queue<TranscriptBlock> | string): void;
|
|
31
|
+
setName(name: string): void;
|
|
32
|
+
private _toggleTranscriber;
|
|
33
|
+
private _maybeReinitTranscriber;
|
|
34
|
+
private _onSegments;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=transcription-manager.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"transcription-manager.d.ts","sourceRoot":"","sources":["../../../src/transcription-manager.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAa,KAAK,KAAK,EAAE,MAAM,eAAe,CAAC;AAItD,OAAO,EAAE,KAAK,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAG9D,OAAO,EAAE,eAAe,EAA0B,MAAM,SAAS,CAAC;AAkBlE;;GAEG;AAEH,qBAAa,oBAAqB,SAAQ,QAAQ;IAChD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAiB;IAC7C,OAAO,CAAC,iBAAiB,CAAC,CAA+B;IACzD,OAAO,CAAC,KAAK,CAAC,CAAqB;IACnC,OAAO,CAAC,cAAc,CAAC,CAAkC;IACzD,OAAO,CAAC,YAAY,CAAC,CAA0B;IAC/C,OAAO,CAAC,QAAQ,CAAS;IACzB,OAAO,CAAC,MAAM,CAAC,CAAqC;gBAExC,UAAU,EAAE,cAAc;cAKb,KAAK;cAIL,MAAM;IAK/B,YAAY,CAAC,SAAS,CAAC,EAAE,OAAO;IAYhC;;;OAGG;IAEG,UAAU,CAAC,OAAO,CAAC,EAAE,OAAO;IAS5B,aAAa,CAAC,KAAK,CAAC,EAAE,gBAAgB;IAQ5C;;;OAGG;IAEH,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC,eAAe,CAAC,GAAG,MAAM;IAsBhD,OAAO,CAAC,IAAI,EAAE,MAAM;YAON,kBAAkB;YAWlB,uBAAuB;YA2BvB,WAAW;CAQ1B"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dxos/plugin-transcription",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.1-main.303c73a",
|
|
4
4
|
"description": "Voice transcription.",
|
|
5
5
|
"homepage": "https://dxos.org",
|
|
6
6
|
"bugs": "https://github.com/dxos/dxos/issues",
|
|
@@ -34,58 +34,58 @@
|
|
|
34
34
|
"dependencies": {
|
|
35
35
|
"@preact/signals-core": "^1.6.0",
|
|
36
36
|
"date-fns": "^3.3.1",
|
|
37
|
-
"effect": "
|
|
37
|
+
"effect": "3.12.12",
|
|
38
38
|
"extendable-media-recorder": "^9.2.19",
|
|
39
39
|
"extendable-media-recorder-wav-encoder": "^7.0.123",
|
|
40
40
|
"react-resize-detector": "^12.0.2",
|
|
41
41
|
"react-router-dom": "^6.4.0",
|
|
42
42
|
"react-use": "^17.5.1",
|
|
43
43
|
"wavefile": "^11.0.0",
|
|
44
|
-
"@dxos/app-framework": "0.8.
|
|
45
|
-
"@dxos/assistant": "0.8.
|
|
46
|
-
"@dxos/artifact": "0.8.
|
|
47
|
-
"@dxos/async": "0.8.
|
|
48
|
-
"@dxos/client": "0.8.
|
|
49
|
-
"@dxos/
|
|
50
|
-
"@dxos/
|
|
51
|
-
"@dxos/
|
|
52
|
-
"@dxos/
|
|
53
|
-
"@dxos/
|
|
54
|
-
"@dxos/
|
|
55
|
-
"@dxos/
|
|
56
|
-
"@dxos/keys": "0.8.
|
|
57
|
-
"@dxos/
|
|
58
|
-
"@dxos/
|
|
59
|
-
"@dxos/
|
|
60
|
-
"@dxos/plugin-graph": "0.8.
|
|
61
|
-
"@dxos/plugin-markdown": "0.8.
|
|
62
|
-
"@dxos/
|
|
63
|
-
"@dxos/
|
|
64
|
-
"@dxos/
|
|
65
|
-
"@dxos/react-
|
|
66
|
-
"@dxos/react-
|
|
67
|
-
"@dxos/react-ui-sfx": "0.8.
|
|
68
|
-
"@dxos/
|
|
69
|
-
"@dxos/
|
|
70
|
-
"@dxos/tracing": "0.8.
|
|
71
|
-
"@dxos/util": "0.8.
|
|
44
|
+
"@dxos/app-framework": "0.8.1-main.303c73a",
|
|
45
|
+
"@dxos/assistant": "0.8.1-main.303c73a",
|
|
46
|
+
"@dxos/artifact": "0.8.1-main.303c73a",
|
|
47
|
+
"@dxos/async": "0.8.1-main.303c73a",
|
|
48
|
+
"@dxos/client": "0.8.1-main.303c73a",
|
|
49
|
+
"@dxos/codec-protobuf": "0.8.1-main.303c73a",
|
|
50
|
+
"@dxos/context": "0.8.1-main.303c73a",
|
|
51
|
+
"@dxos/display-name": "0.8.1-main.303c73a",
|
|
52
|
+
"@dxos/echo-db": "0.8.1-main.303c73a",
|
|
53
|
+
"@dxos/echo-schema": "0.8.1-main.303c73a",
|
|
54
|
+
"@dxos/debug": "0.8.1-main.303c73a",
|
|
55
|
+
"@dxos/invariant": "0.8.1-main.303c73a",
|
|
56
|
+
"@dxos/keys": "0.8.1-main.303c73a",
|
|
57
|
+
"@dxos/log": "0.8.1-main.303c73a",
|
|
58
|
+
"@dxos/plugin-client": "0.8.1-main.303c73a",
|
|
59
|
+
"@dxos/live-object": "0.8.1-main.303c73a",
|
|
60
|
+
"@dxos/plugin-graph": "0.8.1-main.303c73a",
|
|
61
|
+
"@dxos/plugin-markdown": "0.8.1-main.303c73a",
|
|
62
|
+
"@dxos/plugin-space": "0.8.1-main.303c73a",
|
|
63
|
+
"@dxos/protocols": "0.8.1-main.303c73a",
|
|
64
|
+
"@dxos/react-client": "0.8.1-main.303c73a",
|
|
65
|
+
"@dxos/react-edge-client": "0.8.1-main.303c73a",
|
|
66
|
+
"@dxos/react-ui-components": "0.8.1-main.303c73a",
|
|
67
|
+
"@dxos/react-ui-sfx": "0.8.1-main.303c73a",
|
|
68
|
+
"@dxos/react-ui-stack": "0.8.1-main.303c73a",
|
|
69
|
+
"@dxos/schema": "0.8.1-main.303c73a",
|
|
70
|
+
"@dxos/tracing": "0.8.1-main.303c73a",
|
|
71
|
+
"@dxos/util": "0.8.1-main.303c73a"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@types/react": "~18.2.0",
|
|
75
75
|
"react": "~18.2.0",
|
|
76
76
|
"react-dom": "~18.2.0",
|
|
77
77
|
"tailwindcss": "~3.4.1",
|
|
78
|
-
"@dxos/random": "0.8.
|
|
79
|
-
"@dxos/react-ui": "0.8.
|
|
80
|
-
"@dxos/react-ui-theme": "0.8.
|
|
81
|
-
"@dxos/storybook-utils": "0.8.
|
|
82
|
-
"@dxos/test-utils": "0.8.
|
|
78
|
+
"@dxos/random": "0.8.1-main.303c73a",
|
|
79
|
+
"@dxos/react-ui": "0.8.1-main.303c73a",
|
|
80
|
+
"@dxos/react-ui-theme": "0.8.1-main.303c73a",
|
|
81
|
+
"@dxos/storybook-utils": "0.8.1-main.303c73a",
|
|
82
|
+
"@dxos/test-utils": "0.8.1-main.303c73a"
|
|
83
83
|
},
|
|
84
84
|
"optionalDependencies": {
|
|
85
85
|
"react": "~18.2.0",
|
|
86
86
|
"react-dom": "~18.2.0",
|
|
87
|
-
"@dxos/react-ui": "0.8.
|
|
88
|
-
"@dxos/react-ui-theme": "0.8.
|
|
87
|
+
"@dxos/react-ui": "0.8.1-main.303c73a",
|
|
88
|
+
"@dxos/react-ui-theme": "0.8.1-main.303c73a"
|
|
89
89
|
},
|
|
90
90
|
"publishConfig": {
|
|
91
91
|
"access": "public"
|
|
@@ -7,7 +7,7 @@ import React, { useCallback, useMemo, useState, type FC } from 'react';
|
|
|
7
7
|
|
|
8
8
|
import { chain, createIntent, LayoutAction, useIntentDispatcher } from '@dxos/app-framework';
|
|
9
9
|
import { Message } from '@dxos/artifact';
|
|
10
|
-
import {
|
|
10
|
+
import { AIServiceEdgeClient, DEFAULT_LLM_MODEL, MixedStreamParser, type AIServiceClient } from '@dxos/assistant';
|
|
11
11
|
import { create, getSpace, makeRef } from '@dxos/client/echo';
|
|
12
12
|
import { QueueImpl } from '@dxos/echo-db';
|
|
13
13
|
import { createStatic, isInstanceOf } from '@dxos/echo-schema';
|
|
@@ -30,7 +30,7 @@ import { TranscriptBlock, type TranscriptType } from '../types';
|
|
|
30
30
|
export const TranscriptionContainer: FC<{ transcript: TranscriptType }> = ({ transcript }) => {
|
|
31
31
|
const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
|
|
32
32
|
const edge = useEdgeClient();
|
|
33
|
-
const
|
|
33
|
+
const ai = useAiServiceClient();
|
|
34
34
|
const { dispatchPromise: dispatch } = useIntentDispatcher();
|
|
35
35
|
|
|
36
36
|
const queue = useQueue<TranscriptBlock>(edge, transcript.queue ? DXN.parse(transcript.queue) : undefined, {
|
|
@@ -42,7 +42,7 @@ export const TranscriptionContainer: FC<{ transcript: TranscriptType }> = ({ tra
|
|
|
42
42
|
const handleSummarize = useCallback(async () => {
|
|
43
43
|
setIsSummarizing(true);
|
|
44
44
|
try {
|
|
45
|
-
const document = await summarizeTranscript(edge,
|
|
45
|
+
const document = await summarizeTranscript(edge, ai, transcript);
|
|
46
46
|
const space = getSpace(transcript);
|
|
47
47
|
const target = space?.properties[CollectionType.typename]?.target;
|
|
48
48
|
await dispatch(
|
|
@@ -82,14 +82,10 @@ export const TranscriptionContainer: FC<{ transcript: TranscriptType }> = ({ tra
|
|
|
82
82
|
export default TranscriptionContainer;
|
|
83
83
|
|
|
84
84
|
// TODO(dmaretskyi): Extract to the new file once transcript refactoring has settled.
|
|
85
|
-
const summarizeTranscript = async (
|
|
86
|
-
edgeClient: EdgeHttpClient,
|
|
87
|
-
aiService: AIServiceClient,
|
|
88
|
-
transcript: TranscriptType,
|
|
89
|
-
) => {
|
|
85
|
+
const summarizeTranscript = async (edge: EdgeHttpClient, ai: AIServiceClient, transcript: TranscriptType) => {
|
|
90
86
|
invariant(transcript.queue, 'No queue found for transcript');
|
|
91
87
|
|
|
92
|
-
const queue = new QueueImpl(
|
|
88
|
+
const queue = new QueueImpl(edge, DXN.parse(transcript.queue));
|
|
93
89
|
await queue.refresh();
|
|
94
90
|
const content = queue.items
|
|
95
91
|
.filter((block) => isInstanceOf(TranscriptBlock, block))
|
|
@@ -100,7 +96,7 @@ const summarizeTranscript = async (
|
|
|
100
96
|
|
|
101
97
|
log.info('summarizing transcript', { blockCount: queue.items.length });
|
|
102
98
|
const output = await parser.parse(
|
|
103
|
-
await
|
|
99
|
+
await ai.exec({
|
|
104
100
|
model: DEFAULT_LLM_MODEL,
|
|
105
101
|
systemPrompt: SUMMARIZE_PROMPT,
|
|
106
102
|
history: [createStatic(Message, { role: 'user', content: [{ type: 'text', text: content }] })],
|
|
@@ -145,10 +141,10 @@ const SUMMARIZE_PROMPT = `
|
|
|
145
141
|
# Tasks
|
|
146
142
|
At the end of the summary include tasks.
|
|
147
143
|
Extract only the tasks that are:
|
|
148
|
-
- Directly actionable
|
|
149
|
-
- Clearly assigned to a person or team (or can easily be inferred)
|
|
150
|
-
- Strongly implied by the conversation and/or user note (no speculative tasks)
|
|
151
|
-
- Specific enough that someone reading them would know exactly what to do next
|
|
144
|
+
- Directly actionable.
|
|
145
|
+
- Clearly assigned to a person or team (or can easily be inferred).
|
|
146
|
+
- Strongly implied by the conversation and/or user note (no speculative tasks).
|
|
147
|
+
- Specific enough that someone reading them would know exactly what to do next.
|
|
152
148
|
|
|
153
149
|
Format all tasks as markdown checkboxes using the syntax:
|
|
154
150
|
- [ ] Task description
|
|
@@ -163,5 +159,5 @@ const SUMMARIZE_PROMPT = `
|
|
|
163
159
|
const useAiServiceClient = (): AIServiceClient => {
|
|
164
160
|
const config = useConfig();
|
|
165
161
|
const endpoint = config.values.runtime?.services?.ai?.server ?? 'http://localhost:8788'; // TOOD(burdon): Standardize consts.
|
|
166
|
-
return useMemo(() => new
|
|
162
|
+
return useMemo(() => new AIServiceEdgeClient({ endpoint }), [endpoint]);
|
|
167
163
|
};
|
package/src/hooks/index.ts
CHANGED
package/src/index.ts
CHANGED
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import { synchronized } from '@dxos/async';
|
|
6
|
+
import { Resource } from '@dxos/context';
|
|
7
|
+
import { QueueImpl, type Queue } from '@dxos/echo-db';
|
|
8
|
+
import { createStatic } from '@dxos/echo-schema';
|
|
9
|
+
import { invariant } from '@dxos/invariant';
|
|
10
|
+
import { DXN } from '@dxos/keys';
|
|
11
|
+
import { type EdgeHttpClient } from '@dxos/react-edge-client';
|
|
12
|
+
|
|
13
|
+
import { MediaStreamRecorder, Transcriber } from './transcriber';
|
|
14
|
+
import { TranscriptBlock, type TranscriptSegment } from './types';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Length of the chunk in ms.
|
|
18
|
+
*/
|
|
19
|
+
const RECORD_INTERVAL = 200;
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Number of chunks to save before the user starts speaking.
|
|
23
|
+
*/
|
|
24
|
+
const PREFIXED_CHUNKS_AMOUNT = 10;
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Number of chunks to transcribe automatically after.
|
|
28
|
+
* Combined should be mess than 25MB or whisper would fail.
|
|
29
|
+
*/
|
|
30
|
+
const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Manages transcription state.
|
|
34
|
+
*/
|
|
35
|
+
// TODO(mykola): Reconcile with useTranscriber hook.
|
|
36
|
+
export class TranscriptionManager extends Resource {
|
|
37
|
+
private readonly _edgeClient: EdgeHttpClient;
|
|
38
|
+
private _audioStreamTrack?: MediaStreamTrack = undefined;
|
|
39
|
+
private _name?: string = undefined;
|
|
40
|
+
private _mediaRecorder?: MediaStreamRecorder = undefined;
|
|
41
|
+
private _transcriber?: Transcriber = undefined;
|
|
42
|
+
private _enabled = false;
|
|
43
|
+
private _queue?: Queue<TranscriptBlock> = undefined;
|
|
44
|
+
|
|
45
|
+
constructor(edgeClient: EdgeHttpClient) {
|
|
46
|
+
super();
|
|
47
|
+
this._edgeClient = edgeClient;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
protected override async _open() {
|
|
51
|
+
await this._toggleTranscriber();
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
protected override async _close() {
|
|
55
|
+
void this._transcriber?.close();
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
@synchronized
|
|
59
|
+
setRecording(recording?: boolean) {
|
|
60
|
+
if (!this.isOpen || !this._enabled) {
|
|
61
|
+
return;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
if (recording) {
|
|
65
|
+
this._transcriber?.startChunksRecording();
|
|
66
|
+
} else {
|
|
67
|
+
this._transcriber?.stopChunksRecording();
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Enable or disable transcription.
|
|
73
|
+
* @param enabled - Whether to enable transcription.
|
|
74
|
+
*/
|
|
75
|
+
@synchronized
|
|
76
|
+
async setEnabled(enabled?: boolean) {
|
|
77
|
+
if (this._enabled === enabled) {
|
|
78
|
+
return;
|
|
79
|
+
}
|
|
80
|
+
this._enabled = enabled ?? false;
|
|
81
|
+
this.isOpen && (await this._toggleTranscriber());
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
@synchronized
|
|
85
|
+
async setAudioTrack(track?: MediaStreamTrack) {
|
|
86
|
+
if (this._audioStreamTrack === track) {
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
this._audioStreamTrack = track;
|
|
90
|
+
this.isOpen && (await this._toggleTranscriber());
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Set the queue to save the transcription to.
|
|
95
|
+
* @param queue - The queue to save the transcription to or the DXN of the queue.
|
|
96
|
+
*/
|
|
97
|
+
@synchronized
|
|
98
|
+
setQueue(queue?: Queue<TranscriptBlock> | string) {
|
|
99
|
+
switch (typeof queue) {
|
|
100
|
+
case 'string':
|
|
101
|
+
if (this._queue?.dxn.toString() === queue) {
|
|
102
|
+
return;
|
|
103
|
+
}
|
|
104
|
+
this._queue = new QueueImpl<TranscriptBlock>(this._edgeClient, DXN.parse(queue));
|
|
105
|
+
break;
|
|
106
|
+
case 'object':
|
|
107
|
+
if (this._queue === queue) {
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
invariant(queue instanceof QueueImpl);
|
|
111
|
+
this._queue = queue;
|
|
112
|
+
break;
|
|
113
|
+
case 'undefined':
|
|
114
|
+
this._queue = undefined;
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
@synchronized
|
|
120
|
+
setName(name: string) {
|
|
121
|
+
if (this._name === name) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
this._name = name;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
private async _toggleTranscriber() {
|
|
128
|
+
await this._maybeReinitTranscriber();
|
|
129
|
+
|
|
130
|
+
// Open or close transcriber if transcription is enabled or disabled.
|
|
131
|
+
if (this._enabled) {
|
|
132
|
+
await this._transcriber?.open();
|
|
133
|
+
} else {
|
|
134
|
+
await this._transcriber?.close();
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
private async _maybeReinitTranscriber() {
|
|
139
|
+
if (!this._audioStreamTrack || !this._enabled) {
|
|
140
|
+
return;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
// Reinitialize transcriber if queue or media stream track has changed.
|
|
144
|
+
let needReinit = false;
|
|
145
|
+
if (this._audioStreamTrack !== this._mediaRecorder?.mediaStreamTrack) {
|
|
146
|
+
this._mediaRecorder = new MediaStreamRecorder({
|
|
147
|
+
mediaStreamTrack: this._audioStreamTrack,
|
|
148
|
+
interval: RECORD_INTERVAL,
|
|
149
|
+
});
|
|
150
|
+
needReinit = true;
|
|
151
|
+
}
|
|
152
|
+
if (needReinit) {
|
|
153
|
+
await this._transcriber?.close();
|
|
154
|
+
this._transcriber = new Transcriber({
|
|
155
|
+
config: {
|
|
156
|
+
transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
|
|
157
|
+
prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
|
|
158
|
+
},
|
|
159
|
+
recorder: this._mediaRecorder,
|
|
160
|
+
onSegments: (segments) => this._onSegments(segments),
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
private async _onSegments(segments: TranscriptSegment[]) {
|
|
166
|
+
if (!this.isOpen || !this._queue) {
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
const block = createStatic(TranscriptBlock, { author: this._name, segments });
|
|
171
|
+
this._queue.append([block]);
|
|
172
|
+
}
|
|
173
|
+
}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/components/TranscriptContainer.tsx", "../../../src/components/Transcript/Transcript.tsx"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2024 DXOS.org\n//\n\nimport { pipe } from 'effect';\nimport React, { useCallback, useMemo, useState, type FC } from 'react';\n\nimport { chain, createIntent, LayoutAction, useIntentDispatcher } from '@dxos/app-framework';\nimport { Message } from '@dxos/artifact';\nimport { AIServiceClientImpl, DEFAULT_LLM_MODEL, MixedStreamParser, type AIServiceClient } from '@dxos/assistant';\nimport { create, getSpace, makeRef } from '@dxos/client/echo';\nimport { QueueImpl } from '@dxos/echo-db';\nimport { createStatic, isInstanceOf } from '@dxos/echo-schema';\nimport { invariant } from '@dxos/invariant';\nimport { DXN } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { DocumentType } from '@dxos/plugin-markdown/types';\nimport { CollectionType, SpaceAction } from '@dxos/plugin-space/types';\nimport { useConfig } from '@dxos/react-client';\nimport { useEdgeClient, useQueue, type EdgeHttpClient } from '@dxos/react-edge-client';\nimport { IconButton, Toolbar, useTranslation } from '@dxos/react-ui';\nimport { ScrollContainer } from '@dxos/react-ui-components';\nimport { StackItem } from '@dxos/react-ui-stack';\nimport { TextType } from '@dxos/schema';\n\nimport { Transcript } from './Transcript';\nimport { TRANSCRIPTION_PLUGIN } from '../meta';\nimport { TranscriptBlock, type TranscriptType } from '../types';\n\nexport const TranscriptionContainer: FC<{ transcript: TranscriptType }> = ({ transcript }) => {\n const { t } = useTranslation(TRANSCRIPTION_PLUGIN);\n const edge = useEdgeClient();\n const aiService = useAiServiceClient();\n const { dispatchPromise: dispatch } = useIntentDispatcher();\n\n const queue = useQueue<TranscriptBlock>(edge, transcript.queue ? DXN.parse(transcript.queue) : undefined, {\n pollInterval: 1_000,\n });\n\n // TODO(dmaretskyi): Pending state and errors should be handled by the framework!!!\n const [isSummarizing, setIsSummarizing] = useState(false);\n const handleSummarize = useCallback(async () => {\n setIsSummarizing(true);\n try {\n const document = await summarizeTranscript(edge, aiService, transcript);\n const space = getSpace(transcript);\n const target = space?.properties[CollectionType.typename]?.target;\n await dispatch(\n pipe(\n createIntent(SpaceAction.AddObject, { object: document, target }),\n chain(LayoutAction.Open, { part: 'main' }),\n ),\n );\n } finally {\n setIsSummarizing(false);\n }\n }, [transcript, edge]);\n\n // TODO(dmaretskyi): Move action to menu.\n return (\n <StackItem.Content toolbar={true}>\n <StackItem.Heading>\n <Toolbar.Root classNames='flex gap-2'>\n <IconButton\n icon='ph--subtitles--regular'\n iconOnly\n size={5}\n disabled={isSummarizing}\n onClick={handleSummarize}\n label={t('summary button')}\n />\n {isSummarizing && <div className='text-sm'>{t('summarizing label')}</div>}\n </Toolbar.Root>\n </StackItem.Heading>\n <ScrollContainer>\n <Transcript blocks={queue?.items} />\n </ScrollContainer>\n </StackItem.Content>\n );\n};\n\nexport default TranscriptionContainer;\n\n// TODO(dmaretskyi): Extract to the new file once transcript refactoring has settled.\nconst summarizeTranscript = async (\n edgeClient: EdgeHttpClient,\n aiService: AIServiceClient,\n transcript: TranscriptType,\n) => {\n invariant(transcript.queue, 'No queue found for transcript');\n\n const queue = new QueueImpl(edgeClient, DXN.parse(transcript.queue));\n await queue.refresh();\n const content = queue.items\n .filter((block) => isInstanceOf(TranscriptBlock, block))\n .map((block) => `${block.author}: ${block.segments.map((segment) => segment.text).join('\\n')}`)\n .join('\\n\\n');\n\n const parser = new MixedStreamParser();\n\n log.info('summarizing transcript', { blockCount: queue.items.length });\n const output = await parser.parse(\n await aiService.exec({\n model: DEFAULT_LLM_MODEL,\n systemPrompt: SUMMARIZE_PROMPT,\n history: [createStatic(Message, { role: 'user', content: [{ type: 'text', text: content }] })],\n }),\n );\n\n log.info('transcript summary', { output });\n invariant(output[0].content[0].type === 'text', 'Expected text content');\n const summary = output[0].content[0].text;\n\n // TODO(dmaretskyi): .started is missing.\n const name = `Summary ${(transcript.started ?? new Date()).toLocaleString('en-US', {\n year: 'numeric',\n month: 'long',\n day: 'numeric',\n hour: '2-digit',\n minute: '2-digit',\n })}`;\n\n return create(DocumentType, {\n name,\n content: makeRef(create(TextType, { content: summary })),\n threads: [],\n });\n};\n\n// TODO(dmaretskyi): Add example to set consistent structure for the summary.\nconst SUMMARIZE_PROMPT = `\n # Goal\n Create a markdown summary of the transcript provided.\n\n # Formatting\n - Format the summary as a markdown document without extra comments like \"Here is the summary of the transcript:\".\n - Use markdown formatting for headings and bullet points.\n - Format the summary as a list of key points and takeaways.\n - All names of people should be in bold.\n\n # Note Taking\n - Correlate items in the summary with the person of origin to build a coherent narrative.\n - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.\n\n # Tasks\n At the end of the summary include tasks.\n Extract only the tasks that are:\n - Directly actionable\n - Clearly assigned to a person or team (or can easily be inferred)\n - Strongly implied by the conversation and/or user note (no speculative tasks)\n - Specific enough that someone reading them would know exactly what to do next\n\n Format all tasks as markdown checkboxes using the syntax:\n - [ ] Task description\n\n Additional information can be included (indented).\n\n If no actionable tasks are found, omit this tasks section.\n`;\n\n// TODO(dmaretskyi): Extract?\n// Also this conflicts with plugin automation providing the ai client via a capability. Need to reconcile.\nconst useAiServiceClient = (): AIServiceClient => {\n const config = useConfig();\n const endpoint = config.values.runtime?.services?.ai?.server ?? 'http://localhost:8788'; // TOOD(burdon): Standardize consts.\n return useMemo(() => new AIServiceClientImpl({ endpoint }), [endpoint]);\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport { formatDistanceToNow } from 'date-fns/formatDistanceToNow';\nimport React, { type FC } from 'react';\n\nimport { Icon, IconButton, useTranslation } from '@dxos/react-ui';\n\nimport { TRANSCRIPTION_PLUGIN } from '../../meta';\nimport { type TranscriptBlock } from '../../types';\n\n// TODO(burdon): react-ui-list.\n// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).\n\nexport type TranscriptProps = {\n blocks?: TranscriptBlock[];\n};\n\nexport const Transcript: FC<TranscriptProps> = ({ blocks }) => {\n const { t } = useTranslation(TRANSCRIPTION_PLUGIN);\n const hoverButton = 'p-1 min-bs-1 transition-opacity duration-250 opacity-10 group-hover:opacity-100';\n\n return (\n <div role='list' className='flex flex-col grow gap-2 m-1'>\n {blocks?.map((block) => (\n <div\n role='listitem'\n key={block.id}\n className='flex flex-col py-1 border border-transparent hover:border-separator rounded'\n >\n <div className='group flex items-center px-2'>\n <Icon icon='ph--user--regular' />\n <div className='px-2 text-sm text-subdued'>{block.author}</div>\n <div className='grow' />\n <IconButton\n disabled\n icon='ph--x--regular'\n label={t('delete button')}\n iconOnly\n size={4}\n classNames={hoverButton}\n />\n </div>\n\n {block.segments.map((segment, i) => (\n <div key={i} className='group flex flex-col'>\n <div className='px-2'>{segment.text}</div>\n <div className='flex gap-1 items-center justify-between px-2'>\n <div className='grow' />\n <div className='truncate text-xs text-subdued'>\n {formatDistanceToNow(segment.started, { addSuffix: true })}\n </div>\n <IconButton\n disabled\n icon='ph--bookmark-simple--regular'\n label={t('bookmark button')}\n iconOnly\n size={4}\n classNames={hoverButton}\n />\n </div>\n </div>\n ))}\n </div>\n ))}\n </div>\n );\n};\n"],
|
|
5
|
-
"mappings": ";;;;;;AAIA,SAASA,YAAY;AACrB,OAAOC,UAASC,aAAaC,SAASC,gBAAyB;AAE/D,SAASC,OAAOC,cAAcC,cAAcC,2BAA2B;AACvE,SAASC,eAAe;AACxB,SAASC,qBAAqBC,mBAAmBC,yBAA+C;AAChG,SAASC,QAAQC,UAAUC,eAAe;AAC1C,SAASC,iBAAiB;AAC1B,SAASC,cAAcC,oBAAoB;AAC3C,SAASC,iBAAiB;AAC1B,SAASC,WAAW;AACpB,SAASC,WAAW;AACpB,SAASC,oBAAoB;AAC7B,SAASC,gBAAgBC,mBAAmB;AAC5C,SAASC,iBAAiB;AAC1B,SAASC,eAAeC,gBAAqC;AAC7D,SAASC,cAAAA,aAAYC,SAASC,kBAAAA,uBAAsB;AACpD,SAASC,uBAAuB;AAChC,SAASC,iBAAiB;AAC1B,SAASC,gBAAgB;;;ACnBzB,SAASC,2BAA2B;AACpC,OAAOC,WAAwB;AAE/B,SAASC,MAAMC,YAAYC,sBAAsB;AAY1C,IAAMC,aAAkC,CAAC,EAAEC,OAAM,MAAE;AACxD,QAAM,EAAEC,EAAC,IAAKC,eAAeC,oBAAAA;AAC7B,QAAMC,cAAc;AAEpB,SACE,sBAAA,cAACC,OAAAA;IAAIC,MAAK;IAAOC,WAAU;KACxBP,QAAQQ,IAAI,CAACC,UACZ,sBAAA,cAACJ,OAAAA;IACCC,MAAK;IACLI,KAAKD,MAAME;IACXJ,WAAU;KAEV,sBAAA,cAACF,OAAAA;IAAIE,WAAU;KACb,sBAAA,cAACK,MAAAA;IAAKC,MAAK;MACX,sBAAA,cAACR,OAAAA;IAAIE,WAAU;KAA6BE,MAAMK,MAAM,GACxD,sBAAA,cAACT,OAAAA;IAAIE,WAAU;MACf,sBAAA,cAACQ,YAAAA;IACCC,UAAAA;IACAH,MAAK;IACLI,OAAOhB,EAAE,eAAA;IACTiB,UAAAA;IACAC,MAAM;IACNC,YAAYhB;OAIfK,MAAMY,SAASb,IAAI,CAACc,SAASC,MAC5B,sBAAA,cAAClB,OAAAA;IAAIK,KAAKa;IAAGhB,WAAU;KACrB,sBAAA,cAACF,OAAAA;IAAIE,WAAU;KAAQe,QAAQE,IAAI,GACnC,sBAAA,cAACnB,OAAAA;IAAIE,WAAU;KACb,sBAAA,cAACF,OAAAA;IAAIE,WAAU;MACf,sBAAA,cAACF,OAAAA;IAAIE,WAAU;KACZkB,oBAAoBH,QAAQI,SAAS;IAAEC,WAAW;EAAK,CAAA,CAAA,GAE1D,sBAAA,cAACZ,YAAAA;IACCC,UAAAA;IACAH,MAAK;IACLI,OAAOhB,EAAE,iBAAA;IACTiB,UAAAA;IACAC,MAAM;IACNC,YAAYhB;;AAS9B;;;;ADvCO,IAAMwB,yBAA6D,CAAC,EAAEC,WAAU,MAAE;AACvF,QAAM,EAAEC,EAAC,IAAKC,gBAAeC,oBAAAA;AAC7B,QAAMC,OAAOC,cAAAA;AACb,QAAMC,YAAYC,mBAAAA;AAClB,QAAM,EAAEC,iBAAiBC,SAAQ,IAAKC,oBAAAA;AAEtC,QAAMC,QAAQC,SAA0BR,MAAMJ,WAAWW,QAAQE,IAAIC,MAAMd,WAAWW,KAAK,IAAII,QAAW;IACxGC,cAAc;EAChB,CAAA;AAGA,QAAM,CAACC,eAAeC,gBAAAA,IAAoBC,SAAS,KAAA;AACnD,QAAMC,kBAAkBC,YAAY,YAAA;AAClCH,qBAAiB,IAAA;AACjB,QAAI;AACF,YAAMI,WAAW,MAAMC,oBAAoBnB,MAAME,WAAWN,UAAAA;AAC5D,YAAMwB,QAAQC,SAASzB,UAAAA;AACvB,YAAM0B,SAASF,OAAOG,WAAWC,eAAeC,QAAQ,GAAGH;AAC3D,YAAMjB,SACJqB,KACEC,aAAaC,YAAYC,WAAW;QAAEC,QAAQZ;QAAUI;MAAO,CAAA,GAC/DS,MAAMC,aAAaC,MAAM;QAAEC,MAAM;MAAO,CAAA,CAAA,CAAA;IAG9C,UAAA;AACEpB,uBAAiB,KAAA;IACnB;EACF,GAAG;IAAClB;IAAYI;GAAK;AAGrB,SACE,gBAAAmC,OAAA,cAACC,UAAUC,SAAO;IAACC,SAAS;KAC1B,gBAAAH,OAAA,cAACC,UAAUG,SAAO,MAChB,gBAAAJ,OAAA,cAACK,QAAQC,MAAI;IAACC,YAAW;KACvB,gBAAAP,OAAA,cAACQ,aAAAA;IACCC,MAAK;IACLC,UAAAA;IACAC,MAAM;IACNC,UAAUlC;IACVmC,SAAShC;IACTiC,OAAOpD,EAAE,gBAAA;MAEVgB,iBAAiB,gBAAAsB,OAAA,cAACe,OAAAA;IAAIC,WAAU;KAAWtD,EAAE,mBAAA,CAAA,CAAA,CAAA,GAGlD,gBAAAsC,OAAA,cAACiB,iBAAAA,MACC,gBAAAjB,OAAA,cAACkB,YAAAA;IAAWC,QAAQ/C,OAAOgD;;AAInC;AAEA,IAAA,8BAAe5D;AAGf,IAAMwB,sBAAsB,OAC1BqC,YACAtD,WACAN,eAAAA;AAEA6D,YAAU7D,WAAWW,OAAO,iCAAA;;;;;;;;;AAE5B,QAAMA,QAAQ,IAAImD,UAAUF,YAAY/C,IAAIC,MAAMd,WAAWW,KAAK,CAAA;AAClE,QAAMA,MAAMoD,QAAO;AACnB,QAAMC,UAAUrD,MAAMgD,MACnBM,OAAO,CAACC,UAAUC,aAAaC,iBAAiBF,KAAAA,CAAAA,EAChDG,IAAI,CAACH,UAAU,GAAGA,MAAMI,MAAM,KAAKJ,MAAMK,SAASF,IAAI,CAACG,YAAYA,QAAQC,IAAI,EAAEC,KAAK,IAAA,CAAA,EAAO,EAC7FA,KAAK,MAAA;AAER,QAAMC,SAAS,IAAIC,kBAAAA;AAEnBC,MAAIC,KAAK,0BAA0B;IAAEC,YAAYpE,MAAMgD,MAAMqB;EAAO,GAAA;;;;;;AACpE,QAAMC,SAAS,MAAMN,OAAO7D,MAC1B,MAAMR,UAAU4E,KAAK;IACnBC,OAAOC;IACPC,cAAcC;IACdC,SAAS;MAACC,aAAaC,SAAS;QAAEC,MAAM;QAAQ1B,SAAS;UAAC;YAAE2B,MAAM;YAAQlB,MAAMT;UAAQ;;MAAG,CAAA;;EAC7F,CAAA,CAAA;AAGFa,MAAIC,KAAK,sBAAsB;IAAEG;EAAO,GAAA;;;;;;AACxCpB,YAAUoB,OAAO,CAAA,EAAGjB,QAAQ,CAAA,EAAG2B,SAAS,QAAQ,yBAAA;;;;;;;;;AAChD,QAAMC,UAAUX,OAAO,CAAA,EAAGjB,QAAQ,CAAA,EAAGS;AAGrC,QAAMoB,OAAO,YAAY7F,WAAW8F,WAAW,oBAAIC,KAAAA,GAAQC,eAAe,SAAS;IACjFC,MAAM;IACNC,OAAO;IACPC,KAAK;IACLC,MAAM;IACNC,QAAQ;EACV,CAAA,CAAA;AAEA,SAAOC,OAAOC,cAAc;IAC1BV;IACA7B,SAASwC,QAAQF,OAAOG,UAAU;MAAEzC,SAAS4B;IAAQ,CAAA,CAAA;IACrDc,SAAS,CAAA;EACX,CAAA;AACF;AAGA,IAAMpB,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCzB,IAAM/E,qBAAqB,MAAA;AACzB,QAAMoG,SAASC,UAAAA;AACf,QAAMC,WAAWF,OAAOG,OAAOC,SAASC,UAAUC,IAAIC,UAAU;AAChE,SAAOC,QAAQ,MAAM,IAAIC,oBAAoB;IAAEP;EAAS,CAAA,GAAI;IAACA;GAAS;AACxE;",
|
|
6
|
-
"names": ["pipe", "React", "useCallback", "useMemo", "useState", "chain", "createIntent", "LayoutAction", "useIntentDispatcher", "Message", "AIServiceClientImpl", "DEFAULT_LLM_MODEL", "MixedStreamParser", "create", "getSpace", "makeRef", "QueueImpl", "createStatic", "isInstanceOf", "invariant", "DXN", "log", "DocumentType", "CollectionType", "SpaceAction", "useConfig", "useEdgeClient", "useQueue", "IconButton", "Toolbar", "useTranslation", "ScrollContainer", "StackItem", "TextType", "formatDistanceToNow", "React", "Icon", "IconButton", "useTranslation", "Transcript", "blocks", "t", "useTranslation", "TRANSCRIPTION_PLUGIN", "hoverButton", "div", "role", "className", "map", "block", "key", "id", "Icon", "icon", "author", "IconButton", "disabled", "label", "iconOnly", "size", "classNames", "segments", "segment", "i", "text", "formatDistanceToNow", "started", "addSuffix", "TranscriptionContainer", "transcript", "t", "useTranslation", "TRANSCRIPTION_PLUGIN", "edge", "useEdgeClient", "aiService", "useAiServiceClient", "dispatchPromise", "dispatch", "useIntentDispatcher", "queue", "useQueue", "DXN", "parse", "undefined", "pollInterval", "isSummarizing", "setIsSummarizing", "useState", "handleSummarize", "useCallback", "document", "summarizeTranscript", "space", "getSpace", "target", "properties", "CollectionType", "typename", "pipe", "createIntent", "SpaceAction", "AddObject", "object", "chain", "LayoutAction", "Open", "part", "React", "StackItem", "Content", "toolbar", "Heading", "Toolbar", "Root", "classNames", "IconButton", "icon", "iconOnly", "size", "disabled", "onClick", "label", "div", "className", "ScrollContainer", "Transcript", "blocks", "items", "edgeClient", "invariant", "QueueImpl", "refresh", "content", "filter", "block", "isInstanceOf", "TranscriptBlock", "map", "author", "segments", "segment", "text", "join", "parser", "MixedStreamParser", "log", "info", "blockCount", "length", "output", "exec", "model", "DEFAULT_LLM_MODEL", "systemPrompt", "SUMMARIZE_PROMPT", "history", "createStatic", "Message", "role", "type", "summary", "name", "started", "Date", "toLocaleString", "year", "month", "day", "hour", "minute", "create", "DocumentType", "makeRef", "TextType", "threads", "config", "useConfig", "endpoint", "values", "runtime", "services", "ai", "server", "useMemo", "AIServiceClientImpl"]
|
|
7
|
-
}
|
|
@@ -1,78 +0,0 @@
|
|
|
1
|
-
// packages/plugins/experimental/plugin-transcription/src/util/get-time-str.ts
|
|
2
|
-
var getTimeStr = (timestamp) => new Date(timestamp ?? Date.now()).toLocaleTimeString("en-US", {
|
|
3
|
-
timeZone: "UTC",
|
|
4
|
-
hour12: true,
|
|
5
|
-
hour: "numeric",
|
|
6
|
-
minute: "2-digit",
|
|
7
|
-
second: "2-digit"
|
|
8
|
-
});
|
|
9
|
-
|
|
10
|
-
// packages/plugins/experimental/plugin-transcription/src/util/random-queue-dxn.ts
|
|
11
|
-
import { ObjectId } from "@dxos/echo-schema";
|
|
12
|
-
import { DXN, QueueSubspaceTags, SpaceId } from "@dxos/keys";
|
|
13
|
-
var randomQueueDxn = () => new DXN(DXN.kind.QUEUE, [
|
|
14
|
-
QueueSubspaceTags.DATA,
|
|
15
|
-
SpaceId.random(),
|
|
16
|
-
ObjectId.random()
|
|
17
|
-
]);
|
|
18
|
-
|
|
19
|
-
// packages/plugins/experimental/plugin-transcription/src/util/array.ts
|
|
20
|
-
var mergeFloat64Arrays = (arrays) => {
|
|
21
|
-
const newArray = new Float64Array(arrays.reduce((acc, array) => array.length + acc, 0));
|
|
22
|
-
let offset = 0;
|
|
23
|
-
for (const array of arrays) {
|
|
24
|
-
newArray.set(array, offset);
|
|
25
|
-
offset += array.length;
|
|
26
|
-
}
|
|
27
|
-
return newArray;
|
|
28
|
-
};
|
|
29
|
-
|
|
30
|
-
// packages/plugins/experimental/plugin-transcription/src/util/monitor-audio-level.ts
|
|
31
|
-
var monitorAudioLevel = ({ mediaStreamTrack, onMeasure, intervalMs = 1e3 / 10 }) => {
|
|
32
|
-
let timeout = -1;
|
|
33
|
-
let interval = -1;
|
|
34
|
-
const audioContext = new AudioContext();
|
|
35
|
-
const stream = new MediaStream();
|
|
36
|
-
stream.addTrack(mediaStreamTrack);
|
|
37
|
-
const mediaStreamAudioSourceNode = audioContext.createMediaStreamSource(stream);
|
|
38
|
-
const analyserNode = audioContext.createAnalyser();
|
|
39
|
-
mediaStreamAudioSourceNode.connect(analyserNode);
|
|
40
|
-
analyserNode.fftSize = 32;
|
|
41
|
-
const pcmData = new Float32Array(analyserNode.fftSize);
|
|
42
|
-
let peak = 0;
|
|
43
|
-
interval = window.setInterval(() => {
|
|
44
|
-
onMeasure(peak);
|
|
45
|
-
peak = 0;
|
|
46
|
-
}, intervalMs);
|
|
47
|
-
const tick = () => {
|
|
48
|
-
timeout = window.setTimeout(() => {
|
|
49
|
-
analyserNode.getFloatTimeDomainData(pcmData);
|
|
50
|
-
let sumSquares = 0;
|
|
51
|
-
for (const amplitude of pcmData) {
|
|
52
|
-
sumSquares += amplitude * amplitude;
|
|
53
|
-
}
|
|
54
|
-
const current = Math.sqrt(sumSquares / pcmData.length);
|
|
55
|
-
if (current > peak) {
|
|
56
|
-
peak = current;
|
|
57
|
-
}
|
|
58
|
-
tick();
|
|
59
|
-
}, intervalMs);
|
|
60
|
-
};
|
|
61
|
-
tick();
|
|
62
|
-
return () => {
|
|
63
|
-
mediaStreamAudioSourceNode.disconnect();
|
|
64
|
-
analyserNode.disconnect();
|
|
65
|
-
void audioContext.close();
|
|
66
|
-
clearInterval(interval);
|
|
67
|
-
clearTimeout(timeout);
|
|
68
|
-
stream.removeTrack(mediaStreamTrack);
|
|
69
|
-
};
|
|
70
|
-
};
|
|
71
|
-
|
|
72
|
-
export {
|
|
73
|
-
mergeFloat64Arrays,
|
|
74
|
-
getTimeStr,
|
|
75
|
-
monitorAudioLevel,
|
|
76
|
-
randomQueueDxn
|
|
77
|
-
};
|
|
78
|
-
//# sourceMappingURL=chunk-ZMES3U2D.mjs.map
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/util/get-time-str.ts", "../../../src/util/random-queue-dxn.ts", "../../../src/util/array.ts", "../../../src/util/monitor-audio-level.ts"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nexport const getTimeStr = (timestamp?: number) =>\n new Date(timestamp ?? Date.now()).toLocaleTimeString('en-US', {\n timeZone: 'UTC',\n hour12: true,\n hour: 'numeric',\n minute: '2-digit',\n second: '2-digit',\n });\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { ObjectId } from '@dxos/echo-schema';\nimport { DXN, QueueSubspaceTags, SpaceId } from '@dxos/keys';\n\nexport const randomQueueDxn = () =>\n new DXN(DXN.kind.QUEUE, [QueueSubspaceTags.DATA, SpaceId.random(), ObjectId.random()]);\n", "//\n// Copyright 2025 DXOS.org\n//\n\nexport const mergeFloat64Arrays = (arrays: Float64Array[]) => {\n const newArray = new Float64Array(arrays.reduce((acc, array) => array.length + acc, 0));\n let offset = 0;\n for (const array of arrays) {\n newArray.set(array, offset);\n offset += array.length;\n }\n\n return newArray;\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nexport const monitorAudioLevel = ({\n mediaStreamTrack,\n onMeasure,\n intervalMs = 1000 / 10, // 12 fps.\n}: {\n mediaStreamTrack: MediaStreamTrack;\n onMeasure: (value: number) => void;\n intervalMs?: number;\n}) => {\n let timeout = -1;\n let interval = -1;\n const audioContext = new AudioContext();\n const stream = new MediaStream();\n stream.addTrack(mediaStreamTrack);\n const mediaStreamAudioSourceNode = audioContext.createMediaStreamSource(stream);\n const analyserNode = audioContext.createAnalyser();\n mediaStreamAudioSourceNode.connect(analyserNode);\n // Since we just need a rough approximation and will be measuring\n // frequently, we want to drop this down from the default of 2048.\n analyserNode.fftSize = 32;\n\n const pcmData = new Float32Array(analyserNode.fftSize);\n let peak = 0;\n\n interval = window.setInterval(() => {\n onMeasure(peak);\n peak = 0;\n }, intervalMs);\n\n const tick = () => {\n timeout = window.setTimeout(() => {\n analyserNode.getFloatTimeDomainData(pcmData);\n let sumSquares = 0.0;\n for (const amplitude of pcmData) {\n sumSquares += amplitude * amplitude;\n }\n const current = Math.sqrt(sumSquares / pcmData.length);\n if (current > peak) {\n peak = current;\n }\n tick();\n }, intervalMs);\n };\n tick();\n\n return () => {\n mediaStreamAudioSourceNode.disconnect();\n analyserNode.disconnect();\n void audioContext.close();\n clearInterval(interval);\n clearTimeout(timeout);\n stream.removeTrack(mediaStreamTrack);\n };\n};\n"],
|
|
5
|
-
"mappings": ";AAIO,IAAMA,aAAa,CAACC,cACzB,IAAIC,KAAKD,aAAaC,KAAKC,IAAG,CAAA,EAAIC,mBAAmB,SAAS;EAC5DC,UAAU;EACVC,QAAQ;EACRC,MAAM;EACNC,QAAQ;EACRC,QAAQ;AACV,CAAA;;;ACPF,SAASC,gBAAgB;AACzB,SAASC,KAAKC,mBAAmBC,eAAe;AAEzC,IAAMC,iBAAiB,MAC5B,IAAIC,IAAIA,IAAIC,KAAKC,OAAO;EAACC,kBAAkBC;EAAMC,QAAQC,OAAM;EAAIC,SAASD,OAAM;CAAG;;;ACJhF,IAAME,qBAAqB,CAACC,WAAAA;AACjC,QAAMC,WAAW,IAAIC,aAAaF,OAAOG,OAAO,CAACC,KAAKC,UAAUA,MAAMC,SAASF,KAAK,CAAA,CAAA;AACpF,MAAIG,SAAS;AACb,aAAWF,SAASL,QAAQ;AAC1BC,aAASO,IAAIH,OAAOE,MAAAA;AACpBA,cAAUF,MAAMC;EAClB;AAEA,SAAOL;AACT;;;ACTO,IAAMQ,oBAAoB,CAAC,EAChCC,kBACAC,WACAC,aAAa,MAAO,GAAE,MAKvB;AACC,MAAIC,UAAU;AACd,MAAIC,WAAW;AACf,QAAMC,eAAe,IAAIC,aAAAA;AACzB,QAAMC,SAAS,IAAIC,YAAAA;AACnBD,SAAOE,SAAST,gBAAAA;AAChB,QAAMU,6BAA6BL,aAAaM,wBAAwBJ,MAAAA;AACxE,QAAMK,eAAeP,aAAaQ,eAAc;AAChDH,6BAA2BI,QAAQF,YAAAA;AAGnCA,eAAaG,UAAU;AAEvB,QAAMC,UAAU,IAAIC,aAAaL,aAAaG,OAAO;AACrD,MAAIG,OAAO;AAEXd,aAAWe,OAAOC,YAAY,MAAA;AAC5BnB,cAAUiB,IAAAA;AACVA,WAAO;EACT,GAAGhB,UAAAA;AAEH,QAAMmB,OAAO,MAAA;AACXlB,cAAUgB,OAAOG,WAAW,MAAA;AAC1BV,mBAAaW,uBAAuBP,OAAAA;AACpC,UAAIQ,aAAa;AACjB,iBAAWC,aAAaT,SAAS;AAC/BQ,sBAAcC,YAAYA;MAC5B;AACA,YAAMC,UAAUC,KAAKC,KAAKJ,aAAaR,QAAQa,MAAM;AACrD,UAAIH,UAAUR,MAAM;AAClBA,eAAOQ;MACT;AACAL,WAAAA;IACF,GAAGnB,UAAAA;EACL;AACAmB,OAAAA;AAEA,SAAO,MAAA;AACLX,+BAA2BoB,WAAU;AACrClB,iBAAakB,WAAU;AACvB,SAAKzB,aAAa0B,MAAK;AACvBC,kBAAc5B,QAAAA;AACd6B,iBAAa9B,OAAAA;AACbI,WAAO2B,YAAYlC,gBAAAA;EACrB;AACF;",
|
|
6
|
-
"names": ["getTimeStr", "timestamp", "Date", "now", "toLocaleTimeString", "timeZone", "hour12", "hour", "minute", "second", "ObjectId", "DXN", "QueueSubspaceTags", "SpaceId", "randomQueueDxn", "DXN", "kind", "QUEUE", "QueueSubspaceTags", "DATA", "SpaceId", "random", "ObjectId", "mergeFloat64Arrays", "arrays", "newArray", "Float64Array", "reduce", "acc", "array", "length", "offset", "set", "monitorAudioLevel", "mediaStreamTrack", "onMeasure", "intervalMs", "timeout", "interval", "audioContext", "AudioContext", "stream", "MediaStream", "addTrack", "mediaStreamAudioSourceNode", "createMediaStreamSource", "analyserNode", "createAnalyser", "connect", "fftSize", "pcmData", "Float32Array", "peak", "window", "setInterval", "tick", "setTimeout", "getFloatTimeDomainData", "sumSquares", "amplitude", "current", "Math", "sqrt", "length", "disconnect", "close", "clearInterval", "clearTimeout", "removeTrack"]
|
|
7
|
-
}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/util/get-time-str.ts", "../../../src/util/random-queue-dxn.ts", "../../../src/util/array.ts", "../../../src/util/monitor-audio-level.ts"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nexport const getTimeStr = (timestamp?: number) =>\n new Date(timestamp ?? Date.now()).toLocaleTimeString('en-US', {\n timeZone: 'UTC',\n hour12: true,\n hour: 'numeric',\n minute: '2-digit',\n second: '2-digit',\n });\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { ObjectId } from '@dxos/echo-schema';\nimport { DXN, QueueSubspaceTags, SpaceId } from '@dxos/keys';\n\nexport const randomQueueDxn = () =>\n new DXN(DXN.kind.QUEUE, [QueueSubspaceTags.DATA, SpaceId.random(), ObjectId.random()]);\n", "//\n// Copyright 2025 DXOS.org\n//\n\nexport const mergeFloat64Arrays = (arrays: Float64Array[]) => {\n const newArray = new Float64Array(arrays.reduce((acc, array) => array.length + acc, 0));\n let offset = 0;\n for (const array of arrays) {\n newArray.set(array, offset);\n offset += array.length;\n }\n\n return newArray;\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nexport const monitorAudioLevel = ({\n mediaStreamTrack,\n onMeasure,\n intervalMs = 1000 / 10, // 12 fps.\n}: {\n mediaStreamTrack: MediaStreamTrack;\n onMeasure: (value: number) => void;\n intervalMs?: number;\n}) => {\n let timeout = -1;\n let interval = -1;\n const audioContext = new AudioContext();\n const stream = new MediaStream();\n stream.addTrack(mediaStreamTrack);\n const mediaStreamAudioSourceNode = audioContext.createMediaStreamSource(stream);\n const analyserNode = audioContext.createAnalyser();\n mediaStreamAudioSourceNode.connect(analyserNode);\n // Since we just need a rough approximation and will be measuring\n // frequently, we want to drop this down from the default of 2048.\n analyserNode.fftSize = 32;\n\n const pcmData = new Float32Array(analyserNode.fftSize);\n let peak = 0;\n\n interval = window.setInterval(() => {\n onMeasure(peak);\n peak = 0;\n }, intervalMs);\n\n const tick = () => {\n timeout = window.setTimeout(() => {\n analyserNode.getFloatTimeDomainData(pcmData);\n let sumSquares = 0.0;\n for (const amplitude of pcmData) {\n sumSquares += amplitude * amplitude;\n }\n const current = Math.sqrt(sumSquares / pcmData.length);\n if (current > peak) {\n peak = current;\n }\n tick();\n }, intervalMs);\n };\n tick();\n\n return () => {\n mediaStreamAudioSourceNode.disconnect();\n analyserNode.disconnect();\n void audioContext.close();\n clearInterval(interval);\n clearTimeout(timeout);\n stream.removeTrack(mediaStreamTrack);\n };\n};\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;ACIA,yBAAyB;AACzB,kBAAgD;ADDzC,IAAMA,aAAa,CAACC,cACzB,IAAIC,KAAKD,aAAaC,KAAKC,IAAG,CAAA,EAAIC,mBAAmB,SAAS;EAC5DC,UAAU;EACVC,QAAQ;EACRC,MAAM;EACNC,QAAQ;EACRC,QAAQ;AACV,CAAA;ACJK,IAAMC,iBAAiB,MAC5B,IAAIC,gBAAIA,gBAAIC,KAAKC,OAAO;EAACC,8BAAkBC;EAAMC,oBAAQC,OAAM;EAAIC,4BAASD,OAAM;CAAG;ACJhF,IAAME,qBAAqB,CAACC,WAAAA;AACjC,QAAMC,WAAW,IAAIC,aAAaF,OAAOG,OAAO,CAACC,KAAKC,UAAUA,MAAMC,SAASF,KAAK,CAAA,CAAA;AACpF,MAAIG,SAAS;AACb,aAAWF,SAASL,QAAQ;AAC1BC,aAASO,IAAIH,OAAOE,MAAAA;AACpBA,cAAUF,MAAMC;EAClB;AAEA,SAAOL;AACT;ACTO,IAAMQ,oBAAoB,CAAC,EAChCC,kBACAC,WACAC,aAAa,MAAO,GAAE,MAKvB;AACC,MAAIC,UAAU;AACd,MAAIC,WAAW;AACf,QAAMC,eAAe,IAAIC,aAAAA;AACzB,QAAMC,SAAS,IAAIC,YAAAA;AACnBD,SAAOE,SAAST,gBAAAA;AAChB,QAAMU,6BAA6BL,aAAaM,wBAAwBJ,MAAAA;AACxE,QAAMK,eAAeP,aAAaQ,eAAc;AAChDH,6BAA2BI,QAAQF,YAAAA;AAGnCA,eAAaG,UAAU;AAEvB,QAAMC,UAAU,IAAIC,aAAaL,aAAaG,OAAO;AACrD,MAAIG,OAAO;AAEXd,aAAWe,OAAOC,YAAY,MAAA;AAC5BnB,cAAUiB,IAAAA;AACVA,WAAO;EACT,GAAGhB,UAAAA;AAEH,QAAMmB,OAAO,MAAA;AACXlB,cAAUgB,OAAOG,WAAW,MAAA;AAC1BV,mBAAaW,uBAAuBP,OAAAA;AACpC,UAAIQ,aAAa;AACjB,iBAAWC,aAAaT,SAAS;AAC/BQ,sBAAcC,YAAYA;MAC5B;AACA,YAAMC,UAAUC,KAAKC,KAAKJ,aAAaR,QAAQpB,MAAM;AACrD,UAAI8B,UAAUR,MAAM;AAClBA,eAAOQ;MACT;AACAL,WAAAA;IACF,GAAGnB,UAAAA;EACL;AACAmB,OAAAA;AAEA,SAAO,MAAA;AACLX,+BAA2BmB,WAAU;AACrCjB,iBAAaiB,WAAU;AACvB,SAAKxB,aAAayB,MAAK;AACvBC,kBAAc3B,QAAAA;AACd4B,iBAAa7B,OAAAA;AACbI,WAAO0B,YAAYjC,gBAAAA;EACrB;AACF;",
|
|
6
|
-
"names": ["getTimeStr", "timestamp", "Date", "now", "toLocaleTimeString", "timeZone", "hour12", "hour", "minute", "second", "randomQueueDxn", "DXN", "kind", "QUEUE", "QueueSubspaceTags", "DATA", "SpaceId", "random", "ObjectId", "mergeFloat64Arrays", "arrays", "newArray", "Float64Array", "reduce", "acc", "array", "length", "offset", "set", "monitorAudioLevel", "mediaStreamTrack", "onMeasure", "intervalMs", "timeout", "interval", "audioContext", "AudioContext", "stream", "MediaStream", "addTrack", "mediaStreamAudioSourceNode", "createMediaStreamSource", "analyserNode", "createAnalyser", "connect", "fftSize", "pcmData", "Float32Array", "peak", "window", "setInterval", "tick", "setTimeout", "getFloatTimeDomainData", "sumSquares", "amplitude", "current", "Math", "sqrt", "disconnect", "close", "clearInterval", "clearTimeout", "removeTrack"]
|
|
7
|
-
}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/components/TranscriptContainer.tsx", "../../../src/components/Transcript/Transcript.tsx"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2024 DXOS.org\n//\n\nimport { pipe } from 'effect';\nimport React, { useCallback, useMemo, useState, type FC } from 'react';\n\nimport { chain, createIntent, LayoutAction, useIntentDispatcher } from '@dxos/app-framework';\nimport { Message } from '@dxos/artifact';\nimport { AIServiceClientImpl, DEFAULT_LLM_MODEL, MixedStreamParser, type AIServiceClient } from '@dxos/assistant';\nimport { create, getSpace, makeRef } from '@dxos/client/echo';\nimport { QueueImpl } from '@dxos/echo-db';\nimport { createStatic, isInstanceOf } from '@dxos/echo-schema';\nimport { invariant } from '@dxos/invariant';\nimport { DXN } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { DocumentType } from '@dxos/plugin-markdown/types';\nimport { CollectionType, SpaceAction } from '@dxos/plugin-space/types';\nimport { useConfig } from '@dxos/react-client';\nimport { useEdgeClient, useQueue, type EdgeHttpClient } from '@dxos/react-edge-client';\nimport { IconButton, Toolbar, useTranslation } from '@dxos/react-ui';\nimport { ScrollContainer } from '@dxos/react-ui-components';\nimport { StackItem } from '@dxos/react-ui-stack';\nimport { TextType } from '@dxos/schema';\n\nimport { Transcript } from './Transcript';\nimport { TRANSCRIPTION_PLUGIN } from '../meta';\nimport { TranscriptBlock, type TranscriptType } from '../types';\n\nexport const TranscriptionContainer: FC<{ transcript: TranscriptType }> = ({ transcript }) => {\n const { t } = useTranslation(TRANSCRIPTION_PLUGIN);\n const edge = useEdgeClient();\n const aiService = useAiServiceClient();\n const { dispatchPromise: dispatch } = useIntentDispatcher();\n\n const queue = useQueue<TranscriptBlock>(edge, transcript.queue ? DXN.parse(transcript.queue) : undefined, {\n pollInterval: 1_000,\n });\n\n // TODO(dmaretskyi): Pending state and errors should be handled by the framework!!!\n const [isSummarizing, setIsSummarizing] = useState(false);\n const handleSummarize = useCallback(async () => {\n setIsSummarizing(true);\n try {\n const document = await summarizeTranscript(edge, aiService, transcript);\n const space = getSpace(transcript);\n const target = space?.properties[CollectionType.typename]?.target;\n await dispatch(\n pipe(\n createIntent(SpaceAction.AddObject, { object: document, target }),\n chain(LayoutAction.Open, { part: 'main' }),\n ),\n );\n } finally {\n setIsSummarizing(false);\n }\n }, [transcript, edge]);\n\n // TODO(dmaretskyi): Move action to menu.\n return (\n <StackItem.Content toolbar={true}>\n <StackItem.Heading>\n <Toolbar.Root classNames='flex gap-2'>\n <IconButton\n icon='ph--subtitles--regular'\n iconOnly\n size={5}\n disabled={isSummarizing}\n onClick={handleSummarize}\n label={t('summary button')}\n />\n {isSummarizing && <div className='text-sm'>{t('summarizing label')}</div>}\n </Toolbar.Root>\n </StackItem.Heading>\n <ScrollContainer>\n <Transcript blocks={queue?.items} />\n </ScrollContainer>\n </StackItem.Content>\n );\n};\n\nexport default TranscriptionContainer;\n\n// TODO(dmaretskyi): Extract to the new file once transcript refactoring has settled.\nconst summarizeTranscript = async (\n edgeClient: EdgeHttpClient,\n aiService: AIServiceClient,\n transcript: TranscriptType,\n) => {\n invariant(transcript.queue, 'No queue found for transcript');\n\n const queue = new QueueImpl(edgeClient, DXN.parse(transcript.queue));\n await queue.refresh();\n const content = queue.items\n .filter((block) => isInstanceOf(TranscriptBlock, block))\n .map((block) => `${block.author}: ${block.segments.map((segment) => segment.text).join('\\n')}`)\n .join('\\n\\n');\n\n const parser = new MixedStreamParser();\n\n log.info('summarizing transcript', { blockCount: queue.items.length });\n const output = await parser.parse(\n await aiService.exec({\n model: DEFAULT_LLM_MODEL,\n systemPrompt: SUMMARIZE_PROMPT,\n history: [createStatic(Message, { role: 'user', content: [{ type: 'text', text: content }] })],\n }),\n );\n\n log.info('transcript summary', { output });\n invariant(output[0].content[0].type === 'text', 'Expected text content');\n const summary = output[0].content[0].text;\n\n // TODO(dmaretskyi): .started is missing.\n const name = `Summary ${(transcript.started ?? new Date()).toLocaleString('en-US', {\n year: 'numeric',\n month: 'long',\n day: 'numeric',\n hour: '2-digit',\n minute: '2-digit',\n })}`;\n\n return create(DocumentType, {\n name,\n content: makeRef(create(TextType, { content: summary })),\n threads: [],\n });\n};\n\n// TODO(dmaretskyi): Add example to set consistent structure for the summary.\nconst SUMMARIZE_PROMPT = `\n # Goal\n Create a markdown summary of the transcript provided.\n\n # Formatting\n - Format the summary as a markdown document without extra comments like \"Here is the summary of the transcript:\".\n - Use markdown formatting for headings and bullet points.\n - Format the summary as a list of key points and takeaways.\n - All names of people should be in bold.\n\n # Note Taking\n - Correlate items in the summary with the person of origin to build a coherent narrative.\n - Include short quotes verbatim where appropriate. Especially when concerned with design decisions and problem descriptions.\n\n # Tasks\n At the end of the summary include tasks.\n Extract only the tasks that are:\n - Directly actionable\n - Clearly assigned to a person or team (or can easily be inferred)\n - Strongly implied by the conversation and/or user note (no speculative tasks)\n - Specific enough that someone reading them would know exactly what to do next\n\n Format all tasks as markdown checkboxes using the syntax:\n - [ ] Task description\n\n Additional information can be included (indented).\n\n If no actionable tasks are found, omit this tasks section.\n`;\n\n// TODO(dmaretskyi): Extract?\n// Also this conflicts with plugin automation providing the ai client via a capability. Need to reconcile.\nconst useAiServiceClient = (): AIServiceClient => {\n const config = useConfig();\n const endpoint = config.values.runtime?.services?.ai?.server ?? 'http://localhost:8788'; // TOOD(burdon): Standardize consts.\n return useMemo(() => new AIServiceClientImpl({ endpoint }), [endpoint]);\n};\n", "//\n// Copyright 2023 DXOS.org\n//\n\nimport { formatDistanceToNow } from 'date-fns/formatDistanceToNow';\nimport React, { type FC } from 'react';\n\nimport { Icon, IconButton, useTranslation } from '@dxos/react-ui';\n\nimport { TRANSCRIPTION_PLUGIN } from '../../meta';\nimport { type TranscriptBlock } from '../../types';\n\n// TODO(burdon): react-ui-list.\n// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).\n\nexport type TranscriptProps = {\n blocks?: TranscriptBlock[];\n};\n\nexport const Transcript: FC<TranscriptProps> = ({ blocks }) => {\n const { t } = useTranslation(TRANSCRIPTION_PLUGIN);\n const hoverButton = 'p-1 min-bs-1 transition-opacity duration-250 opacity-10 group-hover:opacity-100';\n\n return (\n <div role='list' className='flex flex-col grow gap-2 m-1'>\n {blocks?.map((block) => (\n <div\n role='listitem'\n key={block.id}\n className='flex flex-col py-1 border border-transparent hover:border-separator rounded'\n >\n <div className='group flex items-center px-2'>\n <Icon icon='ph--user--regular' />\n <div className='px-2 text-sm text-subdued'>{block.author}</div>\n <div className='grow' />\n <IconButton\n disabled\n icon='ph--x--regular'\n label={t('delete button')}\n iconOnly\n size={4}\n classNames={hoverButton}\n />\n </div>\n\n {block.segments.map((segment, i) => (\n <div key={i} className='group flex flex-col'>\n <div className='px-2'>{segment.text}</div>\n <div className='flex gap-1 items-center justify-between px-2'>\n <div className='grow' />\n <div className='truncate text-xs text-subdued'>\n {formatDistanceToNow(segment.started, { addSuffix: true })}\n </div>\n <IconButton\n disabled\n icon='ph--bookmark-simple--regular'\n label={t('bookmark button')}\n iconOnly\n size={4}\n classNames={hoverButton}\n />\n </div>\n </div>\n ))}\n </div>\n ))}\n </div>\n );\n};\n"],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,oBAAqB;AACrB,mBAA+D;AAE/D,2BAAuE;AACvE,sBAAwB;AACxB,uBAAgG;AAChG,kBAA0C;AAC1C,qBAA0B;AAC1B,yBAA2C;AAC3C,uBAA0B;AAC1B,kBAAoB;AACpB,iBAAoB;AACpB,mBAA6B;AAC7B,IAAAA,gBAA4C;AAC5C,0BAA0B;AAC1B,+BAA6D;AAC7D,sBAAoD;AACpD,iCAAgC;AAChC,4BAA0B;AAC1B,oBAAyB;ACnBzB,iCAAoC;AACpC,IAAAC,gBAA+B;AAE/B,IAAAC,mBAAiD;AAY1C,IAAMC,aAAkC,CAAC,EAAEC,OAAM,MAAE;AACxD,QAAM,EAAEC,EAAC,QAAKC,iCAAeC,0CAAAA;AAC7B,QAAMC,cAAc;AAEpB,SACE,8BAAAC,QAAA,cAACC,OAAAA;IAAIC,MAAK;IAAOC,WAAU;KACxBR,QAAQS,IAAI,CAACC,UACZ,8BAAAL,QAAA,cAACC,OAAAA;IACCC,MAAK;IACLI,KAAKD,MAAME;IACXJ,WAAU;KAEV,8BAAAH,QAAA,cAACC,OAAAA;IAAIE,WAAU;KACb,8BAAAH,QAAA,cAACQ,uBAAAA;IAAKC,MAAK;MACX,8BAAAT,QAAA,cAACC,OAAAA;IAAIE,WAAU;KAA6BE,MAAMK,MAAM,GACxD,8BAAAV,QAAA,cAACC,OAAAA;IAAIE,WAAU;MACf,8BAAAH,QAAA,cAACW,6BAAAA;IACCC,UAAAA;IACAH,MAAK;IACLI,OAAOjB,EAAE,eAAA;IACTkB,UAAAA;IACAC,MAAM;IACNC,YAAYjB;OAIfM,MAAMY,SAASb,IAAI,CAACc,SAASC,MAC5B,8BAAAnB,QAAA,cAACC,OAAAA;IAAIK,KAAKa;IAAGhB,WAAU;KACrB,8BAAAH,QAAA,cAACC,OAAAA;IAAIE,WAAU;KAAQe,QAAQE,IAAI,GACnC,8BAAApB,QAAA,cAACC,OAAAA;IAAIE,WAAU;KACb,8BAAAH,QAAA,cAACC,OAAAA;IAAIE,WAAU;MACf,8BAAAH,QAAA,cAACC,OAAAA;IAAIE,WAAU;SACZkB,gDAAoBH,QAAQI,SAAS;IAAEC,WAAW;EAAK,CAAA,CAAA,GAE1D,8BAAAvB,QAAA,cAACW,6BAAAA;IACCC,UAAAA;IACAH,MAAK;IACLI,OAAOjB,EAAE,iBAAA;IACTkB,UAAAA;IACAC,MAAM;IACNC,YAAYjB;;AAS9B;;ADvCO,IAAMyB,yBAA6D,CAAC,EAAEC,WAAU,MAAE;AACvF,QAAM,EAAE7B,EAAC,QAAKC,gBAAAA,gBAAeC,0CAAAA;AAC7B,QAAM4B,WAAOC,wCAAAA;AACb,QAAMC,YAAYC,mBAAAA;AAClB,QAAM,EAAEC,iBAAiBC,SAAQ,QAAKC,0CAAAA;AAEtC,QAAMC,YAAQC,mCAA0BR,MAAMD,WAAWQ,QAAQE,gBAAIC,MAAMX,WAAWQ,KAAK,IAAII,QAAW;IACxGC,cAAc;EAChB,CAAA;AAGA,QAAM,CAACC,eAAeC,gBAAAA,QAAoBC,uBAAS,KAAA;AACnD,QAAMC,sBAAkBC,0BAAY,YAAA;AAClCH,qBAAiB,IAAA;AACjB,QAAI;AACF,YAAMI,WAAW,MAAMC,oBAAoBnB,MAAME,WAAWH,UAAAA;AAC5D,YAAMqB,YAAQC,sBAAStB,UAAAA;AACvB,YAAMuB,SAASF,OAAOG,WAAWC,6BAAeC,QAAQ,GAAGH;AAC3D,YAAMjB,aACJqB,wBACEC,mCAAaC,0BAAYC,WAAW;QAAEC,QAAQZ;QAAUI;MAAO,CAAA,OAC/DS,4BAAMC,kCAAaC,MAAM;QAAEC,MAAM;MAAO,CAAA,CAAA,CAAA;IAG9C,UAAA;AACEpB,uBAAiB,KAAA;IACnB;EACF,GAAG;IAACf;IAAYC;GAAK;AAGrB,SACE1B,6BAAAA,QAAA,cAAC6D,gCAAUC,SAAO;IAACC,SAAS;KAC1B/D,6BAAAA,QAAA,cAAC6D,gCAAUG,SAAO,MAChBhE,6BAAAA,QAAA,cAACiE,wBAAQC,MAAI;IAAClD,YAAW;KACvBhB,6BAAAA,QAAA,cAACW,gBAAAA,YAAAA;IACCF,MAAK;IACLK,UAAAA;IACAC,MAAM;IACNH,UAAU2B;IACV4B,SAASzB;IACT7B,OAAOjB,EAAE,gBAAA;MAEV2C,iBAAiBvC,6BAAAA,QAAA,cAACC,OAAAA;IAAIE,WAAU;KAAWP,EAAE,mBAAA,CAAA,CAAA,CAAA,GAGlDI,6BAAAA,QAAA,cAACoE,4CAAAA,MACCpE,6BAAAA,QAAA,cAACN,YAAAA;IAAWC,QAAQsC,OAAOoC;;AAInC;AAEA,IAAA,8BAAe7C;AAGf,IAAMqB,sBAAsB,OAC1ByB,YACA1C,WACAH,eAAAA;AAEA8C,kCAAU9C,WAAWQ,OAAO,iCAAA;;;;;;;;;AAE5B,QAAMA,QAAQ,IAAIuC,yBAAUF,YAAYnC,gBAAIC,MAAMX,WAAWQ,KAAK,CAAA;AAClE,QAAMA,MAAMwC,QAAO;AACnB,QAAMC,UAAUzC,MAAMoC,MACnBM,OAAO,CAACtE,cAAUuE,iCAAaC,uCAAiBxE,KAAAA,CAAAA,EAChDD,IAAI,CAACC,UAAU,GAAGA,MAAMK,MAAM,KAAKL,MAAMY,SAASb,IAAI,CAACc,YAAYA,QAAQE,IAAI,EAAE0D,KAAK,IAAA,CAAA,EAAO,EAC7FA,KAAK,MAAA;AAER,QAAMC,SAAS,IAAIC,mCAAAA;AAEnBC,iBAAIC,KAAK,0BAA0B;IAAEC,YAAYlD,MAAMoC,MAAMe;EAAO,GAAA;;;;;;AACpE,QAAMC,SAAS,MAAMN,OAAO3C,MAC1B,MAAMR,UAAU0D,KAAK;IACnBC,OAAOC;IACPC,cAAcC;IACdC,SAAS;UAACC,iCAAaC,yBAAS;QAAE3F,MAAM;QAAQwE,SAAS;UAAC;YAAEoB,MAAM;YAAQ1E,MAAMsD;UAAQ;;MAAG,CAAA;;EAC7F,CAAA,CAAA;AAGFO,iBAAIC,KAAK,sBAAsB;IAAEG;EAAO,GAAA;;;;;;AACxCd,kCAAUc,OAAO,CAAA,EAAGX,QAAQ,CAAA,EAAGoB,SAAS,QAAQ,yBAAA;;;;;;;;;AAChD,QAAMC,UAAUV,OAAO,CAAA,EAAGX,QAAQ,CAAA,EAAGtD;AAGrC,QAAM4E,OAAO,YAAYvE,WAAWH,WAAW,oBAAI2E,KAAAA,GAAQC,eAAe,SAAS;IACjFC,MAAM;IACNC,OAAO;IACPC,KAAK;IACLC,MAAM;IACNC,QAAQ;EACV,CAAA,CAAA;AAEA,aAAOC,oBAAOC,2BAAc;IAC1BT;IACAtB,aAASgC,yBAAQF,oBAAOG,wBAAU;MAAEjC,SAASqB;IAAQ,CAAA,CAAA;IACrDa,SAAS,CAAA;EACX,CAAA;AACF;AAGA,IAAMlB,mBAAmB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgCzB,IAAM7D,qBAAqB,MAAA;AACzB,QAAMgF,aAASC,+BAAAA;AACf,QAAMC,WAAWF,OAAOG,OAAOC,SAASC,UAAUC,IAAIC,UAAU;AAChE,aAAOC,sBAAQ,MAAM,IAAIC,qCAAoB;IAAEP;EAAS,CAAA,GAAI;IAACA;GAAS;AACxE;",
|
|
6
|
-
"names": ["import_types", "import_react", "import_react_ui", "Transcript", "blocks", "t", "useTranslation", "TRANSCRIPTION_PLUGIN", "hoverButton", "React", "div", "role", "className", "map", "block", "key", "id", "Icon", "icon", "author", "IconButton", "disabled", "label", "iconOnly", "size", "classNames", "segments", "segment", "i", "text", "formatDistanceToNow", "started", "addSuffix", "TranscriptionContainer", "transcript", "edge", "useEdgeClient", "aiService", "useAiServiceClient", "dispatchPromise", "dispatch", "useIntentDispatcher", "queue", "useQueue", "DXN", "parse", "undefined", "pollInterval", "isSummarizing", "setIsSummarizing", "useState", "handleSummarize", "useCallback", "document", "summarizeTranscript", "space", "getSpace", "target", "properties", "CollectionType", "typename", "pipe", "createIntent", "SpaceAction", "AddObject", "object", "chain", "LayoutAction", "Open", "part", "StackItem", "Content", "toolbar", "Heading", "Toolbar", "Root", "onClick", "ScrollContainer", "items", "edgeClient", "invariant", "QueueImpl", "refresh", "content", "filter", "isInstanceOf", "TranscriptBlock", "join", "parser", "MixedStreamParser", "log", "info", "blockCount", "length", "output", "exec", "model", "DEFAULT_LLM_MODEL", "systemPrompt", "SUMMARIZE_PROMPT", "history", "createStatic", "Message", "type", "summary", "name", "Date", "toLocaleString", "year", "month", "day", "hour", "minute", "create", "DocumentType", "makeRef", "TextType", "threads", "config", "useConfig", "endpoint", "values", "runtime", "services", "ai", "server", "useMemo", "AIServiceClientImpl"]
|
|
7
|
-
}
|