@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bcb3aa67d6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/blueprints/index.mjs +31 -0
- package/dist/lib/browser/blueprints/index.mjs.map +7 -0
- package/dist/lib/browser/chunk-HOOPZ7EM.mjs +35 -0
- package/dist/lib/browser/chunk-HOOPZ7EM.mjs.map +7 -0
- package/dist/lib/browser/chunk-J5LGTIGS.mjs +10 -0
- package/dist/lib/browser/chunk-RIUSFL62.mjs +112 -0
- package/dist/lib/browser/chunk-RIUSFL62.mjs.map +7 -0
- package/dist/lib/browser/chunk-YXU2ZO6W.mjs +17 -0
- package/dist/lib/browser/chunk-YXU2ZO6W.mjs.map +7 -0
- package/dist/lib/browser/create-IZPB7VZW.mjs +19 -0
- package/dist/lib/browser/create-IZPB7VZW.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +1178 -97
- package/dist/lib/browser/index.mjs.map +4 -4
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/open-SN5O5I6X.mjs +29 -0
- package/dist/lib/browser/open-SN5O5I6X.mjs.map +7 -0
- package/dist/lib/browser/operations/index.mjs +13 -0
- package/dist/lib/browser/operations/index.mjs.map +7 -0
- package/dist/lib/browser/summarize-5MZENAU5.mjs +71 -0
- package/dist/lib/browser/summarize-5MZENAU5.mjs.map +7 -0
- package/dist/lib/browser/types/index.mjs +4 -6
- package/dist/lib/node-esm/blueprints/index.mjs +32 -0
- package/dist/lib/node-esm/blueprints/index.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-3GSFEN4L.mjs +113 -0
- package/dist/lib/node-esm/chunk-3GSFEN4L.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-BR2R6PCT.mjs +19 -0
- package/dist/lib/node-esm/chunk-BR2R6PCT.mjs.map +7 -0
- package/dist/lib/node-esm/chunk-HSLMI22Q.mjs +11 -0
- package/dist/lib/node-esm/chunk-NVM2QUMX.mjs +37 -0
- package/dist/lib/node-esm/chunk-NVM2QUMX.mjs.map +7 -0
- package/dist/lib/node-esm/create-TQDGXQBL.mjs +20 -0
- package/dist/lib/node-esm/create-TQDGXQBL.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +1178 -97
- package/dist/lib/node-esm/index.mjs.map +4 -4
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/open-4GAAWWKJ.mjs +30 -0
- package/dist/lib/node-esm/open-4GAAWWKJ.mjs.map +7 -0
- package/dist/lib/node-esm/operations/index.mjs +14 -0
- package/dist/lib/node-esm/operations/index.mjs.map +7 -0
- package/dist/lib/node-esm/summarize-RVOSRHEC.mjs +72 -0
- package/dist/lib/node-esm/summarize-RVOSRHEC.mjs.map +7 -0
- package/dist/lib/node-esm/types/index.mjs +4 -6
- package/dist/types/src/TranscriptionPlugin.d.ts +2 -1
- package/dist/types/src/TranscriptionPlugin.d.ts.map +1 -1
- package/dist/types/src/blueprints/index.d.ts +2 -0
- package/dist/types/src/blueprints/index.d.ts.map +1 -0
- package/dist/types/src/blueprints/transcription-blueprint.d.ts +4 -0
- package/dist/types/src/blueprints/transcription-blueprint.d.ts.map +1 -0
- package/dist/types/src/capabilities/blueprint-definition.d.ts +5 -4
- package/dist/types/src/capabilities/blueprint-definition.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts +6 -5
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/operation-handler.d.ts +6 -0
- package/dist/types/src/capabilities/operation-handler.d.ts.map +1 -0
- package/dist/types/src/capabilities/react-surface.d.ts +3 -2
- package/dist/types/src/capabilities/react-surface.d.ts.map +1 -1
- package/dist/types/src/capabilities/transcriber.d.ts +4 -3
- package/dist/types/src/capabilities/transcriber.d.ts.map +1 -1
- package/dist/types/src/components/Transcription/Transcription.d.ts +22 -0
- package/dist/types/src/components/Transcription/Transcription.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts +40 -0
- package/dist/types/src/components/Transcription/Transcription.stories.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/index.d.ts +2 -0
- package/dist/types/src/components/Transcription/index.d.ts.map +1 -0
- package/dist/types/src/components/Transcription/transcription-extension.d.ts +15 -0
- package/dist/types/src/components/Transcription/transcription-extension.d.ts.map +1 -0
- package/dist/types/src/components/index.d.ts +1 -3
- package/dist/types/src/components/index.d.ts.map +1 -1
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts +3 -3
- package/dist/types/src/components/stories/FileTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts +2 -2
- package/dist/types/src/components/stories/KeyWordRecognition.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts +5 -5
- package/dist/types/src/components/stories/MicrophoneTranscription.stories.d.ts.map +1 -1
- package/dist/types/src/components/stories/TranscriptionStory.d.ts +2 -2
- package/dist/types/src/components/stories/TranscriptionStory.d.ts.map +1 -1
- package/dist/types/src/containers/TranscriptionContainer/TranscriptionContainer.d.ts +6 -0
- package/dist/types/src/containers/TranscriptionContainer/TranscriptionContainer.d.ts.map +1 -0
- package/dist/types/src/containers/TranscriptionContainer/index.d.ts +3 -0
- package/dist/types/src/containers/TranscriptionContainer/index.d.ts.map +1 -0
- package/dist/types/src/containers/index.d.ts +3 -0
- package/dist/types/src/containers/index.d.ts.map +1 -0
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts +1 -1
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
- package/dist/types/src/index.d.ts +1 -1
- package/dist/types/src/index.d.ts.map +1 -1
- package/dist/types/src/meta.d.ts +2 -2
- package/dist/types/src/meta.d.ts.map +1 -1
- package/dist/types/src/operations/create.d.ts +5 -0
- package/dist/types/src/operations/create.d.ts.map +1 -0
- package/dist/types/src/operations/definitions.d.ts +905 -0
- package/dist/types/src/operations/definitions.d.ts.map +1 -0
- package/dist/types/src/operations/index.d.ts +4 -0
- package/dist/types/src/operations/index.d.ts.map +1 -0
- package/dist/types/src/operations/open.d.ts +5 -0
- package/dist/types/src/operations/open.d.ts.map +1 -0
- package/dist/types/src/operations/summarize.d.ts +8 -0
- package/dist/types/src/operations/summarize.d.ts.map +1 -0
- package/dist/types/src/segments-normalization/message-normalizer.d.ts +5 -5
- package/dist/types/src/segments-normalization/message-normalizer.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/normalization.d.ts +360 -474
- package/dist/types/src/segments-normalization/normalization.d.ts.map +1 -1
- package/dist/types/src/segments-normalization/utils.d.ts +2 -2
- package/dist/types/src/segments-normalization/utils.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +9 -9
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/media-stream-recorder.d.ts +2 -2
- package/dist/types/src/transcriber/media-stream-recorder.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +4 -4
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcription-manager.d.ts +9 -6
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +7 -7
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/src/types/capabilities.d.ts +18 -0
- package/dist/types/src/types/capabilities.d.ts.map +1 -0
- package/dist/types/src/types/index.d.ts +1 -2
- package/dist/types/src/types/index.d.ts.map +1 -1
- package/dist/types/src/util/index.d.ts +1 -0
- package/dist/types/src/util/index.d.ts.map +1 -1
- package/dist/types/src/util/render-byline.d.ts +4 -0
- package/dist/types/src/util/render-byline.d.ts.map +1 -0
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +105 -70
- package/src/TranscriptionPlugin.tsx +40 -60
- package/src/blueprints/index.ts +5 -0
- package/src/blueprints/transcription-blueprint.ts +30 -0
- package/src/capabilities/blueprint-definition.ts +10 -24
- package/src/capabilities/index.ts +9 -7
- package/src/capabilities/operation-handler.ts +16 -0
- package/src/capabilities/react-surface.tsx +21 -13
- package/src/capabilities/transcriber.ts +51 -43
- package/src/components/{Transcript/Transcript.stories.tsx → Transcription/Transcription.stories.tsx} +49 -45
- package/src/components/Transcription/Transcription.tsx +65 -0
- package/src/components/{Transcript → Transcription}/index.ts +1 -1
- package/src/components/{Transcript/transcript-extension.ts → Transcription/transcription-extension.ts} +13 -9
- package/src/components/index.ts +1 -6
- package/src/components/stories/FileTranscription.stories.tsx +30 -36
- package/src/components/stories/KeyWordRecognition.stories.tsx +6 -6
- package/src/components/stories/MicrophoneTranscription.stories.tsx +53 -74
- package/src/components/stories/TranscriptionStory.tsx +7 -9
- package/src/containers/TranscriptionContainer/TranscriptionContainer.tsx +32 -0
- package/src/containers/TranscriptionContainer/index.ts +7 -0
- package/src/containers/index.ts +7 -0
- package/src/hooks/useQueueModelAdapter.ts +1 -1
- package/src/hooks/useTranscriber.ts +8 -3
- package/src/hooks/useVoiceInput.ts +2 -2
- package/src/index.ts +1 -1
- package/src/meta.ts +3 -3
- package/src/model/model.test.ts +20 -14
- package/src/operations/create.ts +22 -0
- package/src/operations/definitions.ts +102 -0
- package/src/operations/index.ts +13 -0
- package/src/operations/open.ts +31 -0
- package/src/{functions → operations}/summarize.ts +41 -47
- package/src/segments-normalization/message-normalizer.ts +18 -13
- package/src/segments-normalization/normalization.test.ts +6 -11
- package/src/segments-normalization/normalization.ts +34 -72
- package/src/segments-normalization/utils.ts +2 -2
- package/src/testing/testing.ts +28 -28
- package/src/transcriber/media-stream-recorder.ts +3 -3
- package/src/transcriber/transcriber.browser.test.ts +4 -4
- package/src/transcriber/transcriber.ts +7 -7
- package/src/transcriber/transcription-manager.ts +21 -15
- package/src/translations.ts +7 -7
- package/src/{capabilities → types}/capabilities.ts +10 -10
- package/src/types/index.ts +1 -2
- package/src/util/index.ts +1 -0
- package/src/util/render-byline.ts +33 -0
- package/dist/lib/browser/TranscriptContainer-2GSL2XB5.mjs +0 -11
- package/dist/lib/browser/blueprint-definition-UAAJYVLZ.mjs +0 -150
- package/dist/lib/browser/blueprint-definition-UAAJYVLZ.mjs.map +0 -7
- package/dist/lib/browser/chunk-3MKZ2MNQ.mjs +0 -16
- package/dist/lib/browser/chunk-3MKZ2MNQ.mjs.map +0 -7
- package/dist/lib/browser/chunk-535OJKFO.mjs +0 -8
- package/dist/lib/browser/chunk-535OJKFO.mjs.map +0 -7
- package/dist/lib/browser/chunk-7DR6HZEF.mjs +0 -732
- package/dist/lib/browser/chunk-7DR6HZEF.mjs.map +0 -7
- package/dist/lib/browser/chunk-VBUIO677.mjs +0 -29
- package/dist/lib/browser/chunk-VBUIO677.mjs.map +0 -7
- package/dist/lib/browser/chunk-W7WKZTM4.mjs +0 -63
- package/dist/lib/browser/chunk-W7WKZTM4.mjs.map +0 -7
- package/dist/lib/browser/chunk-WH36ENNE.mjs +0 -466
- package/dist/lib/browser/chunk-WH36ENNE.mjs.map +0 -7
- package/dist/lib/browser/intent-resolver-ZTKTEAQX.mjs +0 -25
- package/dist/lib/browser/intent-resolver-ZTKTEAQX.mjs.map +0 -7
- package/dist/lib/browser/react-surface-KA32A72I.mjs +0 -34
- package/dist/lib/browser/react-surface-KA32A72I.mjs.map +0 -7
- package/dist/lib/browser/transcriber-LWOLYQHC.mjs +0 -56
- package/dist/lib/browser/transcriber-LWOLYQHC.mjs.map +0 -7
- package/dist/lib/node-esm/TranscriptContainer-GRLDCESB.mjs +0 -12
- package/dist/lib/node-esm/blueprint-definition-Z3LTZ3ID.mjs +0 -151
- package/dist/lib/node-esm/blueprint-definition-Z3LTZ3ID.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-K4AZOB5Y.mjs +0 -733
- package/dist/lib/node-esm/chunk-K4AZOB5Y.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-OLMSTF3M.mjs +0 -64
- package/dist/lib/node-esm/chunk-OLMSTF3M.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-Q6WD7L2J.mjs +0 -10
- package/dist/lib/node-esm/chunk-Q6WD7L2J.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-SWUVPFKR.mjs +0 -17
- package/dist/lib/node-esm/chunk-SWUVPFKR.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-YFN4FJFK.mjs +0 -467
- package/dist/lib/node-esm/chunk-YFN4FJFK.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-YVNQN2J6.mjs +0 -30
- package/dist/lib/node-esm/chunk-YVNQN2J6.mjs.map +0 -7
- package/dist/lib/node-esm/intent-resolver-55WH2GY6.mjs +0 -26
- package/dist/lib/node-esm/intent-resolver-55WH2GY6.mjs.map +0 -7
- package/dist/lib/node-esm/react-surface-GU7LW2JO.mjs +0 -35
- package/dist/lib/node-esm/react-surface-GU7LW2JO.mjs.map +0 -7
- package/dist/lib/node-esm/transcriber-UGB7ECX6.mjs +0 -57
- package/dist/lib/node-esm/transcriber-UGB7ECX6.mjs.map +0 -7
- package/dist/types/src/capabilities/capabilities.d.ts +0 -17
- package/dist/types/src/capabilities/capabilities.d.ts.map +0 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts +0 -4
- package/dist/types/src/capabilities/intent-resolver.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts +0 -20
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +0 -39
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +0 -28
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/index.d.ts +0 -2
- package/dist/types/src/components/Transcript/index.d.ts.map +0 -1
- package/dist/types/src/components/Transcript/transcript-extension.d.ts +0 -15
- package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +0 -1
- package/dist/types/src/components/TranscriptContainer.d.ts +0 -9
- package/dist/types/src/components/TranscriptContainer.d.ts.map +0 -1
- package/dist/types/src/functions/index.d.ts +0 -3
- package/dist/types/src/functions/index.d.ts.map +0 -1
- package/dist/types/src/functions/open.d.ts +0 -7
- package/dist/types/src/functions/open.d.ts.map +0 -1
- package/dist/types/src/functions/summarize.d.ts +0 -11
- package/dist/types/src/functions/summarize.d.ts.map +0 -1
- package/dist/types/src/types/Transcript.d.ts +0 -30
- package/dist/types/src/types/Transcript.d.ts.map +0 -1
- package/dist/types/src/types/TranscriptAction.d.ts +0 -23
- package/dist/types/src/types/TranscriptAction.d.ts.map +0 -1
- package/src/capabilities/intent-resolver.ts +0 -18
- package/src/components/Transcript/Transcript.tsx +0 -93
- package/src/components/Transcript/deprecated/OldTranscript.tsx +0 -260
- package/src/components/TranscriptContainer.tsx +0 -36
- package/src/functions/index.ts +0 -6
- package/src/functions/open.ts +0 -39
- package/src/types/Transcript.ts +0 -44
- package/src/types/TranscriptAction.ts +0 -24
- /package/dist/lib/browser/{TranscriptContainer-2GSL2XB5.mjs.map → chunk-J5LGTIGS.mjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-GRLDCESB.mjs.map → chunk-HSLMI22Q.mjs.map} +0 -0
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
//
|
|
2
|
+
// Copyright 2025 DXOS.org
|
|
3
|
+
//
|
|
4
|
+
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
6
|
+
|
|
7
|
+
import { Database, Obj } from '@dxos/echo';
|
|
8
|
+
import { QueueService } from '@dxos/functions';
|
|
9
|
+
import { Operation } from '@dxos/operation';
|
|
10
|
+
import { Message } from '@dxos/types';
|
|
11
|
+
|
|
12
|
+
import { renderByline } from '../util';
|
|
13
|
+
import { Open } from './definitions';
|
|
14
|
+
|
|
15
|
+
const handler: Operation.WithHandler<typeof Open> = Open.pipe(
|
|
16
|
+
Operation.withHandler(
|
|
17
|
+
Effect.fn(function* ({ transcript }) {
|
|
18
|
+
const transcriptObj = yield* Database.load(transcript);
|
|
19
|
+
const { dxn } = yield* Effect.promise(() => transcriptObj.queue.load());
|
|
20
|
+
const queue = yield* QueueService.getQueue(dxn);
|
|
21
|
+
yield* Effect.promise(() => queue?.queryObjects());
|
|
22
|
+
const content = queue?.objects
|
|
23
|
+
.filter((message: unknown) => Obj.instanceOf(Message.Message, message))
|
|
24
|
+
.flatMap((message: Message.Message, index: number) => renderByline([])(message, index))
|
|
25
|
+
.join('\n\n');
|
|
26
|
+
return { content };
|
|
27
|
+
}),
|
|
28
|
+
),
|
|
29
|
+
);
|
|
30
|
+
|
|
31
|
+
export default handler;
|
|
@@ -7,67 +7,61 @@ import * as Effect from 'effect/Effect';
|
|
|
7
7
|
import * as Function from 'effect/Function';
|
|
8
8
|
import * as Layer from 'effect/Layer';
|
|
9
9
|
import * as Option from 'effect/Option';
|
|
10
|
-
import * as Schema from 'effect/Schema';
|
|
11
10
|
|
|
12
11
|
import { AiService, ConsolePrinter, ToolExecutionService, ToolResolverService } from '@dxos/ai';
|
|
13
12
|
import { AiSession, GenerationObserver } from '@dxos/assistant';
|
|
14
|
-
import {
|
|
13
|
+
import { FunctionInvocationService, TracingService } from '@dxos/functions';
|
|
14
|
+
import * as Trace from '@dxos/functions/Trace';
|
|
15
|
+
import { Operation } from '@dxos/operation';
|
|
15
16
|
import { trim } from '@dxos/util';
|
|
16
17
|
|
|
18
|
+
import { Summarize } from './definitions';
|
|
19
|
+
|
|
17
20
|
/**
|
|
18
21
|
* Summarize a transcript of a meeting.
|
|
19
22
|
*/
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
}),
|
|
32
|
-
outputSchema: Schema.Struct({
|
|
33
|
-
summary: Schema.String.annotations({
|
|
34
|
-
description: 'The summary of the transcript.',
|
|
35
|
-
}),
|
|
36
|
-
}),
|
|
37
|
-
handler: Effect.fnUntraced(
|
|
38
|
-
function* ({ data: { transcript, notes } }) {
|
|
39
|
-
const result = yield* new AiSession().run({
|
|
40
|
-
prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
|
|
41
|
-
history: [],
|
|
42
|
-
system: systemPrompt,
|
|
43
|
-
observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
|
|
44
|
-
});
|
|
23
|
+
const handler: Operation.WithHandler<typeof Summarize> = Summarize.pipe(
|
|
24
|
+
Operation.withHandler(
|
|
25
|
+
Effect.fnUntraced(
|
|
26
|
+
function* ({ transcript, notes }) {
|
|
27
|
+
const result = yield* new AiSession({
|
|
28
|
+
observer: GenerationObserver.fromPrinter(new ConsolePrinter({ tag: 'summarize' })),
|
|
29
|
+
}).run({
|
|
30
|
+
prompt: `Transcript: ${transcript}\n\nNotes: ${notes}`,
|
|
31
|
+
history: [],
|
|
32
|
+
system: systemPrompt,
|
|
33
|
+
});
|
|
45
34
|
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
35
|
+
const summary = Function.pipe(
|
|
36
|
+
result,
|
|
37
|
+
Array.findLast((msg) => msg.sender.role === 'assistant' && msg.blocks.some((block) => block._tag === 'text')),
|
|
38
|
+
Option.flatMap((msg) =>
|
|
39
|
+
Function.pipe(
|
|
40
|
+
msg.blocks,
|
|
41
|
+
Array.findLast((block) => block._tag === 'text'),
|
|
42
|
+
Option.map((block) => block.text),
|
|
43
|
+
),
|
|
54
44
|
),
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
);
|
|
45
|
+
Option.getOrThrowWith(() => new Error('No summary found')),
|
|
46
|
+
);
|
|
58
47
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
48
|
+
return { summary };
|
|
49
|
+
},
|
|
50
|
+
Effect.provide(
|
|
51
|
+
Layer.mergeAll(
|
|
52
|
+
AiService.model('@anthropic/claude-sonnet-4-0'),
|
|
53
|
+
ToolResolverService.layerEmpty,
|
|
54
|
+
ToolExecutionService.layerEmpty,
|
|
55
|
+
TracingService.layerNoop,
|
|
56
|
+
FunctionInvocationService.layerNotAvailable,
|
|
57
|
+
Trace.writerLayerNoop,
|
|
58
|
+
),
|
|
67
59
|
),
|
|
68
60
|
),
|
|
69
61
|
),
|
|
70
|
-
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
export default handler;
|
|
71
65
|
|
|
72
66
|
const systemPrompt = trim`
|
|
73
67
|
You are a helpful assistant that summarizes transcripts of meetings.
|
|
@@ -6,24 +6,22 @@
|
|
|
6
6
|
// Copyright 2025 DXOS.org
|
|
7
7
|
//
|
|
8
8
|
|
|
9
|
-
import { effect } from '@preact/signals-core';
|
|
10
|
-
|
|
11
9
|
import { DeferredTask, asyncTimeout } from '@dxos/async';
|
|
12
10
|
import { LifecycleState, Resource } from '@dxos/context';
|
|
13
11
|
import { type Queue } from '@dxos/echo-db';
|
|
14
|
-
import { type FunctionExecutor } from '@dxos/functions';
|
|
12
|
+
import { type FunctionExecutor } from '@dxos/functions-runtime';
|
|
15
13
|
import { log } from '@dxos/log';
|
|
16
|
-
import { type
|
|
14
|
+
import { type Message } from '@dxos/types';
|
|
17
15
|
|
|
18
|
-
import { type MessageWithRangeId, sentenceNormalization } from './normalization';
|
|
16
|
+
import { type MessageWithRangeId, type NormalizationOutput, sentenceNormalization } from './normalization';
|
|
19
17
|
import { getActorId } from './utils';
|
|
20
18
|
|
|
21
19
|
const PROCESSING_TIMEOUT = 20_000; // ms
|
|
22
20
|
const MAX_RANGE_ID_COUNT = 10;
|
|
23
21
|
|
|
24
|
-
export type
|
|
22
|
+
export type SegmentsNormalizerProps = {
|
|
25
23
|
functionExecutor: FunctionExecutor;
|
|
26
|
-
queue: Queue<
|
|
24
|
+
queue: Queue<Message.Message>;
|
|
27
25
|
startingCursor: QueueCursor;
|
|
28
26
|
};
|
|
29
27
|
|
|
@@ -36,13 +34,13 @@ export type QueueCursor = {
|
|
|
36
34
|
// TODO(mykola): .
|
|
37
35
|
export class MessageNormalizer extends Resource {
|
|
38
36
|
private readonly _functionExecutor: FunctionExecutor;
|
|
39
|
-
private _queue: Queue<
|
|
37
|
+
private _queue: Queue<Message.Message>;
|
|
40
38
|
private _cursor: QueueCursor;
|
|
41
39
|
private _messagesToProcess: MessageWithRangeId[] = [];
|
|
42
40
|
private _normalizationTask?: DeferredTask;
|
|
43
41
|
private _lastProcessedMessageIds?: string[];
|
|
44
42
|
|
|
45
|
-
constructor({ functionExecutor, queue, startingCursor }:
|
|
43
|
+
constructor({ functionExecutor, queue, startingCursor }: SegmentsNormalizerProps) {
|
|
46
44
|
super();
|
|
47
45
|
this._functionExecutor = functionExecutor;
|
|
48
46
|
this._queue = queue;
|
|
@@ -51,7 +49,8 @@ export class MessageNormalizer extends Resource {
|
|
|
51
49
|
|
|
52
50
|
protected override async _open(): Promise<void> {
|
|
53
51
|
this._normalizationTask = new DeferredTask(this._ctx, () => this._processMessages());
|
|
54
|
-
|
|
52
|
+
|
|
53
|
+
const updateMessages = () => {
|
|
55
54
|
if (this._lifecycleState !== LifecycleState.OPEN) {
|
|
56
55
|
return;
|
|
57
56
|
}
|
|
@@ -62,8 +61,14 @@ export class MessageNormalizer extends Resource {
|
|
|
62
61
|
});
|
|
63
62
|
|
|
64
63
|
this._normalizationTask!.schedule();
|
|
65
|
-
}
|
|
66
|
-
|
|
64
|
+
};
|
|
65
|
+
|
|
66
|
+
// Initial update.
|
|
67
|
+
updateMessages();
|
|
68
|
+
|
|
69
|
+
// Subscribe to queue changes.
|
|
70
|
+
const unsubscribe = this._queue.subscribe(updateMessages);
|
|
71
|
+
this._ctx.onDispose(unsubscribe);
|
|
67
72
|
}
|
|
68
73
|
|
|
69
74
|
// Need to unpack strings from blocks from messages run them through the function and then pack them back into blocks into messages.
|
|
@@ -81,7 +86,7 @@ export class MessageNormalizer extends Resource {
|
|
|
81
86
|
|
|
82
87
|
try {
|
|
83
88
|
// TODO(mykola): Executor should support timeout.
|
|
84
|
-
const response = await asyncTimeout(
|
|
89
|
+
const response: NormalizationOutput = await asyncTimeout(
|
|
85
90
|
this._functionExecutor.invoke(sentenceNormalization, { messages }),
|
|
86
91
|
PROCESSING_TIMEOUT,
|
|
87
92
|
);
|
|
@@ -2,7 +2,6 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { effect } from '@preact/signals-core';
|
|
6
5
|
import { describe, test } from 'vitest';
|
|
7
6
|
|
|
8
7
|
import { scheduleTaskInterval } from '@dxos/async';
|
|
@@ -10,15 +9,15 @@ import { Context } from '@dxos/context';
|
|
|
10
9
|
import { Obj } from '@dxos/echo';
|
|
11
10
|
import { createQueueDXN } from '@dxos/echo/internal';
|
|
12
11
|
import { MemoryQueue } from '@dxos/echo-db';
|
|
13
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
12
|
+
import { FunctionExecutor, ServiceContainer } from '@dxos/functions-runtime';
|
|
14
13
|
import { log } from '@dxos/log';
|
|
15
|
-
import {
|
|
14
|
+
import { type Actor, Message } from '@dxos/types';
|
|
16
15
|
|
|
17
16
|
import { MessageNormalizer } from './message-normalizer';
|
|
18
17
|
import { type MessageWithRangeId, sentenceNormalization } from './normalization';
|
|
19
18
|
import { getActorId } from './utils';
|
|
20
19
|
|
|
21
|
-
const sender:
|
|
20
|
+
const sender: Actor.Actor = {
|
|
22
21
|
identityDid: 'did:key:123',
|
|
23
22
|
};
|
|
24
23
|
|
|
@@ -48,7 +47,7 @@ const messages: MessageWithRangeId[] = [
|
|
|
48
47
|
// No punctuation.
|
|
49
48
|
'in classical physics objects have well-defined properties such as position speed and momentum',
|
|
50
49
|
].map((string, index) =>
|
|
51
|
-
Obj.make(
|
|
50
|
+
Obj.make(Message.Message, {
|
|
52
51
|
created: new Date(Date.now() + 1_000 * index).toISOString(),
|
|
53
52
|
sender,
|
|
54
53
|
blocks: [{ _tag: 'transcript', started: new Date(Date.now() + 1_000 * index).toISOString(), text: string }],
|
|
@@ -99,7 +98,7 @@ describe.skip('SentenceNormalization', () => {
|
|
|
99
98
|
}
|
|
100
99
|
|
|
101
100
|
log.info('sentences', {
|
|
102
|
-
|
|
101
|
+
messages: JSON.stringify(messages, null, 2),
|
|
103
102
|
sentences: JSON.stringify(sentences, null, 2),
|
|
104
103
|
});
|
|
105
104
|
throw new Error('test');
|
|
@@ -108,7 +107,7 @@ describe.skip('SentenceNormalization', () => {
|
|
|
108
107
|
test.only('queue', { timeout: 120_000 }, async () => {
|
|
109
108
|
// Create queue.
|
|
110
109
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
111
|
-
const queue = new MemoryQueue<
|
|
110
|
+
const queue = new MemoryQueue<Message.Message>(createQueueDXN());
|
|
112
111
|
const ctx = new Context();
|
|
113
112
|
let idx = 0;
|
|
114
113
|
scheduleTaskInterval(
|
|
@@ -133,10 +132,6 @@ describe.skip('SentenceNormalization', () => {
|
|
|
133
132
|
|
|
134
133
|
// Start normalizer.
|
|
135
134
|
await normalizer.open();
|
|
136
|
-
effect(() => {
|
|
137
|
-
log.info('normalizer');
|
|
138
|
-
log.info(JSON.stringify(queue.objects, null, 2));
|
|
139
|
-
});
|
|
140
135
|
|
|
141
136
|
await new Promise(() => {});
|
|
142
137
|
});
|
|
@@ -2,85 +2,47 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
|
|
6
|
-
// @ts-nocheck
|
|
7
|
-
|
|
5
|
+
import * as Effect from 'effect/Effect';
|
|
8
6
|
import * as Schema from 'effect/Schema';
|
|
9
7
|
|
|
10
|
-
import { AiService, DEFAULT_EDGE_MODEL } from '@dxos/ai';
|
|
11
|
-
import { AiSession } from '@dxos/assistant';
|
|
12
|
-
import { Obj } from '@dxos/echo';
|
|
13
|
-
import { defineFunction } from '@dxos/functions';
|
|
14
|
-
import { ObjectId } from '@dxos/keys';
|
|
15
8
|
import { log } from '@dxos/log';
|
|
16
|
-
import {
|
|
9
|
+
import { Operation } from '@dxos/operation';
|
|
17
10
|
import { trim } from '@dxos/util';
|
|
18
11
|
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
12
|
+
import {
|
|
13
|
+
SentenceNormalization,
|
|
14
|
+
SentenceNormalizationOutput,
|
|
15
|
+
type MessageWithRangeIdType,
|
|
16
|
+
type SentenceNormalizationInputType,
|
|
17
|
+
} from '../operations/definitions';
|
|
18
|
+
|
|
19
|
+
export type MessageWithRangeId = MessageWithRangeIdType;
|
|
20
|
+
|
|
21
|
+
/** @deprecated Use SentenceNormalizationInputType */
|
|
22
|
+
export type NormalizationInput = SentenceNormalizationInputType;
|
|
23
|
+
|
|
24
|
+
/** @deprecated Use SentenceNormalizationOutput */
|
|
25
|
+
export type NormalizationOutput = Schema.Schema.Type<typeof SentenceNormalizationOutput>;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Sentence normalization for transcription.
|
|
29
|
+
* TODO(dmaretskyi): Reimplement using AiSession.run or LanguageModel.generateObject - runStructured was removed.
|
|
30
|
+
*/
|
|
31
|
+
export const sentenceNormalization = SentenceNormalization.pipe(
|
|
32
|
+
Operation.withHandler(
|
|
33
|
+
Effect.fn(function* ({ messages }) {
|
|
34
|
+
log.info('input', { messages });
|
|
35
|
+
// TODO(dmaretskyi): runStructured was removed from AiSession. Reimplement using new API.
|
|
36
|
+
return yield* Effect.die(
|
|
37
|
+
new Error(
|
|
38
|
+
'Sentence normalization needs to be reimplemented - runStructured was removed from AiSession. Use AiSession.run or LanguageModel.generateObject.',
|
|
39
|
+
),
|
|
40
|
+
);
|
|
25
41
|
}),
|
|
26
|
-
|
|
42
|
+
),
|
|
27
43
|
);
|
|
28
44
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
export const NormalizationInput = Schema.Struct({
|
|
32
|
-
// TODO(mykola): Move to meta in DataType.Message. Use `DataType.Message` instead.
|
|
33
|
-
messages: Schema.Array(MessageWithRangeId).annotations({
|
|
34
|
-
description: 'Messages to normalize into sentences.',
|
|
35
|
-
}),
|
|
36
|
-
});
|
|
37
|
-
export interface NormalizationInput extends Schema.Schema.Type<typeof NormalizationInput> {}
|
|
38
|
-
|
|
39
|
-
export const NormalizationOutput = Schema.Struct({
|
|
40
|
-
sentences: Schema.Array(MessageWithRangeId.pipe(Schema.mutable)).pipe(Schema.mutable).annotations({
|
|
41
|
-
description: 'The sentences of the transcript.',
|
|
42
|
-
}),
|
|
43
|
-
});
|
|
44
|
-
export interface NormalizationOutput extends Schema.Schema.Type<typeof NormalizationOutput> {}
|
|
45
|
-
|
|
46
|
-
export const sentenceNormalization = defineFunction<NormalizationInput, NormalizationOutput>({
|
|
47
|
-
description: 'Post process of transcription for sentence normalization',
|
|
48
|
-
inputSchema: NormalizationInput,
|
|
49
|
-
outputSchema: NormalizationOutput,
|
|
50
|
-
handler: async ({ data: { messages }, context }) => {
|
|
51
|
-
log.info('input', { messages });
|
|
52
|
-
const ai = context.getService(AiService);
|
|
53
|
-
const session = new AiSession({ operationModel: 'configured' });
|
|
54
|
-
|
|
55
|
-
// TODO(dmaretskyi): This got broken after effect-ai transition.
|
|
56
|
-
const response = session.runStructured(NormalizationOutput, {
|
|
57
|
-
generationOptions: {
|
|
58
|
-
model: DEFAULT_EDGE_MODEL,
|
|
59
|
-
},
|
|
60
|
-
client: ai.client,
|
|
61
|
-
tools: [],
|
|
62
|
-
artifacts: [],
|
|
63
|
-
history: [
|
|
64
|
-
Obj.make(DataType.Message, {
|
|
65
|
-
created: new Date().toISOString(),
|
|
66
|
-
sender: {
|
|
67
|
-
role: 'user',
|
|
68
|
-
},
|
|
69
|
-
blocks: messages.map((message) => ({ _tag: 'text', text: JSON.stringify(message) }) as const),
|
|
70
|
-
}),
|
|
71
|
-
],
|
|
72
|
-
prompt,
|
|
73
|
-
}) as any;
|
|
74
|
-
|
|
75
|
-
response.sentences.forEach((sentence) => {
|
|
76
|
-
sentence.id = ObjectId.random();
|
|
77
|
-
});
|
|
78
|
-
log.info('response', { response });
|
|
79
|
-
return response;
|
|
80
|
-
},
|
|
81
|
-
});
|
|
82
|
-
|
|
83
|
-
const prompt = trim`
|
|
45
|
+
const _unusedPrompt = trim`
|
|
84
46
|
You are observing a real-time transcript of a single person speaking.
|
|
85
47
|
The transcription is delivered in chunks of 10 seconds or less. As a result, individual sentences may be split across multiple messages, or multiple sentences may appear within a single message. Additionally, because this is real-time transcription, punctuation and capitalization may be incorrect or missing.
|
|
86
48
|
|
|
@@ -89,7 +51,7 @@ const prompt = trim`
|
|
|
89
51
|
|
|
90
52
|
# Input Format:
|
|
91
53
|
- The input is an array of messages that contain transcription blocks.
|
|
92
|
-
- Each message is a
|
|
54
|
+
- Each message is a Message.Message, as described in the output format.
|
|
93
55
|
|
|
94
56
|
# Output Format:
|
|
95
57
|
- You have been provided with the tool that defines the output format; make sure to query it.
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import { type
|
|
5
|
+
import { type Actor } from '@dxos/types';
|
|
6
6
|
|
|
7
|
-
export const getActorId = (actor:
|
|
7
|
+
export const getActorId = (actor: Actor.Actor) => {
|
|
8
8
|
return actor.identityDid || actor.identityKey || actor.email || actor.name || '';
|
|
9
9
|
};
|
package/src/testing/testing.ts
CHANGED
|
@@ -11,13 +11,13 @@ import { Filter, type Queue } from '@dxos/client/echo';
|
|
|
11
11
|
import { Context } from '@dxos/context';
|
|
12
12
|
import { type Key, Obj, Ref, Type } from '@dxos/echo';
|
|
13
13
|
import { createQueueDXN } from '@dxos/echo/internal';
|
|
14
|
-
import { FunctionExecutor, ServiceContainer } from '@dxos/functions';
|
|
15
14
|
import { IdentityDid } from '@dxos/keys';
|
|
16
15
|
import { log } from '@dxos/log';
|
|
17
16
|
import { faker } from '@dxos/random';
|
|
18
17
|
import { type Space, useQueue } from '@dxos/react-client/echo';
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
18
|
+
import { TestSchema } from '@dxos/schema/testing';
|
|
19
|
+
import { type ContentBlock, Message, Organization, Person } from '@dxos/types';
|
|
20
|
+
import { seedTestData } from '@dxos/types/testing';
|
|
21
21
|
|
|
22
22
|
// TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
|
|
23
23
|
export const TestItem = Schema.Struct({
|
|
@@ -30,15 +30,15 @@ export const TestItem = Schema.Struct({
|
|
|
30
30
|
description: 'Product description',
|
|
31
31
|
}),
|
|
32
32
|
}).pipe(
|
|
33
|
-
Type.
|
|
34
|
-
typename: 'dxos.
|
|
33
|
+
Type.object({
|
|
34
|
+
typename: 'org.dxos.type.test',
|
|
35
35
|
version: '0.1.0',
|
|
36
36
|
}),
|
|
37
37
|
);
|
|
38
38
|
|
|
39
39
|
// TODO(wittjosiah): Make builder generic and reuse for all message types.
|
|
40
40
|
abstract class AbstractMessageBuilder {
|
|
41
|
-
abstract createMessage(numSegments?: number): Promise<
|
|
41
|
+
abstract createMessage(numSegments?: number): Promise<Message.Message>;
|
|
42
42
|
}
|
|
43
43
|
|
|
44
44
|
/**
|
|
@@ -59,19 +59,24 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
59
59
|
super();
|
|
60
60
|
}
|
|
61
61
|
|
|
62
|
-
override async createMessage(numSegments = 1): Promise<
|
|
63
|
-
return Obj.make(
|
|
62
|
+
override async createMessage(numSegments = 1): Promise<Message.Message> {
|
|
63
|
+
return Obj.make(Message.Message, {
|
|
64
64
|
created: this.next().toISOString(),
|
|
65
65
|
sender: faker.helpers.arrayElement(this.users),
|
|
66
66
|
blocks: Array.from({ length: numSegments }).map(() => this.createBlock()),
|
|
67
67
|
});
|
|
68
68
|
}
|
|
69
69
|
|
|
70
|
-
createBlock():
|
|
70
|
+
createBlock(): ContentBlock.Transcript {
|
|
71
71
|
let text = faker.lorem.paragraph();
|
|
72
72
|
if (this._space) {
|
|
73
73
|
const label = faker.commerce.productName();
|
|
74
|
-
const obj = this._space.db.add(
|
|
74
|
+
const obj = this._space.db.add(
|
|
75
|
+
Obj.make(TestItem, {
|
|
76
|
+
title: label,
|
|
77
|
+
description: faker.lorem.paragraph(),
|
|
78
|
+
}),
|
|
79
|
+
);
|
|
75
80
|
const dxn = Ref.make(obj).dxn.toString();
|
|
76
81
|
const words = text.split(' ');
|
|
77
82
|
words.splice(Math.floor(Math.random() * words.length), 0, `[${label}](${dxn})`);
|
|
@@ -93,17 +98,9 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
93
98
|
|
|
94
99
|
// TODO(burdon): Reconcile with BlockBuilder.
|
|
95
100
|
class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
96
|
-
executor = new FunctionExecutor(
|
|
97
|
-
new ServiceContainer().setServices({
|
|
98
|
-
// ai: {
|
|
99
|
-
// client: this.AiService,
|
|
100
|
-
// },
|
|
101
|
-
}),
|
|
102
|
-
);
|
|
103
|
-
|
|
104
101
|
space: Space | undefined;
|
|
105
102
|
currentMessage: number = 0;
|
|
106
|
-
transcriptMessages:
|
|
103
|
+
transcriptMessages: Message.Message[] = [];
|
|
107
104
|
|
|
108
105
|
async connect(space: Space): Promise<void> {
|
|
109
106
|
this.space = space;
|
|
@@ -111,14 +108,18 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
111
108
|
this.transcriptMessages = transcriptMessages;
|
|
112
109
|
}
|
|
113
110
|
|
|
114
|
-
override async createMessage(): Promise<
|
|
111
|
+
override async createMessage(): Promise<Message.Message> {
|
|
115
112
|
if (!this.space) {
|
|
116
113
|
throw new Error('Space not connected');
|
|
117
114
|
}
|
|
118
115
|
|
|
119
|
-
const
|
|
116
|
+
const objects = await this.space.db
|
|
120
117
|
.query(
|
|
121
|
-
Filter.or(
|
|
118
|
+
Filter.or(
|
|
119
|
+
Filter.type(Person.Person),
|
|
120
|
+
Filter.type(Organization.Organization),
|
|
121
|
+
Filter.type(TestSchema.DocumentType),
|
|
122
|
+
),
|
|
122
123
|
)
|
|
123
124
|
.run();
|
|
124
125
|
|
|
@@ -129,7 +130,6 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
129
130
|
|
|
130
131
|
const { message: enhancedMessage } = await processTranscriptMessage({
|
|
131
132
|
input: { message },
|
|
132
|
-
executor: this.executor,
|
|
133
133
|
function: extractionAnthropicFunction,
|
|
134
134
|
});
|
|
135
135
|
|
|
@@ -142,7 +142,7 @@ type UseTestTranscriptionQueue = (
|
|
|
142
142
|
queueId?: Key.ObjectId,
|
|
143
143
|
running?: boolean,
|
|
144
144
|
interval?: number,
|
|
145
|
-
) => Queue<
|
|
145
|
+
) => Queue<Message.Message> | undefined;
|
|
146
146
|
|
|
147
147
|
/**
|
|
148
148
|
* Test transcriptionqueue.
|
|
@@ -155,7 +155,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
|
155
155
|
) => {
|
|
156
156
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
157
157
|
const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
|
|
158
|
-
const queue = useQueue<
|
|
158
|
+
const queue = useQueue<Message.Message>(queueDxn);
|
|
159
159
|
const builder = useMemo(() => new MessageBuilder(space), [space]);
|
|
160
160
|
|
|
161
161
|
useEffect(() => {
|
|
@@ -165,7 +165,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
|
165
165
|
|
|
166
166
|
const i = setInterval(() => {
|
|
167
167
|
void builder.createMessage(Math.ceil(Math.random() * 3)).then(async (message) => {
|
|
168
|
-
await queue.append([Obj.make(
|
|
168
|
+
await queue.append([Obj.make(Message.Message, message)]);
|
|
169
169
|
});
|
|
170
170
|
}, interval);
|
|
171
171
|
return () => clearInterval(i);
|
|
@@ -186,7 +186,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
|
|
|
186
186
|
) => {
|
|
187
187
|
// TODO(dmaretskyi): Use space.queues.create() instead.
|
|
188
188
|
const queueDxn = useMemo(() => (space ? createQueueDXN(space.id, queueId) : undefined), [space, queueId]);
|
|
189
|
-
const queue = useQueue<
|
|
189
|
+
const queue = useQueue<Message.Message>(queueDxn);
|
|
190
190
|
const [builder] = useState(() => new EntityExtractionMessageBuilder());
|
|
191
191
|
|
|
192
192
|
useEffect(() => {
|
|
@@ -203,7 +203,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
|
|
|
203
203
|
ctx,
|
|
204
204
|
async () => {
|
|
205
205
|
const message = await builder.createMessage();
|
|
206
|
-
void queue.append([Obj.make(
|
|
206
|
+
void queue.append([Obj.make(Message.Message, message)]);
|
|
207
207
|
},
|
|
208
208
|
interval,
|
|
209
209
|
);
|
|
@@ -19,7 +19,7 @@ const initializeExtendableMediaRecorder = async () => {
|
|
|
19
19
|
await register(await connect());
|
|
20
20
|
};
|
|
21
21
|
|
|
22
|
-
export type
|
|
22
|
+
export type MediaStreamRecorderProps = {
|
|
23
23
|
mediaStreamTrack: MediaStreamTrack;
|
|
24
24
|
config: {
|
|
25
25
|
interval: number;
|
|
@@ -37,12 +37,12 @@ export class MediaStreamRecorder implements AudioRecorder {
|
|
|
37
37
|
/**
|
|
38
38
|
* Default MediaRecorder implementation do not support wav encoding.
|
|
39
39
|
*/
|
|
40
|
-
private readonly _config:
|
|
40
|
+
private readonly _config: MediaStreamRecorderProps['config'];
|
|
41
41
|
private _mediaRecorder?: IMediaRecorder = undefined;
|
|
42
42
|
private _header?: Uint8Array = undefined;
|
|
43
43
|
private _onChunk?: (chunk: AudioChunk) => void;
|
|
44
44
|
|
|
45
|
-
constructor({ mediaStreamTrack, config }:
|
|
45
|
+
constructor({ mediaStreamTrack, config }: MediaStreamRecorderProps) {
|
|
46
46
|
this._mediaStreamTrack = mediaStreamTrack;
|
|
47
47
|
this._config = config;
|
|
48
48
|
}
|
|
@@ -10,9 +10,9 @@ import { WaveFile } from 'wavefile';
|
|
|
10
10
|
|
|
11
11
|
import { Trigger } from '@dxos/async';
|
|
12
12
|
import { log } from '@dxos/log';
|
|
13
|
-
import { type DataType } from '@dxos/schema';
|
|
14
13
|
import { openAndClose } from '@dxos/test-utils';
|
|
15
14
|
import { TRACE_PROCESSOR, trace } from '@dxos/tracing';
|
|
15
|
+
import { type ContentBlock } from '@dxos/types';
|
|
16
16
|
|
|
17
17
|
import { type AudioChunk, type AudioRecorder, Transcriber } from '../transcriber';
|
|
18
18
|
import { mergeFloat64Arrays } from '../util';
|
|
@@ -94,7 +94,7 @@ describe.skip('Transcriber', () => {
|
|
|
94
94
|
});
|
|
95
95
|
|
|
96
96
|
test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
|
|
97
|
-
const trigger = new Trigger<
|
|
97
|
+
const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
|
|
98
98
|
const recorder = new MockAudioRecorder({
|
|
99
99
|
buffer: await readFile('test.wav'),
|
|
100
100
|
chunkDuration: 3_000,
|
|
@@ -128,7 +128,7 @@ describe.skip('Transcriber', () => {
|
|
|
128
128
|
});
|
|
129
129
|
|
|
130
130
|
test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
|
|
131
|
-
const trigger = new Trigger<
|
|
131
|
+
const trigger = new Trigger<ContentBlock.Transcript[]>({ autoReset: true });
|
|
132
132
|
const recorder = new MockAudioRecorder({
|
|
133
133
|
buffer: await readFile('test.wav'),
|
|
134
134
|
chunkDuration: 3_000,
|
|
@@ -213,7 +213,7 @@ class MockAudioRecorder implements AudioRecorder {
|
|
|
213
213
|
|
|
214
214
|
@trace.span()
|
|
215
215
|
async start(): Promise<void> {
|
|
216
|
-
log.info('start mock audio recorder', {
|
|
216
|
+
log.info('start mock audio recorder', { wavProps: this.wav.fmt });
|
|
217
217
|
const now = Date.now();
|
|
218
218
|
const samples = this.wav.getSamples();
|
|
219
219
|
const sampleRate = this.wavConfig.sampleRate;
|