@dxos/plugin-transcription 0.8.2-main.f11618f → 0.8.2-main.fbd8ed0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{TranscriptContainer-D3EH2SLO.mjs → TranscriptContainer-CXA4ZBHW.mjs} +2 -2
- package/dist/lib/browser/{app-graph-builder-7HN3M24G.mjs → app-graph-builder-REQZ5SC5.mjs} +37 -35
- package/dist/lib/browser/app-graph-builder-REQZ5SC5.mjs.map +7 -0
- package/dist/lib/browser/{chunk-KSUYWWIU.mjs → chunk-3TKXI6TQ.mjs} +6 -6
- package/dist/lib/browser/chunk-3TKXI6TQ.mjs.map +7 -0
- package/dist/lib/browser/{chunk-YRWBY6UK.mjs → chunk-JHWEZRWC.mjs} +19 -17
- package/dist/lib/browser/chunk-JHWEZRWC.mjs.map +7 -0
- package/dist/lib/browser/{chunk-BSZSM6KA.mjs → chunk-K2LS2QOM.mjs} +6 -6
- package/dist/lib/browser/chunk-K2LS2QOM.mjs.map +7 -0
- package/dist/lib/browser/{chunk-IWZ7C3OA.mjs → chunk-LCTMCTYY.mjs} +2 -2
- package/dist/lib/browser/index.mjs +7 -7
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/{intent-resolver-ZWKYRFE7.mjs → intent-resolver-DZ62CJPC.mjs} +3 -3
- package/dist/lib/browser/intent-resolver-DZ62CJPC.mjs.map +7 -0
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{react-surface-F7DBDGHS.mjs → react-surface-PUXZNAZL.mjs} +4 -4
- package/dist/lib/browser/{settings-6YAHKVKX.mjs → settings-JWDS4J3F.mjs} +2 -2
- package/dist/lib/browser/{transcriber-MSBS4VBI.mjs → transcriber-YWWIV6KP.mjs} +3 -3
- package/dist/lib/browser/types/index.mjs +1 -1
- package/dist/lib/node/{TranscriptContainer-WCBZMA5K.cjs → TranscriptContainer-SSJVLRM6.cjs} +7 -7
- package/dist/lib/node/{TranscriptContainer-WCBZMA5K.cjs.map → TranscriptContainer-SSJVLRM6.cjs.map} +2 -2
- package/dist/lib/node/{app-graph-builder-BVWYUGUO.cjs → app-graph-builder-OAPU4GDH.cjs} +50 -48
- package/dist/lib/node/app-graph-builder-OAPU4GDH.cjs.map +7 -0
- package/dist/lib/node/{chunk-IXU57SCA.cjs → chunk-EJWB53WK.cjs} +19 -17
- package/dist/lib/node/chunk-EJWB53WK.cjs.map +7 -0
- package/dist/lib/node/{chunk-JSCC2AQW.cjs → chunk-EVE3H46F.cjs} +9 -9
- package/dist/lib/node/chunk-EVE3H46F.cjs.map +7 -0
- package/dist/lib/node/{chunk-OCHTKDL7.cjs → chunk-L27AK6FQ.cjs} +5 -5
- package/dist/lib/node/{chunk-7XIIUVN3.cjs → chunk-WKZLGVEJ.cjs} +9 -9
- package/dist/lib/node/chunk-WKZLGVEJ.cjs.map +7 -0
- package/dist/lib/node/index.cjs +28 -28
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/{intent-resolver-RM4JOCDV.cjs → intent-resolver-TRBKI24D.cjs} +9 -9
- package/dist/lib/node/intent-resolver-TRBKI24D.cjs.map +7 -0
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/{react-surface-7F37EYZA.cjs → react-surface-SZAMHCX7.cjs} +10 -10
- package/dist/lib/node/{settings-ZMB4XRWS.cjs → settings-AWWMOSCV.cjs} +7 -7
- package/dist/lib/node/{transcriber-55OE4X6I.cjs → transcriber-AFUXFHCR.cjs} +8 -8
- package/dist/lib/node/types/index.cjs +7 -7
- package/dist/lib/node/types/index.cjs.map +1 -1
- package/dist/lib/node-esm/{TranscriptContainer-GBPY6KKA.mjs → TranscriptContainer-M53LI2TX.mjs} +2 -2
- package/dist/lib/node-esm/{app-graph-builder-QK25O5K5.mjs → app-graph-builder-UBP6EOBR.mjs} +37 -35
- package/dist/lib/node-esm/app-graph-builder-UBP6EOBR.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-B2BD7S4X.mjs → chunk-4KC3E5T4.mjs} +6 -6
- package/dist/lib/node-esm/chunk-4KC3E5T4.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-DXJN2JWV.mjs → chunk-MHQP2PAF.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-OS2ITZUL.mjs → chunk-QYG2FOXA.mjs} +19 -17
- package/dist/lib/node-esm/chunk-QYG2FOXA.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-UWF6LM27.mjs → chunk-V5IWBPYO.mjs} +6 -6
- package/dist/lib/node-esm/chunk-V5IWBPYO.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +7 -7
- package/dist/lib/node-esm/index.mjs.map +3 -3
- package/dist/lib/node-esm/{intent-resolver-HDRFXMKP.mjs → intent-resolver-HFVAL274.mjs} +3 -3
- package/dist/lib/node-esm/intent-resolver-HFVAL274.mjs.map +7 -0
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{react-surface-D3L6VBEV.mjs → react-surface-UEJELL7I.mjs} +4 -4
- package/dist/lib/node-esm/{settings-I35TVDUG.mjs → settings-QFNFGFJ5.mjs} +2 -2
- package/dist/lib/node-esm/{transcriber-HH7EMCGH.mjs → transcriber-7RJEFPDI.mjs} +3 -3
- package/dist/lib/node-esm/types/index.mjs +1 -1
- package/dist/types/src/capabilities/app-graph-builder.d.ts +2 -179
- package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts +2 -178
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/capabilities/intent-resolver.d.ts +2 -2
- package/dist/types/src/capabilities/intent-resolver.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts +3 -3
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +2 -2
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +10 -10
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/transcript-extension.d.ts +2 -2
- package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +1 -1
- package/dist/types/src/components/TranscriptionSettings/TranscriptionSettings.d.ts.map +1 -1
- package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
- package/dist/types/src/entity-extraction/entity-extraction.d.ts +3 -3
- package/dist/types/src/entity-extraction/entity-extraction.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioFile.d.ts.map +1 -1
- package/dist/types/src/hooks/useAudioTrack.d.ts.map +1 -1
- package/dist/types/src/hooks/useQueueModelAdapter.d.ts.map +1 -1
- package/dist/types/src/hooks/useTranscriber.d.ts.map +1 -1
- package/dist/types/src/hooks/useVoiceInput.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +11 -10
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +2 -2
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcription-manager.d.ts +2 -2
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/translations.d.ts +2 -20
- package/dist/types/src/translations.d.ts.map +1 -1
- package/dist/types/src/types/schema.d.ts +26 -19
- package/dist/types/src/types/schema.d.ts.map +1 -1
- package/dist/types/src/types/types.d.ts +17 -10
- package/dist/types/src/types/types.d.ts.map +1 -1
- package/dist/types/src/util/array.d.ts.map +1 -1
- package/dist/types/src/util/get-time-str.d.ts.map +1 -1
- package/dist/types/src/util/monitor-audio-level.d.ts.map +1 -1
- package/dist/types/tsconfig.tsbuildinfo +1 -1
- package/package.json +55 -49
- package/src/TranscriptionPlugin.tsx +3 -3
- package/src/capabilities/app-graph-builder.ts +145 -123
- package/src/capabilities/intent-resolver.ts +4 -3
- package/src/components/Transcript/Transcript.stories.tsx +6 -6
- package/src/components/Transcript/Transcript.tsx +3 -3
- package/src/components/Transcript/deprecated/OldTranscript.tsx +19 -13
- package/src/components/Transcript/transcript-extension.ts +3 -3
- package/src/components/TranscriptContainer.tsx +2 -2
- package/src/components/transcription.stories.tsx +17 -15
- package/src/entity-extraction/entity-extraction.ts +4 -4
- package/src/model/model.test.ts +14 -10
- package/src/testing/testing.ts +31 -23
- package/src/transcriber/transcriber.browser.test.ts +3 -3
- package/src/transcriber/transcriber.ts +6 -3
- package/src/transcriber/transcription-manager.ts +8 -8
- package/src/types/schema.ts +10 -8
- package/src/types/types.ts +14 -12
- package/dist/lib/browser/app-graph-builder-7HN3M24G.mjs.map +0 -7
- package/dist/lib/browser/chunk-BSZSM6KA.mjs.map +0 -7
- package/dist/lib/browser/chunk-KSUYWWIU.mjs.map +0 -7
- package/dist/lib/browser/chunk-YRWBY6UK.mjs.map +0 -7
- package/dist/lib/browser/intent-resolver-ZWKYRFE7.mjs.map +0 -7
- package/dist/lib/node/app-graph-builder-BVWYUGUO.cjs.map +0 -7
- package/dist/lib/node/chunk-7XIIUVN3.cjs.map +0 -7
- package/dist/lib/node/chunk-IXU57SCA.cjs.map +0 -7
- package/dist/lib/node/chunk-JSCC2AQW.cjs.map +0 -7
- package/dist/lib/node/intent-resolver-RM4JOCDV.cjs.map +0 -7
- package/dist/lib/node-esm/app-graph-builder-QK25O5K5.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-B2BD7S4X.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-OS2ITZUL.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-UWF6LM27.mjs.map +0 -7
- package/dist/lib/node-esm/intent-resolver-HDRFXMKP.mjs.map +0 -7
- /package/dist/lib/browser/{TranscriptContainer-D3EH2SLO.mjs.map → TranscriptContainer-CXA4ZBHW.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-IWZ7C3OA.mjs.map → chunk-LCTMCTYY.mjs.map} +0 -0
- /package/dist/lib/browser/{react-surface-F7DBDGHS.mjs.map → react-surface-PUXZNAZL.mjs.map} +0 -0
- /package/dist/lib/browser/{settings-6YAHKVKX.mjs.map → settings-JWDS4J3F.mjs.map} +0 -0
- /package/dist/lib/browser/{transcriber-MSBS4VBI.mjs.map → transcriber-YWWIV6KP.mjs.map} +0 -0
- /package/dist/lib/node/{chunk-OCHTKDL7.cjs.map → chunk-L27AK6FQ.cjs.map} +0 -0
- /package/dist/lib/node/{react-surface-7F37EYZA.cjs.map → react-surface-SZAMHCX7.cjs.map} +0 -0
- /package/dist/lib/node/{settings-ZMB4XRWS.cjs.map → settings-AWWMOSCV.cjs.map} +0 -0
- /package/dist/lib/node/{transcriber-55OE4X6I.cjs.map → transcriber-AFUXFHCR.cjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-GBPY6KKA.mjs.map → TranscriptContainer-M53LI2TX.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-DXJN2JWV.mjs.map → chunk-MHQP2PAF.mjs.map} +0 -0
- /package/dist/lib/node-esm/{react-surface-D3L6VBEV.mjs.map → react-surface-UEJELL7I.mjs.map} +0 -0
- /package/dist/lib/node-esm/{settings-I35TVDUG.mjs.map → settings-QFNFGFJ5.mjs.map} +0 -0
- /package/dist/lib/node-esm/{transcriber-HH7EMCGH.mjs.map → transcriber-7RJEFPDI.mjs.map} +0 -0
|
@@ -11,12 +11,12 @@ import { asyncTimeout } from '@dxos/async';
|
|
|
11
11
|
import { type BaseEchoObject, create, ObjectId } from '@dxos/echo-schema';
|
|
12
12
|
import { DXN } from '@dxos/keys';
|
|
13
13
|
import { log } from '@dxos/log';
|
|
14
|
-
import {
|
|
14
|
+
import { DataType } from '@dxos/schema';
|
|
15
15
|
|
|
16
16
|
import SYSTEM_PROMPT from './system-prompt.tpl?raw';
|
|
17
17
|
|
|
18
18
|
type ProcessTranscriptMessageParams = {
|
|
19
|
-
message:
|
|
19
|
+
message: DataType.Message;
|
|
20
20
|
aiService: AIServiceClient;
|
|
21
21
|
context: {
|
|
22
22
|
objects?: BaseEchoObject[];
|
|
@@ -38,7 +38,7 @@ type ProcessTranscriptMessageParams = {
|
|
|
38
38
|
};
|
|
39
39
|
|
|
40
40
|
type ProcessTranscriptMessageResult = {
|
|
41
|
-
message:
|
|
41
|
+
message: DataType.Message;
|
|
42
42
|
timeElapsed: number;
|
|
43
43
|
};
|
|
44
44
|
|
|
@@ -98,7 +98,7 @@ export const processTranscriptMessage = async (
|
|
|
98
98
|
log.info('entity extraction result', { refs: result.references });
|
|
99
99
|
|
|
100
100
|
return {
|
|
101
|
-
message: create(
|
|
101
|
+
message: create(DataType.Message, {
|
|
102
102
|
...params.message,
|
|
103
103
|
blocks: params.message.blocks.map((block, i) =>
|
|
104
104
|
block.type !== 'transcription'
|
package/src/model/model.test.ts
CHANGED
|
@@ -5,11 +5,15 @@
|
|
|
5
5
|
import { EditorView } from '@codemirror/view';
|
|
6
6
|
import { describe, test } from 'vitest';
|
|
7
7
|
|
|
8
|
-
import { type
|
|
8
|
+
import { type DataType } from '@dxos/schema';
|
|
9
9
|
|
|
10
10
|
import { SerializationModel, DocumentAdapter, type ChunkRenderer } from './model';
|
|
11
11
|
|
|
12
|
-
const blockToMarkdown: ChunkRenderer<
|
|
12
|
+
const blockToMarkdown: ChunkRenderer<DataType.Message> = (
|
|
13
|
+
message: DataType.Message,
|
|
14
|
+
index: number,
|
|
15
|
+
debug = true,
|
|
16
|
+
): string[] => {
|
|
13
17
|
return [
|
|
14
18
|
`###### ${message.sender.name}`,
|
|
15
19
|
...message.blocks.filter((block) => block.type === 'transcription').map((block) => block.text),
|
|
@@ -21,12 +25,12 @@ const createDate = () => new Date().toISOString();
|
|
|
21
25
|
|
|
22
26
|
describe('SerializationModel', () => {
|
|
23
27
|
test('basic', ({ expect }) => {
|
|
24
|
-
const model = new SerializationModel<
|
|
28
|
+
const model = new SerializationModel<DataType.Message>(blockToMarkdown);
|
|
25
29
|
expect(model.chunks.length).to.eq(0);
|
|
26
30
|
expect(model.doc.toString()).to.eq('');
|
|
27
31
|
|
|
28
32
|
// Create message.
|
|
29
|
-
const message:
|
|
33
|
+
const message: DataType.Message = {
|
|
30
34
|
id: '1',
|
|
31
35
|
created: createDate(),
|
|
32
36
|
sender: { name: 'Alice' },
|
|
@@ -57,14 +61,14 @@ describe('SerializationModel', () => {
|
|
|
57
61
|
|
|
58
62
|
test('sync - append', ({ expect }) => {
|
|
59
63
|
const view = new EditorView({ extensions: [], doc: '' });
|
|
60
|
-
const model = new SerializationModel<
|
|
64
|
+
const model = new SerializationModel<DataType.Message>(blockToMarkdown);
|
|
61
65
|
const adapter = new DocumentAdapter(view);
|
|
62
66
|
expect(adapter.lineCount()).to.eq(0);
|
|
63
67
|
expect(view.state.doc.toString()).to.eq('');
|
|
64
68
|
|
|
65
69
|
// Append message.
|
|
66
70
|
{
|
|
67
|
-
const message:
|
|
71
|
+
const message: DataType.Message = {
|
|
68
72
|
id: '1',
|
|
69
73
|
created: createDate(),
|
|
70
74
|
sender: { name: 'Alice' },
|
|
@@ -83,7 +87,7 @@ describe('SerializationModel', () => {
|
|
|
83
87
|
|
|
84
88
|
// Append message.
|
|
85
89
|
{
|
|
86
|
-
const message:
|
|
90
|
+
const message: DataType.Message = {
|
|
87
91
|
id: '2',
|
|
88
92
|
created: createDate(),
|
|
89
93
|
sender: { name: 'Bob' },
|
|
@@ -103,13 +107,13 @@ describe('SerializationModel', () => {
|
|
|
103
107
|
|
|
104
108
|
test('sync - append, update, delete', ({ expect }) => {
|
|
105
109
|
const view = new EditorView({ extensions: [], doc: '' });
|
|
106
|
-
const model = new SerializationModel<
|
|
110
|
+
const model = new SerializationModel<DataType.Message>(blockToMarkdown);
|
|
107
111
|
const adapter = new DocumentAdapter(view);
|
|
108
112
|
expect(adapter.lineCount()).to.eq(0);
|
|
109
113
|
|
|
110
114
|
// Append message.
|
|
111
115
|
{
|
|
112
|
-
const message:
|
|
116
|
+
const message: DataType.Message = {
|
|
113
117
|
id: '1',
|
|
114
118
|
created: createDate(),
|
|
115
119
|
sender: { name: 'Alice' },
|
|
@@ -142,7 +146,7 @@ describe('SerializationModel', () => {
|
|
|
142
146
|
|
|
143
147
|
// Append message.
|
|
144
148
|
{
|
|
145
|
-
const message:
|
|
149
|
+
const message: DataType.Message = {
|
|
146
150
|
id: '2',
|
|
147
151
|
created: createDate(),
|
|
148
152
|
sender: { name: 'Bob' },
|
package/src/testing/testing.ts
CHANGED
|
@@ -2,39 +2,45 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
+
import { Schema, SchemaAST } from 'effect';
|
|
5
6
|
import { useEffect, useMemo, useState } from 'react';
|
|
6
7
|
|
|
7
8
|
import { AIServiceEdgeClient } from '@dxos/assistant';
|
|
8
9
|
import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
|
|
9
10
|
import { scheduleTaskInterval } from '@dxos/async';
|
|
10
|
-
import {
|
|
11
|
+
import { Filter, type Queue } from '@dxos/client/echo';
|
|
11
12
|
import { Context } from '@dxos/context';
|
|
12
|
-
import {
|
|
13
|
+
import { Type } from '@dxos/echo';
|
|
14
|
+
import { create, createQueueDxn, ObjectId } from '@dxos/echo-schema';
|
|
13
15
|
import { IdentityDid } from '@dxos/keys';
|
|
14
16
|
import { log } from '@dxos/log';
|
|
15
17
|
import { faker } from '@dxos/random';
|
|
16
18
|
import { live, makeRef, useQueue, type Space } from '@dxos/react-client/echo';
|
|
17
|
-
import {
|
|
19
|
+
import { DataType } from '@dxos/schema';
|
|
18
20
|
import { Testing, seedTestData } from '@dxos/schema/testing';
|
|
19
21
|
|
|
20
22
|
import { processTranscriptMessage } from '../entity-extraction';
|
|
21
23
|
|
|
22
24
|
// TODO(burdon): Reconcile with plugin-markdown. Move to @dxos/schema/testing.
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
[
|
|
27
|
-
|
|
25
|
+
export const TestItem = Schema.Struct({
|
|
26
|
+
title: Schema.String.annotations({
|
|
27
|
+
[SchemaAST.TitleAnnotationId]: 'Title',
|
|
28
|
+
[SchemaAST.DescriptionAnnotationId]: 'Product title',
|
|
29
|
+
}),
|
|
30
|
+
description: Schema.String.annotations({
|
|
31
|
+
[SchemaAST.TitleAnnotationId]: 'Description',
|
|
32
|
+
[SchemaAST.DescriptionAnnotationId]: 'Product description',
|
|
28
33
|
}),
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
34
|
+
}).pipe(
|
|
35
|
+
Type.def({
|
|
36
|
+
typename: 'dxos.org/type/Test',
|
|
37
|
+
version: '0.1.0',
|
|
32
38
|
}),
|
|
33
|
-
|
|
39
|
+
);
|
|
34
40
|
|
|
35
41
|
// TODO(wittjosiah): Make builder generic and reuse for all message types.
|
|
36
42
|
abstract class AbstractMessageBuilder {
|
|
37
|
-
abstract createMessage(numSegments?: number): Promise<
|
|
43
|
+
abstract createMessage(numSegments?: number): Promise<DataType.Message>;
|
|
38
44
|
}
|
|
39
45
|
|
|
40
46
|
/**
|
|
@@ -55,7 +61,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
55
61
|
super();
|
|
56
62
|
}
|
|
57
63
|
|
|
58
|
-
override async createMessage(numSegments = 1): Promise<
|
|
64
|
+
override async createMessage(numSegments = 1): Promise<DataType.Message> {
|
|
59
65
|
return {
|
|
60
66
|
id: ObjectId.random().toString(),
|
|
61
67
|
created: this.next().toISOString(),
|
|
@@ -64,7 +70,7 @@ export class MessageBuilder extends AbstractMessageBuilder {
|
|
|
64
70
|
};
|
|
65
71
|
}
|
|
66
72
|
|
|
67
|
-
createBlock():
|
|
73
|
+
createBlock(): DataType.MessageBlock.Transcription {
|
|
68
74
|
let text = faker.lorem.paragraph();
|
|
69
75
|
if (this._space) {
|
|
70
76
|
const label = faker.commerce.productName();
|
|
@@ -96,7 +102,7 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
96
102
|
|
|
97
103
|
space: Space | undefined;
|
|
98
104
|
currentMessage: number = 0;
|
|
99
|
-
transcriptMessages:
|
|
105
|
+
transcriptMessages: DataType.Message[] = [];
|
|
100
106
|
|
|
101
107
|
async connect(space: Space) {
|
|
102
108
|
this.space = space;
|
|
@@ -104,13 +110,15 @@ class EntityExtractionMessageBuilder extends AbstractMessageBuilder {
|
|
|
104
110
|
this.transcriptMessages = transcriptMessages;
|
|
105
111
|
}
|
|
106
112
|
|
|
107
|
-
override async createMessage(): Promise<
|
|
113
|
+
override async createMessage(): Promise<DataType.Message> {
|
|
108
114
|
if (!this.space) {
|
|
109
115
|
throw new Error('Space not connected');
|
|
110
116
|
}
|
|
111
117
|
|
|
112
118
|
const { objects } = await this.space.db
|
|
113
|
-
.query(
|
|
119
|
+
.query(
|
|
120
|
+
Filter.or(Filter.type(DataType.Person), Filter.type(DataType.Organization), Filter.type(Testing.DocumentType)),
|
|
121
|
+
)
|
|
114
122
|
.run();
|
|
115
123
|
|
|
116
124
|
log.info('context', { objects });
|
|
@@ -135,7 +143,7 @@ type UseTestTranscriptionQueue = (
|
|
|
135
143
|
queueId?: ObjectId,
|
|
136
144
|
running?: boolean,
|
|
137
145
|
interval?: number,
|
|
138
|
-
) => Queue<
|
|
146
|
+
) => Queue<DataType.Message> | undefined;
|
|
139
147
|
|
|
140
148
|
/**
|
|
141
149
|
* Test transcriptionqueue.
|
|
@@ -147,7 +155,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
|
147
155
|
interval = 1_000,
|
|
148
156
|
) => {
|
|
149
157
|
const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
|
|
150
|
-
const queue = useQueue<
|
|
158
|
+
const queue = useQueue<DataType.Message>(queueDxn);
|
|
151
159
|
const builder = useMemo(() => new MessageBuilder(space), [space]);
|
|
152
160
|
|
|
153
161
|
useEffect(() => {
|
|
@@ -157,7 +165,7 @@ export const useTestTranscriptionQueue: UseTestTranscriptionQueue = (
|
|
|
157
165
|
|
|
158
166
|
const i = setInterval(() => {
|
|
159
167
|
void builder.createMessage(Math.ceil(Math.random() * 3)).then((message) => {
|
|
160
|
-
queue.append([create(
|
|
168
|
+
queue.append([create(DataType.Message, message)]);
|
|
161
169
|
});
|
|
162
170
|
}, interval);
|
|
163
171
|
return () => clearInterval(i);
|
|
@@ -177,7 +185,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
|
|
|
177
185
|
interval = 1_000,
|
|
178
186
|
) => {
|
|
179
187
|
const queueDxn = useMemo(() => (space ? createQueueDxn(space.id, queueId) : undefined), [space, queueId]);
|
|
180
|
-
const queue = useQueue<
|
|
188
|
+
const queue = useQueue<DataType.Message>(queueDxn);
|
|
181
189
|
const [builder] = useState(() => new EntityExtractionMessageBuilder());
|
|
182
190
|
|
|
183
191
|
useEffect(() => {
|
|
@@ -194,7 +202,7 @@ export const useTestTranscriptionQueueWithEntityExtraction: UseTestTranscription
|
|
|
194
202
|
ctx,
|
|
195
203
|
async () => {
|
|
196
204
|
const message = await builder.createMessage();
|
|
197
|
-
queue.append([create(
|
|
205
|
+
queue.append([create(DataType.Message, message)]);
|
|
198
206
|
},
|
|
199
207
|
interval,
|
|
200
208
|
);
|
|
@@ -9,7 +9,7 @@ import { WaveFile } from 'wavefile';
|
|
|
9
9
|
|
|
10
10
|
import { Trigger } from '@dxos/async';
|
|
11
11
|
import { log } from '@dxos/log';
|
|
12
|
-
import { type
|
|
12
|
+
import { type DataType } from '@dxos/schema';
|
|
13
13
|
import { openAndClose } from '@dxos/test-utils';
|
|
14
14
|
import { trace, TRACE_PROCESSOR } from '@dxos/tracing';
|
|
15
15
|
|
|
@@ -93,7 +93,7 @@ describe.skip('Transcriber', () => {
|
|
|
93
93
|
});
|
|
94
94
|
|
|
95
95
|
test.skip('transcription of audio recording', { timeout: 10_000 }, async () => {
|
|
96
|
-
const trigger = new Trigger<
|
|
96
|
+
const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
|
|
97
97
|
const recorder = new MockAudioRecorder({
|
|
98
98
|
buffer: await readFile('test.wav'),
|
|
99
99
|
chunkDuration: 3_000,
|
|
@@ -127,7 +127,7 @@ describe.skip('Transcriber', () => {
|
|
|
127
127
|
});
|
|
128
128
|
|
|
129
129
|
test.skip('transcription of audio recording with overlapping chunks', { timeout: 20_000 }, async () => {
|
|
130
|
-
const trigger = new Trigger<
|
|
130
|
+
const trigger = new Trigger<DataType.MessageBlock.Transcription[]>({ autoReset: true });
|
|
131
131
|
const recorder = new MockAudioRecorder({
|
|
132
132
|
buffer: await readFile('test.wav'),
|
|
133
133
|
chunkDuration: 3_000,
|
|
@@ -7,7 +7,7 @@ import { WaveFile } from 'wavefile';
|
|
|
7
7
|
import { DeferredTask, Trigger } from '@dxos/async';
|
|
8
8
|
import { type Context, LifecycleState, Resource } from '@dxos/context';
|
|
9
9
|
import { log } from '@dxos/log';
|
|
10
|
-
import { type
|
|
10
|
+
import { type DataType } from '@dxos/schema';
|
|
11
11
|
import { trace } from '@dxos/tracing';
|
|
12
12
|
|
|
13
13
|
import { type AudioRecorder, type AudioChunk } from './audio-recorder';
|
|
@@ -65,7 +65,7 @@ export type TranscribeConfig = {
|
|
|
65
65
|
export type TranscriberParams = {
|
|
66
66
|
config: TranscribeConfig;
|
|
67
67
|
recorder: AudioRecorder;
|
|
68
|
-
onSegments: (segments:
|
|
68
|
+
onSegments: (segments: DataType.MessageBlock.Transcription[]) => Promise<void>;
|
|
69
69
|
};
|
|
70
70
|
|
|
71
71
|
/**
|
|
@@ -216,7 +216,10 @@ export class Transcriber extends Resource {
|
|
|
216
216
|
return segments;
|
|
217
217
|
}
|
|
218
218
|
|
|
219
|
-
private _alignSegments(
|
|
219
|
+
private _alignSegments(
|
|
220
|
+
segments: WhisperSegment[],
|
|
221
|
+
originalChunks: AudioChunk[],
|
|
222
|
+
): DataType.MessageBlock.Transcription[] {
|
|
220
223
|
// Absolute zero for all relative timestamps in the segments.
|
|
221
224
|
const zeroTimestamp = originalChunks.at(0)!.timestamp;
|
|
222
225
|
|
|
@@ -9,7 +9,7 @@ import { create } from '@dxos/echo-schema';
|
|
|
9
9
|
import { type DXN } from '@dxos/keys';
|
|
10
10
|
import { log } from '@dxos/log';
|
|
11
11
|
import { type EdgeHttpClient } from '@dxos/react-edge-client';
|
|
12
|
-
import {
|
|
12
|
+
import { DataType } from '@dxos/schema';
|
|
13
13
|
|
|
14
14
|
import { MediaStreamRecorder } from './media-stream-recorder';
|
|
15
15
|
import { Transcriber } from './transcriber';
|
|
@@ -30,7 +30,7 @@ const PREFIXED_CHUNKS_AMOUNT = 10;
|
|
|
30
30
|
*/
|
|
31
31
|
const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
|
|
32
32
|
|
|
33
|
-
export type TranscriptMessageEnricher = (message:
|
|
33
|
+
export type TranscriptMessageEnricher = (message: DataType.Message) => Promise<DataType.Message>;
|
|
34
34
|
|
|
35
35
|
export type TranscriptionManagerOptions = {
|
|
36
36
|
edgeClient: EdgeHttpClient;
|
|
@@ -51,7 +51,7 @@ export class TranscriptionManager extends Resource {
|
|
|
51
51
|
private _identityDid?: string = undefined;
|
|
52
52
|
private _mediaRecorder?: MediaStreamRecorder = undefined;
|
|
53
53
|
private _transcriber?: Transcriber = undefined;
|
|
54
|
-
private _queue?: Queue<
|
|
54
|
+
private _queue?: Queue<DataType.Message> = undefined;
|
|
55
55
|
private _enabled = false;
|
|
56
56
|
|
|
57
57
|
constructor(options: TranscriptionManagerOptions) {
|
|
@@ -71,7 +71,7 @@ export class TranscriptionManager extends Resource {
|
|
|
71
71
|
setQueue(queueDxn: DXN): TranscriptionManager {
|
|
72
72
|
if (this._queue?.dxn.toString() !== queueDxn.toString()) {
|
|
73
73
|
log.info('setQueue', { queueDxn: queueDxn.toString() });
|
|
74
|
-
this._queue = new QueueImpl<
|
|
74
|
+
this._queue = new QueueImpl<DataType.Message>(this._edgeClient, queueDxn);
|
|
75
75
|
}
|
|
76
76
|
return this;
|
|
77
77
|
}
|
|
@@ -129,7 +129,7 @@ export class TranscriptionManager extends Resource {
|
|
|
129
129
|
if (this._enabled) {
|
|
130
130
|
await this._transcriber?.open();
|
|
131
131
|
// TODO(burdon): Started and stopped blocks appear twice.
|
|
132
|
-
const block = create(
|
|
132
|
+
const block = create(DataType.Message, {
|
|
133
133
|
created: new Date().toISOString(),
|
|
134
134
|
blocks: [{ type: 'transcription', text: 'Started', started: new Date().toISOString() }],
|
|
135
135
|
sender: { role: 'assistant' },
|
|
@@ -137,7 +137,7 @@ export class TranscriptionManager extends Resource {
|
|
|
137
137
|
this._queue?.append([block]);
|
|
138
138
|
} else {
|
|
139
139
|
await this._transcriber?.close();
|
|
140
|
-
const block = create(
|
|
140
|
+
const block = create(DataType.Message, {
|
|
141
141
|
created: new Date().toISOString(),
|
|
142
142
|
blocks: [{ type: 'transcription', text: 'Stopped', started: new Date().toISOString() }],
|
|
143
143
|
sender: { role: 'assistant' },
|
|
@@ -173,12 +173,12 @@ export class TranscriptionManager extends Resource {
|
|
|
173
173
|
}
|
|
174
174
|
}
|
|
175
175
|
|
|
176
|
-
private async _onSegments(segments:
|
|
176
|
+
private async _onSegments(segments: DataType.MessageBlock.Transcription[]) {
|
|
177
177
|
if (!this.isOpen || !this._queue) {
|
|
178
178
|
return;
|
|
179
179
|
}
|
|
180
180
|
|
|
181
|
-
let block = create(
|
|
181
|
+
let block = create(DataType.Message, {
|
|
182
182
|
created: new Date().toISOString(),
|
|
183
183
|
blocks: segments,
|
|
184
184
|
sender: { identityDid: this._identityDid },
|
package/src/types/schema.ts
CHANGED
|
@@ -2,17 +2,19 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
5
|
+
import { Schema } from 'effect';
|
|
6
|
+
|
|
7
|
+
import { Expando, Ref, TypedObject } from '@dxos/echo-schema';
|
|
6
8
|
|
|
7
9
|
/**
|
|
8
10
|
* Root transcript object created when the user starts a transcription.
|
|
9
11
|
*/
|
|
10
|
-
export const TranscriptSchema =
|
|
11
|
-
name:
|
|
12
|
+
export const TranscriptSchema = Schema.Struct({
|
|
13
|
+
name: Schema.String,
|
|
12
14
|
|
|
13
15
|
// TODO(burdon): Use Date or string?
|
|
14
|
-
started:
|
|
15
|
-
ended:
|
|
16
|
+
started: Schema.optional(Schema.String),
|
|
17
|
+
ended: Schema.optional(Schema.String),
|
|
16
18
|
|
|
17
19
|
/**
|
|
18
20
|
* Queue containing TranscriptBlock objects.
|
|
@@ -32,8 +34,8 @@ export class TranscriptType extends TypedObject({
|
|
|
32
34
|
* First message in queue.
|
|
33
35
|
* Contains metadata for the recording and transcript.
|
|
34
36
|
*/
|
|
35
|
-
const TranscriptHeader =
|
|
36
|
-
started:
|
|
37
|
+
const TranscriptHeader = Schema.Struct({
|
|
38
|
+
started: Schema.optional(Schema.String),
|
|
37
39
|
});
|
|
38
40
|
|
|
39
|
-
export type TranscriptHeader =
|
|
41
|
+
export type TranscriptHeader = Schema.Schema.Type<typeof TranscriptHeader>;
|
package/src/types/types.ts
CHANGED
|
@@ -2,8 +2,10 @@
|
|
|
2
2
|
// Copyright 2025 DXOS.org
|
|
3
3
|
//
|
|
4
4
|
|
|
5
|
-
import {
|
|
6
|
-
|
|
5
|
+
import { Schema } from 'effect';
|
|
6
|
+
|
|
7
|
+
import { isInstanceOf } from '@dxos/echo-schema';
|
|
8
|
+
import { isLiveObject, SpaceId } from '@dxos/react-client/echo';
|
|
7
9
|
|
|
8
10
|
import { TranscriptType } from './schema';
|
|
9
11
|
import { TRANSCRIPTION_PLUGIN } from '../meta';
|
|
@@ -19,12 +21,12 @@ export const TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';
|
|
|
19
21
|
export namespace TranscriptionAction {
|
|
20
22
|
const TRANSCRIPTION_ACTION = `${TRANSCRIPTION_PLUGIN}/action`;
|
|
21
23
|
|
|
22
|
-
export class Create extends
|
|
23
|
-
input:
|
|
24
|
-
name:
|
|
25
|
-
spaceId:
|
|
24
|
+
export class Create extends Schema.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {
|
|
25
|
+
input: Schema.Struct({
|
|
26
|
+
name: Schema.optional(Schema.String),
|
|
27
|
+
spaceId: SpaceId,
|
|
26
28
|
}),
|
|
27
|
-
output:
|
|
29
|
+
output: Schema.Struct({
|
|
28
30
|
object: TranscriptType,
|
|
29
31
|
}),
|
|
30
32
|
}) {}
|
|
@@ -34,11 +36,11 @@ export const isTranscript = (object: unknown): object is typeof TranscriptType =
|
|
|
34
36
|
return isLiveObject(object) && isInstanceOf(TranscriptType, object);
|
|
35
37
|
};
|
|
36
38
|
|
|
37
|
-
// TODO(burdon): Create with decode consistently:
|
|
38
|
-
export const TranscriptionSettingsSchema =
|
|
39
|
-
|
|
40
|
-
entityExtraction:
|
|
39
|
+
// TODO(burdon): Create with decode consistently: Schema.decodeSync(TranscriptionSettingsSchema)({}))
|
|
40
|
+
export const TranscriptionSettingsSchema = Schema.mutable(
|
|
41
|
+
Schema.Struct({
|
|
42
|
+
entityExtraction: Schema.optional(Schema.Boolean).pipe(Schema.withConstructorDefault(() => true)),
|
|
41
43
|
}),
|
|
42
44
|
);
|
|
43
45
|
|
|
44
|
-
export type TranscriptionSettingsProps =
|
|
46
|
+
export type TranscriptionSettingsProps = Schema.Schema.Type<typeof TranscriptionSettingsSchema>;
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"version": 3,
|
|
3
|
-
"sources": ["../../../src/capabilities/app-graph-builder.ts", "../../../src/entity-extraction/entity-extraction.ts", "raw-loader:./system-prompt.tpl?raw"],
|
|
4
|
-
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport type { Schema } from 'effect';\n\nimport { Capabilities, contributes, createIntent, type PluginsContext } from '@dxos/app-framework';\nimport { AIServiceEdgeClient, type AIServiceClient } from '@dxos/assistant';\nimport { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';\nimport { Filter, fullyQualifiedId, getSpace, makeRef, type Space } from '@dxos/client/echo';\nimport { getSchemaTypename, isInstanceOf, type BaseEchoObject } from '@dxos/echo-schema';\nimport { invariant } from '@dxos/invariant';\nimport { DXN } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { ClientCapabilities } from '@dxos/plugin-client';\nimport { PLANK_COMPANION_TYPE, ATTENDABLE_PATH_SEPARATOR } from '@dxos/plugin-deck/types';\nimport { createExtension, type Node } from '@dxos/plugin-graph';\nimport { DocumentType } from '@dxos/plugin-markdown/types';\nimport { MeetingCapabilities, type CallState, type MediaState } from '@dxos/plugin-meeting';\nimport { MeetingType, type MeetingCallProperties } from '@dxos/plugin-meeting/types';\nimport { type buf } from '@dxos/protocols/buf';\nimport { type TranscriptionPayloadSchema } from '@dxos/protocols/buf/dxos/edge/calls_pb';\nimport { type MessageType, Contact, Organization } from '@dxos/schema';\n\nimport { TranscriptionCapabilities } from './capabilities';\nimport { processTranscriptMessage } from '../entity-extraction';\nimport { TRANSCRIPTION_PLUGIN } from '../meta';\nimport { TranscriptionManager } from '../transcriber';\nimport { TranscriptionAction, type TranscriptionSettingsProps, TranscriptType } from '../types';\n\n// TODO(wittjosiah): Factor out.\n// TODO(wittjosiah): Can we stop using protobuf for this?\ntype TranscriptionPayload = buf.MessageInitShape<typeof TranscriptionPayloadSchema>;\n\nconst getMeetingTranscript = async (\n context: PluginsContext,\n meeting: MeetingType,\n): Promise<TranscriptType | undefined> => {\n const { dispatchPromise: dispatch } = context.requestCapability(Capabilities.IntentDispatcher);\n\n const typename = getSchemaTypename(TranscriptType)!;\n const existing = await meeting.artifacts[typename]?.load();\n if (existing instanceof TranscriptType) {\n return existing;\n }\n\n const space = getSpace(meeting);\n invariant(space);\n const { data } = await dispatch(createIntent(TranscriptionAction.Create, { spaceId: space.id }));\n meeting.artifacts[typename] = makeRef(data!.object);\n return data?.object;\n};\n\n// TODO(wittjosiah): Introduce a sound that plays when transcription is activated/deactivated.\nexport default (context: PluginsContext) =>\n contributes(Capabilities.AppGraphBuilder, [\n createExtension({\n id: `${TRANSCRIPTION_PLUGIN}/meeting-transcript`,\n filter: (node): node is Node<MeetingType> => isInstanceOf(MeetingType, node.data),\n actions: ({ node }) => {\n const meeting = node.data;\n const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);\n return [\n {\n id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,\n data: async () => {\n // NOTE: We are not saving the state of the transcription manager here.\n // We expect the state to be updated through `onCallStateUpdated` once it is propagated through Swarm.\n // This is done to avoid race conditions and to not handle optimistic updates.\n const transcript = await getMeetingTranscript(context, meeting);\n invariant(transcript, 'Failed to create transcript');\n\n const callManager = context.requestCapability(MeetingCapabilities.CallManager);\n callManager.setActivity(getSchemaTypename(TranscriptType)!, {\n queueDxn: transcript.queue.dxn.toString(),\n enabled: !state.enabled,\n });\n },\n properties: {\n label: state.enabled\n ? ['stop transcription label', { ns: TRANSCRIPTION_PLUGIN }]\n : ['start transcription label', { ns: TRANSCRIPTION_PLUGIN }],\n icon: 'ph--subtitles--regular',\n disposition: 'toolbar',\n classNames: state.enabled ? 'bg-callAlert' : '',\n },\n },\n ];\n },\n connector: ({ node }) => {\n const meeting = node.data;\n const client = context.requestCapability(ClientCapabilities.Client);\n const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);\n const settings = context\n .requestCapability(Capabilities.SettingsStore)\n .getStore<TranscriptionSettingsProps>(TRANSCRIPTION_PLUGIN)!.value;\n\n // TODO(dmaretskyi): Request via capability.\n const aiClient: AIServiceClient | undefined = new AIServiceEdgeClient({\n endpoint: AI_SERVICE_ENDPOINT.REMOTE,\n });\n\n return [\n {\n id: `${fullyQualifiedId(meeting)}${ATTENDABLE_PATH_SEPARATOR}${getSchemaTypename(TranscriptType)}`,\n type: PLANK_COMPANION_TYPE,\n data: meeting.artifacts[getSchemaTypename(TranscriptType)!]?.target,\n properties: {\n label: ['transcript companion label', { ns: TRANSCRIPTION_PLUGIN }],\n icon: 'ph--subtitles--regular',\n position: 'hoist',\n disposition: 'hidden',\n schema: TranscriptType,\n getIntent: ({ space }: { space: Space }) =>\n createIntent(TranscriptionAction.Create, { spaceId: space.id }),\n\n // TODO(burdon): Is this the right place to inisitalize the state? Change to a capability?\n onJoin: async ({ meeting }: { meeting: MeetingType; roomId: string }) => {\n const identity = client.halo.identity.get();\n invariant(identity);\n const space = getSpace(meeting);\n invariant(space);\n\n let messageEnricher;\n if (aiClient && settings.entityExtraction) {\n messageEnricher = createEntityExtractionEnricher({\n aiClient,\n // TODO(dmaretskyi): Have those be discovered from the schema graph or contributed by capabilities?\n // This forced me to add a dependency on markdown plugin.\n // This will be replaced with a vector search index anyway, so its not a big deal.\n contextTypes: [DocumentType, Contact, Organization],\n space,\n });\n }\n\n // TODO(burdon): The TranscriptionManager singleton is part of the state and should just be updated here.\n state.transcriptionManager = await new TranscriptionManager({\n edgeClient: client.edge,\n messageEnricher,\n })\n .setIdentityDid(identity.did)\n .open();\n },\n onLeave: async () => {\n await state.transcriptionManager?.close();\n state.transcriptionManager = undefined;\n state.enabled = false;\n },\n onCallStateUpdated: async (callState: CallState) => {\n const typename = getSchemaTypename(TranscriptType);\n const transcription = typename ? callState.activities?.[typename] : undefined;\n if (!transcription?.payload) {\n return;\n }\n\n const payload: TranscriptionPayload = transcription.payload;\n state.enabled = !!payload.enabled;\n if (payload.queueDxn) {\n // NOTE: Must set queue before enabling transcription.\n state.transcriptionManager?.setQueue(DXN.parse(payload.queueDxn));\n }\n await state.transcriptionManager?.setEnabled(payload.enabled);\n },\n onMediaStateUpdated: async ([mediaState, isSpeaking]: [MediaState, boolean]) => {\n void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);\n void state.transcriptionManager?.setRecording(isSpeaking);\n },\n } satisfies MeetingCallProperties,\n },\n ];\n },\n }),\n ]);\n\ntype EntityExtractionEnricherFactoryOptions = {\n aiClient: AIServiceClient;\n contextTypes: Schema.Schema.AnyNoContext[];\n space: Space;\n};\n\nconst createEntityExtractionEnricher = ({ aiClient, contextTypes, space }: EntityExtractionEnricherFactoryOptions) => {\n return async (message: MessageType) => {\n const { objects } = await space.db.query(Filter.or(...contextTypes.map((s) => Filter.schema(s)))).run();\n log.info('context', { objects });\n\n const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({\n aiService: aiClient,\n message,\n context: { objects: await Promise.all(objects.map(processContextObject)) },\n options: { timeout: ENTITY_EXTRACTOR_TIMEOUT, fallbackToRaw: true },\n });\n\n log.info('entity extraction time', { timeElapsed });\n return enhancedMessage;\n };\n};\n\nconst processContextObject = async (object: BaseEchoObject): Promise<any> => {\n // TODO(dmaretskyi): Documents need special processing is the content is behind a ref.\n // TODO(dmaretskyi): Think about a way to handle this serialization with a decorator.\n // if (isInstanceOf(DocumentType, object)) {\n // return {\n // ...object,\n // content: await object.content.load(),\n // };\n // }\n\n return object;\n};\n\nconst ENTITY_EXTRACTOR_TIMEOUT = 25_000;\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { Schema } from 'effect';\n\nimport { createTemplate, Message, structuredOutputParser } from '@dxos/artifact';\nimport type { AIServiceClient } from '@dxos/assistant';\nimport { MixedStreamParser } from '@dxos/assistant';\nimport { asyncTimeout } from '@dxos/async';\nimport { type BaseEchoObject, create, ObjectId } from '@dxos/echo-schema';\nimport { DXN } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { MessageType } from '@dxos/schema';\n\nimport SYSTEM_PROMPT from './system-prompt.tpl?raw';\n\ntype ProcessTranscriptMessageParams = {\n message: MessageType;\n aiService: AIServiceClient;\n context: {\n objects?: BaseEchoObject[];\n };\n\n options?: {\n /**\n * Timeout for the entity extraction.\n */\n timeout?: number;\n\n /**\n * Fallback to raw text if the entity extraction fails.\n * Otherwise the function will throw an error.\n * @default false\n */\n fallbackToRaw?: boolean;\n };\n};\n\ntype ProcessTranscriptMessageResult = {\n message: MessageType;\n timeElapsed: number;\n};\n\n/**\n * Process Handlebars template.\n */\nconst createSystemPrompt = (): string => {\n const template = createTemplate(SYSTEM_PROMPT);\n return template({});\n};\n\nconst ReferencedQuotes = Schema.Struct({\n references: Schema.Array(\n Schema.Struct({\n quote: Schema.String,\n id: Schema.String,\n }),\n ).annotations({\n description:\n 'The references to the context objects that are mentioned in the transcript. quote should match the original transcript text exactly, while id is the id of the context object.',\n }),\n});\ninterface ReferencedQuotes extends Schema.Schema.Type<typeof ReferencedQuotes> {}\n\n// TODO(dmaretskyi): Move context to a vector search index.\nexport const processTranscriptMessage = async (\n params: ProcessTranscriptMessageParams,\n): Promise<ProcessTranscriptMessageResult> => {\n try {\n const outputParser = structuredOutputParser(ReferencedQuotes);\n const runParser = async (): Promise<ProcessTranscriptMessageResult> => {\n const startTime = performance.now();\n const result = outputParser.getResult(\n await new MixedStreamParser().parse(\n await params.aiService.execStream({\n model: '@anthropic/claude-3-5-haiku-20241022',\n systemPrompt: createSystemPrompt(),\n history: [\n create(Message, {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `<context>${JSON.stringify(params.context.objects)}</context>`,\n },\n {\n type: 'text',\n text: `<transcript>${JSON.stringify(params.message.blocks)}</transcript>`,\n },\n ],\n }),\n ],\n tools: [outputParser.tool],\n }),\n ),\n );\n log.info('entity extraction result', { refs: result.references });\n\n return {\n message: create(MessageType, {\n ...params.message,\n blocks: params.message.blocks.map((block, i) =>\n block.type !== 'transcription'\n ? block\n : {\n ...block,\n text: postprocessText(block.text, result),\n },\n ),\n }),\n timeElapsed: performance.now() - startTime,\n };\n };\n\n if (params.options?.timeout && params.options.timeout > 0) {\n return await asyncTimeout(runParser(), params.options.timeout);\n } else {\n return await runParser();\n }\n } catch (error) {\n if (params.options?.fallbackToRaw) {\n log.warn('failed to process transcript message, falling back to raw text', { error });\n return {\n message: params.message,\n timeElapsed: 0,\n };\n } else {\n throw error;\n }\n }\n};\n\n/**\n * Finds and replaces all quotes with DXNs references.\n */\n// TODO(dmaretskyi): Lookup and verifiy ids from provided context.\nexport const postprocessText = (text: string, quotes: ReferencedQuotes) => {\n for (const quote of quotes.references) {\n if (!ObjectId.isValid(quote.id)) {\n continue;\n }\n\n // Use a case-insensitive regular expression to replace the quote\n const regex = new RegExp(quote.quote.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&'), 'gi');\n text = text.replace(regex, `[${quote.quote}][${DXN.fromLocalObjectId(quote.id).toString()}]`);\n }\n return text;\n};\n", "You are an AI assistant specialized in processing and enhancing transcripts with contextual information.\nYour task is to analyze a given transcript and extract references to the context objects.\n\nThe context and transcript are provided in the user message in `context` and `transcript` XML tags.\n\nHere are your instructions:\n\n1. Read the transcript and context information provided above.\n\n2. Process each segment of the transcript separately.\n\n3. For each segment:\n a. Analyze the content and identify relevant information from the context.\n b. Extract a number of references to the context objects that are mentioned in the segment.\n c. Output the references in the format `{ \"quote\": \"earnings report\", \"id\": \"01JT0JP9AX0XKGZX4MV4B69VT6\" }`.\n d. If there are no references, just output an empty array.\n\n4. Maintain the exact order and structure of the original transcript segments.\n\nOutput Format:\nThe final output should be a JSON array of references.\n\nExample output structure:\n```json\n{\n \"references\": [\n { \"quote\": \"earnings report\", \"id\": \"01JT0JP9AX0XKGZX4MV4B69VT6\" },\n { \"quote\": \"John\", \"id\": \"01JT0VM0VN3ZKN7GDBM8CV9BA7\" }\n ]\n}\n```\n\nRemember:\n- Only quote the text that is in the transcript.\n- Analyze each segment carefully before processing it.\n- Only reference objects that appear in the context and exactly by their id, If the object is not in the context, do not insert a reference. \n\nNow, process the transcript and context provided below according to these instructions."],
|
|
5
|
-
"mappings": ";;;;;;;;;;;;;;;;AAMA,SAASA,cAAcC,aAAaC,oBAAyC;AAC7E,SAASC,2BAAiD;AAC1D,SAASC,2BAA2B;AACpC,SAASC,QAAQC,kBAAkBC,UAAUC,eAA2B;AACxE,SAASC,mBAAmBC,oBAAyC;AACrE,SAASC,iBAAiB;AAC1B,SAASC,OAAAA,YAAW;AACpB,SAASC,OAAAA,YAAW;AACpB,SAASC,0BAA0B;AACnC,SAASC,sBAAsBC,iCAAiC;AAChE,SAASC,uBAAkC;AAC3C,SAASC,oBAAoB;AAC7B,SAASC,2BAA4D;AACrE,SAASC,mBAA+C;AAGxD,SAA2BC,SAASC,oBAAoB;;;AClBxD,SAASC,cAAc;AAEvB,SAASC,gBAAgBC,SAASC,8BAA8B;AAEhE,SAASC,yBAAyB;AAClC,SAASC,oBAAoB;AAC7B,SAA8BC,QAAQC,gBAAgB;AACtD,SAASC,WAAW;AACpB,SAASC,WAAW;AACpB,SAASC,mBAAmB;;;ACb5B;;;;AD+CA,IAAMC,qBAAqB,MAAA;AACzB,QAAMC,WAAWC,eAAeC,qBAAAA;AAChC,SAAOF,SAAS,CAAC,CAAA;AACnB;AAEA,IAAMG,mBAAmBC,OAAOC,OAAO;EACrCC,YAAYF,OAAOG,MACjBH,OAAOC,OAAO;IACZG,OAAOJ,OAAOK;IACdC,IAAIN,OAAOK;EACb,CAAA,CAAA,EACAE,YAAY;IACZC,aACE;EACJ,CAAA;AACF,CAAA;AAIO,IAAMC,2BAA2B,OACtCC,WAAAA;AAEA,MAAI;AACF,UAAMC,eAAeC,uBAAuBb,gBAAAA;AAC5C,UAAMc,YAAY,YAAA;AAChB,YAAMC,YAAYC,YAAYC,IAAG;AACjC,YAAMC,SAASN,aAAaO,UAC1B,MAAM,IAAIC,kBAAAA,EAAoBC,MAC5B,MAAMV,OAAOW,UAAUC,WAAW;QAChCC,OAAO;QACPC,cAAc7B,mBAAAA;QACd8B,SAAS;UACPC,OAAOC,SAAS;YACdC,MAAM;YACNC,SAAS;cACP;gBACEC,MAAM;gBACNC,MAAM,YAAYC,KAAKC,UAAUvB,OAAOwB,QAAQC,OAAO,CAAA;cACzD;cACA;gBACEL,MAAM;gBACNC,MAAM,eAAeC,KAAKC,UAAUvB,OAAO0B,QAAQC,MAAM,CAAA;cAC3D;;UAEJ,CAAA;;QAEFC,OAAO;UAAC3B,aAAa4B;;MACvB,CAAA,CAAA,CAAA;AAGJC,UAAIC,KAAK,4BAA4B;QAAEC,MAAMzB,OAAOf;MAAW,GAAA;;;;;;AAE/D,aAAO;QACLkC,SAASV,OAAOiB,aAAa;UAC3B,GAAGjC,OAAO0B;UACVC,QAAQ3B,OAAO0B,QAAQC,OAAOO,IAAI,CAACC,OAAOC,MACxCD,MAAMf,SAAS,kBACXe,QACA;YACE,GAAGA;YACHd,MAAMgB,gBAAgBF,MAAMd,MAAMd,MAAAA;UACpC,CAAA;QAER,CAAA;QACA+B,aAAajC,YAAYC,IAAG,IAAKF;MACnC;IACF;AAEA,QAAIJ,OAAOuC,SAASC,WAAWxC,OAAOuC,QAAQC,UAAU,GAAG;AACzD,aAAO,MAAMC,aAAatC,UAAAA,GAAaH,OAAOuC,QAAQC,OAAO;IAC/D,OAAO;AACL,aAAO,MAAMrC,UAAAA;IACf;EACF,SAASuC,OAAO;AACd,QAAI1C,OAAOuC,SAASI,eAAe;AACjCb,UAAIc,KAAK,kEAAkE;QAAEF;MAAM,GAAA;;;;;;AACnF,aAAO;QACLhB,SAAS1B,OAAO0B;QAChBY,aAAa;MACf;IACF,OAAO;AACL,YAAMI;IACR;EACF;AACF;AAMO,IAAML,kBAAkB,CAAChB,MAAcwB,WAAAA;AAC5C,aAAWnD,SAASmD,OAAOrD,YAAY;AACrC,QAAI,CAACsD,SAASC,QAAQrD,MAAME,EAAE,GAAG;AAC/B;IACF;AAGA,UAAMoD,QAAQ,IAAIC,OAAOvD,MAAMA,MAAMwD,QAAQ,uBAAuB,MAAA,GAAS,IAAA;AAC7E7B,WAAOA,KAAK6B,QAAQF,OAAO,IAAItD,MAAMA,KAAK,KAAKyD,IAAIC,kBAAkB1D,MAAME,EAAE,EAAEyD,SAAQ,CAAA,GAAK;EAC9F;AACA,SAAOhC;AACT;;;;ADlHA,IAAMiC,uBAAuB,OAC3BC,SACAC,YAAAA;AAEA,QAAM,EAAEC,iBAAiBC,SAAQ,IAAKH,QAAQI,kBAAkBC,aAAaC,gBAAgB;AAE7F,QAAMC,WAAWC,kBAAkBC,cAAAA;AACnC,QAAMC,WAAW,MAAMT,QAAQU,UAAUJ,QAAAA,GAAWK,KAAAA;AACpD,MAAIF,oBAAoBD,gBAAgB;AACtC,WAAOC;EACT;AAEA,QAAMG,QAAQC,SAASb,OAAAA;AACvBc,YAAUF,OAAAA,QAAAA;;;;;;;;;AACV,QAAM,EAAEG,KAAI,IAAK,MAAMb,SAASc,aAAaC,oBAAoBC,QAAQ;IAAEC,SAASP,MAAMQ;EAAG,CAAA,CAAA;AAC7FpB,UAAQU,UAAUJ,QAAAA,IAAYe,QAAQN,KAAMO,MAAM;AAClD,SAAOP,MAAMO;AACf;AAGA,IAAA,4BAAe,CAACvB,YACdwB,YAAYnB,aAAaoB,iBAAiB;EACxCC,gBAAgB;IACdL,IAAI,GAAGM,oBAAAA;IACPC,QAAQ,CAACC,SAAoCC,aAAaC,aAAaF,KAAKb,IAAI;IAChFgB,SAAS,CAAC,EAAEH,KAAI,MAAE;AAChB,YAAM5B,UAAU4B,KAAKb;AACrB,YAAMiB,QAAQjC,QAAQI,kBAAkB8B,0BAA0BC,yBAAyB;AAC3F,aAAO;QACL;UACEd,IAAI,GAAGe,iBAAiBnC,OAAAA,CAAAA;UACxBe,MAAM,YAAA;AAIJ,kBAAMqB,aAAa,MAAMtC,qBAAqBC,SAASC,OAAAA;AACvDc,sBAAUsB,YAAY,+BAAA;;;;;;;;;AAEtB,kBAAMC,cAActC,QAAQI,kBAAkBmC,oBAAoBC,WAAW;AAC7EF,wBAAYG,YAAYjC,kBAAkBC,cAAAA,GAAkB;cAC1DiC,UAAUL,WAAWM,MAAMC,IAAIC,SAAQ;cACvCC,SAAS,CAACb,MAAMa;YAClB,CAAA;UACF;UACAC,YAAY;YACVC,OAAOf,MAAMa,UACT;cAAC;cAA4B;gBAAEG,IAAItB;cAAqB;gBACxD;cAAC;cAA6B;gBAAEsB,IAAItB;cAAqB;;YAC7DuB,MAAM;YACNC,aAAa;YACbC,YAAYnB,MAAMa,UAAU,iBAAiB;UAC/C;QACF;;IAEJ;IACAO,WAAW,CAAC,EAAExB,KAAI,MAAE;AAClB,YAAM5B,UAAU4B,KAAKb;AACrB,YAAMsC,SAAStD,QAAQI,kBAAkBmD,mBAAmBC,MAAM;AAClE,YAAMvB,QAAQjC,QAAQI,kBAAkB8B,0BAA0BC,yBAAyB;AAC3F,YAAMsB,WAAWzD,QACdI,kBAAkBC,aAAaqD,aAAa,EAC5CC,SAAqChC,oBAAAA,EAAuBiC;AAG/D,YAAMC,WAAwC,IAAIC,oBAAoB;QACpEC,UAAUC,oBAAoBC;MAChC,CAAA;AAEA,aAAO;QACL;UACE5C,IAAI,GAAGe,iBAAiBnC,OAAAA,CAAAA,GAAWiE,yBAAAA,GAA4B1D,kBAAkBC,cAAAA,CAAAA;UACjF0D,MAAMC;UACNpD,MAAMf,QAAQU,UAAUH,kBAAkBC,cAAAA,CAAAA,GAAmB4D;UAC7DtB,YAAY;YACVC,OAAO;cAAC;cAA8B;gBAAEC,IAAItB;cAAqB;;YACjEuB,MAAM;YACNoB,UAAU;YACVnB,aAAa;YACboB,QAAQ9D;YACR+D,WAAW,CAAC,EAAE3D,MAAK,MACjBI,aAAaC,oBAAoBC,QAAQ;cAAEC,SAASP,MAAMQ;YAAG,CAAA;;YAG/DoD,QAAQ,OAAO,EAAExE,SAAAA,SAAO,MAA4C;AAClE,oBAAMyE,WAAWpB,OAAOqB,KAAKD,SAASE,IAAG;AACzC7D,wBAAU2D,UAAAA,QAAAA;;;;;;;;;AACV,oBAAM7D,QAAQC,SAASb,QAAAA;AACvBc,wBAAUF,OAAAA,QAAAA;;;;;;;;;AAEV,kBAAIgE;AACJ,kBAAIhB,YAAYJ,SAASqB,kBAAkB;AACzCD,kCAAkBE,+BAA+B;kBAC/ClB;;;;kBAIAmB,cAAc;oBAACC;oBAAcC;oBAASC;;kBACtCtE;gBACF,CAAA;cACF;AAGAoB,oBAAMmD,uBAAuB,MAAM,IAAIC,qBAAqB;gBAC1DC,YAAYhC,OAAOiC;gBACnBV;cACF,CAAA,EACGW,eAAed,SAASe,GAAG,EAC3BC,KAAI;YACT;YACAC,SAAS,YAAA;AACP,oBAAM1D,MAAMmD,sBAAsBQ,MAAAA;AAClC3D,oBAAMmD,uBAAuBS;AAC7B5D,oBAAMa,UAAU;YAClB;YACAgD,oBAAoB,OAAOC,cAAAA;AACzB,oBAAMxF,WAAWC,kBAAkBC,cAAAA;AACnC,oBAAMuF,gBAAgBzF,WAAWwF,UAAUE,aAAa1F,QAAAA,IAAYsF;AACpE,kBAAI,CAACG,eAAeE,SAAS;AAC3B;cACF;AAEA,oBAAMA,UAAgCF,cAAcE;AACpDjE,oBAAMa,UAAU,CAAC,CAACoD,QAAQpD;AAC1B,kBAAIoD,QAAQxD,UAAU;AAEpBT,sBAAMmD,sBAAsBe,SAASC,KAAIC,MAAMH,QAAQxD,QAAQ,CAAA;cACjE;AACA,oBAAMT,MAAMmD,sBAAsBkB,WAAWJ,QAAQpD,OAAO;YAC9D;YACAyD,qBAAqB,OAAO,CAACC,YAAYC,UAAAA,MAAkC;AACzE,mBAAKxE,MAAMmD,sBAAsBsB,cAAcF,WAAWG,UAAU;AACpE,mBAAK1E,MAAMmD,sBAAsBwB,aAAaH,UAAAA;YAChD;UACF;QACF;;IAEJ;EACF,CAAA;CACD;AAQH,IAAM1B,iCAAiC,CAAC,EAAElB,UAAUmB,cAAcnE,MAAK,MAA0C;AAC/G,SAAO,OAAOgG,YAAAA;AACZ,UAAM,EAAEC,QAAO,IAAK,MAAMjG,MAAMkG,GAAGC,MAAMC,OAAOC,GAAE,GAAIlC,aAAamC,IAAI,CAACC,MAAMH,OAAO1C,OAAO6C,CAAAA,CAAAA,CAAAA,CAAAA,EAAMC,IAAG;AACrGC,IAAAA,KAAIC,KAAK,WAAW;MAAET;IAAQ,GAAA;;;;;;AAE9B,UAAM,EAAED,SAASW,iBAAiBC,YAAW,IAAK,MAAMC,yBAAyB;MAC/EC,WAAW9D;MACXgD;MACA7G,SAAS;QAAE8G,SAAS,MAAMc,QAAQC,IAAIf,QAAQK,IAAIW,oBAAAA,CAAAA;MAAuB;MACzEC,SAAS;QAAEC,SAASC;QAA0BC,eAAe;MAAK;IACpE,CAAA;AAEAZ,IAAAA,KAAIC,KAAK,0BAA0B;MAAEE;IAAY,GAAA;;;;;;AACjD,WAAOD;EACT;AACF;AAEA,IAAMM,uBAAuB,OAAOvG,WAAAA;AAUlC,SAAOA;AACT;AAEA,IAAM0G,2BAA2B;",
|
|
6
|
-
"names": ["Capabilities", "contributes", "createIntent", "AIServiceEdgeClient", "AI_SERVICE_ENDPOINT", "Filter", "fullyQualifiedId", "getSpace", "makeRef", "getSchemaTypename", "isInstanceOf", "invariant", "DXN", "log", "ClientCapabilities", "PLANK_COMPANION_TYPE", "ATTENDABLE_PATH_SEPARATOR", "createExtension", "DocumentType", "MeetingCapabilities", "MeetingType", "Contact", "Organization", "Schema", "createTemplate", "Message", "structuredOutputParser", "MixedStreamParser", "asyncTimeout", "create", "ObjectId", "DXN", "log", "MessageType", "createSystemPrompt", "template", "createTemplate", "SYSTEM_PROMPT", "ReferencedQuotes", "Schema", "Struct", "references", "Array", "quote", "String", "id", "annotations", "description", "processTranscriptMessage", "params", "outputParser", "structuredOutputParser", "runParser", "startTime", "performance", "now", "result", "getResult", "MixedStreamParser", "parse", "aiService", "execStream", "model", "systemPrompt", "history", "create", "Message", "role", "content", "type", "text", "JSON", "stringify", "context", "objects", "message", "blocks", "tools", "tool", "log", "info", "refs", "MessageType", "map", "block", "i", "postprocessText", "timeElapsed", "options", "timeout", "asyncTimeout", "error", "fallbackToRaw", "warn", "quotes", "ObjectId", "isValid", "regex", "RegExp", "replace", "DXN", "fromLocalObjectId", "toString", "getMeetingTranscript", "context", "meeting", "dispatchPromise", "dispatch", "requestCapability", "Capabilities", "IntentDispatcher", "typename", "getSchemaTypename", "TranscriptType", "existing", "artifacts", "load", "space", "getSpace", "invariant", "data", "createIntent", "TranscriptionAction", "Create", "spaceId", "id", "makeRef", "object", "contributes", "AppGraphBuilder", "createExtension", "TRANSCRIPTION_PLUGIN", "filter", "node", "isInstanceOf", "MeetingType", "actions", "state", "TranscriptionCapabilities", "MeetingTranscriptionState", "fullyQualifiedId", "transcript", "callManager", "MeetingCapabilities", "CallManager", "setActivity", "queueDxn", "queue", "dxn", "toString", "enabled", "properties", "label", "ns", "icon", "disposition", "classNames", "connector", "client", "ClientCapabilities", "Client", "settings", "SettingsStore", "getStore", "value", "aiClient", "AIServiceEdgeClient", "endpoint", "AI_SERVICE_ENDPOINT", "REMOTE", "ATTENDABLE_PATH_SEPARATOR", "type", "PLANK_COMPANION_TYPE", "target", "position", "schema", "getIntent", "onJoin", "identity", "halo", "get", "messageEnricher", "entityExtraction", "createEntityExtractionEnricher", "contextTypes", "DocumentType", "Contact", "Organization", "transcriptionManager", "TranscriptionManager", "edgeClient", "edge", "setIdentityDid", "did", "open", "onLeave", "close", "undefined", "onCallStateUpdated", "callState", "transcription", "activities", "payload", "setQueue", "DXN", "parse", "setEnabled", "onMediaStateUpdated", "mediaState", "isSpeaking", "setAudioTrack", "audioTrack", "setRecording", "message", "objects", "db", "query", "Filter", "or", "map", "s", "run", "log", "info", "enhancedMessage", "timeElapsed", "processTranscriptMessage", "aiService", "Promise", "all", "processContextObject", "options", "timeout", "ENTITY_EXTRACTOR_TIMEOUT", "fallbackToRaw"]
|
|
7
|
-
}
|