@dxos/plugin-transcription 0.8.2-main.2f9c567 → 0.8.2-main.5885341
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/lib/browser/{TranscriptContainer-3LU3XBEB.mjs → TranscriptContainer-D3EH2SLO.mjs} +2 -2
- package/dist/lib/browser/{app-graph-builder-YXELPN6F.mjs → app-graph-builder-7HN3M24G.mjs} +8 -8
- package/dist/lib/browser/app-graph-builder-7HN3M24G.mjs.map +7 -0
- package/dist/lib/browser/{chunk-HNIKXVCG.mjs → chunk-BSZSM6KA.mjs} +6 -6
- package/dist/lib/browser/chunk-BSZSM6KA.mjs.map +7 -0
- package/dist/lib/browser/{chunk-HRDAILQX.mjs → chunk-IWZ7C3OA.mjs} +2 -2
- package/dist/lib/browser/{chunk-GWY2Q2NE.mjs → chunk-KSUYWWIU.mjs} +6 -6
- package/dist/lib/browser/chunk-KSUYWWIU.mjs.map +7 -0
- package/dist/lib/browser/{chunk-TGOKX32G.mjs → chunk-YRWBY6UK.mjs} +15 -17
- package/dist/lib/browser/chunk-YRWBY6UK.mjs.map +7 -0
- package/dist/lib/browser/index.mjs +6 -6
- package/dist/lib/browser/index.mjs.map +3 -3
- package/dist/lib/browser/{intent-resolver-BBB3DWV4.mjs → intent-resolver-ZWKYRFE7.mjs} +2 -2
- package/dist/lib/browser/meta.json +1 -1
- package/dist/lib/browser/{react-surface-R277DACJ.mjs → react-surface-F7DBDGHS.mjs} +4 -4
- package/dist/lib/browser/{settings-I226DZ27.mjs → settings-6YAHKVKX.mjs} +2 -2
- package/dist/lib/browser/{transcriber-QRSS6X6S.mjs → transcriber-MSBS4VBI.mjs} +3 -3
- package/dist/lib/browser/types/index.mjs +1 -1
- package/dist/lib/node/{TranscriptContainer-DICOMZLM.cjs → TranscriptContainer-WCBZMA5K.cjs} +7 -7
- package/dist/lib/node/{TranscriptContainer-DICOMZLM.cjs.map → TranscriptContainer-WCBZMA5K.cjs.map} +2 -2
- package/dist/lib/node/{app-graph-builder-I6LYVIYY.cjs → app-graph-builder-BVWYUGUO.cjs} +19 -19
- package/dist/lib/node/app-graph-builder-BVWYUGUO.cjs.map +7 -0
- package/dist/lib/node/{chunk-RK4BEN5E.cjs → chunk-7XIIUVN3.cjs} +9 -9
- package/dist/lib/node/chunk-7XIIUVN3.cjs.map +7 -0
- package/dist/lib/node/{chunk-34WFY7M7.cjs → chunk-IXU57SCA.cjs} +16 -18
- package/dist/lib/node/chunk-IXU57SCA.cjs.map +7 -0
- package/dist/lib/node/{chunk-4JOJCZPJ.cjs → chunk-JSCC2AQW.cjs} +9 -9
- package/dist/lib/node/chunk-JSCC2AQW.cjs.map +7 -0
- package/dist/lib/node/{chunk-K553L7CZ.cjs → chunk-OCHTKDL7.cjs} +5 -5
- package/dist/lib/node/index.cjs +27 -27
- package/dist/lib/node/index.cjs.map +3 -3
- package/dist/lib/node/{intent-resolver-7XY3WITN.cjs → intent-resolver-RM4JOCDV.cjs} +7 -7
- package/dist/lib/node/meta.json +1 -1
- package/dist/lib/node/{react-surface-HUSWAORU.cjs → react-surface-7F37EYZA.cjs} +10 -10
- package/dist/lib/node/{settings-NDMU3EWQ.cjs → settings-ZMB4XRWS.cjs} +7 -7
- package/dist/lib/node/{transcriber-UZJMG44C.cjs → transcriber-55OE4X6I.cjs} +8 -8
- package/dist/lib/node/types/index.cjs +7 -7
- package/dist/lib/node/types/index.cjs.map +1 -1
- package/dist/lib/node-esm/{TranscriptContainer-K4E4P5T6.mjs → TranscriptContainer-GBPY6KKA.mjs} +2 -2
- package/dist/lib/node-esm/{app-graph-builder-6FY2UQOH.mjs → app-graph-builder-QK25O5K5.mjs} +8 -8
- package/dist/lib/node-esm/app-graph-builder-QK25O5K5.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-6C34FO4C.mjs → chunk-B2BD7S4X.mjs} +6 -6
- package/dist/lib/node-esm/chunk-B2BD7S4X.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-3I7AC3D3.mjs → chunk-DXJN2JWV.mjs} +2 -2
- package/dist/lib/node-esm/{chunk-KUQ3ZTRX.mjs → chunk-OS2ITZUL.mjs} +15 -17
- package/dist/lib/node-esm/chunk-OS2ITZUL.mjs.map +7 -0
- package/dist/lib/node-esm/{chunk-CKANLRS4.mjs → chunk-UWF6LM27.mjs} +6 -6
- package/dist/lib/node-esm/chunk-UWF6LM27.mjs.map +7 -0
- package/dist/lib/node-esm/index.mjs +6 -6
- package/dist/lib/node-esm/index.mjs.map +3 -3
- package/dist/lib/node-esm/{intent-resolver-ACGAPLDZ.mjs → intent-resolver-HDRFXMKP.mjs} +2 -2
- package/dist/lib/node-esm/meta.json +1 -1
- package/dist/lib/node-esm/{react-surface-G2OY4MWL.mjs → react-surface-D3L6VBEV.mjs} +4 -4
- package/dist/lib/node-esm/{settings-O7X2UQQG.mjs → settings-I35TVDUG.mjs} +2 -2
- package/dist/lib/node-esm/{transcriber-NVVFH7IF.mjs → transcriber-HH7EMCGH.mjs} +3 -3
- package/dist/lib/node-esm/types/index.mjs +1 -1
- package/dist/types/src/capabilities/app-graph-builder.d.ts.map +1 -1
- package/dist/types/src/capabilities/index.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.d.ts +3 -3
- package/dist/types/src/components/Transcript/Transcript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts +2 -2
- package/dist/types/src/components/Transcript/Transcript.stories.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts +9 -9
- package/dist/types/src/components/Transcript/deprecated/OldTranscript.d.ts.map +1 -1
- package/dist/types/src/components/Transcript/transcript-extension.d.ts +2 -2
- package/dist/types/src/components/Transcript/transcript-extension.d.ts.map +1 -1
- package/dist/types/src/components/transcription.stories.d.ts.map +1 -1
- package/dist/types/src/entity-extraction/entity-extraction.d.ts +3 -3
- package/dist/types/src/entity-extraction/entity-extraction.d.ts.map +1 -1
- package/dist/types/src/testing/testing.d.ts +9 -10
- package/dist/types/src/testing/testing.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcriber.d.ts +2 -2
- package/dist/types/src/transcriber/transcriber.d.ts.map +1 -1
- package/dist/types/src/transcriber/transcription-manager.d.ts +2 -2
- package/dist/types/src/transcriber/transcription-manager.d.ts.map +1 -1
- package/dist/types/src/types/schema.d.ts +15 -16
- package/dist/types/src/types/schema.d.ts.map +1 -1
- package/dist/types/src/types/types.d.ts +10 -10
- package/dist/types/src/types/types.d.ts.map +1 -1
- package/package.json +48 -50
- package/src/TranscriptionPlugin.tsx +2 -2
- package/src/capabilities/app-graph-builder.ts +3 -3
- package/src/components/Transcript/Transcript.stories.tsx +5 -5
- package/src/components/Transcript/Transcript.tsx +3 -3
- package/src/components/Transcript/deprecated/OldTranscript.tsx +13 -19
- package/src/components/Transcript/transcript-extension.ts +3 -3
- package/src/components/TranscriptContainer.tsx +2 -2
- package/src/components/transcription.stories.tsx +8 -14
- package/src/entity-extraction/entity-extraction.ts +4 -4
- package/src/model/model.test.ts +10 -14
- package/src/testing/testing.ts +21 -34
- package/src/transcriber/transcriber.browser.test.ts +3 -3
- package/src/transcriber/transcriber.ts +3 -6
- package/src/transcriber/transcription-manager.ts +8 -8
- package/src/types/schema.ts +8 -10
- package/src/types/types.ts +10 -12
- package/dist/lib/browser/app-graph-builder-YXELPN6F.mjs.map +0 -7
- package/dist/lib/browser/chunk-GWY2Q2NE.mjs.map +0 -7
- package/dist/lib/browser/chunk-HNIKXVCG.mjs.map +0 -7
- package/dist/lib/browser/chunk-TGOKX32G.mjs.map +0 -7
- package/dist/lib/node/app-graph-builder-I6LYVIYY.cjs.map +0 -7
- package/dist/lib/node/chunk-34WFY7M7.cjs.map +0 -7
- package/dist/lib/node/chunk-4JOJCZPJ.cjs.map +0 -7
- package/dist/lib/node/chunk-RK4BEN5E.cjs.map +0 -7
- package/dist/lib/node-esm/app-graph-builder-6FY2UQOH.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-6C34FO4C.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-CKANLRS4.mjs.map +0 -7
- package/dist/lib/node-esm/chunk-KUQ3ZTRX.mjs.map +0 -7
- /package/dist/lib/browser/{TranscriptContainer-3LU3XBEB.mjs.map → TranscriptContainer-D3EH2SLO.mjs.map} +0 -0
- /package/dist/lib/browser/{chunk-HRDAILQX.mjs.map → chunk-IWZ7C3OA.mjs.map} +0 -0
- /package/dist/lib/browser/{intent-resolver-BBB3DWV4.mjs.map → intent-resolver-ZWKYRFE7.mjs.map} +0 -0
- /package/dist/lib/browser/{react-surface-R277DACJ.mjs.map → react-surface-F7DBDGHS.mjs.map} +0 -0
- /package/dist/lib/browser/{settings-I226DZ27.mjs.map → settings-6YAHKVKX.mjs.map} +0 -0
- /package/dist/lib/browser/{transcriber-QRSS6X6S.mjs.map → transcriber-MSBS4VBI.mjs.map} +0 -0
- /package/dist/lib/node/{chunk-K553L7CZ.cjs.map → chunk-OCHTKDL7.cjs.map} +0 -0
- /package/dist/lib/node/{intent-resolver-7XY3WITN.cjs.map → intent-resolver-RM4JOCDV.cjs.map} +0 -0
- /package/dist/lib/node/{react-surface-HUSWAORU.cjs.map → react-surface-7F37EYZA.cjs.map} +0 -0
- /package/dist/lib/node/{settings-NDMU3EWQ.cjs.map → settings-ZMB4XRWS.cjs.map} +0 -0
- /package/dist/lib/node/{transcriber-UZJMG44C.cjs.map → transcriber-55OE4X6I.cjs.map} +0 -0
- /package/dist/lib/node-esm/{TranscriptContainer-K4E4P5T6.mjs.map → TranscriptContainer-GBPY6KKA.mjs.map} +0 -0
- /package/dist/lib/node-esm/{chunk-3I7AC3D3.mjs.map → chunk-DXJN2JWV.mjs.map} +0 -0
- /package/dist/lib/node-esm/{intent-resolver-ACGAPLDZ.mjs.map → intent-resolver-HDRFXMKP.mjs.map} +0 -0
- /package/dist/lib/node-esm/{react-surface-G2OY4MWL.mjs.map → react-surface-D3L6VBEV.mjs.map} +0 -0
- /package/dist/lib/node-esm/{settings-O7X2UQQG.mjs.map → settings-I35TVDUG.mjs.map} +0 -0
- /package/dist/lib/node-esm/{transcriber-NVVFH7IF.mjs.map → transcriber-HH7EMCGH.mjs.map} +0 -0
package/dist/lib/browser/{TranscriptContainer-3LU3XBEB.mjs → TranscriptContainer-D3EH2SLO.mjs}
RENAMED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
TranscriptContainer_default,
|
|
3
3
|
TranscriptionContainer
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-BSZSM6KA.mjs";
|
|
5
5
|
import "./chunk-LTIQFX4W.mjs";
|
|
6
6
|
import "./chunk-6ZR5MDK3.mjs";
|
|
7
7
|
export {
|
|
8
8
|
TranscriptionContainer,
|
|
9
9
|
TranscriptContainer_default as default
|
|
10
10
|
};
|
|
11
|
-
//# sourceMappingURL=TranscriptContainer-
|
|
11
|
+
//# sourceMappingURL=TranscriptContainer-D3EH2SLO.mjs.map
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import {
|
|
2
2
|
TranscriptionManager
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-KSUYWWIU.mjs";
|
|
4
4
|
import "./chunk-KLD7O5GJ.mjs";
|
|
5
5
|
import {
|
|
6
6
|
TranscriptType,
|
|
7
7
|
TranscriptionAction
|
|
8
|
-
} from "./chunk-
|
|
8
|
+
} from "./chunk-YRWBY6UK.mjs";
|
|
9
9
|
import {
|
|
10
10
|
TranscriptionCapabilities
|
|
11
11
|
} from "./chunk-LTIQFX4W.mjs";
|
|
@@ -28,7 +28,7 @@ import { createExtension } from "@dxos/plugin-graph";
|
|
|
28
28
|
import { DocumentType } from "@dxos/plugin-markdown/types";
|
|
29
29
|
import { MeetingCapabilities } from "@dxos/plugin-meeting";
|
|
30
30
|
import { MeetingType } from "@dxos/plugin-meeting/types";
|
|
31
|
-
import {
|
|
31
|
+
import { Contact, Organization } from "@dxos/schema";
|
|
32
32
|
|
|
33
33
|
// packages/plugins/plugin-transcription/src/entity-extraction/entity-extraction.ts
|
|
34
34
|
import { Schema } from "effect";
|
|
@@ -38,7 +38,7 @@ import { asyncTimeout } from "@dxos/async";
|
|
|
38
38
|
import { create, ObjectId } from "@dxos/echo-schema";
|
|
39
39
|
import { DXN } from "@dxos/keys";
|
|
40
40
|
import { log } from "@dxos/log";
|
|
41
|
-
import {
|
|
41
|
+
import { MessageType } from "@dxos/schema";
|
|
42
42
|
|
|
43
43
|
// raw-loader:./system-prompt.tpl?raw
|
|
44
44
|
var system_prompt_default = 'You are an AI assistant specialized in processing and enhancing transcripts with contextual information.\nYour task is to analyze a given transcript and extract references to the context objects.\n\nThe context and transcript are provided in the user message in `context` and `transcript` XML tags.\n\nHere are your instructions:\n\n1. Read the transcript and context information provided above.\n\n2. Process each segment of the transcript separately.\n\n3. For each segment:\n a. Analyze the content and identify relevant information from the context.\n b. Extract a number of references to the context objects that are mentioned in the segment.\n c. Output the references in the format `{ "quote": "earnings report", "id": "01JT0JP9AX0XKGZX4MV4B69VT6" }`.\n d. If there are no references, just output an empty array.\n\n4. Maintain the exact order and structure of the original transcript segments.\n\nOutput Format:\nThe final output should be a JSON array of references.\n\nExample output structure:\n```json\n{\n "references": [\n { "quote": "earnings report", "id": "01JT0JP9AX0XKGZX4MV4B69VT6" },\n { "quote": "John", "id": "01JT0VM0VN3ZKN7GDBM8CV9BA7" }\n ]\n}\n```\n\nRemember:\n- Only quote the text that is in the transcript.\n- Analyze each segment carefully before processing it.\n- Only reference objects that appear in the context and exactly by their id, If the object is not in the context, do not insert a reference. \n\nNow, process the transcript and context provided below according to these instructions.';
|
|
@@ -93,7 +93,7 @@ var processTranscriptMessage = async (params) => {
|
|
|
93
93
|
C: (f, a) => f(...a)
|
|
94
94
|
});
|
|
95
95
|
return {
|
|
96
|
-
message: create(
|
|
96
|
+
message: create(MessageType, {
|
|
97
97
|
...params.message,
|
|
98
98
|
blocks: params.message.blocks.map((block, i) => block.type !== "transcription" ? block : {
|
|
99
99
|
...block,
|
|
@@ -267,8 +267,8 @@ var app_graph_builder_default = (context) => contributes(Capabilities.AppGraphBu
|
|
|
267
267
|
// This will be replaced with a vector search index anyway, so its not a big deal.
|
|
268
268
|
contextTypes: [
|
|
269
269
|
DocumentType,
|
|
270
|
-
|
|
271
|
-
|
|
270
|
+
Contact,
|
|
271
|
+
Organization
|
|
272
272
|
],
|
|
273
273
|
space
|
|
274
274
|
});
|
|
@@ -346,4 +346,4 @@ var ENTITY_EXTRACTOR_TIMEOUT = 25e3;
|
|
|
346
346
|
export {
|
|
347
347
|
app_graph_builder_default as default
|
|
348
348
|
};
|
|
349
|
-
//# sourceMappingURL=app-graph-builder-
|
|
349
|
+
//# sourceMappingURL=app-graph-builder-7HN3M24G.mjs.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/capabilities/app-graph-builder.ts", "../../../src/entity-extraction/entity-extraction.ts", "raw-loader:./system-prompt.tpl?raw"],
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport type { Schema } from 'effect';\n\nimport { Capabilities, contributes, createIntent, type PluginsContext } from '@dxos/app-framework';\nimport { AIServiceEdgeClient, type AIServiceClient } from '@dxos/assistant';\nimport { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';\nimport { Filter, fullyQualifiedId, getSpace, makeRef, type Space } from '@dxos/client/echo';\nimport { getSchemaTypename, isInstanceOf, type BaseEchoObject } from '@dxos/echo-schema';\nimport { invariant } from '@dxos/invariant';\nimport { DXN } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { ClientCapabilities } from '@dxos/plugin-client';\nimport { PLANK_COMPANION_TYPE, ATTENDABLE_PATH_SEPARATOR } from '@dxos/plugin-deck/types';\nimport { createExtension, type Node } from '@dxos/plugin-graph';\nimport { DocumentType } from '@dxos/plugin-markdown/types';\nimport { MeetingCapabilities, type CallState, type MediaState } from '@dxos/plugin-meeting';\nimport { MeetingType, type MeetingCallProperties } from '@dxos/plugin-meeting/types';\nimport { type buf } from '@dxos/protocols/buf';\nimport { type TranscriptionPayloadSchema } from '@dxos/protocols/buf/dxos/edge/calls_pb';\nimport { type MessageType, Contact, Organization } from '@dxos/schema';\n\nimport { TranscriptionCapabilities } from './capabilities';\nimport { processTranscriptMessage } from '../entity-extraction';\nimport { TRANSCRIPTION_PLUGIN } from '../meta';\nimport { TranscriptionManager } from '../transcriber';\nimport { TranscriptionAction, type TranscriptionSettingsProps, TranscriptType } from '../types';\n\n// TODO(wittjosiah): Factor out.\n// TODO(wittjosiah): Can we stop using protobuf for this?\ntype TranscriptionPayload = buf.MessageInitShape<typeof TranscriptionPayloadSchema>;\n\nconst getMeetingTranscript = async (\n context: PluginsContext,\n meeting: MeetingType,\n): Promise<TranscriptType | undefined> => {\n const { dispatchPromise: dispatch } = context.requestCapability(Capabilities.IntentDispatcher);\n\n const typename = getSchemaTypename(TranscriptType)!;\n const existing = await meeting.artifacts[typename]?.load();\n if (existing instanceof TranscriptType) {\n return existing;\n }\n\n const space = getSpace(meeting);\n invariant(space);\n const { data } = await dispatch(createIntent(TranscriptionAction.Create, { spaceId: space.id }));\n meeting.artifacts[typename] = makeRef(data!.object);\n return data?.object;\n};\n\n// TODO(wittjosiah): Introduce a sound that plays when transcription is activated/deactivated.\nexport default (context: PluginsContext) =>\n contributes(Capabilities.AppGraphBuilder, [\n createExtension({\n id: `${TRANSCRIPTION_PLUGIN}/meeting-transcript`,\n filter: (node): node is Node<MeetingType> => isInstanceOf(MeetingType, node.data),\n actions: ({ node }) => {\n const meeting = node.data;\n const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);\n return [\n {\n id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,\n data: async () => {\n // NOTE: We are not saving the state of the transcription manager here.\n // We expect the state to be updated through `onCallStateUpdated` once it is propagated through Swarm.\n // This is done to avoid race conditions and to not handle optimistic updates.\n const transcript = await getMeetingTranscript(context, meeting);\n invariant(transcript, 'Failed to create transcript');\n\n const callManager = context.requestCapability(MeetingCapabilities.CallManager);\n callManager.setActivity(getSchemaTypename(TranscriptType)!, {\n queueDxn: transcript.queue.dxn.toString(),\n enabled: !state.enabled,\n });\n },\n properties: {\n label: state.enabled\n ? ['stop transcription label', { ns: TRANSCRIPTION_PLUGIN }]\n : ['start transcription label', { ns: TRANSCRIPTION_PLUGIN }],\n icon: 'ph--subtitles--regular',\n disposition: 'toolbar',\n classNames: state.enabled ? 'bg-callAlert' : '',\n },\n },\n ];\n },\n connector: ({ node }) => {\n const meeting = node.data;\n const client = context.requestCapability(ClientCapabilities.Client);\n const state = context.requestCapability(TranscriptionCapabilities.MeetingTranscriptionState);\n const settings = context\n .requestCapability(Capabilities.SettingsStore)\n .getStore<TranscriptionSettingsProps>(TRANSCRIPTION_PLUGIN)!.value;\n\n // TODO(dmaretskyi): Request via capability.\n const aiClient: AIServiceClient | undefined = new AIServiceEdgeClient({\n endpoint: AI_SERVICE_ENDPOINT.REMOTE,\n });\n\n return [\n {\n id: `${fullyQualifiedId(meeting)}${ATTENDABLE_PATH_SEPARATOR}${getSchemaTypename(TranscriptType)}`,\n type: PLANK_COMPANION_TYPE,\n data: meeting.artifacts[getSchemaTypename(TranscriptType)!]?.target,\n properties: {\n label: ['transcript companion label', { ns: TRANSCRIPTION_PLUGIN }],\n icon: 'ph--subtitles--regular',\n position: 'hoist',\n disposition: 'hidden',\n schema: TranscriptType,\n getIntent: ({ space }: { space: Space }) =>\n createIntent(TranscriptionAction.Create, { spaceId: space.id }),\n\n // TODO(burdon): Is this the right place to inisitalize the state? Change to a capability?\n onJoin: async ({ meeting }: { meeting: MeetingType; roomId: string }) => {\n const identity = client.halo.identity.get();\n invariant(identity);\n const space = getSpace(meeting);\n invariant(space);\n\n let messageEnricher;\n if (aiClient && settings.entityExtraction) {\n messageEnricher = createEntityExtractionEnricher({\n aiClient,\n // TODO(dmaretskyi): Have those be discovered from the schema graph or contributed by capabilities?\n // This forced me to add a dependency on markdown plugin.\n // This will be replaced with a vector search index anyway, so its not a big deal.\n contextTypes: [DocumentType, Contact, Organization],\n space,\n });\n }\n\n // TODO(burdon): The TranscriptionManager singleton is part of the state and should just be updated here.\n state.transcriptionManager = await new TranscriptionManager({\n edgeClient: client.edge,\n messageEnricher,\n })\n .setIdentityDid(identity.did)\n .open();\n },\n onLeave: async () => {\n await state.transcriptionManager?.close();\n state.transcriptionManager = undefined;\n state.enabled = false;\n },\n onCallStateUpdated: async (callState: CallState) => {\n const typename = getSchemaTypename(TranscriptType);\n const transcription = typename ? callState.activities?.[typename] : undefined;\n if (!transcription?.payload) {\n return;\n }\n\n const payload: TranscriptionPayload = transcription.payload;\n state.enabled = !!payload.enabled;\n if (payload.queueDxn) {\n // NOTE: Must set queue before enabling transcription.\n state.transcriptionManager?.setQueue(DXN.parse(payload.queueDxn));\n }\n await state.transcriptionManager?.setEnabled(payload.enabled);\n },\n onMediaStateUpdated: async ([mediaState, isSpeaking]: [MediaState, boolean]) => {\n void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);\n void state.transcriptionManager?.setRecording(isSpeaking);\n },\n } satisfies MeetingCallProperties,\n },\n ];\n },\n }),\n ]);\n\ntype EntityExtractionEnricherFactoryOptions = {\n aiClient: AIServiceClient;\n contextTypes: Schema.Schema.AnyNoContext[];\n space: Space;\n};\n\nconst createEntityExtractionEnricher = ({ aiClient, contextTypes, space }: EntityExtractionEnricherFactoryOptions) => {\n return async (message: MessageType) => {\n const { objects } = await space.db.query(Filter.or(...contextTypes.map((s) => Filter.schema(s)))).run();\n log.info('context', { objects });\n\n const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({\n aiService: aiClient,\n message,\n context: { objects: await Promise.all(objects.map(processContextObject)) },\n options: { timeout: ENTITY_EXTRACTOR_TIMEOUT, fallbackToRaw: true },\n });\n\n log.info('entity extraction time', { timeElapsed });\n return enhancedMessage;\n };\n};\n\nconst processContextObject = async (object: BaseEchoObject): Promise<any> => {\n // TODO(dmaretskyi): Documents need special processing is the content is behind a ref.\n // TODO(dmaretskyi): Think about a way to handle this serialization with a decorator.\n // if (isInstanceOf(DocumentType, object)) {\n // return {\n // ...object,\n // content: await object.content.load(),\n // };\n // }\n\n return object;\n};\n\nconst ENTITY_EXTRACTOR_TIMEOUT = 25_000;\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { Schema } from 'effect';\n\nimport { createTemplate, Message, structuredOutputParser } from '@dxos/artifact';\nimport type { AIServiceClient } from '@dxos/assistant';\nimport { MixedStreamParser } from '@dxos/assistant';\nimport { asyncTimeout } from '@dxos/async';\nimport { type BaseEchoObject, create, ObjectId } from '@dxos/echo-schema';\nimport { DXN } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { MessageType } from '@dxos/schema';\n\nimport SYSTEM_PROMPT from './system-prompt.tpl?raw';\n\ntype ProcessTranscriptMessageParams = {\n message: MessageType;\n aiService: AIServiceClient;\n context: {\n objects?: BaseEchoObject[];\n };\n\n options?: {\n /**\n * Timeout for the entity extraction.\n */\n timeout?: number;\n\n /**\n * Fallback to raw text if the entity extraction fails.\n * Otherwise the function will throw an error.\n * @default false\n */\n fallbackToRaw?: boolean;\n };\n};\n\ntype ProcessTranscriptMessageResult = {\n message: MessageType;\n timeElapsed: number;\n};\n\n/**\n * Process Handlebars template.\n */\nconst createSystemPrompt = (): string => {\n const template = createTemplate(SYSTEM_PROMPT);\n return template({});\n};\n\nconst ReferencedQuotes = Schema.Struct({\n references: Schema.Array(\n Schema.Struct({\n quote: Schema.String,\n id: Schema.String,\n }),\n ).annotations({\n description:\n 'The references to the context objects that are mentioned in the transcript. quote should match the original transcript text exactly, while id is the id of the context object.',\n }),\n});\ninterface ReferencedQuotes extends Schema.Schema.Type<typeof ReferencedQuotes> {}\n\n// TODO(dmaretskyi): Move context to a vector search index.\nexport const processTranscriptMessage = async (\n params: ProcessTranscriptMessageParams,\n): Promise<ProcessTranscriptMessageResult> => {\n try {\n const outputParser = structuredOutputParser(ReferencedQuotes);\n const runParser = async (): Promise<ProcessTranscriptMessageResult> => {\n const startTime = performance.now();\n const result = outputParser.getResult(\n await new MixedStreamParser().parse(\n await params.aiService.execStream({\n model: '@anthropic/claude-3-5-haiku-20241022',\n systemPrompt: createSystemPrompt(),\n history: [\n create(Message, {\n role: 'user',\n content: [\n {\n type: 'text',\n text: `<context>${JSON.stringify(params.context.objects)}</context>`,\n },\n {\n type: 'text',\n text: `<transcript>${JSON.stringify(params.message.blocks)}</transcript>`,\n },\n ],\n }),\n ],\n tools: [outputParser.tool],\n }),\n ),\n );\n log.info('entity extraction result', { refs: result.references });\n\n return {\n message: create(MessageType, {\n ...params.message,\n blocks: params.message.blocks.map((block, i) =>\n block.type !== 'transcription'\n ? block\n : {\n ...block,\n text: postprocessText(block.text, result),\n },\n ),\n }),\n timeElapsed: performance.now() - startTime,\n };\n };\n\n if (params.options?.timeout && params.options.timeout > 0) {\n return await asyncTimeout(runParser(), params.options.timeout);\n } else {\n return await runParser();\n }\n } catch (error) {\n if (params.options?.fallbackToRaw) {\n log.warn('failed to process transcript message, falling back to raw text', { error });\n return {\n message: params.message,\n timeElapsed: 0,\n };\n } else {\n throw error;\n }\n }\n};\n\n/**\n * Finds and replaces all quotes with DXNs references.\n */\n// TODO(dmaretskyi): Lookup and verifiy ids from provided context.\nexport const postprocessText = (text: string, quotes: ReferencedQuotes) => {\n for (const quote of quotes.references) {\n if (!ObjectId.isValid(quote.id)) {\n continue;\n }\n\n // Use a case-insensitive regular expression to replace the quote\n const regex = new RegExp(quote.quote.replace(/[.*+?^${}()|[\\]\\\\]/g, '\\\\$&'), 'gi');\n text = text.replace(regex, `[${quote.quote}][${DXN.fromLocalObjectId(quote.id).toString()}]`);\n }\n return text;\n};\n", "You are an AI assistant specialized in processing and enhancing transcripts with contextual information.\nYour task is to analyze a given transcript and extract references to the context objects.\n\nThe context and transcript are provided in the user message in `context` and `transcript` XML tags.\n\nHere are your instructions:\n\n1. Read the transcript and context information provided above.\n\n2. Process each segment of the transcript separately.\n\n3. For each segment:\n a. Analyze the content and identify relevant information from the context.\n b. Extract a number of references to the context objects that are mentioned in the segment.\n c. Output the references in the format `{ \"quote\": \"earnings report\", \"id\": \"01JT0JP9AX0XKGZX4MV4B69VT6\" }`.\n d. If there are no references, just output an empty array.\n\n4. Maintain the exact order and structure of the original transcript segments.\n\nOutput Format:\nThe final output should be a JSON array of references.\n\nExample output structure:\n```json\n{\n \"references\": [\n { \"quote\": \"earnings report\", \"id\": \"01JT0JP9AX0XKGZX4MV4B69VT6\" },\n { \"quote\": \"John\", \"id\": \"01JT0VM0VN3ZKN7GDBM8CV9BA7\" }\n ]\n}\n```\n\nRemember:\n- Only quote the text that is in the transcript.\n- Analyze each segment carefully before processing it.\n- Only reference objects that appear in the context and exactly by their id, If the object is not in the context, do not insert a reference. \n\nNow, process the transcript and context provided below according to these instructions."],
|
|
5
|
+
"mappings": ";;;;;;;;;;;;;;;;AAMA,SAASA,cAAcC,aAAaC,oBAAyC;AAC7E,SAASC,2BAAiD;AAC1D,SAASC,2BAA2B;AACpC,SAASC,QAAQC,kBAAkBC,UAAUC,eAA2B;AACxE,SAASC,mBAAmBC,oBAAyC;AACrE,SAASC,iBAAiB;AAC1B,SAASC,OAAAA,YAAW;AACpB,SAASC,OAAAA,YAAW;AACpB,SAASC,0BAA0B;AACnC,SAASC,sBAAsBC,iCAAiC;AAChE,SAASC,uBAAkC;AAC3C,SAASC,oBAAoB;AAC7B,SAASC,2BAA4D;AACrE,SAASC,mBAA+C;AAGxD,SAA2BC,SAASC,oBAAoB;;;AClBxD,SAASC,cAAc;AAEvB,SAASC,gBAAgBC,SAASC,8BAA8B;AAEhE,SAASC,yBAAyB;AAClC,SAASC,oBAAoB;AAC7B,SAA8BC,QAAQC,gBAAgB;AACtD,SAASC,WAAW;AACpB,SAASC,WAAW;AACpB,SAASC,mBAAmB;;;ACb5B;;;;AD+CA,IAAMC,qBAAqB,MAAA;AACzB,QAAMC,WAAWC,eAAeC,qBAAAA;AAChC,SAAOF,SAAS,CAAC,CAAA;AACnB;AAEA,IAAMG,mBAAmBC,OAAOC,OAAO;EACrCC,YAAYF,OAAOG,MACjBH,OAAOC,OAAO;IACZG,OAAOJ,OAAOK;IACdC,IAAIN,OAAOK;EACb,CAAA,CAAA,EACAE,YAAY;IACZC,aACE;EACJ,CAAA;AACF,CAAA;AAIO,IAAMC,2BAA2B,OACtCC,WAAAA;AAEA,MAAI;AACF,UAAMC,eAAeC,uBAAuBb,gBAAAA;AAC5C,UAAMc,YAAY,YAAA;AAChB,YAAMC,YAAYC,YAAYC,IAAG;AACjC,YAAMC,SAASN,aAAaO,UAC1B,MAAM,IAAIC,kBAAAA,EAAoBC,MAC5B,MAAMV,OAAOW,UAAUC,WAAW;QAChCC,OAAO;QACPC,cAAc7B,mBAAAA;QACd8B,SAAS;UACPC,OAAOC,SAAS;YACdC,MAAM;YACNC,SAAS;cACP;gBACEC,MAAM;gBACNC,MAAM,YAAYC,KAAKC,UAAUvB,OAAOwB,QAAQC,OAAO,CAAA;cACzD;cACA;gBACEL,MAAM;gBACNC,MAAM,eAAeC,KAAKC,UAAUvB,OAAO0B,QAAQC,MAAM,CAAA;cAC3D;;UAEJ,CAAA;;QAEFC,OAAO;UAAC3B,aAAa4B;;MACvB,CAAA,CAAA,CAAA;AAGJC,UAAIC,KAAK,4BAA4B;QAAEC,MAAMzB,OAAOf;MAAW,GAAA;;;;;;AAE/D,aAAO;QACLkC,SAASV,OAAOiB,aAAa;UAC3B,GAAGjC,OAAO0B;UACVC,QAAQ3B,OAAO0B,QAAQC,OAAOO,IAAI,CAACC,OAAOC,MACxCD,MAAMf,SAAS,kBACXe,QACA;YACE,GAAGA;YACHd,MAAMgB,gBAAgBF,MAAMd,MAAMd,MAAAA;UACpC,CAAA;QAER,CAAA;QACA+B,aAAajC,YAAYC,IAAG,IAAKF;MACnC;IACF;AAEA,QAAIJ,OAAOuC,SAASC,WAAWxC,OAAOuC,QAAQC,UAAU,GAAG;AACzD,aAAO,MAAMC,aAAatC,UAAAA,GAAaH,OAAOuC,QAAQC,OAAO;IAC/D,OAAO;AACL,aAAO,MAAMrC,UAAAA;IACf;EACF,SAASuC,OAAO;AACd,QAAI1C,OAAOuC,SAASI,eAAe;AACjCb,UAAIc,KAAK,kEAAkE;QAAEF;MAAM,GAAA;;;;;;AACnF,aAAO;QACLhB,SAAS1B,OAAO0B;QAChBY,aAAa;MACf;IACF,OAAO;AACL,YAAMI;IACR;EACF;AACF;AAMO,IAAML,kBAAkB,CAAChB,MAAcwB,WAAAA;AAC5C,aAAWnD,SAASmD,OAAOrD,YAAY;AACrC,QAAI,CAACsD,SAASC,QAAQrD,MAAME,EAAE,GAAG;AAC/B;IACF;AAGA,UAAMoD,QAAQ,IAAIC,OAAOvD,MAAMA,MAAMwD,QAAQ,uBAAuB,MAAA,GAAS,IAAA;AAC7E7B,WAAOA,KAAK6B,QAAQF,OAAO,IAAItD,MAAMA,KAAK,KAAKyD,IAAIC,kBAAkB1D,MAAME,EAAE,EAAEyD,SAAQ,CAAA,GAAK;EAC9F;AACA,SAAOhC;AACT;;;;ADlHA,IAAMiC,uBAAuB,OAC3BC,SACAC,YAAAA;AAEA,QAAM,EAAEC,iBAAiBC,SAAQ,IAAKH,QAAQI,kBAAkBC,aAAaC,gBAAgB;AAE7F,QAAMC,WAAWC,kBAAkBC,cAAAA;AACnC,QAAMC,WAAW,MAAMT,QAAQU,UAAUJ,QAAAA,GAAWK,KAAAA;AACpD,MAAIF,oBAAoBD,gBAAgB;AACtC,WAAOC;EACT;AAEA,QAAMG,QAAQC,SAASb,OAAAA;AACvBc,YAAUF,OAAAA,QAAAA;;;;;;;;;AACV,QAAM,EAAEG,KAAI,IAAK,MAAMb,SAASc,aAAaC,oBAAoBC,QAAQ;IAAEC,SAASP,MAAMQ;EAAG,CAAA,CAAA;AAC7FpB,UAAQU,UAAUJ,QAAAA,IAAYe,QAAQN,KAAMO,MAAM;AAClD,SAAOP,MAAMO;AACf;AAGA,IAAA,4BAAe,CAACvB,YACdwB,YAAYnB,aAAaoB,iBAAiB;EACxCC,gBAAgB;IACdL,IAAI,GAAGM,oBAAAA;IACPC,QAAQ,CAACC,SAAoCC,aAAaC,aAAaF,KAAKb,IAAI;IAChFgB,SAAS,CAAC,EAAEH,KAAI,MAAE;AAChB,YAAM5B,UAAU4B,KAAKb;AACrB,YAAMiB,QAAQjC,QAAQI,kBAAkB8B,0BAA0BC,yBAAyB;AAC3F,aAAO;QACL;UACEd,IAAI,GAAGe,iBAAiBnC,OAAAA,CAAAA;UACxBe,MAAM,YAAA;AAIJ,kBAAMqB,aAAa,MAAMtC,qBAAqBC,SAASC,OAAAA;AACvDc,sBAAUsB,YAAY,+BAAA;;;;;;;;;AAEtB,kBAAMC,cAActC,QAAQI,kBAAkBmC,oBAAoBC,WAAW;AAC7EF,wBAAYG,YAAYjC,kBAAkBC,cAAAA,GAAkB;cAC1DiC,UAAUL,WAAWM,MAAMC,IAAIC,SAAQ;cACvCC,SAAS,CAACb,MAAMa;YAClB,CAAA;UACF;UACAC,YAAY;YACVC,OAAOf,MAAMa,UACT;cAAC;cAA4B;gBAAEG,IAAItB;cAAqB;gBACxD;cAAC;cAA6B;gBAAEsB,IAAItB;cAAqB;;YAC7DuB,MAAM;YACNC,aAAa;YACbC,YAAYnB,MAAMa,UAAU,iBAAiB;UAC/C;QACF;;IAEJ;IACAO,WAAW,CAAC,EAAExB,KAAI,MAAE;AAClB,YAAM5B,UAAU4B,KAAKb;AACrB,YAAMsC,SAAStD,QAAQI,kBAAkBmD,mBAAmBC,MAAM;AAClE,YAAMvB,QAAQjC,QAAQI,kBAAkB8B,0BAA0BC,yBAAyB;AAC3F,YAAMsB,WAAWzD,QACdI,kBAAkBC,aAAaqD,aAAa,EAC5CC,SAAqChC,oBAAAA,EAAuBiC;AAG/D,YAAMC,WAAwC,IAAIC,oBAAoB;QACpEC,UAAUC,oBAAoBC;MAChC,CAAA;AAEA,aAAO;QACL;UACE5C,IAAI,GAAGe,iBAAiBnC,OAAAA,CAAAA,GAAWiE,yBAAAA,GAA4B1D,kBAAkBC,cAAAA,CAAAA;UACjF0D,MAAMC;UACNpD,MAAMf,QAAQU,UAAUH,kBAAkBC,cAAAA,CAAAA,GAAmB4D;UAC7DtB,YAAY;YACVC,OAAO;cAAC;cAA8B;gBAAEC,IAAItB;cAAqB;;YACjEuB,MAAM;YACNoB,UAAU;YACVnB,aAAa;YACboB,QAAQ9D;YACR+D,WAAW,CAAC,EAAE3D,MAAK,MACjBI,aAAaC,oBAAoBC,QAAQ;cAAEC,SAASP,MAAMQ;YAAG,CAAA;;YAG/DoD,QAAQ,OAAO,EAAExE,SAAAA,SAAO,MAA4C;AAClE,oBAAMyE,WAAWpB,OAAOqB,KAAKD,SAASE,IAAG;AACzC7D,wBAAU2D,UAAAA,QAAAA;;;;;;;;;AACV,oBAAM7D,QAAQC,SAASb,QAAAA;AACvBc,wBAAUF,OAAAA,QAAAA;;;;;;;;;AAEV,kBAAIgE;AACJ,kBAAIhB,YAAYJ,SAASqB,kBAAkB;AACzCD,kCAAkBE,+BAA+B;kBAC/ClB;;;;kBAIAmB,cAAc;oBAACC;oBAAcC;oBAASC;;kBACtCtE;gBACF,CAAA;cACF;AAGAoB,oBAAMmD,uBAAuB,MAAM,IAAIC,qBAAqB;gBAC1DC,YAAYhC,OAAOiC;gBACnBV;cACF,CAAA,EACGW,eAAed,SAASe,GAAG,EAC3BC,KAAI;YACT;YACAC,SAAS,YAAA;AACP,oBAAM1D,MAAMmD,sBAAsBQ,MAAAA;AAClC3D,oBAAMmD,uBAAuBS;AAC7B5D,oBAAMa,UAAU;YAClB;YACAgD,oBAAoB,OAAOC,cAAAA;AACzB,oBAAMxF,WAAWC,kBAAkBC,cAAAA;AACnC,oBAAMuF,gBAAgBzF,WAAWwF,UAAUE,aAAa1F,QAAAA,IAAYsF;AACpE,kBAAI,CAACG,eAAeE,SAAS;AAC3B;cACF;AAEA,oBAAMA,UAAgCF,cAAcE;AACpDjE,oBAAMa,UAAU,CAAC,CAACoD,QAAQpD;AAC1B,kBAAIoD,QAAQxD,UAAU;AAEpBT,sBAAMmD,sBAAsBe,SAASC,KAAIC,MAAMH,QAAQxD,QAAQ,CAAA;cACjE;AACA,oBAAMT,MAAMmD,sBAAsBkB,WAAWJ,QAAQpD,OAAO;YAC9D;YACAyD,qBAAqB,OAAO,CAACC,YAAYC,UAAAA,MAAkC;AACzE,mBAAKxE,MAAMmD,sBAAsBsB,cAAcF,WAAWG,UAAU;AACpE,mBAAK1E,MAAMmD,sBAAsBwB,aAAaH,UAAAA;YAChD;UACF;QACF;;IAEJ;EACF,CAAA;CACD;AAQH,IAAM1B,iCAAiC,CAAC,EAAElB,UAAUmB,cAAcnE,MAAK,MAA0C;AAC/G,SAAO,OAAOgG,YAAAA;AACZ,UAAM,EAAEC,QAAO,IAAK,MAAMjG,MAAMkG,GAAGC,MAAMC,OAAOC,GAAE,GAAIlC,aAAamC,IAAI,CAACC,MAAMH,OAAO1C,OAAO6C,CAAAA,CAAAA,CAAAA,CAAAA,EAAMC,IAAG;AACrGC,IAAAA,KAAIC,KAAK,WAAW;MAAET;IAAQ,GAAA;;;;;;AAE9B,UAAM,EAAED,SAASW,iBAAiBC,YAAW,IAAK,MAAMC,yBAAyB;MAC/EC,WAAW9D;MACXgD;MACA7G,SAAS;QAAE8G,SAAS,MAAMc,QAAQC,IAAIf,QAAQK,IAAIW,oBAAAA,CAAAA;MAAuB;MACzEC,SAAS;QAAEC,SAASC;QAA0BC,eAAe;MAAK;IACpE,CAAA;AAEAZ,IAAAA,KAAIC,KAAK,0BAA0B;MAAEE;IAAY,GAAA;;;;;;AACjD,WAAOD;EACT;AACF;AAEA,IAAMM,uBAAuB,OAAOvG,WAAAA;AAUlC,SAAOA;AACT;AAEA,IAAM0G,2BAA2B;",
|
|
6
|
+
"names": ["Capabilities", "contributes", "createIntent", "AIServiceEdgeClient", "AI_SERVICE_ENDPOINT", "Filter", "fullyQualifiedId", "getSpace", "makeRef", "getSchemaTypename", "isInstanceOf", "invariant", "DXN", "log", "ClientCapabilities", "PLANK_COMPANION_TYPE", "ATTENDABLE_PATH_SEPARATOR", "createExtension", "DocumentType", "MeetingCapabilities", "MeetingType", "Contact", "Organization", "Schema", "createTemplate", "Message", "structuredOutputParser", "MixedStreamParser", "asyncTimeout", "create", "ObjectId", "DXN", "log", "MessageType", "createSystemPrompt", "template", "createTemplate", "SYSTEM_PROMPT", "ReferencedQuotes", "Schema", "Struct", "references", "Array", "quote", "String", "id", "annotations", "description", "processTranscriptMessage", "params", "outputParser", "structuredOutputParser", "runParser", "startTime", "performance", "now", "result", "getResult", "MixedStreamParser", "parse", "aiService", "execStream", "model", "systemPrompt", "history", "create", "Message", "role", "content", "type", "text", "JSON", "stringify", "context", "objects", "message", "blocks", "tools", "tool", "log", "info", "refs", "MessageType", "map", "block", "i", "postprocessText", "timeElapsed", "options", "timeout", "asyncTimeout", "error", "fallbackToRaw", "warn", "quotes", "ObjectId", "isValid", "regex", "RegExp", "replace", "DXN", "fromLocalObjectId", "toString", "getMeetingTranscript", "context", "meeting", "dispatchPromise", "dispatch", "requestCapability", "Capabilities", "IntentDispatcher", "typename", "getSchemaTypename", "TranscriptType", "existing", "artifacts", "load", "space", "getSpace", "invariant", "data", "createIntent", "TranscriptionAction", "Create", "spaceId", "id", "makeRef", "object", "contributes", "AppGraphBuilder", "createExtension", "TRANSCRIPTION_PLUGIN", "filter", "node", "isInstanceOf", "MeetingType", "actions", "state", "TranscriptionCapabilities", "MeetingTranscriptionState", "fullyQualifiedId", "transcript", "callManager", "MeetingCapabilities", "CallManager", "setActivity", "queueDxn", "queue", "dxn", "toString", "enabled", "properties", "label", "ns", "icon", "disposition", "classNames", "connector", "client", "ClientCapabilities", "Client", "settings", "SettingsStore", "getStore", "value", "aiClient", "AIServiceEdgeClient", "endpoint", "AI_SERVICE_ENDPOINT", "REMOTE", "ATTENDABLE_PATH_SEPARATOR", "type", "PLANK_COMPANION_TYPE", "target", "position", "schema", "getIntent", "onJoin", "identity", "halo", "get", "messageEnricher", "entityExtraction", "createEntityExtractionEnricher", "contextTypes", "DocumentType", "Contact", "Organization", "transcriptionManager", "TranscriptionManager", "edgeClient", "edge", "setIdentityDid", "did", "open", "onLeave", "close", "undefined", "onCallStateUpdated", "callState", "transcription", "activities", "payload", "setQueue", "DXN", "parse", "setEnabled", "onMediaStateUpdated", "mediaState", "isSpeaking", "setAudioTrack", "audioTrack", "setRecording", "message", "objects", "db", "query", "Filter", "or", "map", "s", "run", "log", "info", "enhancedMessage", "timeElapsed", "processTranscriptMessage", "aiService", "Promise", "all", "processContextObject", "options", "timeout", "ENTITY_EXTRACTOR_TIMEOUT", "fallbackToRaw"]
|
|
7
|
+
}
|
|
@@ -582,12 +582,12 @@ import { useCapabilities } from "@dxos/app-framework";
|
|
|
582
582
|
|
|
583
583
|
// packages/plugins/plugin-transcription/src/capabilities/index.ts
|
|
584
584
|
import { lazy } from "@dxos/app-framework";
|
|
585
|
-
var AppGraphBuilder = lazy(() => import("./app-graph-builder-
|
|
586
|
-
var IntentResolver = lazy(() => import("./intent-resolver-
|
|
585
|
+
var AppGraphBuilder = lazy(() => import("./app-graph-builder-7HN3M24G.mjs"));
|
|
586
|
+
var IntentResolver = lazy(() => import("./intent-resolver-ZWKYRFE7.mjs"));
|
|
587
587
|
var MeetingTranscriptionState = lazy(() => import("./meeting-transcription-state-MNA2XROE.mjs"));
|
|
588
|
-
var ReactSurface = lazy(() => import("./react-surface-
|
|
589
|
-
var Transcriber = lazy(() => import("./transcriber-
|
|
590
|
-
var Settings = lazy(() => import("./settings-
|
|
588
|
+
var ReactSurface = lazy(() => import("./react-surface-F7DBDGHS.mjs"));
|
|
589
|
+
var Transcriber = lazy(() => import("./transcriber-MSBS4VBI.mjs"));
|
|
590
|
+
var Settings = lazy(() => import("./settings-6YAHKVKX.mjs"));
|
|
591
591
|
|
|
592
592
|
// packages/plugins/plugin-transcription/src/hooks/useTranscriber.ts
|
|
593
593
|
var useTranscriber = ({ audioStreamTrack, onSegments, config }) => {
|
|
@@ -717,4 +717,4 @@ export {
|
|
|
717
717
|
Transcriber,
|
|
718
718
|
Settings
|
|
719
719
|
};
|
|
720
|
-
//# sourceMappingURL=chunk-
|
|
720
|
+
//# sourceMappingURL=chunk-BSZSM6KA.mjs.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/components/TranscriptContainer.tsx", "../../../src/components/Transcript/Transcript.tsx", "../../../src/components/Transcript/transcript-extension.ts", "../../../src/model/model.ts", "../../../src/hooks/useAudioFile.ts", "../../../src/hooks/useAudioTrack.ts", "../../../src/hooks/useQueueModelAdapter.ts", "../../../src/hooks/useTranscriber.ts", "../../../src/capabilities/index.ts", "../../../src/hooks/useVoiceInput.ts"],
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2024 DXOS.org\n//\n\nimport React, { type FC } from 'react';\n\nimport { fullyQualifiedId, getSpace } from '@dxos/client/echo';\nimport { useMembers, useQueue } from '@dxos/react-client/echo';\nimport { StackItem } from '@dxos/react-ui-stack';\nimport { type MessageType } from '@dxos/schema';\n\nimport { Transcript, renderMarkdown } from './Transcript';\nimport { useQueueModelAdapter } from '../hooks';\nimport { type TranscriptType } from '../types';\n\nexport const TranscriptionContainer: FC<{ role: string; transcript: TranscriptType }> = ({ role, transcript }) => {\n const attendableId = fullyQualifiedId(transcript);\n const space = getSpace(transcript);\n const members = useMembers(space?.key).map((member) => member.identity);\n const queue = useQueue<MessageType>(transcript.queue.dxn, { pollInterval: 1_000 });\n const model = useQueueModelAdapter(renderMarkdown(members), queue);\n\n return (\n <StackItem.Content role={role} classNames='container-max-width pli-2'>\n <Transcript attendableId={attendableId} space={space} model={model} transcript={transcript} />\n </StackItem.Content>\n );\n};\n\nexport default TranscriptionContainer;\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport React from 'react';\n\nimport { type Space } from '@dxos/react-client/echo';\nimport { type Identity } from '@dxos/react-client/halo';\nimport { IconButton, type ThemedClassName, useThemeContext } from '@dxos/react-ui';\nimport {\n createBasicExtensions,\n createMarkdownExtensions,\n createRenderer,\n createThemeExtensions,\n decorateMarkdown,\n editorWidth,\n preview,\n useTextEditor,\n} from '@dxos/react-ui-editor';\nimport { mx } from '@dxos/react-ui-theme';\nimport { type MessageType } from '@dxos/schema';\nimport { isNotFalsy } from '@dxos/util';\n\nimport { transcript } from './transcript-extension';\nimport { type SerializationModel } from '../../model';\nimport { type TranscriptType } from '../../types';\n\nexport const renderMarkdown =\n (identities: Identity[]) =>\n (message: MessageType, index: number, debug = false): string[] => {\n if (message.sender.role === 'assistant') {\n // Start/stop block.\n return [message.blocks.find((block) => block.type === 'transcription')?.text ?? '', ''];\n }\n\n // TODO(burdon): Use link/reference markup for users (with popover).\n // TODO(burdon): Color and avatar.\n const identity = identities.find((identity) => identity.did === message.sender.identityDid);\n const name =\n identity?.profile?.displayName ??\n message.sender.contact?.target?.fullName ??\n message.sender.name ??\n message.sender.email ??\n message.sender.identityDid;\n const blocks = message.blocks.filter((block) => block.type === 'transcription');\n return [\n `###### ${name}` + (debug ? ` [${index}]:${message.id}` : ''),\n blocks.map((block) => block.text.trim()).join(' '),\n '',\n ];\n };\n\nexport type TranscriptProps = ThemedClassName<{\n transcript?: TranscriptType;\n space?: Space;\n model: SerializationModel<MessageType>;\n // TODO(wittjosiah): Move to container.\n attendableId?: string;\n ignoreAttention?: boolean;\n}>;\n\n/**\n * Transcript component implemented using the editor.\n */\nexport const Transcript = ({\n classNames,\n transcript: object,\n space,\n model,\n attendableId,\n ignoreAttention,\n}: TranscriptProps) => {\n const { themeMode } = useThemeContext();\n const { parentRef } = useTextEditor(() => {\n return {\n extensions: [\n createBasicExtensions({ readOnly: true, lineWrapping: true }),\n createMarkdownExtensions({ themeMode }),\n createThemeExtensions({ themeMode }),\n decorateMarkdown(),\n space && preview(),\n transcript({\n model,\n started: object?.started ? new Date(object.started) : undefined,\n renderButton: createRenderer(({ onClick }) => (\n <IconButton icon='ph--arrow-line-down--regular' iconOnly label='Scroll to bottom' onClick={onClick} />\n )),\n }),\n ].filter(isNotFalsy),\n };\n }, [space, model]);\n\n return (\n <div\n ref={parentRef}\n className={mx('flex grow overflow-hidden', editorWidth, classNames)}\n data-popover-collision-boundary={true}\n />\n );\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { type Extension, RangeSetBuilder } from '@codemirror/state';\nimport { EditorView, GutterMarker, ViewPlugin, type ViewUpdate, gutter } from '@codemirror/view';\nimport { format } from 'date-fns/format';\nimport { intervalToDuration } from 'date-fns/intervalToDuration';\n\nimport { type CleanupFn, addEventListener, combine } from '@dxos/async';\nimport { type RenderCallback } from '@dxos/react-ui-editor';\nimport { type MessageType } from '@dxos/schema';\n\nimport { DocumentAdapter, type SerializationModel } from '../../model';\n\n/**\n * Data structure that maps Chunks queue to lines with transcript state.\n */\nexport type TranscriptOptions = {\n model: SerializationModel<MessageType>;\n started?: Date;\n renderButton?: RenderCallback<{ onClick: () => void }>;\n};\n\n/**\n * Scrolling transcript with timestamps.\n */\nexport const transcript = ({ model, started, renderButton }: TranscriptOptions): Extension => {\n return [\n // Show timestamps in the gutter.\n gutter({\n class: 'cm-timestamp-gutter',\n lineMarkerChange: (update) => update.docChanged || update.viewportChanged,\n markers: (view) => {\n const start = getStartTime(started, model.chunks[0]);\n const builder = new RangeSetBuilder<GutterMarker>();\n for (const { from, to } of view.visibleRanges) {\n let line = view.state.doc.lineAt(from);\n while (line.from <= to) {\n const block = model.getChunkAtLine(line.number)?.blocks[0];\n const timestamp = block?.type === 'transcription' && block.started;\n if (timestamp) {\n builder.add(line.from, line.from, new TimestampMarker(line.number, new Date(timestamp), start));\n }\n\n if (line.to + 1 > view.state.doc.length) {\n break;\n }\n\n line = view.state.doc.lineAt(line.to + 1);\n }\n }\n\n return builder.finish();\n },\n }),\n\n // Listen for model updates.\n ViewPlugin.fromClass(\n class {\n private readonly _controls?: HTMLDivElement;\n private readonly _cleanup: CleanupFn;\n private readonly _adapter: DocumentAdapter;\n private _initialized = false;\n\n constructor(view: EditorView) {\n this._adapter = new DocumentAdapter(view);\n\n const scroller = view.scrollDOM;\n let isAutoScrolling = false;\n let hasScrolled = false;\n\n let timeout: NodeJS.Timeout | undefined;\n const scrollToBottom = (smooth = false) => {\n scroller.style.scrollBehavior = smooth ? 'smooth' : '';\n\n // Temporarily hide scrollbar to prevent flicker.\n scroller.classList.add('cm-hide-scrollbar');\n isAutoScrolling = true;\n clearTimeout(timeout);\n timeout = setTimeout(() => {\n this._controls?.classList.add('opacity-0');\n scroller.classList.remove('cm-hide-scrollbar');\n isAutoScrolling = false;\n }, 500);\n\n // Scroll to bottom.\n view.dispatch({\n effects: EditorView.scrollIntoView(view.state.doc.length, { y: 'end' }),\n });\n };\n\n // Scroll button.\n if (renderButton) {\n this._controls = document.createElement('div');\n this._controls.classList.add('cm-controls', 'transition-opacity', 'duration-300', 'opacity-0');\n view.dom.appendChild(this._controls);\n renderButton(\n this._controls,\n {\n onClick: () => {\n scrollToBottom(false);\n },\n },\n view,\n );\n }\n\n // Event listeners.\n this._cleanup = combine(\n addEventListener(view.scrollDOM, 'scroll', () => {\n if (!isAutoScrolling) {\n hasScrolled = true;\n this._controls?.classList.remove('opacity-0');\n }\n }),\n\n model.update.on(() => {\n // Check if clamped to bottom.\n const autoScroll =\n scroller.scrollHeight - scroller.scrollTop - scroller.clientHeight === 0 || !hasScrolled;\n\n // Sync.\n model.sync(this._adapter);\n\n // Scroll.\n if (autoScroll) {\n scrollToBottom(true);\n }\n }),\n );\n }\n\n update(update: ViewUpdate) {\n // Initial sync.\n if (!this._initialized) {\n this._initialized = true;\n setTimeout(() => {\n model.sync(this._adapter);\n });\n }\n }\n\n destroy() {\n this._controls?.remove();\n this._cleanup();\n }\n },\n ),\n\n EditorView.theme({\n '.cm-scroller': {\n overflowY: 'scroll',\n },\n '.cm-hide-scrollbar': {\n scrollbarWidth: 'none',\n '-ms-overflow-style': 'none',\n },\n '.cm-hide-scrollbar::-webkit-scrollbar': {\n display: 'none',\n },\n '.cm-controls': {\n position: 'absolute',\n bottom: '0.5rem',\n right: '0.5rem',\n zIndex: 1000,\n },\n '.cm-line': {\n paddingRight: '1rem',\n },\n '.cm-timestamp-gutter': {\n width: '6rem',\n paddingRight: '1rem',\n },\n '.cm-timestamp-gutter > .cm-gutterElement > div': {\n display: 'inline-flex',\n textAlign: 'right',\n padding: '3px',\n },\n }),\n ];\n};\n\n/**\n * Gutter marker that displays a timestamp.\n */\nclass TimestampMarker extends GutterMarker {\n constructor(\n readonly _line: number,\n readonly _timestamp: Date,\n readonly _started?: Date,\n ) {\n super();\n }\n\n override eq(other: this) {\n return other._timestamp === this._timestamp;\n }\n\n override toDOM(view: EditorView) {\n const el = document.createElement('div');\n el.className = 'text-sm text-subdued hover:bg-hoverSurface cursor-pointer';\n el.textContent = formatTimestamp(this._timestamp, this._started);\n // TODO(burdon): Click to bookmark or copy hyperlink.\n el.onclick = () => {\n const pos = view.state.doc.line(this._line).from;\n view.dispatch({\n effects: EditorView.scrollIntoView(pos, { y: 'start' }),\n });\n };\n\n return el;\n }\n}\n\nconst getStartTime = (started?: Date, message?: MessageType): Date | undefined => {\n if (started) {\n return started;\n }\n\n if (message?.blocks[0]?.type === 'transcription' && message.blocks[0].started) {\n return new Date(message.blocks[0].started);\n }\n\n return undefined;\n};\n\nconst formatTimestamp = (timestamp: Date, relative?: Date) => {\n if (relative) {\n const pad = (n = 0) => String(n).padStart(2, '0');\n const { hours, minutes, seconds } = intervalToDuration({ start: relative, end: timestamp });\n return `${pad(hours)}:${pad(minutes)}:${pad(seconds)}`;\n } else {\n return format(timestamp, 'HH:mm:ss');\n }\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { Text } from '@codemirror/state';\nimport { type EditorView } from '@codemirror/view';\n\nimport { Event } from '@dxos/async';\nimport { invariant } from '@dxos/invariant';\nimport { log } from '@dxos/log';\n\nexport type Chunk = { id: string };\n\n/**\n * Covert block to a set of markdown lines.\n */\nexport type ChunkRenderer<T extends Chunk> = (chunk: T, index: number, debug?: boolean) => string[];\n\n/**\n * Abstract representation of the document model.\n */\nexport interface ChunkDocument {\n lineCount(): number;\n replaceLines(from: number, remove: number, insert: string[]): void;\n}\n\n/**\n * Codemirror document adapter.\n */\nexport class DocumentAdapter implements ChunkDocument {\n constructor(private readonly _view: EditorView) {}\n\n /** The document must always have at least one line. */\n lineCount(): number {\n return this._view.state.doc.lines - 1;\n }\n\n replaceLines(from: number, remove: number, insert: string[]): void {\n log('replaceLines', { count: this._view.state.doc.lines, from, remove, insert });\n\n const numLines = this._view.state.doc.lines;\n const posFrom = this._view.state.doc.line(from).from;\n const posTo = remove === 0 ? posFrom : this._view.state.doc.line(Math.min(numLines, from + remove)).from;\n\n // Don't remove the last line.\n let text = insert.join('\\n');\n if (posTo === this._view.state.doc.length) {\n text += '\\n';\n }\n\n this._view.dispatch({\n changes: { from: posFrom, to: posTo, insert: text },\n });\n }\n}\n\ntype ChunkChange<T extends Chunk> =\n | {\n type: 'append';\n index: number;\n chunk: T;\n }\n | {\n type: 'update';\n index: number;\n chunk: T;\n }\n | {\n type: 'delete';\n index: number;\n id: string;\n };\n\n/**\n * Each Chunk can be converted into a set of lines.\n * Chunks can be added, updated or deleted from the Queue.\n * A Document is an immutable projection that represents a flat list of lines corresponding to the Queue.\n *\n * Ideally we would implement a custom virtual Text model for the View, but this currently isn't possible in Codemirror.\n * Instead this model tracks changes and syncs them with the ChunkDocument.\n */\nexport class SerializationModel<T extends Chunk> {\n /** Ordered set of chunks. */\n private readonly _chunks: T[] = [];\n\n /** Map of chunk IDs to their current line counts. */\n private readonly _chunkLineCounts = new Map<string, number>();\n\n /** Map of line numbers to chunk IDs. */\n private readonly _lineToChunk = new Map<number, T>();\n\n /** Track chunk changes since last sync. */\n private readonly _changes: Array<ChunkChange<T>> = [];\n\n /** Emits when the document is updated. */\n public readonly update = new Event<void>();\n\n constructor(\n private readonly _renderer: ChunkRenderer<T>,\n initialChunks: T[] = [],\n ) {\n initialChunks.forEach((chunk) => {\n this.appendChunk(chunk);\n });\n }\n\n toJSON() {\n return {\n chunks: this._chunks.length,\n changes: this._changes.length,\n };\n }\n\n get doc() {\n return Text.of([...this._chunks.flatMap((chunk, index) => this._renderer(chunk, index)), '']);\n }\n\n get chunks() {\n return this._chunks;\n }\n\n /**\n * Get the block ID at the given line number.\n */\n getChunkAtLine(line: number): T | undefined {\n return this._lineToChunk.get(line);\n }\n\n reset(): this {\n this._chunks.length = 0;\n this._chunkLineCounts.clear();\n this._lineToChunk.clear();\n this.update.emit();\n return this;\n }\n\n appendChunk(chunk: T): this {\n const index = this._chunks.length;\n this._chunks.push(chunk);\n this._changes.push({ type: 'append', index, chunk });\n this.update.emit();\n return this;\n }\n\n updateChunk(chunk: T): this {\n const index = this._chunks.findIndex((c) => c.id === chunk.id);\n invariant(index !== -1);\n this._chunks[index] = chunk;\n this._changes.push({ type: 'update', index, chunk });\n this.update.emit();\n return this;\n }\n\n deleteBlock(id: string): this {\n const index = this._chunks.findIndex((c) => c.id === id);\n invariant(index !== -1);\n this._chunks.splice(index, 1);\n this._changes.push({ type: 'delete', index, id });\n this.update.emit();\n return this;\n }\n\n /**\n * Syncs the tracked changes with the ChunkDocument.\n */\n sync(document: ChunkDocument): this {\n log('sync', { changes: this._changes });\n\n // Process each change in order.\n for (const change of this._changes) {\n switch (change.type) {\n case 'append': {\n // For appends, convert block to lines and add at the end.\n const lines = this._renderer(change.chunk, change.index);\n const lastLine = document.lineCount();\n document.replaceLines(lastLine + 1, 0, lines);\n break;\n }\n\n case 'update': {\n // For updates, find the block's line range and replace with new lines.\n let lineStart = 1;\n for (let i = 0; i < change.index; i++) {\n lineStart += this._chunkLineCounts.get(this._chunks[i].id)!;\n }\n const previousLineCount = this._chunkLineCounts.get(change.chunk.id)!;\n const newLines = this._renderer(change.chunk, change.index);\n document.replaceLines(lineStart, previousLineCount, newLines);\n break;\n }\n\n case 'delete': {\n // For deletes, find the block's line range and remove those lines.\n let lineStart = 1;\n for (let i = 0; i < change.index; i++) {\n lineStart += this._chunkLineCounts.get(this._chunks[i].id)!;\n }\n const previousLineCount = this._chunkLineCounts.get(change.id)!;\n document.replaceLines(lineStart, previousLineCount, []);\n break;\n }\n }\n }\n\n // Clear the line-to-block mapping.\n this._lineToChunk.clear();\n\n // Update all block line counts and line-to-block mapping after processing changes.\n let currentLine = 1;\n for (const [index, chunk] of this._chunks.entries()) {\n const lineCount = this._renderer(chunk, index).length;\n this._chunkLineCounts.set(chunk.id, lineCount);\n this._lineToChunk.set(currentLine, chunk);\n currentLine += lineCount;\n }\n\n // Clean up deleted blocks.\n const chunkIds = new Set(this._chunks.map((chunk) => chunk.id));\n for (const id of this._chunkLineCounts.keys()) {\n if (!chunkIds.has(id)) {\n this._chunkLineCounts.delete(id);\n }\n }\n\n // Clear changes after sync.\n this._changes.length = 0;\n return this;\n }\n}\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { useEffect, useState } from 'react';\n\nimport { log } from '@dxos/log';\n\nexport type UseAudioState = {\n audio?: HTMLAudioElement;\n stream?: MediaStream;\n track?: MediaStreamTrack;\n};\n\nexport const useAudioFile = (audioUrl: string): UseAudioState => {\n const [{ audio, stream, track }, setStream] = useState<UseAudioState>({});\n useEffect(() => {\n const t = setTimeout(async () => {\n const response = await fetch(audioUrl);\n const blob = await response.blob();\n const audio = new Audio();\n audio.src = URL.createObjectURL(blob);\n await new Promise<void>((resolve, reject) => {\n audio.addEventListener(\n 'error',\n (err) => {\n log.error('error', { err });\n reject(err);\n },\n { once: true },\n );\n audio.addEventListener(\n 'canplay',\n () => {\n log.info('starting...');\n resolve();\n },\n { once: true },\n );\n audio.load();\n });\n\n const ctx = new AudioContext();\n\n const destination = ctx.createMediaStreamDestination();\n destination.channelCount = 1;\n\n const source = ctx.createMediaElementSource(audio);\n source.connect(destination);\n\n setStream({\n audio,\n stream: destination.stream,\n track: destination.stream.getAudioTracks()[0],\n });\n });\n\n return () => clearTimeout(t);\n }, [audioUrl]);\n\n return { audio, stream, track };\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { useEffect, useState } from 'react';\n\n/**\n *\n */\n// TODO(burdon): Reconcile with react-ui-sfx and plugin-calls.\nexport const useAudioTrack = (active?: boolean): MediaStreamTrack | undefined => {\n const [track, setTrack] = useState<MediaStreamTrack>();\n useEffect(() => {\n if (!active) {\n track?.stop();\n setTrack(undefined);\n return;\n }\n\n const initAudio = async () => {\n const audio = new Audio();\n audio.srcObject = await navigator.mediaDevices.getUserMedia({ audio: true });\n const [track] = audio.srcObject.getAudioTracks();\n if (track) {\n setTrack(track);\n }\n };\n\n if (active) {\n void initAudio();\n }\n }, [active]);\n\n return track;\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { useMemo, useEffect, useState } from 'react';\n\nimport { type Queue } from '@dxos/react-client/echo';\n\nimport { type Chunk, type ChunkRenderer, SerializationModel } from '../model';\n\n/**\n * Model adapter for a queue.\n */\nexport const useQueueModelAdapter = <T extends Chunk>(\n renderer: ChunkRenderer<T>,\n queue: Queue<T> | undefined,\n initialChunks: T[] = [],\n): SerializationModel<T> => {\n const model = useMemo(() => new SerializationModel<T>(renderer, initialChunks), [queue]);\n const [loaded, setLoaded] = useState(false);\n\n // Set initial blocks.\n useEffect(() => {\n if (!queue) {\n return;\n }\n\n const update = () => {\n for (const block of queue?.items ?? []) {\n model.appendChunk(block);\n }\n\n setLoaded(true);\n };\n\n // TODO(burdon): This is a hack to ensure the queue is loaded.\n let i: NodeJS.Timeout | undefined;\n if (queue.isLoading) {\n i = setInterval(() => {\n if (!queue?.isLoading) {\n clearInterval(i);\n update();\n }\n }, 1_000);\n } else {\n update();\n }\n\n return () => clearTimeout(i);\n }, [model, queue]);\n\n // TODO(burdon): Can we listen for queue events?\n useEffect(() => {\n if (!loaded || !queue || model.chunks.length === queue.items.length) {\n return;\n }\n\n const chunk = queue.items[queue.items.length - 1];\n model.appendChunk(chunk);\n }, [model, loaded, queue?.items.length]);\n\n return model;\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { useEffect, useMemo } from 'react';\n\nimport { useCapabilities } from '@dxos/app-framework';\n\nimport { TranscriptionCapabilities } from '../capabilities';\nimport { type Transcriber } from '../transcriber';\n\n/**\n * Records audio while user is speaking and transcribes it after user is done speaking.\n */\nexport const useTranscriber = ({\n audioStreamTrack,\n onSegments,\n config,\n}: Partial<TranscriptionCapabilities.GetTranscriberProps>) => {\n const [getTranscriber] = useCapabilities(TranscriptionCapabilities.Transcriber);\n\n // Initialize audio transcription.\n const transcriber = useMemo<Transcriber | undefined>(() => {\n if (!onSegments || !audioStreamTrack || !getTranscriber) {\n return undefined;\n }\n\n return getTranscriber({ audioStreamTrack, onSegments, config });\n }, [audioStreamTrack, onSegments, getTranscriber, config]);\n\n useEffect(() => {\n return () => {\n void transcriber?.close();\n };\n }, [transcriber]);\n\n return transcriber;\n};\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { lazy } from '@dxos/app-framework';\n\nexport const AppGraphBuilder = lazy(() => import('./app-graph-builder'));\nexport const IntentResolver = lazy(() => import('./intent-resolver'));\nexport const MeetingTranscriptionState = lazy(() => import('./meeting-transcription-state'));\nexport const ReactSurface = lazy(() => import('./react-surface'));\nexport const Transcriber = lazy(() => import('./transcriber'));\nexport const Settings = lazy(() => import('./settings'));\n\nexport * from './capabilities';\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { useState, useEffect, useCallback } from 'react';\n\nimport { scheduleMicroTask } from '@dxos/async';\nimport { Context } from '@dxos/context';\nimport { log } from '@dxos/log';\nimport { useSoundEffect } from '@dxos/react-ui-sfx';\n\nimport { useAudioTrack } from './useAudioTrack';\nimport { useTranscriber } from './useTranscriber';\nimport { type TranscriberParams } from '../transcriber';\n\nexport type UseVoiceInputProps = {\n active?: boolean;\n onUpdate: (text: string) => void;\n};\n\n/**\n * Hook for voice input.\n */\nexport const useVoiceInput = ({ active, onUpdate }: UseVoiceInputProps) => {\n const soundStart = useSoundEffect('StartRecording');\n const soundStop = useSoundEffect('StopRecording');\n\n const handleSegments = useCallback<TranscriberParams['onSegments']>(async (segments) => {\n const text = segments.map((str) => str.text.trim().replace(/[^\\w\\s]/g, '')).join(' ');\n onUpdate(text);\n }, []);\n\n // Audio/transcription.\n const [recording, setRecording] = useState(false);\n const track = useAudioTrack(active);\n const transcriber = useTranscriber({\n audioStreamTrack: track,\n onSegments: handleSegments,\n });\n\n // Start/stop transcription.\n useEffect(() => {\n const ctx = new Context();\n scheduleMicroTask(ctx, async () => {\n if (active && transcriber) {\n await transcriber.open();\n log.info('starting...');\n setRecording(true);\n void soundStart.play();\n transcriber.startChunksRecording();\n } else if (!active && transcriber?.isOpen) {\n transcriber?.stopChunksRecording();\n await transcriber?.close();\n log.info('stopped');\n setRecording(false);\n void soundStop.play();\n }\n });\n\n return () => {\n void ctx.dispose();\n };\n }, [active, transcriber]);\n\n return { recording };\n};\n"],
|
|
5
|
+
"mappings": ";;;;;AAIA,OAAOA,YAAwB;AAE/B,SAASC,kBAAkBC,gBAAgB;AAC3C,SAASC,YAAYC,gBAAgB;AACrC,SAASC,iBAAiB;;;ACJ1B,OAAOC,WAAW;AAIlB,SAASC,YAAkCC,uBAAuB;AAClE,SACEC,uBACAC,0BACAC,gBACAC,uBACAC,kBACAC,aACAC,SACAC,qBACK;AACP,SAASC,UAAU;AAEnB,SAASC,kBAAkB;;;ACjB3B,SAAyBC,uBAAuB;AAChD,SAASC,YAAYC,cAAcC,YAA6BC,cAAc;AAC9E,SAASC,cAAc;AACvB,SAASC,0BAA0B;AAEnC,SAAyBC,kBAAkBC,eAAe;;;ACL1D,SAASC,YAAY;AAGrB,SAASC,aAAa;AACtB,SAASC,iBAAiB;AAC1B,SAASC,WAAW;;AAoBb,IAAMC,kBAAN,MAAMA;EACXC,YAA6BC,OAAmB;SAAnBA,QAAAA;EAAoB;;EAGjDC,YAAoB;AAClB,WAAO,KAAKD,MAAME,MAAMC,IAAIC,QAAQ;EACtC;EAEAC,aAAaC,MAAcC,QAAgBC,QAAwB;AACjEX,QAAI,gBAAgB;MAAEY,OAAO,KAAKT,MAAME,MAAMC,IAAIC;MAAOE;MAAMC;MAAQC;IAAO,GAAA;;;;;;AAE9E,UAAME,WAAW,KAAKV,MAAME,MAAMC,IAAIC;AACtC,UAAMO,UAAU,KAAKX,MAAME,MAAMC,IAAIS,KAAKN,IAAAA,EAAMA;AAChD,UAAMO,QAAQN,WAAW,IAAII,UAAU,KAAKX,MAAME,MAAMC,IAAIS,KAAKE,KAAKC,IAAIL,UAAUJ,OAAOC,MAAAA,CAAAA,EAASD;AAGpG,QAAIU,OAAOR,OAAOS,KAAK,IAAA;AACvB,QAAIJ,UAAU,KAAKb,MAAME,MAAMC,IAAIe,QAAQ;AACzCF,cAAQ;IACV;AAEA,SAAKhB,MAAMmB,SAAS;MAClBC,SAAS;QAAEd,MAAMK;QAASU,IAAIR;QAAOL,QAAQQ;MAAK;IACpD,CAAA;EACF;AACF;AA2BO,IAAMM,qBAAN,MAAMA;EAgBXvB,YACmBwB,WACjBC,gBAAqB,CAAA,GACrB;SAFiBD,YAAAA;SAfFE,UAAe,CAAA;SAGfC,mBAAmB,oBAAIC,IAAAA;SAGvBC,eAAe,oBAAID,IAAAA;SAGnBE,WAAkC,CAAA;SAGnCC,SAAS,IAAInC,MAAAA;AAM3B6B,kBAAcO,QAAQ,CAACC,UAAAA;AACrB,WAAKC,YAAYD,KAAAA;IACnB,CAAA;EACF;EAEAE,SAAS;AACP,WAAO;MACLC,QAAQ,KAAKV,QAAQP;MACrBE,SAAS,KAAKS,SAASX;IACzB;EACF;EAEA,IAAIf,MAAM;AACR,WAAOT,KAAK0C,GAAG;SAAI,KAAKX,QAAQY,QAAQ,CAACL,OAAOM,UAAU,KAAKf,UAAUS,OAAOM,KAAAA,CAAAA;MAAS;KAAG;EAC9F;EAEA,IAAIH,SAAS;AACX,WAAO,KAAKV;EACd;;;;EAKAc,eAAe3B,MAA6B;AAC1C,WAAO,KAAKgB,aAAaY,IAAI5B,IAAAA;EAC/B;EAEA6B,QAAc;AACZ,SAAKhB,QAAQP,SAAS;AACtB,SAAKQ,iBAAiBgB,MAAK;AAC3B,SAAKd,aAAac,MAAK;AACvB,SAAKZ,OAAOa,KAAI;AAChB,WAAO;EACT;EAEAV,YAAYD,OAAgB;AAC1B,UAAMM,QAAQ,KAAKb,QAAQP;AAC3B,SAAKO,QAAQmB,KAAKZ,KAAAA;AAClB,SAAKH,SAASe,KAAK;MAAEC,MAAM;MAAUP;MAAON;IAAM,CAAA;AAClD,SAAKF,OAAOa,KAAI;AAChB,WAAO;EACT;EAEAG,YAAYd,OAAgB;AAC1B,UAAMM,QAAQ,KAAKb,QAAQsB,UAAU,CAACC,MAAMA,EAAEC,OAAOjB,MAAMiB,EAAE;AAC7DrD,cAAU0C,UAAU,IAAC,QAAA;;;;;;;;;AACrB,SAAKb,QAAQa,KAAAA,IAASN;AACtB,SAAKH,SAASe,KAAK;MAAEC,MAAM;MAAUP;MAAON;IAAM,CAAA;AAClD,SAAKF,OAAOa,KAAI;AAChB,WAAO;EACT;EAEAO,YAAYD,IAAkB;AAC5B,UAAMX,QAAQ,KAAKb,QAAQsB,UAAU,CAACC,MAAMA,EAAEC,OAAOA,EAAAA;AACrDrD,cAAU0C,UAAU,IAAC,QAAA;;;;;;;;;AACrB,SAAKb,QAAQ0B,OAAOb,OAAO,CAAA;AAC3B,SAAKT,SAASe,KAAK;MAAEC,MAAM;MAAUP;MAAOW;IAAG,CAAA;AAC/C,SAAKnB,OAAOa,KAAI;AAChB,WAAO;EACT;;;;EAKAS,KAAKC,WAA+B;AAClCxD,QAAI,QAAQ;MAAEuB,SAAS,KAAKS;IAAS,GAAA;;;;;;AAGrC,eAAWyB,UAAU,KAAKzB,UAAU;AAClC,cAAQyB,OAAOT,MAAI;QACjB,KAAK,UAAU;AAEb,gBAAMzC,QAAQ,KAAKmB,UAAU+B,OAAOtB,OAAOsB,OAAOhB,KAAK;AACvD,gBAAMiB,WAAWF,UAASpD,UAAS;AACnCoD,UAAAA,UAAShD,aAAakD,WAAW,GAAG,GAAGnD,KAAAA;AACvC;QACF;QAEA,KAAK,UAAU;AAEb,cAAIoD,YAAY;AAChB,mBAASC,IAAI,GAAGA,IAAIH,OAAOhB,OAAOmB,KAAK;AACrCD,yBAAa,KAAK9B,iBAAiBc,IAAI,KAAKf,QAAQgC,CAAAA,EAAGR,EAAE;UAC3D;AACA,gBAAMS,oBAAoB,KAAKhC,iBAAiBc,IAAIc,OAAOtB,MAAMiB,EAAE;AACnE,gBAAMU,WAAW,KAAKpC,UAAU+B,OAAOtB,OAAOsB,OAAOhB,KAAK;AAC1De,UAAAA,UAAShD,aAAamD,WAAWE,mBAAmBC,QAAAA;AACpD;QACF;QAEA,KAAK,UAAU;AAEb,cAAIH,YAAY;AAChB,mBAASC,IAAI,GAAGA,IAAIH,OAAOhB,OAAOmB,KAAK;AACrCD,yBAAa,KAAK9B,iBAAiBc,IAAI,KAAKf,QAAQgC,CAAAA,EAAGR,EAAE;UAC3D;AACA,gBAAMS,oBAAoB,KAAKhC,iBAAiBc,IAAIc,OAAOL,EAAE;AAC7DI,UAAAA,UAAShD,aAAamD,WAAWE,mBAAmB,CAAA,CAAE;AACtD;QACF;MACF;IACF;AAGA,SAAK9B,aAAac,MAAK;AAGvB,QAAIkB,cAAc;AAClB,eAAW,CAACtB,OAAON,KAAAA,KAAU,KAAKP,QAAQoC,QAAO,GAAI;AACnD,YAAM5D,YAAY,KAAKsB,UAAUS,OAAOM,KAAAA,EAAOpB;AAC/C,WAAKQ,iBAAiBoC,IAAI9B,MAAMiB,IAAIhD,SAAAA;AACpC,WAAK2B,aAAakC,IAAIF,aAAa5B,KAAAA;AACnC4B,qBAAe3D;IACjB;AAGA,UAAM8D,WAAW,IAAIC,IAAI,KAAKvC,QAAQwC,IAAI,CAACjC,UAAUA,MAAMiB,EAAE,CAAA;AAC7D,eAAWA,MAAM,KAAKvB,iBAAiBwC,KAAI,GAAI;AAC7C,UAAI,CAACH,SAASI,IAAIlB,EAAAA,GAAK;AACrB,aAAKvB,iBAAiB0C,OAAOnB,EAAAA;MAC/B;IACF;AAGA,SAAKpB,SAASX,SAAS;AACvB,WAAO;EACT;AACF;;;ADzMO,IAAMmD,aAAa,CAAC,EAAEC,OAAOC,SAASC,aAAY,MAAqB;AAC5E,SAAO;;IAELC,OAAO;MACLC,OAAO;MACPC,kBAAkB,CAACC,WAAWA,OAAOC,cAAcD,OAAOE;MAC1DC,SAAS,CAACC,SAAAA;AACR,cAAMC,QAAQC,aAAaX,SAASD,MAAMa,OAAO,CAAA,CAAE;AACnD,cAAMC,UAAU,IAAIC,gBAAAA;AACpB,mBAAW,EAAEC,MAAMC,GAAE,KAAMP,KAAKQ,eAAe;AAC7C,cAAIC,OAAOT,KAAKU,MAAMC,IAAIC,OAAON,IAAAA;AACjC,iBAAOG,KAAKH,QAAQC,IAAI;AACtB,kBAAMM,QAAQvB,MAAMwB,eAAeL,KAAKM,MAAM,GAAGC,OAAO,CAAA;AACxD,kBAAMC,YAAYJ,OAAOK,SAAS,mBAAmBL,MAAMtB;AAC3D,gBAAI0B,WAAW;AACbb,sBAAQe,IAAIV,KAAKH,MAAMG,KAAKH,MAAM,IAAIc,gBAAgBX,KAAKM,QAAQ,IAAIM,KAAKJ,SAAAA,GAAYhB,KAAAA,CAAAA;YAC1F;AAEA,gBAAIQ,KAAKF,KAAK,IAAIP,KAAKU,MAAMC,IAAIW,QAAQ;AACvC;YACF;AAEAb,mBAAOT,KAAKU,MAAMC,IAAIC,OAAOH,KAAKF,KAAK,CAAA;UACzC;QACF;AAEA,eAAOH,QAAQmB,OAAM;MACvB;IACF,CAAA;;IAGAC,WAAWC,UACT,MAAA;MAMEC,YAAY1B,MAAkB;AAFtB2B,4BAAe;AAGrB,aAAKC,WAAW,IAAIC,gBAAgB7B,IAAAA;AAEpC,cAAM8B,WAAW9B,KAAK+B;AACtB,YAAIC,kBAAkB;AACtB,YAAIC,cAAc;AAElB,YAAIC;AACJ,cAAMC,iBAAiB,CAACC,SAAS,UAAK;AACpCN,mBAASO,MAAMC,iBAAiBF,SAAS,WAAW;AAGpDN,mBAASS,UAAUpB,IAAI,mBAAA;AACvBa,4BAAkB;AAClBQ,uBAAaN,OAAAA;AACbA,oBAAUO,WAAW,MAAA;AACnB,iBAAKC,WAAWH,UAAUpB,IAAI,WAAA;AAC9BW,qBAASS,UAAUI,OAAO,mBAAA;AAC1BX,8BAAkB;UACpB,GAAG,GAAA;AAGHhC,eAAK4C,SAAS;YACZC,SAASC,WAAWC,eAAe/C,KAAKU,MAAMC,IAAIW,QAAQ;cAAE0B,GAAG;YAAM,CAAA;UACvE,CAAA;QACF;AAGA,YAAIxD,cAAc;AAChB,eAAKkD,YAAYO,SAASC,cAAc,KAAA;AACxC,eAAKR,UAAUH,UAAUpB,IAAI,eAAe,sBAAsB,gBAAgB,WAAA;AAClFnB,eAAKmD,IAAIC,YAAY,KAAKV,SAAS;AACnClD,uBACE,KAAKkD,WACL;YACEW,SAAS,MAAA;AACPlB,6BAAe,KAAA;YACjB;UACF,GACAnC,IAAAA;QAEJ;AAGA,aAAKsD,WAAWC,QACdC,iBAAiBxD,KAAK+B,WAAW,UAAU,MAAA;AACzC,cAAI,CAACC,iBAAiB;AACpBC,0BAAc;AACd,iBAAKS,WAAWH,UAAUI,OAAO,WAAA;UACnC;QACF,CAAA,GAEArD,MAAMM,OAAO6D,GAAG,MAAA;AAEd,gBAAMC,aACJ5B,SAAS6B,eAAe7B,SAAS8B,YAAY9B,SAAS+B,iBAAiB,KAAK,CAAC5B;AAG/E3C,gBAAMwE,KAAK,KAAKlC,QAAQ;AAGxB,cAAI8B,YAAY;AACdvB,2BAAe,IAAA;UACjB;QACF,CAAA,CAAA;MAEJ;MAEAvC,OAAOA,QAAoB;AAEzB,YAAI,CAAC,KAAK+B,cAAc;AACtB,eAAKA,eAAe;AACpBc,qBAAW,MAAA;AACTnD,kBAAMwE,KAAK,KAAKlC,QAAQ;UAC1B,CAAA;QACF;MACF;MAEAmC,UAAU;AACR,aAAKrB,WAAWC,OAAAA;AAChB,aAAKW,SAAQ;MACf;IACF,CAAA;IAGFR,WAAWkB,MAAM;MACf,gBAAgB;QACdC,WAAW;MACb;MACA,sBAAsB;QACpBC,gBAAgB;QAChB,sBAAsB;MACxB;MACA,yCAAyC;QACvCC,SAAS;MACX;MACA,gBAAgB;QACdC,UAAU;QACVC,QAAQ;QACRC,OAAO;QACPC,QAAQ;MACV;MACA,YAAY;QACVC,cAAc;MAChB;MACA,wBAAwB;QACtBC,OAAO;QACPD,cAAc;MAChB;MACA,kDAAkD;QAChDL,SAAS;QACTO,WAAW;QACXC,SAAS;MACX;IACF,CAAA;;AAEJ;AAKA,IAAMvD,kBAAN,cAA8BwD,aAAAA;EAC5BlD,YACWmD,OACAC,YACAC,UACT;AACA,UAAK;SAJIF,QAAAA;SACAC,aAAAA;SACAC,WAAAA;EAGX;EAESC,GAAGC,OAAa;AACvB,WAAOA,MAAMH,eAAe,KAAKA;EACnC;EAESI,MAAMlF,MAAkB;AAC/B,UAAMmF,KAAKlC,SAASC,cAAc,KAAA;AAClCiC,OAAGC,YAAY;AACfD,OAAGE,cAAcC,gBAAgB,KAAKR,YAAY,KAAKC,QAAQ;AAE/DI,OAAGI,UAAU,MAAA;AACX,YAAMC,MAAMxF,KAAKU,MAAMC,IAAIF,KAAK,KAAKoE,KAAK,EAAEvE;AAC5CN,WAAK4C,SAAS;QACZC,SAASC,WAAWC,eAAeyC,KAAK;UAAExC,GAAG;QAAQ,CAAA;MACvD,CAAA;IACF;AAEA,WAAOmC;EACT;AACF;AAEA,IAAMjF,eAAe,CAACX,SAAgBkG,YAAAA;AACpC,MAAIlG,SAAS;AACX,WAAOA;EACT;AAEA,MAAIkG,SAASzE,OAAO,CAAA,GAAIE,SAAS,mBAAmBuE,QAAQzE,OAAO,CAAA,EAAGzB,SAAS;AAC7E,WAAO,IAAI8B,KAAKoE,QAAQzE,OAAO,CAAA,EAAGzB,OAAO;EAC3C;AAEA,SAAOmG;AACT;AAEA,IAAMJ,kBAAkB,CAACrE,WAAiB0E,aAAAA;AACxC,MAAIA,UAAU;AACZ,UAAMC,MAAM,CAACC,IAAI,MAAMC,OAAOD,CAAAA,EAAGE,SAAS,GAAG,GAAA;AAC7C,UAAM,EAAEC,OAAOC,SAASC,QAAO,IAAKC,mBAAmB;MAAElG,OAAO0F;MAAUS,KAAKnF;IAAU,CAAA;AACzF,WAAO,GAAG2E,IAAII,KAAAA,CAAAA,IAAUJ,IAAIK,OAAAA,CAAAA,IAAYL,IAAIM,OAAAA,CAAAA;EAC9C,OAAO;AACL,WAAOG,OAAOpF,WAAW,UAAA;EAC3B;AACF;;;ADhNO,IAAMqF,iBACX,CAACC,eACD,CAACC,SAAsBC,OAAeC,QAAQ,UAAK;AACjD,MAAIF,QAAQG,OAAOC,SAAS,aAAa;AAEvC,WAAO;MAACJ,QAAQK,OAAOC,KAAK,CAACC,UAAUA,MAAMC,SAAS,eAAA,GAAkBC,QAAQ;MAAI;;EACtF;AAIA,QAAMC,WAAWX,WAAWO,KAAK,CAACI,cAAaA,UAASC,QAAQX,QAAQG,OAAOS,WAAW;AAC1F,QAAMC,OACJH,UAAUI,SAASC,eACnBf,QAAQG,OAAOa,SAASC,QAAQC,YAChClB,QAAQG,OAAOU,QACfb,QAAQG,OAAOgB,SACfnB,QAAQG,OAAOS;AACjB,QAAMP,SAASL,QAAQK,OAAOe,OAAO,CAACb,UAAUA,MAAMC,SAAS,eAAA;AAC/D,SAAO;IACL,UAAUK,IAAAA,MAAUX,QAAQ,KAAKD,KAAAA,KAAUD,QAAQqB,EAAE,KAAK;IAC1DhB,OAAOiB,IAAI,CAACf,UAAUA,MAAME,KAAKc,KAAI,CAAA,EAAIC,KAAK,GAAA;IAC9C;;AAEJ;AAcK,IAAMC,aAAa,CAAC,EACzBC,YACAC,YAAYC,QACZC,OACAC,OACAC,cACAC,gBAAe,MACC;AAChB,QAAM,EAAEC,UAAS,IAAKC,gBAAAA;AACtB,QAAM,EAAEC,UAAS,IAAKC,cAAc,MAAA;AAClC,WAAO;MACLC,YAAY;QACVC,sBAAsB;UAAEC,UAAU;UAAMC,cAAc;QAAK,CAAA;QAC3DC,yBAAyB;UAAER;QAAU,CAAA;QACrCS,sBAAsB;UAAET;QAAU,CAAA;QAClCU,iBAAAA;QACAd,SAASe,QAAAA;QACTjB,WAAW;UACTG;UACAe,SAASjB,QAAQiB,UAAU,IAAIC,KAAKlB,OAAOiB,OAAO,IAAIE;UACtDC,cAAcC,eAAe,CAAC,EAAEC,QAAO,MACrC,sBAAA,cAACC,YAAAA;YAAWC,MAAK;YAA+BC,UAAAA;YAASC,OAAM;YAAmBJ;;QAEtF,CAAA;QACA9B,OAAOmC,UAAAA;IACX;EACF,GAAG;IAAC1B;IAAOC;GAAM;AAEjB,SACE,sBAAA,cAAC0B,OAAAA;IACCC,KAAKtB;IACLuB,WAAWC,GAAG,6BAA6BC,aAAalC,UAAAA;IACxDmC,mCAAiC;;AAGvC;;;AG/FA,SAASC,WAAWC,gBAAgB;AAEpC,SAASC,OAAAA,YAAW;;AAQb,IAAMC,eAAe,CAACC,aAAAA;AAC3B,QAAM,CAAC,EAAEC,OAAOC,QAAQC,MAAK,GAAIC,SAAAA,IAAaP,SAAwB,CAAC,CAAA;AACvED,YAAU,MAAA;AACR,UAAMS,IAAIC,WAAW,YAAA;AACnB,YAAMC,WAAW,MAAMC,MAAMR,QAAAA;AAC7B,YAAMS,OAAO,MAAMF,SAASE,KAAI;AAChC,YAAMR,SAAQ,IAAIS,MAAAA;AAClBT,MAAAA,OAAMU,MAAMC,IAAIC,gBAAgBJ,IAAAA;AAChC,YAAM,IAAIK,QAAc,CAACC,SAASC,WAAAA;AAChCf,QAAAA,OAAMgB,iBACJ,SACA,CAACC,QAAAA;AACCpB,UAAAA,KAAIqB,MAAM,SAAS;YAAED;UAAI,GAAA;;;;;;AACzBF,iBAAOE,GAAAA;QACT,GACA;UAAEE,MAAM;QAAK,CAAA;AAEfnB,QAAAA,OAAMgB,iBACJ,WACA,MAAA;AACEnB,UAAAA,KAAIuB,KAAK,eAAA,QAAA;;;;;;AACTN,kBAAAA;QACF,GACA;UAAEK,MAAM;QAAK,CAAA;AAEfnB,QAAAA,OAAMqB,KAAI;MACZ,CAAA;AAEA,YAAMC,MAAM,IAAIC,aAAAA;AAEhB,YAAMC,cAAcF,IAAIG,6BAA4B;AACpDD,kBAAYE,eAAe;AAE3B,YAAMC,SAASL,IAAIM,yBAAyB5B,MAAAA;AAC5C2B,aAAOE,QAAQL,WAAAA;AAEfrB,gBAAU;QACRH,OAAAA;QACAC,QAAQuB,YAAYvB;QACpBC,OAAOsB,YAAYvB,OAAO6B,eAAc,EAAG,CAAA;MAC7C,CAAA;IACF,CAAA;AAEA,WAAO,MAAMC,aAAa3B,CAAAA;EAC5B,GAAG;IAACL;GAAS;AAEb,SAAO;IAAEC;IAAOC;IAAQC;EAAM;AAChC;;;ACzDA,SAAS8B,aAAAA,YAAWC,YAAAA,iBAAgB;AAM7B,IAAMC,gBAAgB,CAACC,WAAAA;AAC5B,QAAM,CAACC,OAAOC,QAAAA,IAAYC,UAAAA;AAC1BC,EAAAA,WAAU,MAAA;AACR,QAAI,CAACJ,QAAQ;AACXC,aAAOI,KAAAA;AACPH,eAASI,MAAAA;AACT;IACF;AAEA,UAAMC,YAAY,YAAA;AAChB,YAAMC,QAAQ,IAAIC,MAAAA;AAClBD,YAAME,YAAY,MAAMC,UAAUC,aAAaC,aAAa;QAAEL,OAAO;MAAK,CAAA;AAC1E,YAAM,CAACP,MAAAA,IAASO,MAAME,UAAUI,eAAc;AAC9C,UAAIb,QAAO;AACTC,iBAASD,MAAAA;MACX;IACF;AAEA,QAAID,QAAQ;AACV,WAAKO,UAAAA;IACP;EACF,GAAG;IAACP;GAAO;AAEX,SAAOC;AACT;;;AC9BA,SAASc,SAASC,aAAAA,YAAWC,YAAAA,iBAAgB;AAStC,IAAMC,uBAAuB,CAClCC,UACAC,OACAC,gBAAqB,CAAA,MAAE;AAEvB,QAAMC,QAAQC,QAAQ,MAAM,IAAIC,mBAAsBL,UAAUE,aAAAA,GAAgB;IAACD;GAAM;AACvF,QAAM,CAACK,QAAQC,SAAAA,IAAaC,UAAS,KAAA;AAGrCC,EAAAA,WAAU,MAAA;AACR,QAAI,CAACR,OAAO;AACV;IACF;AAEA,UAAMS,SAAS,MAAA;AACb,iBAAWC,SAASV,OAAOW,SAAS,CAAA,GAAI;AACtCT,cAAMU,YAAYF,KAAAA;MACpB;AAEAJ,gBAAU,IAAA;IACZ;AAGA,QAAIO;AACJ,QAAIb,MAAMc,WAAW;AACnBD,UAAIE,YAAY,MAAA;AACd,YAAI,CAACf,OAAOc,WAAW;AACrBE,wBAAcH,CAAAA;AACdJ,iBAAAA;QACF;MACF,GAAG,GAAA;IACL,OAAO;AACLA,aAAAA;IACF;AAEA,WAAO,MAAMQ,aAAaJ,CAAAA;EAC5B,GAAG;IAACX;IAAOF;GAAM;AAGjBQ,EAAAA,WAAU,MAAA;AACR,QAAI,CAACH,UAAU,CAACL,SAASE,MAAMgB,OAAOC,WAAWnB,MAAMW,MAAMQ,QAAQ;AACnE;IACF;AAEA,UAAMC,QAAQpB,MAAMW,MAAMX,MAAMW,MAAMQ,SAAS,CAAA;AAC/CjB,UAAMU,YAAYQ,KAAAA;EACpB,GAAG;IAAClB;IAAOG;IAAQL,OAAOW,MAAMQ;GAAO;AAEvC,SAAOjB;AACT;;;AC1DA,SAASmB,aAAAA,YAAWC,WAAAA,gBAAe;AAEnC,SAASC,uBAAuB;;;ACFhC,SAASC,YAAY;AAEd,IAAMC,kBAAkBC,KAAK,MAAM,OAAO,kCAAA,CAAA;AAC1C,IAAMC,iBAAiBD,KAAK,MAAM,OAAO,gCAAA,CAAA;AACzC,IAAME,4BAA4BF,KAAK,MAAM,OAAO,4CAAA,CAAA;AACpD,IAAMG,eAAeH,KAAK,MAAM,OAAO,8BAAA,CAAA;AACvC,IAAMI,cAAcJ,KAAK,MAAM,OAAO,4BAAA,CAAA;AACtC,IAAMK,WAAWL,KAAK,MAAM,OAAO,yBAAA,CAAA;;;ADGnC,IAAMM,iBAAiB,CAAC,EAC7BC,kBACAC,YACAC,OAAM,MACiD;AACvD,QAAM,CAACC,cAAAA,IAAkBC,gBAAgBC,0BAA0BC,WAAW;AAG9E,QAAMC,cAAcC,SAAiC,MAAA;AACnD,QAAI,CAACP,cAAc,CAACD,oBAAoB,CAACG,gBAAgB;AACvD,aAAOM;IACT;AAEA,WAAON,eAAe;MAAEH;MAAkBC;MAAYC;IAAO,CAAA;EAC/D,GAAG;IAACF;IAAkBC;IAAYE;IAAgBD;GAAO;AAEzDQ,EAAAA,WAAU,MAAA;AACR,WAAO,MAAA;AACL,WAAKH,aAAaI,MAAAA;IACpB;EACF,GAAG;IAACJ;GAAY;AAEhB,SAAOA;AACT;;;AEjCA,SAASK,YAAAA,WAAUC,aAAAA,YAAWC,mBAAmB;AAEjD,SAASC,yBAAyB;AAClC,SAASC,eAAe;AACxB,SAASC,OAAAA,YAAW;AACpB,SAASC,sBAAsB;;AAcxB,IAAMC,gBAAgB,CAAC,EAAEC,QAAQC,SAAQ,MAAsB;AACpE,QAAMC,aAAaC,eAAe,gBAAA;AAClC,QAAMC,YAAYD,eAAe,eAAA;AAEjC,QAAME,iBAAiBC,YAA6C,OAAOC,aAAAA;AACzE,UAAMC,OAAOD,SAASE,IAAI,CAACC,QAAQA,IAAIF,KAAKG,KAAI,EAAGC,QAAQ,YAAY,EAAA,CAAA,EAAKC,KAAK,GAAA;AACjFZ,aAASO,IAAAA;EACX,GAAG,CAAA,CAAE;AAGL,QAAM,CAACM,WAAWC,YAAAA,IAAgBC,UAAS,KAAA;AAC3C,QAAMC,QAAQC,cAAclB,MAAAA;AAC5B,QAAMmB,cAAcC,eAAe;IACjCC,kBAAkBJ;IAClBK,YAAYjB;EACd,CAAA;AAGAkB,EAAAA,WAAU,MAAA;AACR,UAAMC,MAAM,IAAIC,QAAAA,QAAAA;;;;AAChBC,sBAAkBF,KAAK,YAAA;AACrB,UAAIxB,UAAUmB,aAAa;AACzB,cAAMA,YAAYQ,KAAI;AACtBC,QAAAA,KAAIC,KAAK,eAAA,QAAA;;;;;;AACTd,qBAAa,IAAA;AACb,aAAKb,WAAW4B,KAAI;AACpBX,oBAAYY,qBAAoB;MAClC,WAAW,CAAC/B,UAAUmB,aAAaa,QAAQ;AACzCb,qBAAac,oBAAAA;AACb,cAAMd,aAAae,MAAAA;AACnBN,QAAAA,KAAIC,KAAK,WAAA,QAAA;;;;;;AACTd,qBAAa,KAAA;AACb,aAAKX,UAAU0B,KAAI;MACrB;IACF,CAAA;AAEA,WAAO,MAAA;AACL,WAAKN,IAAIW,QAAO;IAClB;EACF,GAAG;IAACnC;IAAQmB;GAAY;AAExB,SAAO;IAAEL;EAAU;AACrB;;;ATlDO,IAAMsB,yBAA2E,CAAC,EAAEC,MAAMC,YAAAA,YAAU,MAAE;AAC3G,QAAMC,eAAeC,iBAAiBF,WAAAA;AACtC,QAAMG,QAAQC,SAASJ,WAAAA;AACvB,QAAMK,UAAUC,WAAWH,OAAOI,GAAAA,EAAKC,IAAI,CAACC,WAAWA,OAAOC,QAAQ;AACtE,QAAMC,QAAQC,SAAsBZ,YAAWW,MAAME,KAAK;IAAEC,cAAc;EAAM,CAAA;AAChF,QAAMC,QAAQC,qBAAqBC,eAAeZ,OAAAA,GAAUM,KAAAA;AAE5D,SACE,gBAAAO,OAAA,cAACC,UAAUC,SAAO;IAACrB;IAAYsB,YAAW;KACxC,gBAAAH,OAAA,cAACI,YAAAA;IAAWrB;IAA4BE;IAAcY;IAAcf,YAAYA;;AAGtF;AAEA,IAAA,8BAAeF;",
|
|
6
|
+
"names": ["React", "fullyQualifiedId", "getSpace", "useMembers", "useQueue", "StackItem", "React", "IconButton", "useThemeContext", "createBasicExtensions", "createMarkdownExtensions", "createRenderer", "createThemeExtensions", "decorateMarkdown", "editorWidth", "preview", "useTextEditor", "mx", "isNotFalsy", "RangeSetBuilder", "EditorView", "GutterMarker", "ViewPlugin", "gutter", "format", "intervalToDuration", "addEventListener", "combine", "Text", "Event", "invariant", "log", "DocumentAdapter", "constructor", "_view", "lineCount", "state", "doc", "lines", "replaceLines", "from", "remove", "insert", "count", "numLines", "posFrom", "line", "posTo", "Math", "min", "text", "join", "length", "dispatch", "changes", "to", "SerializationModel", "_renderer", "initialChunks", "_chunks", "_chunkLineCounts", "Map", "_lineToChunk", "_changes", "update", "forEach", "chunk", "appendChunk", "toJSON", "chunks", "of", "flatMap", "index", "getChunkAtLine", "get", "reset", "clear", "emit", "push", "type", "updateChunk", "findIndex", "c", "id", "deleteBlock", "splice", "sync", "document", "change", "lastLine", "lineStart", "i", "previousLineCount", "newLines", "currentLine", "entries", "set", "chunkIds", "Set", "map", "keys", "has", "delete", "transcript", "model", "started", "renderButton", "gutter", "class", "lineMarkerChange", "update", "docChanged", "viewportChanged", "markers", "view", "start", "getStartTime", "chunks", "builder", "RangeSetBuilder", "from", "to", "visibleRanges", "line", "state", "doc", "lineAt", "block", "getChunkAtLine", "number", "blocks", "timestamp", "type", "add", "TimestampMarker", "Date", "length", "finish", "ViewPlugin", "fromClass", "constructor", "_initialized", "_adapter", "DocumentAdapter", "scroller", "scrollDOM", "isAutoScrolling", "hasScrolled", "timeout", "scrollToBottom", "smooth", "style", "scrollBehavior", "classList", "clearTimeout", "setTimeout", "_controls", "remove", "dispatch", "effects", "EditorView", "scrollIntoView", "y", "document", "createElement", "dom", "appendChild", "onClick", "_cleanup", "combine", "addEventListener", "on", "autoScroll", "scrollHeight", "scrollTop", "clientHeight", "sync", "destroy", "theme", "overflowY", "scrollbarWidth", "display", "position", "bottom", "right", "zIndex", "paddingRight", "width", "textAlign", "padding", "GutterMarker", "_line", "_timestamp", "_started", "eq", "other", "toDOM", "el", "className", "textContent", "formatTimestamp", "onclick", "pos", "message", "undefined", "relative", "pad", "n", "String", "padStart", "hours", "minutes", "seconds", "intervalToDuration", "end", "format", "renderMarkdown", "identities", "message", "index", "debug", "sender", "role", "blocks", "find", "block", "type", "text", "identity", "did", "identityDid", "name", "profile", "displayName", "contact", "target", "fullName", "email", "filter", "id", "map", "trim", "join", "Transcript", "classNames", "transcript", "object", "space", "model", "attendableId", "ignoreAttention", "themeMode", "useThemeContext", "parentRef", "useTextEditor", "extensions", "createBasicExtensions", "readOnly", "lineWrapping", "createMarkdownExtensions", "createThemeExtensions", "decorateMarkdown", "preview", "started", "Date", "undefined", "renderButton", "createRenderer", "onClick", "IconButton", "icon", "iconOnly", "label", "isNotFalsy", "div", "ref", "className", "mx", "editorWidth", "data-popover-collision-boundary", "useEffect", "useState", "log", "useAudioFile", "audioUrl", "audio", "stream", "track", "setStream", "t", "setTimeout", "response", "fetch", "blob", "Audio", "src", "URL", "createObjectURL", "Promise", "resolve", "reject", "addEventListener", "err", "error", "once", "info", "load", "ctx", "AudioContext", "destination", "createMediaStreamDestination", "channelCount", "source", "createMediaElementSource", "connect", "getAudioTracks", "clearTimeout", "useEffect", "useState", "useAudioTrack", "active", "track", "setTrack", "useState", "useEffect", "stop", "undefined", "initAudio", "audio", "Audio", "srcObject", "navigator", "mediaDevices", "getUserMedia", "getAudioTracks", "useMemo", "useEffect", "useState", "useQueueModelAdapter", "renderer", "queue", "initialChunks", "model", "useMemo", "SerializationModel", "loaded", "setLoaded", "useState", "useEffect", "update", "block", "items", "appendChunk", "i", "isLoading", "setInterval", "clearInterval", "clearTimeout", "chunks", "length", "chunk", "useEffect", "useMemo", "useCapabilities", "lazy", "AppGraphBuilder", "lazy", "IntentResolver", "MeetingTranscriptionState", "ReactSurface", "Transcriber", "Settings", "useTranscriber", "audioStreamTrack", "onSegments", "config", "getTranscriber", "useCapabilities", "TranscriptionCapabilities", "Transcriber", "transcriber", "useMemo", "undefined", "useEffect", "close", "useState", "useEffect", "useCallback", "scheduleMicroTask", "Context", "log", "useSoundEffect", "useVoiceInput", "active", "onUpdate", "soundStart", "useSoundEffect", "soundStop", "handleSegments", "useCallback", "segments", "text", "map", "str", "trim", "replace", "join", "recording", "setRecording", "useState", "track", "useAudioTrack", "transcriber", "useTranscriber", "audioStreamTrack", "onSegments", "useEffect", "ctx", "Context", "scheduleMicroTask", "open", "log", "info", "play", "startChunksRecording", "isOpen", "stopChunksRecording", "close", "dispose", "TranscriptionContainer", "role", "transcript", "attendableId", "fullyQualifiedId", "space", "getSpace", "members", "useMembers", "key", "map", "member", "identity", "queue", "useQueue", "dxn", "pollInterval", "model", "useQueueModelAdapter", "renderMarkdown", "React", "StackItem", "Content", "classNames", "Transcript"]
|
|
7
|
+
}
|
|
@@ -18,10 +18,10 @@ var TranscriptionSettings = ({ settings }) => {
|
|
|
18
18
|
|
|
19
19
|
// packages/plugins/plugin-transcription/src/components/index.ts
|
|
20
20
|
import { lazy } from "react";
|
|
21
|
-
var TranscriptContainer = lazy(() => import("./TranscriptContainer-
|
|
21
|
+
var TranscriptContainer = lazy(() => import("./TranscriptContainer-D3EH2SLO.mjs"));
|
|
22
22
|
|
|
23
23
|
export {
|
|
24
24
|
TranscriptionSettings,
|
|
25
25
|
TranscriptContainer
|
|
26
26
|
};
|
|
27
|
-
//# sourceMappingURL=chunk-
|
|
27
|
+
//# sourceMappingURL=chunk-IWZ7C3OA.mjs.map
|
|
@@ -3,7 +3,7 @@ import {
|
|
|
3
3
|
} from "./chunk-KLD7O5GJ.mjs";
|
|
4
4
|
import {
|
|
5
5
|
TRANSCRIPTION_URL
|
|
6
|
-
} from "./chunk-
|
|
6
|
+
} from "./chunk-YRWBY6UK.mjs";
|
|
7
7
|
|
|
8
8
|
// packages/plugins/plugin-transcription/src/transcriber/media-stream-recorder.ts
|
|
9
9
|
import { MediaRecorder, register } from "extendable-media-recorder";
|
|
@@ -314,7 +314,7 @@ import { Resource as Resource2 } from "@dxos/context";
|
|
|
314
314
|
import { QueueImpl } from "@dxos/echo-db";
|
|
315
315
|
import { create } from "@dxos/echo-schema";
|
|
316
316
|
import { log as log3 } from "@dxos/log";
|
|
317
|
-
import {
|
|
317
|
+
import { MessageType } from "@dxos/schema";
|
|
318
318
|
function _ts_decorate3(decorators, target, key, desc) {
|
|
319
319
|
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
320
320
|
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
@@ -397,7 +397,7 @@ var TranscriptionManager = class extends Resource2 {
|
|
|
397
397
|
await this._maybeReinitTranscriber();
|
|
398
398
|
if (this._enabled) {
|
|
399
399
|
await this._transcriber?.open();
|
|
400
|
-
const block = create(
|
|
400
|
+
const block = create(MessageType, {
|
|
401
401
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
402
402
|
blocks: [
|
|
403
403
|
{
|
|
@@ -415,7 +415,7 @@ var TranscriptionManager = class extends Resource2 {
|
|
|
415
415
|
]);
|
|
416
416
|
} else {
|
|
417
417
|
await this._transcriber?.close();
|
|
418
|
-
const block = create(
|
|
418
|
+
const block = create(MessageType, {
|
|
419
419
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
420
420
|
blocks: [
|
|
421
421
|
{
|
|
@@ -461,7 +461,7 @@ var TranscriptionManager = class extends Resource2 {
|
|
|
461
461
|
if (!this.isOpen || !this._queue) {
|
|
462
462
|
return;
|
|
463
463
|
}
|
|
464
|
-
let block = create(
|
|
464
|
+
let block = create(MessageType, {
|
|
465
465
|
created: (/* @__PURE__ */ new Date()).toISOString(),
|
|
466
466
|
blocks: segments,
|
|
467
467
|
sender: {
|
|
@@ -488,4 +488,4 @@ export {
|
|
|
488
488
|
Transcriber,
|
|
489
489
|
TranscriptionManager
|
|
490
490
|
};
|
|
491
|
-
//# sourceMappingURL=chunk-
|
|
491
|
+
//# sourceMappingURL=chunk-KSUYWWIU.mjs.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/transcriber/media-stream-recorder.ts", "../../../src/transcriber/transcriber.ts", "../../../src/transcriber/transcription-manager.ts"],
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport { type IBlobEvent, type IMediaRecorder, MediaRecorder, register } from 'extendable-media-recorder';\nimport { WaveFile } from 'wavefile';\n\nimport { synchronized } from '@dxos/async';\nimport { invariant } from '@dxos/invariant';\nimport { log } from '@dxos/log';\nimport { trace } from '@dxos/tracing';\n\nimport { type WavConfig, type AudioChunk, type AudioRecorder } from './audio-recorder';\n\nlet initializingPromise: Promise<void> | undefined;\n\nconst initializeExtendableMediaRecorder = async () => {\n const { connect } = await import('extendable-media-recorder-wav-encoder');\n await register(await connect());\n};\n\n/**\n * Recorder that uses the MediaContext API and AudioNode API to record audio.\n *\n * It records MediaStream using https://www.npmjs.com/package/extendable-media-recorder.\n */\n@trace.resource()\nexport class MediaStreamRecorder implements AudioRecorder {\n private readonly _mediaStreamTrack: MediaStreamTrack;\n /**\n * Default MediaRecorder implementation do not support wav encoding.\n */\n private readonly _interval: number;\n private _mediaRecorder?: IMediaRecorder = undefined;\n private _header?: Uint8Array = undefined;\n private _onChunk?: (chunk: AudioChunk) => void;\n\n constructor({ mediaStreamTrack, interval }: { mediaStreamTrack: MediaStreamTrack; interval: number }) {\n this._mediaStreamTrack = mediaStreamTrack;\n this._interval = interval;\n }\n\n get wavConfig(): WavConfig {\n const settings = this._mediaStreamTrack.getSettings();\n return {\n channels: settings.channelCount ?? 1,\n sampleRate: settings.sampleRate ?? 16000,\n bitDepthCode: settings.sampleSize ? String(settings.sampleSize) : '16',\n };\n }\n\n get mediaStreamTrack() {\n return this._mediaStreamTrack;\n }\n\n setOnChunk(onChunk: (chunk: AudioChunk) => void): void {\n this._onChunk = onChunk;\n }\n\n async start() {\n await (initializingPromise ??= initializeExtendableMediaRecorder()).catch((err) =>\n log.info('initializeExtendableMediaRecorder', { err }),\n );\n\n if (!this._mediaRecorder) {\n const stream = new MediaStream([this._mediaStreamTrack]);\n this._mediaRecorder = new MediaRecorder(stream, { mimeType: 'audio/wav' });\n }\n\n if (this._mediaRecorder.state === 'recording') {\n return;\n }\n invariant(this._onChunk, 'MediaStreamRecorder: onChunk is not set');\n this._mediaRecorder.ondataavailable = (event) => this._ondataavailable(event);\n this._mediaRecorder.start(this._interval);\n }\n\n async stop() {\n if (this._mediaRecorder?.state !== 'recording') {\n return;\n }\n this._mediaRecorder.stop();\n }\n\n @synchronized\n private async _ondataavailable(event: IBlobEvent) {\n const blob = event.data;\n const uint8Array = new Uint8Array(await blob.arrayBuffer());\n\n // First chunk from the MediaRecorder has a header.\n const isHeader =\n uint8Array[0] === 0x52 && // R\n uint8Array[1] === 0x49 && // I\n uint8Array[2] === 0x46 && // F\n uint8Array[3] === 0x46; // F\n let wav: WaveFile;\n\n if (isHeader) {\n wav = new WaveFile(uint8Array);\n this._header = uint8Array.slice(0, 44);\n } else if (this._header) {\n wav = new WaveFile(new Uint8Array([...this._header, ...uint8Array]));\n } else {\n log.warn('MediaStreamRecorder: no header');\n return;\n }\n\n this._onChunk!({\n timestamp: Date.now(),\n data: wav.getSamples(),\n });\n }\n}\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { WaveFile } from 'wavefile';\n\nimport { DeferredTask, Trigger } from '@dxos/async';\nimport { type Context, LifecycleState, Resource } from '@dxos/context';\nimport { log } from '@dxos/log';\nimport { type TranscriptionContentBlock } from '@dxos/schema';\nimport { trace } from '@dxos/tracing';\n\nimport { type AudioRecorder, type AudioChunk } from './audio-recorder';\nimport { TRANSCRIPTION_URL } from '../types';\nimport { mergeFloat64Arrays } from '../util';\n\ntype WhisperWord = {\n word: string;\n\n /**\n * Time in seconds from the start of the audio.\n */\n start: number;\n\n /**\n * Time in seconds from the start of the audio.\n */\n end: number;\n};\n\ntype WhisperSegment = {\n text: string;\n\n /**\n * Time in seconds from the start of the audio.\n */\n start: number;\n\n /**\n * Time in seconds from the start of the audio.\n */\n end: number;\n\n /**\n * Probability of no speech in the segment.\n */\n no_speech_prob: number;\n\n words: WhisperWord[];\n};\n\nexport type TranscribeConfig = {\n /**\n * Number of chunks to transcribe automatically after.\n */\n transcribeAfterChunksAmount: number;\n\n /**\n * Number of chunks to save before the user starts speaking.\n * This is needed to catch the beginning of the user's speech.\n */\n prefixBufferChunksAmount: number;\n};\n\nexport type TranscriberParams = {\n config: TranscribeConfig;\n recorder: AudioRecorder;\n onSegments: (segments: TranscriptionContentBlock[]) => Promise<void>;\n};\n\n/**\n * Handles transcription of audio chunks.\n * If user is not speaking, the last `minChunksAmount` chunks are saved and transcribed.\n * If user is speaking, the chunks are added to the buffer until the user is done talking.\n */\nexport class Transcriber extends Resource {\n private _audioChunks: AudioChunk[] = [];\n private _lastUsedTimestamp = 0;\n\n private readonly _openTrigger = new Trigger({ autoReset: true });\n\n private readonly _config: TranscribeConfig;\n private readonly _recorder: AudioRecorder;\n private readonly _onSegments: TranscriberParams['onSegments'];\n\n private _recording = false;\n private _transcribeTask?: DeferredTask = undefined;\n\n constructor({ config, recorder, onSegments }: TranscriberParams) {\n super();\n this._config = config;\n this._recorder = recorder;\n this._onSegments = onSegments;\n }\n\n protected override async _open(ctx: Context) {\n log.info('opening');\n this._recorder.setOnChunk((chunk) => this._saveAudioChunk(chunk));\n await this._recorder.start();\n this._transcribeTask = new DeferredTask(ctx, async () => this._transcribe());\n this._openTrigger.wake();\n }\n\n protected override async _close() {\n log.info('closing');\n this._recording = false;\n this._transcribeTask = undefined;\n await this._recorder.stop();\n }\n\n startChunksRecording() {\n log.info('starting');\n if (this._lifecycleState !== LifecycleState.OPEN) {\n return;\n }\n\n this._recording = true;\n }\n\n stopChunksRecording() {\n if (this._lifecycleState !== LifecycleState.OPEN || !this._recording) {\n return;\n }\n\n this._recording = false;\n this._transcribeTask?.schedule();\n log.info('stopped');\n }\n\n private _saveAudioChunk(chunk: AudioChunk) {\n log('saving audio chunk', { chunk });\n this._audioChunks.push(chunk);\n\n // Clean the buffer if the user is not speaking and the transcription task is not scheduled.\n if (\n !this._recording &&\n (!this._transcribeTask || !this._transcribeTask.scheduled) &&\n this._audioChunks.length >= this._config.prefixBufferChunksAmount\n ) {\n this._audioChunks = this._dropOldChunks();\n } else if (this._audioChunks.length >= this._config.transcribeAfterChunksAmount && this._recording) {\n this._transcribeTask?.schedule();\n }\n }\n\n private async _transcribe() {\n log('transcribing', { chunks: this._audioChunks.length });\n const chunks = this._audioChunks;\n this._audioChunks = this._dropOldChunks();\n if (chunks.length === 0) {\n return;\n }\n\n const audio = await this._mergeAudioChunks(chunks);\n const segments = await this._fetchTranscription(audio);\n if (!Array.isArray(segments) || segments.length === 0) {\n return;\n }\n\n const alignedSegments = this._alignSegments(segments, chunks);\n if (alignedSegments.length > 0) {\n await this._onSegments(alignedSegments);\n }\n }\n\n private _dropOldChunks() {\n return this._config.prefixBufferChunksAmount > 0\n ? this._audioChunks.slice(-this._config.prefixBufferChunksAmount)\n : [];\n }\n\n @trace.span({ showInBrowserTimeline: true })\n private async _mergeAudioChunks(chunks: AudioChunk[]) {\n const file = new WaveFile();\n const wavConfig = this._recorder.wavConfig;\n\n file.fromScratch(\n wavConfig.channels,\n wavConfig.sampleRate,\n wavConfig.bitDepthCode,\n mergeFloat64Arrays(chunks.map(({ data }) => data)),\n );\n\n return file.toBase64();\n }\n\n @trace.span({ showInBrowserTimeline: true })\n private async _fetchTranscription(audio: string) {\n if (audio.length === 0) {\n this._audioChunks = [];\n throw new Error('No audio to send for transcribing');\n }\n\n const response = await fetch(`${TRANSCRIPTION_URL}/transcribe`, {\n method: 'POST',\n body: JSON.stringify({ audio }),\n headers: {\n 'Content-Type': 'application/json',\n },\n });\n\n if (!response.ok) {\n this._audioChunks = [];\n throw new Error(`Transcription failed: ${response.statusText}`);\n }\n\n const { segments } = (await response.json()) as {\n segments: WhisperSegment[];\n };\n\n log.info('transcription response', {\n segments: segments.length,\n string: segments.map((segments) => segments.text).join(' '),\n });\n\n return segments;\n }\n\n private _alignSegments(segments: WhisperSegment[], originalChunks: AudioChunk[]): TranscriptionContentBlock[] {\n // Absolute zero for all relative timestamps in the segments.\n const zeroTimestamp = originalChunks.at(0)!.timestamp;\n\n const filteredSegments = segments\n // Use segments that end after the last used timestamp.\n // Segments could overlap, so we need to filter words that starts after the last used timestamp.\n .filter((segment) => zeroTimestamp + segment.end * 1_000 >= this._lastUsedTimestamp)\n // Use words that starts after the last used timestamp.\n .map((segment) => {\n const words = segment.words.filter((word) => zeroTimestamp + word.start * 1_000 >= this._lastUsedTimestamp);\n return {\n ...segment,\n words,\n text: words.map((word) => word.word).join(''),\n start: words.at(0)?.start ?? segment.end,\n };\n })\n .filter((segment) => segment.words.length > 0);\n\n if (filteredSegments.length === 0) {\n return [];\n }\n\n // Update last timestamp.\n this._lastUsedTimestamp = zeroTimestamp + filteredSegments.at(-1)!.end * 1_000;\n\n // Add absolute timestamp to each segment.\n return filteredSegments.map((segment) => ({\n type: 'transcription',\n started: new Date(zeroTimestamp + segment.start * 1_000).toISOString(),\n text: segment.text.trim(),\n }));\n }\n}\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { synchronized } from '@dxos/async';\nimport { Resource } from '@dxos/context';\nimport { QueueImpl, type Queue } from '@dxos/echo-db';\nimport { create } from '@dxos/echo-schema';\nimport { type DXN } from '@dxos/keys';\nimport { log } from '@dxos/log';\nimport { type EdgeHttpClient } from '@dxos/react-edge-client';\nimport { MessageType, type TranscriptionContentBlock } from '@dxos/schema';\n\nimport { MediaStreamRecorder } from './media-stream-recorder';\nimport { Transcriber } from './transcriber';\n\n/**\n * Length of the chunk in ms.\n */\nconst RECORD_INTERVAL = 200;\n\n/**\n * Number of chunks to save before the user starts speaking.\n */\nconst PREFIXED_CHUNKS_AMOUNT = 10;\n\n/**\n * Number of chunks to transcribe automatically after.\n * Combined should be mess than 25MB or whisper would fail.\n */\nconst TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;\n\nexport type TranscriptMessageEnricher = (message: MessageType) => Promise<MessageType>;\n\nexport type TranscriptionManagerOptions = {\n edgeClient: EdgeHttpClient;\n\n /**\n * Enrich the message before it is written to the transcription queue.\n */\n messageEnricher?: TranscriptMessageEnricher;\n};\n\n/**\n * Manages transcription state for a meeting.\n */\nexport class TranscriptionManager extends Resource {\n private readonly _edgeClient: EdgeHttpClient;\n private readonly _messageEnricher?: TranscriptMessageEnricher;\n private _audioStreamTrack?: MediaStreamTrack = undefined;\n private _identityDid?: string = undefined;\n private _mediaRecorder?: MediaStreamRecorder = undefined;\n private _transcriber?: Transcriber = undefined;\n private _queue?: Queue<MessageType> = undefined;\n private _enabled = false;\n\n constructor(options: TranscriptionManagerOptions) {\n super();\n this._edgeClient = options.edgeClient;\n this._messageEnricher = options.messageEnricher;\n }\n\n protected override async _open() {\n await this._toggleTranscriber();\n }\n\n protected override async _close() {\n void this._transcriber?.close();\n }\n\n setQueue(queueDxn: DXN): TranscriptionManager {\n if (this._queue?.dxn.toString() !== queueDxn.toString()) {\n log.info('setQueue', { queueDxn: queueDxn.toString() });\n this._queue = new QueueImpl<MessageType>(this._edgeClient, queueDxn);\n }\n return this;\n }\n\n setIdentityDid(did: string): TranscriptionManager {\n if (this._identityDid !== did) {\n this._identityDid = did;\n }\n return this;\n }\n\n setRecording(recording?: boolean): TranscriptionManager {\n if (!this.isOpen || !this._enabled) {\n return this;\n }\n\n if (recording) {\n this._transcriber?.startChunksRecording();\n } else {\n this._transcriber?.stopChunksRecording();\n }\n return this;\n }\n\n /**\n * Enable or disable transcription.\n * @param enabled - Whether to enable transcription.\n */\n @synchronized\n async setEnabled(enabled?: boolean) {\n if (this._enabled === enabled) {\n return;\n }\n\n this._enabled = enabled ?? false;\n // TODO(burdon): Why is toggle called here?\n this.isOpen && (await this._toggleTranscriber());\n }\n\n @synchronized\n async setAudioTrack(track?: MediaStreamTrack) {\n if (this._audioStreamTrack === track) {\n return;\n }\n\n this._audioStreamTrack = track;\n this.isOpen && (await this._toggleTranscriber());\n }\n\n // TODO(burdon): Change this to setEnables (explicit), not toggle.\n private async _toggleTranscriber() {\n await this._maybeReinitTranscriber();\n\n // Open or close transcriber if transcription is enabled or disabled.\n if (this._enabled) {\n await this._transcriber?.open();\n // TODO(burdon): Started and stopped blocks appear twice.\n const block = create(MessageType, {\n created: new Date().toISOString(),\n blocks: [{ type: 'transcription', text: 'Started', started: new Date().toISOString() }],\n sender: { role: 'assistant' },\n });\n this._queue?.append([block]);\n } else {\n await this._transcriber?.close();\n const block = create(MessageType, {\n created: new Date().toISOString(),\n blocks: [{ type: 'transcription', text: 'Stopped', started: new Date().toISOString() }],\n sender: { role: 'assistant' },\n });\n this._queue?.append([block]);\n }\n }\n\n private async _maybeReinitTranscriber() {\n if (!this._audioStreamTrack || !this._enabled) {\n return;\n }\n\n // Reinitialize transcriber if queue or media stream track has changed.\n let needReinit = false;\n if (this._audioStreamTrack !== this._mediaRecorder?.mediaStreamTrack) {\n this._mediaRecorder = new MediaStreamRecorder({\n mediaStreamTrack: this._audioStreamTrack,\n interval: RECORD_INTERVAL,\n });\n needReinit = true;\n }\n if (needReinit) {\n await this._transcriber?.close();\n this._transcriber = new Transcriber({\n config: {\n transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,\n prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,\n },\n recorder: this._mediaRecorder,\n onSegments: (segments) => this._onSegments(segments),\n });\n }\n }\n\n private async _onSegments(segments: TranscriptionContentBlock[]) {\n if (!this.isOpen || !this._queue) {\n return;\n }\n\n let block = create(MessageType, {\n created: new Date().toISOString(),\n blocks: segments,\n sender: { identityDid: this._identityDid },\n });\n\n if (this._messageEnricher) {\n block = await this._messageEnricher(block);\n }\n\n this._queue.append([block]);\n }\n}\n"],
|
|
5
|
+
"mappings": ";;;;;;;;AAIA,SAA+CA,eAAeC,gBAAgB;AAC9E,SAASC,gBAAgB;AAEzB,SAASC,oBAAoB;AAC7B,SAASC,iBAAiB;AAC1B,SAASC,WAAW;AACpB,SAASC,aAAa;;;;;;;;AAItB,IAAIC;AAEJ,IAAMC,oCAAoC,YAAA;AACxC,QAAM,EAAEC,QAAO,IAAK,MAAM,OAAO,uCAAA;AACjC,QAAMR,SAAS,MAAMQ,QAAAA,CAAAA;AACvB;AAQO,IAAMC,sBAAN,MAAMA;EAUXC,YAAY,EAAEC,kBAAkBC,SAAQ,GAA8D;AAJ9FC,0BAAkCC;AAClCC,mBAAuBD;AAI7B,SAAKE,oBAAoBL;AACzB,SAAKM,YAAYL;EACnB;EAEA,IAAIM,YAAuB;AACzB,UAAMC,WAAW,KAAKH,kBAAkBI,YAAW;AACnD,WAAO;MACLC,UAAUF,SAASG,gBAAgB;MACnCC,YAAYJ,SAASI,cAAc;MACnCC,cAAcL,SAASM,aAAaC,OAAOP,SAASM,UAAU,IAAI;IACpE;EACF;EAEA,IAAId,mBAAmB;AACrB,WAAO,KAAKK;EACd;EAEAW,WAAWC,SAA4C;AACrD,SAAKC,WAAWD;EAClB;EAEA,MAAME,QAAQ;AACZ,WAAOxB,wBAAwBC,kCAAAA,GAAqCwB,MAAM,CAACC,QACzE5B,IAAI6B,KAAK,qCAAqC;MAAED;IAAI,GAAA;;;;;;AAGtD,QAAI,CAAC,KAAKnB,gBAAgB;AACxB,YAAMqB,SAAS,IAAIC,YAAY;QAAC,KAAKnB;OAAkB;AACvD,WAAKH,iBAAiB,IAAId,cAAcmC,QAAQ;QAAEE,UAAU;MAAY,CAAA;IAC1E;AAEA,QAAI,KAAKvB,eAAewB,UAAU,aAAa;AAC7C;IACF;AACAlC,cAAU,KAAK0B,UAAU,2CAAA;;;;;;;;;AACzB,SAAKhB,eAAeyB,kBAAkB,CAACC,UAAU,KAAKC,iBAAiBD,KAAAA;AACvE,SAAK1B,eAAeiB,MAAM,KAAKb,SAAS;EAC1C;EAEA,MAAMwB,OAAO;AACX,QAAI,KAAK5B,gBAAgBwB,UAAU,aAAa;AAC9C;IACF;AACA,SAAKxB,eAAe4B,KAAI;EAC1B;EAEA,MACcD,iBAAiBD,OAAmB;AAChD,UAAMG,OAAOH,MAAMI;AACnB,UAAMC,aAAa,IAAIC,WAAW,MAAMH,KAAKI,YAAW,CAAA;AAGxD,UAAMC,WACJH,WAAW,CAAA,MAAO;IAClBA,WAAW,CAAA,MAAO;IAClBA,WAAW,CAAA,MAAO;IAClBA,WAAW,CAAA,MAAO;AACpB,QAAII;AAEJ,QAAID,UAAU;AACZC,YAAM,IAAI/C,SAAS2C,UAAAA;AACnB,WAAK7B,UAAU6B,WAAWK,MAAM,GAAG,EAAA;IACrC,WAAW,KAAKlC,SAAS;AACvBiC,YAAM,IAAI/C,SAAS,IAAI4C,WAAW;WAAI,KAAK9B;WAAY6B;OAAW,CAAA;IACpE,OAAO;AACLxC,UAAI8C,KAAK,kCAAA,QAAA;;;;;;AACT;IACF;AAEA,SAAKrB,SAAU;MACbsB,WAAWC,KAAKC,IAAG;MACnBV,MAAMK,IAAIM,WAAU;IACtB,CAAA;EACF;AACF;;EA5BGpD;GAzDUO,oBAAAA,WAAAA,oBAAAA,IAAAA;AAAAA,sBAAAA,aAAAA;EADZJ,MAAMkD,SAAQ;GACF9C,mBAAAA;;;ACvBb,SAAS+C,YAAAA,iBAAgB;AAEzB,SAASC,cAAcC,eAAe;AACtC,SAAuBC,gBAAgBC,gBAAgB;AACvD,SAASC,OAAAA,YAAW;AAEpB,SAASC,SAAAA,cAAa;;;;;;;;AAiEf,IAAMC,cAAN,cAA0BC,SAAAA;EAa/BC,YAAY,EAAEC,QAAQC,UAAUC,WAAU,GAAuB;AAC/D,UAAK;AAbCC,wBAA6B,CAAA;AAC7BC,8BAAqB;AAEZC,wBAAe,IAAIC,QAAQ;MAAEC,WAAW;IAAK,CAAA;AAMtDC,sBAAa;AACbC,2BAAiCC;AAIvC,SAAKC,UAAUX;AACf,SAAKY,YAAYX;AACjB,SAAKY,cAAcX;EACrB;EAEA,MAAyBY,MAAMC,KAAc;AAC3CC,IAAAA,KAAIC,KAAK,WAAA,QAAA;;;;;;AACT,SAAKL,UAAUM,WAAW,CAACC,UAAU,KAAKC,gBAAgBD,KAAAA,CAAAA;AAC1D,UAAM,KAAKP,UAAUS,MAAK;AAC1B,SAAKZ,kBAAkB,IAAIa,aAAaP,KAAK,YAAY,KAAKQ,YAAW,CAAA;AACzE,SAAKlB,aAAamB,KAAI;EACxB;EAEA,MAAyBC,SAAS;AAChCT,IAAAA,KAAIC,KAAK,WAAA,QAAA;;;;;;AACT,SAAKT,aAAa;AAClB,SAAKC,kBAAkBC;AACvB,UAAM,KAAKE,UAAUc,KAAI;EAC3B;EAEAC,uBAAuB;AACrBX,IAAAA,KAAIC,KAAK,YAAA,QAAA;;;;;;AACT,QAAI,KAAKW,oBAAoBC,eAAeC,MAAM;AAChD;IACF;AAEA,SAAKtB,aAAa;EACpB;EAEAuB,sBAAsB;AACpB,QAAI,KAAKH,oBAAoBC,eAAeC,QAAQ,CAAC,KAAKtB,YAAY;AACpE;IACF;AAEA,SAAKA,aAAa;AAClB,SAAKC,iBAAiBuB,SAAAA;AACtBhB,IAAAA,KAAIC,KAAK,WAAA,QAAA;;;;;;EACX;EAEQG,gBAAgBD,OAAmB;AACzCH,IAAAA,KAAI,sBAAsB;MAAEG;IAAM,GAAA;;;;;;AAClC,SAAKhB,aAAa8B,KAAKd,KAAAA;AAGvB,QACE,CAAC,KAAKX,eACL,CAAC,KAAKC,mBAAmB,CAAC,KAAKA,gBAAgByB,cAChD,KAAK/B,aAAagC,UAAU,KAAKxB,QAAQyB,0BACzC;AACA,WAAKjC,eAAe,KAAKkC,eAAc;IACzC,WAAW,KAAKlC,aAAagC,UAAU,KAAKxB,QAAQ2B,+BAA+B,KAAK9B,YAAY;AAClG,WAAKC,iBAAiBuB,SAAAA;IACxB;EACF;EAEA,MAAcT,cAAc;AAC1BP,IAAAA,KAAI,gBAAgB;MAAEuB,QAAQ,KAAKpC,aAAagC;IAAO,GAAA;;;;;;AACvD,UAAMI,SAAS,KAAKpC;AACpB,SAAKA,eAAe,KAAKkC,eAAc;AACvC,QAAIE,OAAOJ,WAAW,GAAG;AACvB;IACF;AAEA,UAAMK,QAAQ,MAAM,KAAKC,kBAAkBF,MAAAA;AAC3C,UAAMG,WAAW,MAAM,KAAKC,oBAAoBH,KAAAA;AAChD,QAAI,CAACI,MAAMC,QAAQH,QAAAA,KAAaA,SAASP,WAAW,GAAG;AACrD;IACF;AAEA,UAAMW,kBAAkB,KAAKC,eAAeL,UAAUH,MAAAA;AACtD,QAAIO,gBAAgBX,SAAS,GAAG;AAC9B,YAAM,KAAKtB,YAAYiC,eAAAA;IACzB;EACF;EAEQT,iBAAiB;AACvB,WAAO,KAAK1B,QAAQyB,2BAA2B,IAC3C,KAAKjC,aAAa6C,MAAM,CAAC,KAAKrC,QAAQyB,wBAAwB,IAC9D,CAAA;EACN;EAEA,MACcK,kBAAkBF,QAAsB;AACpD,UAAMU,OAAO,IAAIC,UAAAA;AACjB,UAAMC,YAAY,KAAKvC,UAAUuC;AAEjCF,SAAKG,YACHD,UAAUE,UACVF,UAAUG,YACVH,UAAUI,cACVC,mBAAmBjB,OAAOkB,IAAI,CAAC,EAAEC,KAAI,MAAOA,IAAAA,CAAAA,CAAAA;AAG9C,WAAOT,KAAKU,SAAQ;EACtB;EAEA,MACchB,oBAAoBH,OAAe;AAC/C,QAAIA,MAAML,WAAW,GAAG;AACtB,WAAKhC,eAAe,CAAA;AACpB,YAAM,IAAIyD,MAAM,mCAAA;IAClB;AAEA,UAAMC,WAAW,MAAMC,MAAM,GAAGC,iBAAAA,eAAgC;MAC9DC,QAAQ;MACRC,MAAMC,KAAKC,UAAU;QAAE3B;MAAM,CAAA;MAC7B4B,SAAS;QACP,gBAAgB;MAClB;IACF,CAAA;AAEA,QAAI,CAACP,SAASQ,IAAI;AAChB,WAAKlE,eAAe,CAAA;AACpB,YAAM,IAAIyD,MAAM,yBAAyBC,SAASS,UAAU,EAAE;IAChE;AAEA,UAAM,EAAE5B,SAAQ,IAAM,MAAMmB,SAASU,KAAI;AAIzCvD,IAAAA,KAAIC,KAAK,0BAA0B;MACjCyB,UAAUA,SAASP;MACnBqC,QAAQ9B,SAASe,IAAI,CAACf,cAAaA,UAAS+B,IAAI,EAAEC,KAAK,GAAA;IACzD,GAAA;;;;;;AAEA,WAAOhC;EACT;EAEQK,eAAeL,UAA4BiC,gBAA2D;AAE5G,UAAMC,gBAAgBD,eAAeE,GAAG,CAAA,EAAIC;AAE5C,UAAMC,mBAAmBrC,SAGtBsC,OAAO,CAACC,YAAYL,gBAAgBK,QAAQC,MAAM,OAAS,KAAK9E,kBAAkB,EAElFqD,IAAI,CAACwB,YAAAA;AACJ,YAAME,QAAQF,QAAQE,MAAMH,OAAO,CAACI,SAASR,gBAAgBQ,KAAK/D,QAAQ,OAAS,KAAKjB,kBAAkB;AAC1G,aAAO;QACL,GAAG6E;QACHE;QACAV,MAAMU,MAAM1B,IAAI,CAAC2B,SAASA,KAAKA,IAAI,EAAEV,KAAK,EAAA;QAC1CrD,OAAO8D,MAAMN,GAAG,CAAA,GAAIxD,SAAS4D,QAAQC;MACvC;IACF,CAAA,EACCF,OAAO,CAACC,YAAYA,QAAQE,MAAMhD,SAAS,CAAA;AAE9C,QAAI4C,iBAAiB5C,WAAW,GAAG;AACjC,aAAO,CAAA;IACT;AAGA,SAAK/B,qBAAqBwE,gBAAgBG,iBAAiBF,GAAG,EAAC,EAAIK,MAAM;AAGzE,WAAOH,iBAAiBtB,IAAI,CAACwB,aAAa;MACxCI,MAAM;MACNC,SAAS,IAAIC,KAAKX,gBAAgBK,QAAQ5D,QAAQ,GAAA,EAAOmE,YAAW;MACpEf,MAAMQ,QAAQR,KAAKgB,KAAI;IACzB,EAAA;EACF;AACF;;EAjFGC,OAAMC,KAAK;IAAEC,uBAAuB;EAAK,CAAA;GAhG/B/F,YAAAA,WAAAA,qBAAAA,IAAAA;;EA+GV6F,OAAMC,KAAK;IAAEC,uBAAuB;EAAK,CAAA;GA/G/B/F,YAAAA,WAAAA,uBAAAA,IAAAA;;;ACvEb,SAASgG,gBAAAA,qBAAoB;AAC7B,SAASC,YAAAA,iBAAgB;AACzB,SAASC,iBAA6B;AACtC,SAASC,cAAc;AAEvB,SAASC,OAAAA,YAAW;AAEpB,SAASC,mBAAmD;;;;;;;;AAQ5D,IAAMC,kBAAkB;AAKxB,IAAMC,yBAAyB;AAM/B,IAAMC,iCAAiC;AAgBhC,IAAMC,uBAAN,cAAmCC,UAAAA;EAUxCC,YAAYC,SAAsC;AAChD,UAAK;AARCC,6BAAuCC;AACvCC,wBAAwBD;AACxBE,0BAAuCF;AACvCG,wBAA6BH;AAC7BI,kBAA8BJ;AAC9BK,oBAAW;AAIjB,SAAKC,cAAcR,QAAQS;AAC3B,SAAKC,mBAAmBV,QAAQW;EAClC;EAEA,MAAyBC,QAAQ;AAC/B,UAAM,KAAKC,mBAAkB;EAC/B;EAEA,MAAyBC,SAAS;AAChC,SAAK,KAAKT,cAAcU,MAAAA;EAC1B;EAEAC,SAASC,UAAqC;AAC5C,QAAI,KAAKX,QAAQY,IAAIC,SAAAA,MAAeF,SAASE,SAAQ,GAAI;AACvDC,MAAAA,KAAIC,KAAK,YAAY;QAAEJ,UAAUA,SAASE,SAAQ;MAAG,GAAA;;;;;;AACrD,WAAKb,SAAS,IAAIgB,UAAuB,KAAKd,aAAaS,QAAAA;IAC7D;AACA,WAAO;EACT;EAEAM,eAAeC,KAAmC;AAChD,QAAI,KAAKrB,iBAAiBqB,KAAK;AAC7B,WAAKrB,eAAeqB;IACtB;AACA,WAAO;EACT;EAEAC,aAAaC,WAA2C;AACtD,QAAI,CAAC,KAAKC,UAAU,CAAC,KAAKpB,UAAU;AAClC,aAAO;IACT;AAEA,QAAImB,WAAW;AACb,WAAKrB,cAAcuB,qBAAAA;IACrB,OAAO;AACL,WAAKvB,cAAcwB,oBAAAA;IACrB;AACA,WAAO;EACT;;;;;EAMA,MACMC,WAAWC,SAAmB;AAClC,QAAI,KAAKxB,aAAawB,SAAS;AAC7B;IACF;AAEA,SAAKxB,WAAWwB,WAAW;AAE3B,SAAKJ,UAAW,MAAM,KAAKd,mBAAkB;EAC/C;EAEA,MACMmB,cAAcC,OAA0B;AAC5C,QAAI,KAAKhC,sBAAsBgC,OAAO;AACpC;IACF;AAEA,SAAKhC,oBAAoBgC;AACzB,SAAKN,UAAW,MAAM,KAAKd,mBAAkB;EAC/C;;EAGA,MAAcA,qBAAqB;AACjC,UAAM,KAAKqB,wBAAuB;AAGlC,QAAI,KAAK3B,UAAU;AACjB,YAAM,KAAKF,cAAc8B,KAAAA;AAEzB,YAAMC,QAAQC,OAAOC,aAAa;QAChCC,UAAS,oBAAIC,KAAAA,GAAOC,YAAW;QAC/BC,QAAQ;UAAC;YAAEC,MAAM;YAAiBC,MAAM;YAAWC,UAAS,oBAAIL,KAAAA,GAAOC,YAAW;UAAG;;QACrFK,QAAQ;UAAEC,MAAM;QAAY;MAC9B,CAAA;AACA,WAAKzC,QAAQ0C,OAAO;QAACZ;OAAM;IAC7B,OAAO;AACL,YAAM,KAAK/B,cAAcU,MAAAA;AACzB,YAAMqB,QAAQC,OAAOC,aAAa;QAChCC,UAAS,oBAAIC,KAAAA,GAAOC,YAAW;QAC/BC,QAAQ;UAAC;YAAEC,MAAM;YAAiBC,MAAM;YAAWC,UAAS,oBAAIL,KAAAA,GAAOC,YAAW;UAAG;;QACrFK,QAAQ;UAAEC,MAAM;QAAY;MAC9B,CAAA;AACA,WAAKzC,QAAQ0C,OAAO;QAACZ;OAAM;IAC7B;EACF;EAEA,MAAcF,0BAA0B;AACtC,QAAI,CAAC,KAAKjC,qBAAqB,CAAC,KAAKM,UAAU;AAC7C;IACF;AAGA,QAAI0C,aAAa;AACjB,QAAI,KAAKhD,sBAAsB,KAAKG,gBAAgB8C,kBAAkB;AACpE,WAAK9C,iBAAiB,IAAI+C,oBAAoB;QAC5CD,kBAAkB,KAAKjD;QACvBmD,UAAU1D;MACZ,CAAA;AACAuD,mBAAa;IACf;AACA,QAAIA,YAAY;AACd,YAAM,KAAK5C,cAAcU,MAAAA;AACzB,WAAKV,eAAe,IAAIgD,YAAY;QAClCC,QAAQ;UACNC,6BAA6B3D;UAC7B4D,0BAA0B7D;QAC5B;QACA8D,UAAU,KAAKrD;QACfsD,YAAY,CAACC,aAAa,KAAKC,YAAYD,QAAAA;MAC7C,CAAA;IACF;EACF;EAEA,MAAcC,YAAYD,UAAuC;AAC/D,QAAI,CAAC,KAAKhC,UAAU,CAAC,KAAKrB,QAAQ;AAChC;IACF;AAEA,QAAI8B,QAAQC,OAAOC,aAAa;MAC9BC,UAAS,oBAAIC,KAAAA,GAAOC,YAAW;MAC/BC,QAAQiB;MACRb,QAAQ;QAAEe,aAAa,KAAK1D;MAAa;IAC3C,CAAA;AAEA,QAAI,KAAKO,kBAAkB;AACzB0B,cAAQ,MAAM,KAAK1B,iBAAiB0B,KAAAA;IACtC;AAEA,SAAK9B,OAAO0C,OAAO;MAACZ;KAAM;EAC5B;AACF;;EA1FG0B;GAxDUjE,qBAAAA,WAAAA,cAAAA,IAAAA;;EAmEViE;GAnEUjE,qBAAAA,WAAAA,iBAAAA,IAAAA;",
|
|
6
|
+
"names": ["MediaRecorder", "register", "WaveFile", "synchronized", "invariant", "log", "trace", "initializingPromise", "initializeExtendableMediaRecorder", "connect", "MediaStreamRecorder", "constructor", "mediaStreamTrack", "interval", "_mediaRecorder", "undefined", "_header", "_mediaStreamTrack", "_interval", "wavConfig", "settings", "getSettings", "channels", "channelCount", "sampleRate", "bitDepthCode", "sampleSize", "String", "setOnChunk", "onChunk", "_onChunk", "start", "catch", "err", "info", "stream", "MediaStream", "mimeType", "state", "ondataavailable", "event", "_ondataavailable", "stop", "blob", "data", "uint8Array", "Uint8Array", "arrayBuffer", "isHeader", "wav", "slice", "warn", "timestamp", "Date", "now", "getSamples", "resource", "WaveFile", "DeferredTask", "Trigger", "LifecycleState", "Resource", "log", "trace", "Transcriber", "Resource", "constructor", "config", "recorder", "onSegments", "_audioChunks", "_lastUsedTimestamp", "_openTrigger", "Trigger", "autoReset", "_recording", "_transcribeTask", "undefined", "_config", "_recorder", "_onSegments", "_open", "ctx", "log", "info", "setOnChunk", "chunk", "_saveAudioChunk", "start", "DeferredTask", "_transcribe", "wake", "_close", "stop", "startChunksRecording", "_lifecycleState", "LifecycleState", "OPEN", "stopChunksRecording", "schedule", "push", "scheduled", "length", "prefixBufferChunksAmount", "_dropOldChunks", "transcribeAfterChunksAmount", "chunks", "audio", "_mergeAudioChunks", "segments", "_fetchTranscription", "Array", "isArray", "alignedSegments", "_alignSegments", "slice", "file", "WaveFile", "wavConfig", "fromScratch", "channels", "sampleRate", "bitDepthCode", "mergeFloat64Arrays", "map", "data", "toBase64", "Error", "response", "fetch", "TRANSCRIPTION_URL", "method", "body", "JSON", "stringify", "headers", "ok", "statusText", "json", "string", "text", "join", "originalChunks", "zeroTimestamp", "at", "timestamp", "filteredSegments", "filter", "segment", "end", "words", "word", "type", "started", "Date", "toISOString", "trim", "trace", "span", "showInBrowserTimeline", "synchronized", "Resource", "QueueImpl", "create", "log", "MessageType", "RECORD_INTERVAL", "PREFIXED_CHUNKS_AMOUNT", "TRANSCRIBE_AFTER_CHUNKS_AMOUNT", "TranscriptionManager", "Resource", "constructor", "options", "_audioStreamTrack", "undefined", "_identityDid", "_mediaRecorder", "_transcriber", "_queue", "_enabled", "_edgeClient", "edgeClient", "_messageEnricher", "messageEnricher", "_open", "_toggleTranscriber", "_close", "close", "setQueue", "queueDxn", "dxn", "toString", "log", "info", "QueueImpl", "setIdentityDid", "did", "setRecording", "recording", "isOpen", "startChunksRecording", "stopChunksRecording", "setEnabled", "enabled", "setAudioTrack", "track", "_maybeReinitTranscriber", "open", "block", "create", "MessageType", "created", "Date", "toISOString", "blocks", "type", "text", "started", "sender", "role", "append", "needReinit", "mediaStreamTrack", "MediaStreamRecorder", "interval", "Transcriber", "config", "transcribeAfterChunksAmount", "prefixBufferChunksAmount", "recorder", "onSegments", "segments", "_onSegments", "identityDid", "synchronized"]
|
|
7
|
+
}
|
|
@@ -3,13 +3,12 @@ import {
|
|
|
3
3
|
} from "./chunk-6ZR5MDK3.mjs";
|
|
4
4
|
|
|
5
5
|
// packages/plugins/plugin-transcription/src/types/schema.ts
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
name: Schema.String,
|
|
6
|
+
import { Expando, Ref, S, TypedObject } from "@dxos/echo-schema";
|
|
7
|
+
var TranscriptSchema = S.Struct({
|
|
8
|
+
name: S.String,
|
|
10
9
|
// TODO(burdon): Use Date or string?
|
|
11
|
-
started:
|
|
12
|
-
ended:
|
|
10
|
+
started: S.optional(S.String),
|
|
11
|
+
ended: S.optional(S.String),
|
|
13
12
|
/**
|
|
14
13
|
* Queue containing TranscriptBlock objects.
|
|
15
14
|
*/
|
|
@@ -21,24 +20,23 @@ var TranscriptType = class extends TypedObject({
|
|
|
21
20
|
version: "0.1.0"
|
|
22
21
|
})(TranscriptSchema.fields) {
|
|
23
22
|
};
|
|
24
|
-
var TranscriptHeader =
|
|
25
|
-
started:
|
|
23
|
+
var TranscriptHeader = S.Struct({
|
|
24
|
+
started: S.optional(S.String)
|
|
26
25
|
});
|
|
27
26
|
|
|
28
27
|
// packages/plugins/plugin-transcription/src/types/types.ts
|
|
29
|
-
import {
|
|
30
|
-
import { SpaceIdSchema, isInstanceOf } from "@dxos/echo-schema";
|
|
28
|
+
import { SpaceIdSchema, S as S2, isInstanceOf } from "@dxos/echo-schema";
|
|
31
29
|
import { isLiveObject } from "@dxos/react-client/echo";
|
|
32
30
|
var TRANSCRIPTION_URL = "https://calls-service.dxos.workers.dev";
|
|
33
31
|
var TranscriptionAction;
|
|
34
32
|
(function(TranscriptionAction2) {
|
|
35
33
|
const TRANSCRIPTION_ACTION = `${TRANSCRIPTION_PLUGIN}/action`;
|
|
36
|
-
class Create extends
|
|
37
|
-
input:
|
|
38
|
-
name:
|
|
34
|
+
class Create extends S2.TaggedClass()(`${TRANSCRIPTION_ACTION}/create`, {
|
|
35
|
+
input: S2.Struct({
|
|
36
|
+
name: S2.optional(S2.String),
|
|
39
37
|
spaceId: SpaceIdSchema
|
|
40
38
|
}),
|
|
41
|
-
output:
|
|
39
|
+
output: S2.Struct({
|
|
42
40
|
object: TranscriptType
|
|
43
41
|
})
|
|
44
42
|
}) {
|
|
@@ -48,8 +46,8 @@ var TranscriptionAction;
|
|
|
48
46
|
var isTranscript = (object) => {
|
|
49
47
|
return isLiveObject(object) && isInstanceOf(TranscriptType, object);
|
|
50
48
|
};
|
|
51
|
-
var TranscriptionSettingsSchema =
|
|
52
|
-
entityExtraction:
|
|
49
|
+
var TranscriptionSettingsSchema = S2.mutable(S2.Struct({
|
|
50
|
+
entityExtraction: S2.optional(S2.Boolean).pipe(S2.withConstructorDefault(() => true))
|
|
53
51
|
}));
|
|
54
52
|
|
|
55
53
|
export {
|
|
@@ -60,4 +58,4 @@ export {
|
|
|
60
58
|
isTranscript,
|
|
61
59
|
TranscriptionSettingsSchema
|
|
62
60
|
};
|
|
63
|
-
//# sourceMappingURL=chunk-
|
|
61
|
+
//# sourceMappingURL=chunk-YRWBY6UK.mjs.map
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": 3,
|
|
3
|
+
"sources": ["../../../src/types/schema.ts", "../../../src/types/types.ts"],
|
|
4
|
+
"sourcesContent": ["//\n// Copyright 2025 DXOS.org\n//\n\nimport { Expando, Ref, S, TypedObject } from '@dxos/echo-schema';\n\n/**\n * Root transcript object created when the user starts a transcription.\n */\nexport const TranscriptSchema = S.Struct({\n name: S.String,\n\n // TODO(burdon): Use Date or string?\n started: S.optional(S.String),\n ended: S.optional(S.String),\n\n /**\n * Queue containing TranscriptBlock objects.\n */\n // TODO(wittjosiah): Should be a ref to a queue.\n queue: Ref(Expando),\n});\n\nexport class TranscriptType extends TypedObject({\n typename: 'dxos.org/type/Transcript',\n version: '0.1.0',\n})(TranscriptSchema.fields) {}\n\n// TODO(burdon): Do these need to be kept in sync with EDGE?\n\n/**\n * First message in queue.\n * Contains metadata for the recording and transcript.\n */\nconst TranscriptHeader = S.Struct({\n started: S.optional(S.String),\n});\n\nexport type TranscriptHeader = S.Schema.Type<typeof TranscriptHeader>;\n", "//\n// Copyright 2025 DXOS.org\n//\n\nimport { SpaceIdSchema, S, isInstanceOf } from '@dxos/echo-schema';\nimport { isLiveObject } from '@dxos/react-client/echo';\n\nimport { TranscriptType } from './schema';\nimport { TRANSCRIPTION_PLUGIN } from '../meta';\n\n// TODO(burdon): Move to separate proto.\n\n/**\n * Endpoint to the calls service.\n */\n// TODO(burdon): Reconfigure/move.\nexport const TRANSCRIPTION_URL = 'https://calls-service.dxos.workers.dev';\n\nexport namespace TranscriptionAction {\n const TRANSCRIPTION_ACTION = `${TRANSCRIPTION_PLUGIN}/action`;\n\n export class Create extends S.TaggedClass<Create>()(`${TRANSCRIPTION_ACTION}/create`, {\n input: S.Struct({\n name: S.optional(S.String),\n spaceId: SpaceIdSchema,\n }),\n output: S.Struct({\n object: TranscriptType,\n }),\n }) {}\n}\n\nexport const isTranscript = (object: unknown): object is typeof TranscriptType => {\n return isLiveObject(object) && isInstanceOf(TranscriptType, object);\n};\n\n// TODO(burdon): Create with decode consistently: S.decodeSync(TranscriptionSettingsSchema)({}))\nexport const TranscriptionSettingsSchema = S.mutable(\n S.Struct({\n entityExtraction: S.optional(S.Boolean).pipe(S.withConstructorDefault(() => true)),\n }),\n);\n\nexport type TranscriptionSettingsProps = S.Schema.Type<typeof TranscriptionSettingsSchema>;\n"],
|
|
5
|
+
"mappings": ";;;;;AAIA,SAASA,SAASC,KAAKC,GAAGC,mBAAmB;AAKtC,IAAMC,mBAAmBC,EAAEC,OAAO;EACvCC,MAAMF,EAAEG;;EAGRC,SAASJ,EAAEK,SAASL,EAAEG,MAAM;EAC5BG,OAAON,EAAEK,SAASL,EAAEG,MAAM;;;;;EAM1BI,OAAOC,IAAIC,OAAAA;AACb,CAAA;AAEO,IAAMC,iBAAN,cAA6BC,YAAY;EAC9CC,UAAU;EACVC,SAAS;AACX,CAAA,EAAGd,iBAAiBe,MAAM,EAAA;AAAG;AAQ7B,IAAMC,mBAAmBf,EAAEC,OAAO;EAChCG,SAASJ,EAAEK,SAASL,EAAEG,MAAM;AAC9B,CAAA;;;AChCA,SAASa,eAAeC,KAAAA,IAAGC,oBAAoB;AAC/C,SAASC,oBAAoB;AAWtB,IAAMC,oBAAoB;;UAEhBC,sBAAAA;AACf,QAAMC,uBAAuB,GAAGC,oBAAAA;EAEzB,MAAMC,eAAeC,GAAEC,YAAW,EAAW,GAAGJ,oBAAAA,WAA+B;IACpFK,OAAOF,GAAEG,OAAO;MACdC,MAAMJ,GAAEK,SAASL,GAAEM,MAAM;MACzBC,SAASC;IACX,CAAA;IACAC,QAAQT,GAAEG,OAAO;MACfO,QAAQC;IACV,CAAA;EACF,CAAA,EAAA;EAAI;uBARSZ,SAAAA;AASf,GAZiBH,wBAAAA,sBAAAA,CAAAA,EAAAA;AAcV,IAAMgB,eAAe,CAACF,WAAAA;AAC3B,SAAOG,aAAaH,MAAAA,KAAWI,aAAaH,gBAAgBD,MAAAA;AAC9D;AAGO,IAAMK,8BAA8Bf,GAAEgB,QAC3ChB,GAAEG,OAAO;EACPc,kBAAkBjB,GAAEK,SAASL,GAAEkB,OAAO,EAAEC,KAAKnB,GAAEoB,uBAAuB,MAAM,IAAA,CAAA;AAC9E,CAAA,CAAA;",
|
|
6
|
+
"names": ["Expando", "Ref", "S", "TypedObject", "TranscriptSchema", "S", "Struct", "name", "String", "started", "optional", "ended", "queue", "Ref", "Expando", "TranscriptType", "TypedObject", "typename", "version", "fields", "TranscriptHeader", "SpaceIdSchema", "S", "isInstanceOf", "isLiveObject", "TRANSCRIPTION_URL", "TranscriptionAction", "TRANSCRIPTION_ACTION", "TRANSCRIPTION_PLUGIN", "Create", "S", "TaggedClass", "input", "Struct", "name", "optional", "String", "spaceId", "SpaceIdSchema", "output", "object", "TranscriptType", "isTranscript", "isLiveObject", "isInstanceOf", "TranscriptionSettingsSchema", "mutable", "entityExtraction", "Boolean", "pipe", "withConstructorDefault"]
|
|
7
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import {
|
|
2
2
|
TranscriptContainer,
|
|
3
3
|
TranscriptionSettings
|
|
4
|
-
} from "./chunk-
|
|
4
|
+
} from "./chunk-IWZ7C3OA.mjs";
|
|
5
5
|
import {
|
|
6
6
|
AppGraphBuilder,
|
|
7
7
|
IntentResolver,
|
|
@@ -17,16 +17,16 @@ import {
|
|
|
17
17
|
useQueueModelAdapter,
|
|
18
18
|
useTranscriber,
|
|
19
19
|
useVoiceInput
|
|
20
|
-
} from "./chunk-
|
|
20
|
+
} from "./chunk-BSZSM6KA.mjs";
|
|
21
21
|
import {
|
|
22
22
|
MediaStreamRecorder,
|
|
23
23
|
Transcriber,
|
|
24
24
|
TranscriptionManager
|
|
25
|
-
} from "./chunk-
|
|
25
|
+
} from "./chunk-KSUYWWIU.mjs";
|
|
26
26
|
import "./chunk-KLD7O5GJ.mjs";
|
|
27
27
|
import {
|
|
28
28
|
TranscriptType
|
|
29
|
-
} from "./chunk-
|
|
29
|
+
} from "./chunk-YRWBY6UK.mjs";
|
|
30
30
|
import {
|
|
31
31
|
TranscriptionCapabilities
|
|
32
32
|
} from "./chunk-LTIQFX4W.mjs";
|
|
@@ -41,7 +41,7 @@ import { QueueImpl } from "@dxos/echo-db";
|
|
|
41
41
|
import { isInstanceOf } from "@dxos/echo-schema";
|
|
42
42
|
import { ClientCapabilities, ClientEvents } from "@dxos/plugin-client";
|
|
43
43
|
import { getSpace } from "@dxos/react-client/echo";
|
|
44
|
-
import {
|
|
44
|
+
import { MessageType } from "@dxos/schema";
|
|
45
45
|
|
|
46
46
|
// packages/plugins/plugin-transcription/src/translations.ts
|
|
47
47
|
var translations_default = [
|
|
@@ -91,7 +91,7 @@ var TranscriptionPlugin = () => definePlugin(meta, [
|
|
|
91
91
|
const members = space?.members.get().map((member) => member.identity) ?? [];
|
|
92
92
|
const queue = new QueueImpl(client.edge, transcript.queue.dxn);
|
|
93
93
|
await queue.refresh();
|
|
94
|
-
const content = queue.items.filter((message) => isInstanceOf(
|
|
94
|
+
const content = queue.items.filter((message) => isInstanceOf(MessageType, message)).flatMap((message, index) => renderMarkdown(members)(message, index)).join("\n\n");
|
|
95
95
|
return content;
|
|
96
96
|
}
|
|
97
97
|
}
|