@dxos/plugin-transcription 0.8.3 → 0.8.4-main.1068cf700f

package/package.json

@@ -1,27 +1,42 @@
 {
   "name": "@dxos/plugin-transcription",
-  "version": "0.8.3",
+  "version": "0.8.4-main.1068cf700f",
   "description": "Voice transcription.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
   "license": "MIT",
   "author": "DXOS.org",
   "type": "module",
   "exports": {
     ".": {
-      "types": "./dist/types/src/index.d.ts",
       "browser": "./dist/lib/browser/index.mjs",
-      "node": "./dist/lib/node-esm/index.mjs"
+      "node": "./dist/lib/node-esm/index.mjs",
+      "source": "./src/index.ts",
+      "types": "./dist/types/src/index.d.ts"
+    },
+    "./blueprints": {
+      "browser": "./dist/lib/browser/blueprints/index.mjs",
+      "node": "./dist/lib/node-esm/blueprints/index.mjs",
+      "source": "./src/blueprints/index.ts",
+      "types": "./dist/types/src/blueprints/index.d.ts"
     },
     "./types": {
-      "types": "./dist/types/src/types/index.d.ts",
       "browser": "./dist/lib/browser/types/index.mjs",
-      "node": "./dist/lib/node-esm/types/index.mjs"
+      "node": "./dist/lib/node-esm/types/index.mjs",
+      "source": "./src/types/index.ts",
+      "types": "./dist/types/src/types/index.d.ts"
     }
   },
   "types": "dist/types/src/index.d.ts",
   "typesVersions": {
     "*": {
+      "blueprints": [
+        "dist/types/src/blueprints/index.d.ts"
+      ],
       "types": [
         "dist/types/src/types/index.d.ts"
       ]
@@ -32,85 +47,86 @@
     "src"
   ],
   "dependencies": {
-    "@codemirror/state": "^6.4.1",
-    "@codemirror/view": "^6.34.1",
-    "@preact-signals/safe-react": "^0.9.0",
-    "@preact/signals-core": "^1.9.0",
+    "@codemirror/state": "^6.5.2",
+    "@codemirror/view": "^6.38.4",
     "date-fns": "^3.3.1",
     "extendable-media-recorder": "^9.2.19",
     "extendable-media-recorder-wav-encoder": "^7.0.123",
     "main-thread-scheduling": "^14.1.1",
-    "react-resize-detector": "^12.0.2",
+    "react-resize-detector": "^11.0.1",
     "react-router-dom": "^6.4.0",
     "react-use": "^17.5.1",
     "wavefile": "^11.0.0",
-    "@dxos/ai": "0.8.3",
-    "@dxos/app-framework": "0.8.3",
-    "@dxos/artifact": "0.8.3",
-    "@dxos/assistant": "0.8.3",
-    "@dxos/async": "0.8.3",
-    "@dxos/client": "0.8.3",
-    "@dxos/codec-protobuf": "0.8.3",
-    "@dxos/context": "0.8.3",
-    "@dxos/debug": "0.8.3",
-    "@dxos/display-name": "0.8.3",
-    "@dxos/echo": "0.8.3",
-    "@dxos/echo-db": "0.8.3",
-    "@dxos/echo-schema": "0.8.3",
-    "@dxos/invariant": "0.8.3",
-    "@dxos/functions": "0.8.3",
-    "@dxos/live-object": "0.8.3",
-    "@dxos/keys": "0.8.3",
-    "@dxos/local-storage": "0.8.3",
-    "@dxos/log": "0.8.3",
-    "@dxos/plugin-client": "0.8.3",
-    "@dxos/plugin-deck": "0.8.3",
-    "@dxos/plugin-graph": "0.8.3",
-    "@dxos/plugin-markdown": "0.8.3",
-    "@dxos/plugin-thread": "0.8.3",
-    "@dxos/protocols": "0.8.3",
-    "@dxos/react-client": "0.8.3",
-    "@dxos/react-edge-client": "0.8.3",
-    "@dxos/plugin-space": "0.8.3",
-    "@dxos/react-ui-attention": "0.8.3",
-    "@dxos/react-ui-editor": "0.8.3",
-    "@dxos/react-ui-components": "0.8.3",
-    "@dxos/react-ui-stack": "0.8.3",
-    "@dxos/react-ui-grid": "0.8.3",
-    "@dxos/schema": "0.8.3",
-    "@dxos/tracing": "0.8.3",
-    "@dxos/react-ui-sfx": "0.8.3",
-    "@dxos/react-ui-form": "0.8.3",
-    "@dxos/util": "0.8.3"
+    "@dxos/ai": "0.8.4-main.1068cf700f",
+    "@dxos/app-framework": "0.8.4-main.1068cf700f",
+    "@dxos/assistant": "0.8.4-main.1068cf700f",
+    "@dxos/async": "0.8.4-main.1068cf700f",
+    "@dxos/app-toolkit": "0.8.4-main.1068cf700f",
+    "@dxos/blueprints": "0.8.4-main.1068cf700f",
+    "@dxos/client": "0.8.4-main.1068cf700f",
+    "@dxos/codec-protobuf": "0.8.4-main.1068cf700f",
+    "@dxos/context": "0.8.4-main.1068cf700f",
+    "@dxos/debug": "0.8.4-main.1068cf700f",
+    "@dxos/display-name": "0.8.4-main.1068cf700f",
+    "@dxos/echo": "0.8.4-main.1068cf700f",
+    "@dxos/echo-db": "0.8.4-main.1068cf700f",
+    "@dxos/functions": "0.8.4-main.1068cf700f",
+    "@dxos/functions-runtime": "0.8.4-main.1068cf700f",
+    "@dxos/invariant": "0.8.4-main.1068cf700f",
+    "@dxos/log": "0.8.4-main.1068cf700f",
+    "@dxos/operation": "0.8.4-main.1068cf700f",
+    "@dxos/keys": "0.8.4-main.1068cf700f",
+    "@dxos/plugin-client": "0.8.4-main.1068cf700f",
+    "@dxos/plugin-deck": "0.8.4-main.1068cf700f",
+    "@dxos/plugin-graph": "0.8.4-main.1068cf700f",
+    "@dxos/plugin-markdown": "0.8.4-main.1068cf700f",
+    "@dxos/plugin-space": "0.8.4-main.1068cf700f",
+    "@dxos/plugin-thread": "0.8.4-main.1068cf700f",
+    "@dxos/protocols": "0.8.4-main.1068cf700f",
+    "@dxos/react-client": "0.8.4-main.1068cf700f",
+    "@dxos/react-edge-client": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-attention": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-components": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-form": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-editor": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-grid": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-mosaic": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-sfx": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-stack": "0.8.4-main.1068cf700f",
+    "@dxos/schema": "0.8.4-main.1068cf700f",
+    "@dxos/tracing": "0.8.4-main.1068cf700f",
+    "@dxos/ui-editor": "0.8.4-main.1068cf700f",
+    "@dxos/types": "0.8.4-main.1068cf700f",
+    "@dxos/util": "0.8.4-main.1068cf700f"
   },
   "devDependencies": {
-    "@effect-rx/rx-react": "^0.34.1",
-    "@effect/platform": "0.80.12",
+    "@effect-atom/atom-react": "^0.5.0",
+    "@effect/platform": "0.94.4",
     "@types/dom-speech-recognition": "^0.0.6",
-    "@types/react": "~18.2.0",
-    "effect": "3.14.21",
-    "react": "~18.2.0",
-    "react-dom": "~18.2.0",
+    "@types/react": "~19.2.7",
+    "effect": "3.19.16",
+    "react": "~19.2.3",
+    "react-dom": "~19.2.3",
     "tailwindcss": "~3.4.1",
-    "@dxos/plugin-preview": "0.8.3",
-    "@dxos/react-ui": "0.8.3",
-    "@dxos/random": "0.8.3",
-    "@dxos/plugin-storybook-layout": "0.8.3",
-    "@dxos/react-ui-form": "0.8.3",
-    "@dxos/react-ui-syntax-highlighter": "0.8.3",
-    "@dxos/plugin-theme": "0.8.3",
-    "@dxos/react-ui-theme": "0.8.3",
-    "@dxos/storybook-utils": "0.8.3",
-    "@dxos/test-utils": "0.8.3"
+    "@dxos/plugin-preview": "0.8.4-main.1068cf700f",
+    "@dxos/plugin-testing": "0.8.4-main.1068cf700f",
+    "@dxos/plugin-theme": "0.8.4-main.1068cf700f",
+    "@dxos/random": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-form": "0.8.4-main.1068cf700f",
+    "@dxos/react-ui-syntax-highlighter": "0.8.4-main.1068cf700f",
+    "@dxos/storybook-utils": "0.8.4-main.1068cf700f",
+    "@dxos/test-utils": "0.8.4-main.1068cf700f",
+    "@dxos/ui-theme": "0.8.4-main.1068cf700f"
   },
-  "optionalDependencies": {
-    "@effect-rx/rx-react": "^0.34.1",
-    "@effect/platform": "0.80.12",
-    "effect": "^3.13.3",
-    "react": "~18.2.0",
-    "react-dom": "~18.2.0",
-    "@dxos/react-ui-theme": "0.8.3",
-    "@dxos/react-ui": "0.8.3"
+  "peerDependencies": {
+    "@effect-atom/atom-react": "^0.5.0",
+    "@effect/platform": "0.94.4",
+    "effect": "3.19.16",
+    "react": "~19.2.3",
+    "react-dom": "~19.2.3",
+    "@dxos/react-ui": "0.8.4-main.1068cf700f",
+    "@dxos/ui-theme": "0.8.4-main.1068cf700f"
   },
   "publishConfig": {
     "access": "public"

package/src/TranscriptionPlugin.tsx

@@ -2,66 +2,48 @@
 // Copyright 2023 DXOS.org
 //
 
-import { Capabilities, Events, contributes, defineModule, definePlugin } from '@dxos/app-framework';
+import { Plugin } from '@dxos/app-framework';
+import { AppActivationEvents, AppPlugin } from '@dxos/app-toolkit';
 import { Obj } from '@dxos/echo';
-import { ClientCapabilities, ClientEvents } from '@dxos/plugin-client';
 import { getSpace } from '@dxos/react-client/echo';
-import { DataType } from '@dxos/schema';
+import { Message, Transcript } from '@dxos/types';
 
-import { IntentResolver, ReactSurface, Transcriber } from './capabilities';
-import { renderMarkdown } from './components';
+import { BlueprintDefinition, OperationResolver, ReactSurface, Transcriber } from './capabilities';
 import { meta } from './meta';
-import translations from './translations';
-import { TranscriptType } from './types';
+import { translations } from './translations';
+import { renderByline } from './util';
 
-export const TranscriptionPlugin = () =>
-  definePlugin(meta, [
-    defineModule({
-      id: `${meta.id}/module/translations`,
-      activatesOn: Events.SetupTranslations,
-      activate: () => contributes(Capabilities.Translations, translations),
-    }),
-    defineModule({
-      id: `${meta.id}/module/metadata`,
-      activatesOn: Events.SetupMetadata,
-      activate: (context) =>
-        contributes(Capabilities.Metadata, {
-          id: TranscriptType.typename,
-          metadata: {
-            icon: 'ph--subtitles--regular',
-            // TODO(wittjosiah): Factor out. Artifact? Separate capability?
-            getTextContent: async (transcript: TranscriptType) => {
-              const space = getSpace(transcript);
-              const members = space?.members.get().map((member) => member.identity) ?? [];
-              const queue = space?.queues.get<DataType.Message>(transcript.queue.dxn);
-              await queue?.refresh();
-              const content = queue?.objects
-                .filter((message) => Obj.instanceOf(DataType.Message, message))
-                .flatMap((message, index) => renderMarkdown(members)(message, index))
-                .join('\n\n');
-              return content;
-            },
-          },
-        }),
-    }),
-    defineModule({
-      id: `${meta.id}/module/schema`,
-      activatesOn: ClientEvents.SetupSchema,
-      activate: () => [contributes(ClientCapabilities.Schema, [TranscriptType])],
-    }),
-    defineModule({
-      id: `${meta.id}/module/react-surface`,
-      activatesOn: Events.SetupReactSurface,
-      activate: ReactSurface,
-    }),
-    defineModule({
-      id: `${meta.id}/module/intent-resolver`,
-      activatesOn: Events.SetupIntentResolver,
-      activate: IntentResolver,
-    }),
-    defineModule({
-      id: `${meta.id}/module/transcription`,
-      activatesOn: Events.SetupAppGraph,
-      activate: Transcriber,
-    }),
-  ]);
+export const TranscriptionPlugin = Plugin.define(meta).pipe(
+  AppPlugin.addBlueprintDefinitionModule({ activate: BlueprintDefinition }),
+  AppPlugin.addMetadataModule({
+    metadata: {
+      id: Transcript.Transcript.typename,
+      metadata: {
+        icon: 'ph--subtitles--regular',
+        iconHue: 'sky',
+        // TODO(wittjosiah): Factor out. Artifact? Separate capability?
+        getTextContent: async (transcript: Transcript.Transcript) => {
+          const space = getSpace(transcript);
+          const members = space?.members.get().map((member) => member.identity) ?? [];
+          const queue = space?.queues.get<Message.Message>(transcript.queue.dxn);
+          await queue?.refresh();
+          const content = queue?.objects
+            .filter((message) => Obj.instanceOf(Message.Message, message))
+            .flatMap((message, index) => renderByline(members)(message, index))
+            .join('\n\n');
+          return content;
+        },
+      },
+    },
+  }),
+  AppPlugin.addOperationResolverModule({ activate: OperationResolver }),
+  AppPlugin.addSchemaModule({ schema: [Transcript.Transcript] }),
+  AppPlugin.addSurfaceModule({ activate: ReactSurface }),
+  AppPlugin.addTranslationsModule({ translations }),
+  Plugin.addModule({
+    id: 'transcription',
+    activatesOn: AppActivationEvents.SetupAppGraph,
+    activate: Transcriber,
+  }),
+  Plugin.make,
+);

package/src/blueprints/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * as TranscriptionBlueprint from './transcription-blueprint';

package/src/blueprints/transcription-blueprint.ts

@@ -0,0 +1,25 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Blueprint, Template } from '@dxos/blueprints';
+import { type FunctionDefinition } from '@dxos/functions';
+import { trim } from '@dxos/util';
+
+import { open, summarize } from '../functions';
+
+export const functions: FunctionDefinition[] = [open, summarize];
+
+export const Key = 'dxos.org/blueprint/transcription';
+
+export const make = () =>
+  Blueprint.make({
+    key: Key,
+    name: 'Transcription',
+    tools: Blueprint.toolDefinitions({ functions }),
+    instructions: Template.make({
+      source: trim`
+        You can open and summarize a meeting transcript.
+      `,
+    }),
+  });

package/src/capabilities/blueprint-definition/blueprint-definition.ts

@@ -0,0 +1,24 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Capability } from '@dxos/app-framework';
+import { AppCapabilities } from '@dxos/app-toolkit';
+
+import { TranscriptionBlueprint } from '../../blueprints';
+
+export type BlueprintCapabilities = [
+  Capability.Capability<typeof AppCapabilities.Functions>,
+  Capability.Capability<typeof AppCapabilities.BlueprintDefinition>,
+];
+
+const blueprintDefinition = Capability.makeModule<[], BlueprintCapabilities>(() =>
+  Effect.succeed([
+    Capability.contributes(AppCapabilities.Functions, TranscriptionBlueprint.functions),
+    Capability.contributes(AppCapabilities.BlueprintDefinition, TranscriptionBlueprint.make()),
+  ]),
+);
+
+export default blueprintDefinition;

package/src/capabilities/blueprint-definition/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const BlueprintDefinition = Capability.lazy('BlueprintDefinition', () => import('./blueprint-definition'));

package/src/capabilities/index.ts

@@ -2,10 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import { lazy } from '@dxos/app-framework';
-
-export const IntentResolver = lazy(() => import('./intent-resolver'));
-export const ReactSurface = lazy(() => import('./react-surface'));
-export const Transcriber = lazy(() => import('./transcriber'));
-
-export * from './capabilities';
+export * from './blueprint-definition';
+export * from './operation-resolver';
+export * from './react-surface';
+export * from './transcriber';

package/src/capabilities/operation-handler/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const OperationResolver = Capability.lazy('OperationResolver', () => import('./operation-handler'));

package/src/capabilities/operation-handler/operation-handler.ts

@@ -0,0 +1,25 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
+import { OperationResolver } from '@dxos/operation';
+import { Transcript } from '@dxos/types';
+
+import { TranscriptOperation } from '../../types';
+
+export default Capability.makeModule(() =>
+  Effect.succeed(
+    Capability.contributes(Capabilities.OperationResolver, [
+      OperationResolver.make({
+        operation: TranscriptOperation.Create,
+        handler: ({ space }) =>
+          Effect.succeed({
+            object: Transcript.make(space.queues.create().dxn),
+          }),
+      }),
+    ]),
+  ),
+);

package/src/capabilities/operation-resolver/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const OperationResolver = Capability.lazy('OperationResolver', () => import('./operation-resolver'));

package/src/capabilities/operation-resolver/operation-resolver.ts

@@ -0,0 +1,25 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
+import { OperationResolver } from '@dxos/operation';
+import { Transcript } from '@dxos/types';
+
+import { TranscriptOperation } from '../../types';
+
+export default Capability.makeModule(() =>
+  Effect.succeed(
+    Capability.contributes(Capabilities.OperationResolver, [
+      OperationResolver.make({
+        operation: TranscriptOperation.Create,
+        handler: ({ space }) =>
+          Effect.succeed({
+            object: Transcript.make(space.queues.create().dxn),
+          }),
+      }),
+    ]),
+  ),
+);

package/src/capabilities/react-surface/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const ReactSurface = Capability.lazy('ReactSurface', () => import('./react-surface'));

package/src/capabilities/react-surface/react-surface.tsx

@@ -0,0 +1,28 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import React from 'react';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
+import { Surface } from '@dxos/app-framework/ui';
+import { Obj } from '@dxos/echo';
+import { Transcript } from '@dxos/types';
+
+import { TranscriptionContainer } from '../../components';
+import { meta } from '../../meta';
+
+export default Capability.makeModule(() =>
+  Effect.succeed(
+    Capability.contributes(Capabilities.ReactSurface, [
+      Surface.create({
+        id: `${meta.id}/article/transcript`,
+        role: ['article', 'section'],
+        filter: (data): data is { subject: Transcript.Transcript } =>
+          Obj.instanceOf(Transcript.Transcript, data.subject),
+        component: ({ data, role }) => <TranscriptionContainer role={role} subject={data.subject} />,
+      }),
+    ]),
+  ),
+);

package/src/capabilities/transcriber/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const Transcriber = Capability.lazy('Transcriber', () => import('./transcriber'));

package/src/capabilities/transcriber/transcriber.ts

@@ -0,0 +1,85 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Capabilities, Capability } from '@dxos/app-framework';
+import { ClientCapabilities } from '@dxos/plugin-client';
+
+import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../../transcriber';
+import { TranscriptionCapabilities } from '../../types';
+
+// TODO(burdon): Move to config?
+
+/**
+ * Length of the chunk in ms.
+ */
+const RECORD_INTERVAL = 200;
+
+/**
+ * Number of chunks to save before the user starts speaking.
+ */
+const PREFIXED_CHUNKS_AMOUNT = 10;
+
+/**
+ * Number of chunks to transcribe automatically after.
+ * Combined should be mess than 25MB or whisper would fail.
+ */
+const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
+
+/**
+ * Records audio while user is speaking and transcribes it after user is done speaking.
+ */
+export default Capability.makeModule(
+  Effect.fnUntraced(function* () {
+    // Get context for lazy capability access in callbacks.
+    const capabilities = yield* Capability.Service;
+    const registry = yield* Capability.get(Capabilities.AtomRegistry);
+
+    const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({
+      audioStreamTrack,
+      onSegments,
+      transcriberConfig,
+      recorderConfig,
+    }) => {
+      // Initialize audio transcription.
+      return new Transcriber({
+        config: {
+          transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
+          prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
+          ...transcriberConfig,
+        },
+        recorder: new MediaStreamRecorder({
+          mediaStreamTrack: audioStreamTrack,
+          config: {
+            interval: RECORD_INTERVAL,
+            ...recorderConfig,
+          },
+        }),
+        onSegments,
+      });
+    };
+
+    const getTranscriptionManager: TranscriptionCapabilities.GetTranscriptionManager = ({ messageEnricher }) => {
+      const client = capabilities.get(ClientCapabilities.Client);
+      const transcriptionManager = new TranscriptionManager({
+        edgeClient: client.edge,
+        messageEnricher,
+        registry,
+      });
+
+      const identity = client.halo.identity.get();
+      if (identity) {
+        transcriptionManager.setIdentityDid(identity.did);
+      }
+
+      return transcriptionManager;
+    };
+
+    return [
+      Capability.contributes(TranscriptionCapabilities.Transcriber, getTranscriber),
+      Capability.contributes(TranscriptionCapabilities.TranscriptionManager, getTranscriptionManager),
+    ];
+  }),
+);