@dxos/plugin-transcription 0.8.4-main.ae835ea → 0.8.4-main.bc674ce

package/package.json

@@ -1,29 +1,42 @@
 {
   "name": "@dxos/plugin-transcription",
-  "version": "0.8.4-main.ae835ea",
+  "version": "0.8.4-main.bc674ce",
   "description": "Voice transcription.",
   "homepage": "https://dxos.org",
   "bugs": "https://github.com/dxos/dxos/issues",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dxos/dxos"
+  },
   "license": "MIT",
   "author": "DXOS.org",
   "type": "module",
   "exports": {
     ".": {
-      "source": "./src/index.ts",
-      "types": "./dist/types/src/index.d.ts",
       "browser": "./dist/lib/browser/index.mjs",
-      "node": "./dist/lib/node-esm/index.mjs"
+      "node": "./dist/lib/node-esm/index.mjs",
+      "source": "./src/index.ts",
+      "types": "./dist/types/src/index.d.ts"
+    },
+    "./blueprints": {
+      "browser": "./dist/lib/browser/blueprints/index.mjs",
+      "node": "./dist/lib/node-esm/blueprints/index.mjs",
+      "source": "./src/blueprints/index.ts",
+      "types": "./dist/types/src/blueprints/index.d.ts"
     },
     "./types": {
-      "source": "./src/types/index.ts",
-      "types": "./dist/types/src/types/index.d.ts",
       "browser": "./dist/lib/browser/types/index.mjs",
-      "node": "./dist/lib/node-esm/types/index.mjs"
+      "node": "./dist/lib/node-esm/types/index.mjs",
+      "source": "./src/types/index.ts",
+      "types": "./dist/types/src/types/index.d.ts"
     }
   },
   "types": "dist/types/src/index.d.ts",
   "typesVersions": {
     "*": {
+      "blueprints": [
+        "dist/types/src/blueprints/index.d.ts"
+      ],
       "types": [
         "dist/types/src/types/index.d.ts"
       ]
@@ -36,8 +49,6 @@
   "dependencies": {
     "@codemirror/state": "^6.5.2",
     "@codemirror/view": "^6.38.4",
-    "@preact-signals/safe-react": "^0.9.0",
-    "@preact/signals-core": "^1.12.1",
     "date-fns": "^3.3.1",
     "extendable-media-recorder": "^9.2.19",
     "extendable-media-recorder-wav-encoder": "^7.0.123",
@@ -46,72 +57,75 @@
     "react-router-dom": "^6.4.0",
     "react-use": "^17.5.1",
     "wavefile": "^11.0.0",
-    "@dxos/ai": "0.8.4-main.ae835ea",
-    "@dxos/app-framework": "0.8.4-main.ae835ea",
-    "@dxos/assistant": "0.8.4-main.ae835ea",
-    "@dxos/client": "0.8.4-main.ae835ea",
-    "@dxos/async": "0.8.4-main.ae835ea",
-    "@dxos/blueprints": "0.8.4-main.ae835ea",
-    "@dxos/codec-protobuf": "0.8.4-main.ae835ea",
-    "@dxos/context": "0.8.4-main.ae835ea",
-    "@dxos/display-name": "0.8.4-main.ae835ea",
-    "@dxos/echo": "0.8.4-main.ae835ea",
-    "@dxos/debug": "0.8.4-main.ae835ea",
-    "@dxos/echo-db": "0.8.4-main.ae835ea",
-    "@dxos/functions": "0.8.4-main.ae835ea",
-    "@dxos/invariant": "0.8.4-main.ae835ea",
-    "@dxos/keys": "0.8.4-main.ae835ea",
-    "@dxos/local-storage": "0.8.4-main.ae835ea",
-    "@dxos/live-object": "0.8.4-main.ae835ea",
-    "@dxos/log": "0.8.4-main.ae835ea",
-    "@dxos/plugin-client": "0.8.4-main.ae835ea",
-    "@dxos/plugin-deck": "0.8.4-main.ae835ea",
-    "@dxos/plugin-markdown": "0.8.4-main.ae835ea",
-    "@dxos/plugin-space": "0.8.4-main.ae835ea",
-    "@dxos/protocols": "0.8.4-main.ae835ea",
-    "@dxos/plugin-thread": "0.8.4-main.ae835ea",
-    "@dxos/plugin-graph": "0.8.4-main.ae835ea",
-    "@dxos/react-client": "0.8.4-main.ae835ea",
-    "@dxos/react-edge-client": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-attention": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-components": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-editor": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-form": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-grid": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-sfx": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-stack": "0.8.4-main.ae835ea",
-    "@dxos/tracing": "0.8.4-main.ae835ea",
-    "@dxos/util": "0.8.4-main.ae835ea",
-    "@dxos/schema": "0.8.4-main.ae835ea"
+    "@dxos/ai": "0.8.4-main.bc674ce",
+    "@dxos/app-framework": "0.8.4-main.bc674ce",
+    "@dxos/async": "0.8.4-main.bc674ce",
+    "@dxos/assistant": "0.8.4-main.bc674ce",
+    "@dxos/blueprints": "0.8.4-main.bc674ce",
+    "@dxos/client": "0.8.4-main.bc674ce",
+    "@dxos/codec-protobuf": "0.8.4-main.bc674ce",
+    "@dxos/context": "0.8.4-main.bc674ce",
+    "@dxos/display-name": "0.8.4-main.bc674ce",
+    "@dxos/debug": "0.8.4-main.bc674ce",
+    "@dxos/echo": "0.8.4-main.bc674ce",
+    "@dxos/functions": "0.8.4-main.bc674ce",
+    "@dxos/echo-db": "0.8.4-main.bc674ce",
+    "@dxos/functions-runtime": "0.8.4-main.bc674ce",
+    "@dxos/keys": "0.8.4-main.bc674ce",
+    "@dxos/log": "0.8.4-main.bc674ce",
+    "@dxos/invariant": "0.8.4-main.bc674ce",
+    "@dxos/operation": "0.8.4-main.bc674ce",
+    "@dxos/plugin-client": "0.8.4-main.bc674ce",
+    "@dxos/plugin-deck": "0.8.4-main.bc674ce",
+    "@dxos/plugin-graph": "0.8.4-main.bc674ce",
+    "@dxos/plugin-markdown": "0.8.4-main.bc674ce",
+    "@dxos/plugin-thread": "0.8.4-main.bc674ce",
+    "@dxos/protocols": "0.8.4-main.bc674ce",
+    "@dxos/react-client": "0.8.4-main.bc674ce",
+    "@dxos/react-edge-client": "0.8.4-main.bc674ce",
+    "@dxos/plugin-space": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-attention": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-components": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-editor": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-form": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-mosaic": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-sfx": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-grid": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-stack": "0.8.4-main.bc674ce",
+    "@dxos/schema": "0.8.4-main.bc674ce",
+    "@dxos/types": "0.8.4-main.bc674ce",
+    "@dxos/ui-editor": "0.8.4-main.bc674ce",
+    "@dxos/tracing": "0.8.4-main.bc674ce",
+    "@dxos/util": "0.8.4-main.bc674ce"
   },
   "devDependencies": {
-    "@effect-rx/rx-react": "0.42.4",
-    "@effect/platform": "0.92.1",
+    "@effect-atom/atom-react": "^0.4.6",
+    "@effect/platform": "0.93.6",
     "@types/dom-speech-recognition": "^0.0.6",
-    "@types/react": "~19.2.2",
-    "effect": "3.18.3",
-    "react": "~19.2.0",
-    "react-dom": "~19.2.0",
+    "@types/react": "~19.2.7",
+    "effect": "3.19.11",
+    "react": "~19.2.3",
+    "react-dom": "~19.2.3",
     "tailwindcss": "~3.4.1",
-    "@dxos/plugin-preview": "0.8.4-main.ae835ea",
-    "@dxos/plugin-storybook-layout": "0.8.4-main.ae835ea",
-    "@dxos/plugin-theme": "0.8.4-main.ae835ea",
-    "@dxos/random": "0.8.4-main.ae835ea",
-    "@dxos/react-ui": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-form": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-syntax-highlighter": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-theme": "0.8.4-main.ae835ea",
-    "@dxos/storybook-utils": "0.8.4-main.ae835ea",
-    "@dxos/test-utils": "0.8.4-main.ae835ea"
+    "@dxos/plugin-preview": "0.8.4-main.bc674ce",
+    "@dxos/plugin-testing": "0.8.4-main.bc674ce",
+    "@dxos/plugin-theme": "0.8.4-main.bc674ce",
+    "@dxos/random": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-form": "0.8.4-main.bc674ce",
+    "@dxos/react-ui": "0.8.4-main.bc674ce",
+    "@dxos/storybook-utils": "0.8.4-main.bc674ce",
+    "@dxos/react-ui-syntax-highlighter": "0.8.4-main.bc674ce",
+    "@dxos/ui-theme": "0.8.4-main.bc674ce",
+    "@dxos/test-utils": "0.8.4-main.bc674ce"
   },
   "peerDependencies": {
-    "@effect-rx/rx-react": "^0.34.1",
-    "@effect/platform": "^0.80.12",
-    "effect": "^3.13.3",
-    "react": "^19.0.0",
-    "react-dom": "^19.0.0",
-    "@dxos/react-ui": "0.8.4-main.ae835ea",
-    "@dxos/react-ui-theme": "0.8.4-main.ae835ea"
+    "@effect-atom/atom-react": "^0.4.6",
+    "@effect/platform": "0.93.6",
+    "effect": "3.19.11",
+    "react": "~19.2.3",
+    "react-dom": "~19.2.3",
+    "@dxos/react-ui": "0.8.4-main.bc674ce",
+    "@dxos/ui-theme": "0.8.4-main.bc674ce"
   },
   "publishConfig": {
     "access": "public"

package/src/TranscriptionPlugin.tsx

@@ -2,71 +2,47 @@
 // Copyright 2023 DXOS.org
 //
 
-import { Capabilities, Events, contributes, defineModule, definePlugin } from '@dxos/app-framework';
+import { Common, Plugin } from '@dxos/app-framework';
 import { Obj } from '@dxos/echo';
-import { ClientCapabilities, ClientEvents } from '@dxos/plugin-client';
 import { getSpace } from '@dxos/react-client/echo';
-import { DataType } from '@dxos/schema';
+import { Message, Transcript } from '@dxos/types';
 
-import { BlueprintDefinition, IntentResolver, ReactSurface, Transcriber } from './capabilities';
-import { renderByline } from './components';
+import { BlueprintDefinition, OperationResolver, ReactSurface, Transcriber } from './capabilities';
 import { meta } from './meta';
 import { translations } from './translations';
-import { Transcript } from './types';
+import { renderByline } from './util';
 
-export const TranscriptionPlugin = definePlugin(meta, () => [
-  defineModule({
-    id: `${meta.id}/module/translations`,
-    activatesOn: Events.SetupTranslations,
-    activate: () => contributes(Capabilities.Translations, translations),
-  }),
-  defineModule({
-    id: `${meta.id}/module/metadata`,
-    activatesOn: Events.SetupMetadata,
-    activate: () =>
-      contributes(Capabilities.Metadata, {
-        id: Transcript.Transcript.typename,
-        metadata: {
-          icon: 'ph--subtitles--regular',
-          iconHue: 'sky',
-          // TODO(wittjosiah): Factor out. Artifact? Separate capability?
-          getTextContent: async (transcript: Transcript.Transcript) => {
-            const space = getSpace(transcript);
-            const members = space?.members.get().map((member) => member.identity) ?? [];
-            const queue = space?.queues.get<DataType.Message>(transcript.queue.dxn);
-            await queue?.refresh();
-            const content = queue?.objects
-              .filter((message) => Obj.instanceOf(DataType.Message, message))
-              .flatMap((message, index) => renderByline(members)(message, index))
-              .join('\n\n');
-            return content;
-          },
+export const TranscriptionPlugin = Plugin.define(meta).pipe(
+  Common.Plugin.addTranslationsModule({ translations }),
+  Common.Plugin.addMetadataModule({
+    metadata: {
+      id: Transcript.Transcript.typename,
+      metadata: {
+        icon: 'ph--subtitles--regular',
+        iconHue: 'sky',
+        // TODO(wittjosiah): Factor out. Artifact? Separate capability?
+        getTextContent: async (transcript: Transcript.Transcript) => {
+          const space = getSpace(transcript);
+          const members = space?.members.get().map((member) => member.identity) ?? [];
+          const queue = space?.queues.get<Message.Message>(transcript.queue.dxn);
+          await queue?.refresh();
+          const content = queue?.objects
+            .filter((message) => Obj.instanceOf(Message.Message, message))
+            .flatMap((message, index) => renderByline(members)(message, index))
+            .join('\n\n');
+          return content;
         },
-      }),
-  }),
-  defineModule({
-    id: `${meta.id}/module/schema`,
-    activatesOn: ClientEvents.SetupSchema,
-    activate: () => [contributes(ClientCapabilities.Schema, [Transcript.Transcript])],
-  }),
-  defineModule({
-    id: `${meta.id}/module/react-surface`,
-    activatesOn: Events.SetupReactSurface,
-    activate: ReactSurface,
+      },
+    },
   }),
-  defineModule({
-    id: `${meta.id}/module/intent-resolver`,
-    activatesOn: Events.SetupIntentResolver,
-    activate: IntentResolver,
-  }),
-  defineModule({
-    id: `${meta.id}/module/transcription`,
-    activatesOn: Events.SetupAppGraph,
+  Common.Plugin.addSchemaModule({ schema: [Transcript.Transcript] }),
+  Common.Plugin.addSurfaceModule({ activate: ReactSurface }),
+  Common.Plugin.addOperationResolverModule({ activate: OperationResolver }),
+  Plugin.addModule({
+    id: 'transcription',
+    activatesOn: Common.ActivationEvent.SetupAppGraph,
     activate: Transcriber,
   }),
-  defineModule({
-    id: `${meta.id}/module/blueprint`,
-    activatesOn: Events.SetupArtifactDefinition,
-    activate: BlueprintDefinition,
-  }),
-]);
+  Common.Plugin.addBlueprintDefinitionModule({ activate: BlueprintDefinition }),
+  Plugin.make,
+);

package/src/blueprints/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * as TranscriptionBlueprint from './transcription-blueprint';

package/src/blueprints/transcription-blueprint.ts

@@ -0,0 +1,25 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Blueprint, Template } from '@dxos/blueprints';
+import { type FunctionDefinition } from '@dxos/functions';
+import { trim } from '@dxos/util';
+
+import { open, summarize } from '../functions';
+
+export const functions: FunctionDefinition[] = [open, summarize];
+
+export const Key = 'dxos.org/blueprint/transcription';
+
+export const make = () =>
+  Blueprint.make({
+    key: Key,
+    name: 'Transcription',
+    tools: Blueprint.toolDefinitions({ functions }),
+    instructions: Template.make({
+      source: trim`
+        You can open and summarize a meeting transcript.
+      `,
+    }),
+  });

package/src/capabilities/blueprint-definition/blueprint-definition.ts

@@ -0,0 +1,23 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Capability, Common } from '@dxos/app-framework';
+
+import { TranscriptionBlueprint } from '../../blueprints';
+
+export type BlueprintCapabilities = [
+  Capability.Capability<typeof Common.Capability.Functions>,
+  Capability.Capability<typeof Common.Capability.BlueprintDefinition>,
+];
+
+const blueprintDefinition = Capability.makeModule<[], BlueprintCapabilities>(() =>
+  Effect.succeed([
+    Capability.contributes(Common.Capability.Functions, TranscriptionBlueprint.functions),
+    Capability.contributes(Common.Capability.BlueprintDefinition, TranscriptionBlueprint.make()),
+  ]),
+);
+
+export default blueprintDefinition;

package/src/capabilities/blueprint-definition/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const BlueprintDefinition = Capability.lazy('BlueprintDefinition', () => import('./blueprint-definition'));

package/src/capabilities/index.ts

@@ -2,11 +2,7 @@
 // Copyright 2025 DXOS.org
 //
 
-import { lazy } from '@dxos/app-framework';
-
-export const BlueprintDefinition = lazy(() => import('./blueprint-definition'));
-export const IntentResolver = lazy(() => import('./intent-resolver'));
-export const ReactSurface = lazy(() => import('./react-surface'));
-export const Transcriber = lazy(() => import('./transcriber'));
-
-export * from './capabilities';
+export * from './blueprint-definition';
+export * from './operation-resolver';
+export * from './react-surface';
+export * from './transcriber';

package/src/capabilities/operation-handler/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const OperationResolver = Capability.lazy('OperationResolver', () => import('./operation-handler'));

package/src/capabilities/operation-handler/operation-handler.ts

@@ -0,0 +1,25 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Capability, Common } from '@dxos/app-framework';
+import { OperationResolver } from '@dxos/operation';
+import { Transcript } from '@dxos/types';
+
+import { TranscriptOperation } from '../../types';
+
+export default Capability.makeModule(() =>
+  Effect.succeed(
+    Capability.contributes(Common.Capability.OperationResolver, [
+      OperationResolver.make({
+        operation: TranscriptOperation.Create,
+        handler: ({ space }) =>
+          Effect.succeed({
+            object: Transcript.make(space.queues.create().dxn),
+          }),
+      }),
+    ]),
+  ),
+);

package/src/capabilities/operation-resolver/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const OperationResolver = Capability.lazy('OperationResolver', () => import('./operation-resolver'));

package/src/capabilities/operation-resolver/operation-resolver.ts

@@ -0,0 +1,25 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Capability, Common } from '@dxos/app-framework';
+import { OperationResolver } from '@dxos/operation';
+import { Transcript } from '@dxos/types';
+
+import { TranscriptOperation } from '../../types';
+
+export default Capability.makeModule(() =>
+  Effect.succeed(
+    Capability.contributes(Common.Capability.OperationResolver, [
+      OperationResolver.make({
+        operation: TranscriptOperation.Create,
+        handler: ({ space }) =>
+          Effect.succeed({
+            object: Transcript.make(space.queues.create().dxn),
+          }),
+      }),
+    ]),
+  ),
+);

package/src/capabilities/react-surface/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const ReactSurface = Capability.lazy('ReactSurface', () => import('./react-surface'));

package/src/capabilities/react-surface/react-surface.tsx

@@ -0,0 +1,27 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+import React from 'react';
+
+import { Capability, Common } from '@dxos/app-framework';
+import { Obj } from '@dxos/echo';
+import { Transcript } from '@dxos/types';
+
+import { TranscriptionContainer } from '../../components';
+import { meta } from '../../meta';
+
+export default Capability.makeModule(() =>
+  Effect.succeed(
+    Capability.contributes(Common.Capability.ReactSurface, [
+      Common.createSurface({
+        id: `${meta.id}/article/transcript`,
+        role: ['article', 'section'],
+        filter: (data): data is { subject: Transcript.Transcript } =>
+          Obj.instanceOf(Transcript.Transcript, data.subject),
+        component: ({ data, role }) => <TranscriptionContainer role={role} subject={data.subject} />,
+      }),
+    ]),
+  ),
+);

package/src/capabilities/transcriber/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capability } from '@dxos/app-framework';
+
+export const Transcriber = Capability.lazy('Transcriber', () => import('./transcriber'));

package/src/capabilities/transcriber/transcriber.ts

@@ -0,0 +1,85 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import * as Effect from 'effect/Effect';
+
+import { Capability, Common } from '@dxos/app-framework';
+import { ClientCapabilities } from '@dxos/plugin-client';
+
+import { MediaStreamRecorder, Transcriber, TranscriptionManager } from '../../transcriber';
+import { TranscriptionCapabilities } from '../../types';
+
+// TODO(burdon): Move to config?
+
+/**
+ * Length of the chunk in ms.
+ */
+const RECORD_INTERVAL = 200;
+
+/**
+ * Number of chunks to save before the user starts speaking.
+ */
+const PREFIXED_CHUNKS_AMOUNT = 10;
+
+/**
+ * Number of chunks to transcribe automatically after.
+ * Combined should be mess than 25MB or whisper would fail.
+ */
+const TRANSCRIBE_AFTER_CHUNKS_AMOUNT = 50;
+
+/**
+ * Records audio while user is speaking and transcribes it after user is done speaking.
+ */
+export default Capability.makeModule(
+  Effect.fnUntraced(function* () {
+    // Get context for lazy capability access in callbacks.
+    const capabilities = yield* Capability.Service;
+    const registry = yield* Capability.get(Common.Capability.AtomRegistry);
+
+    const getTranscriber: TranscriptionCapabilities.GetTranscriber = ({
+      audioStreamTrack,
+      onSegments,
+      transcriberConfig,
+      recorderConfig,
+    }) => {
+      // Initialize audio transcription.
+      return new Transcriber({
+        config: {
+          transcribeAfterChunksAmount: TRANSCRIBE_AFTER_CHUNKS_AMOUNT,
+          prefixBufferChunksAmount: PREFIXED_CHUNKS_AMOUNT,
+          ...transcriberConfig,
+        },
+        recorder: new MediaStreamRecorder({
+          mediaStreamTrack: audioStreamTrack,
+          config: {
+            interval: RECORD_INTERVAL,
+            ...recorderConfig,
+          },
+        }),
+        onSegments,
+      });
+    };
+
+    const getTranscriptionManager: TranscriptionCapabilities.GetTranscriptionManager = ({ messageEnricher }) => {
+      const client = capabilities.get(ClientCapabilities.Client);
+      const transcriptionManager = new TranscriptionManager({
+        edgeClient: client.edge,
+        messageEnricher,
+        registry,
+      });
+
+      const identity = client.halo.identity.get();
+      if (identity) {
+        transcriptionManager.setIdentityDid(identity.did);
+      }
+
+      return transcriptionManager;
+    };
+
+    return [
+      Capability.contributes(TranscriptionCapabilities.Transcriber, getTranscriber),
+      Capability.contributes(TranscriptionCapabilities.TranscriptionManager, getTranscriptionManager),
+    ];
+  }),
+);