@dxos/plugin-transcription 0.7.5-staging.2ff1350

package/dist/types/src/types/schema.d.ts

@@ -0,0 +1,63 @@
+import { S } from '@dxos/echo-schema';
+/**
+ * Root transcript object created when the user starts a transcription.
+ */
+export declare const TranscriptSchema: S.Struct<{
+    name: typeof S.String;
+    started: S.optional<typeof S.Date>;
+    ended: S.optional<typeof S.Date>;
+    /**
+     * Queue containing TranscriptBlock objects.
+     */
+    queue: S.optional<typeof S.String>;
+}>;
+declare const TranscriptType_base: import("@dxos/echo-schema").TypedObjectPrototype<import("packages/core/echo/echo-schema/dist/types/src/object/common").TypedObjectFields<{
+    readonly name: typeof S.String;
+    readonly started: S.optional<typeof S.Date>;
+    readonly ended: S.optional<typeof S.Date>;
+    readonly queue: S.optional<typeof S.String>;
+}, import("packages/core/echo/echo-schema/dist/types/src/object/common").TypedObjectOptions>, S.Struct.Encoded<{
+    readonly name: typeof S.String;
+    readonly started: S.optional<typeof S.Date>;
+    readonly ended: S.optional<typeof S.Date>;
+    readonly queue: S.optional<typeof S.String>;
+}>>;
+export declare class TranscriptType extends TranscriptType_base {
+}
+/**
+ * First message in queue.
+ * Contains metadata for the recording and transcript.
+ */
+declare const TranscriptHeader: S.Struct<{
+    started: S.optional<typeof S.Date>;
+}>;
+export type TranscriptHeader = S.Schema.Type<typeof TranscriptHeader>;
+/**
+ * Transcription fragment.
+ */
+declare const TranscriptSegment: S.Struct<{
+    started: typeof S.Date;
+    text: typeof S.String;
+}>;
+export type TranscriptSegment = S.Schema.Type<typeof TranscriptSegment>;
+/**
+ * Transcription block (from single speaker).
+ */
+export declare const TranscriptBlock: S.Schema<{
+    readonly id: string;
+    author?: string | undefined;
+    segments: {
+        readonly started: Date;
+        readonly text: string;
+    }[];
+}, {
+    readonly id: string;
+    author?: string | undefined;
+    segments: {
+        readonly started: Date;
+        readonly text: string;
+    }[];
+}, never>;
+export type TranscriptBlock = S.Schema.Type<typeof TranscriptBlock>;
+export {};
+//# sourceMappingURL=schema.d.ts.map

package/dist/types/src/types/schema.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../../../src/types/schema.ts"],"names":[],"mappings":"AAIA,OAAO,EAAwB,CAAC,EAAe,MAAM,mBAAmB,CAAC;AAEzE;;GAEG;AACH,eAAO,MAAM,gBAAgB;;;;IAO3B;;OAEG;;EAEH,CAAC;;;;;;;;;;;;AAEH,qBAAa,cAAe,SAAQ,mBAGT;CAAG;AAI9B;;;GAGG;AACH,QAAA,MAAM,gBAAgB;;EAEpB,CAAC;AAEH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,gBAAgB,CAAC,CAAC;AAEtE;;GAEG;AACH,QAAA,MAAM,iBAAiB;;;EAIrB,CAAC;AAEH,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,iBAAiB,CAAC,CAAC;AAExE;;GAEG;AACH,eAAO,MAAM,eAAe;;;;;;;;;;;;;;SAIiC,CAAC;AAE9D,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,eAAe,CAAC,CAAC"}

package/dist/types/src/types/types.d.ts

@@ -0,0 +1,28 @@
+import { S } from '@dxos/echo-schema';
+import { TranscriptType } from './schema';
+/**
+ * Endpoint to the calls service.
+ */
+export declare const TRANSCRIPTION_URL = "https://calls-service.dxos.workers.dev";
+export declare namespace TranscriptionAction {
+    const Create_base: S.TaggedClass<Create, "dxos.org/plugin/transcription/action/create", {
+        readonly _tag: S.tag<"dxos.org/plugin/transcription/action/create">;
+    } & {
+        input: S.Struct<{
+            name: S.optional<typeof S.String>;
+        }>;
+        output: S.Struct<{
+            object: S.Struct<{
+                name: typeof S.String;
+                started: S.optional<typeof S.Date>;
+                ended: S.optional<typeof S.Date>;
+                queue: S.optional<typeof S.String>;
+            }>;
+        }>;
+    }>;
+    export class Create extends Create_base {
+    }
+    export {};
+}
+export declare const isTranscript: (object: unknown) => object is typeof TranscriptType;
+//# sourceMappingURL=types.d.ts.map

package/dist/types/src/types/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../../src/types/types.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,CAAC,EAAgB,MAAM,mBAAmB,CAAC;AAGpD,OAAO,EAAoB,cAAc,EAAE,MAAM,UAAU,CAAC;AAK5D;;GAEG;AAEH,eAAO,MAAM,iBAAiB,2CAA2C,CAAC;AAE1E,yBAAiB,mBAAmB,CAAC;;;;;;;;;;;;;;;;IAGnC,MAAM,OAAO,MAAO,SAAQ,WAO1B;KAAG;;CACN;AAED,eAAO,MAAM,YAAY,WAAY,OAAO,KAAG,MAAM,IAAI,OAAO,cAE/D,CAAC"}

package/dist/types/src/util/array.d.ts

@@ -0,0 +1,2 @@
+export declare const mergeFloat64Arrays: (arrays: Float64Array[]) => Float64Array<ArrayBuffer>;
+//# sourceMappingURL=array.d.ts.map

package/dist/types/src/util/array.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"array.d.ts","sourceRoot":"","sources":["../../../../src/util/array.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,kBAAkB,WAAY,YAAY,EAAE,8BASxD,CAAC"}

package/dist/types/src/util/get-time-str.d.ts

@@ -0,0 +1,2 @@
+export declare const getTimeStr: (timestamp?: number) => string;
+//# sourceMappingURL=get-time-str.d.ts.map

package/dist/types/src/util/get-time-str.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"get-time-str.d.ts","sourceRoot":"","sources":["../../../../src/util/get-time-str.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,UAAU,eAAgB,MAAM,WAOzC,CAAC"}

package/dist/types/src/util/index.d.ts

@@ -0,0 +1,5 @@
+export * from './array';
+export * from './get-time-str';
+export * from './monitor-audio-level';
+export * from './random-queue-dxn';
+//# sourceMappingURL=index.d.ts.map

package/dist/types/src/util/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/util/index.ts"],"names":[],"mappings":"AAIA,cAAc,SAAS,CAAC;AACxB,cAAc,gBAAgB,CAAC;AAC/B,cAAc,uBAAuB,CAAC;AACtC,cAAc,oBAAoB,CAAC"}

package/dist/types/src/util/monitor-audio-level.d.ts

@@ -0,0 +1,6 @@
+export declare const monitorAudioLevel: ({ mediaStreamTrack, onMeasure, intervalMs, }: {
+    mediaStreamTrack: MediaStreamTrack;
+    onMeasure: (value: number) => void;
+    intervalMs?: number;
+}) => () => void;
+//# sourceMappingURL=monitor-audio-level.d.ts.map

package/dist/types/src/util/monitor-audio-level.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"monitor-audio-level.d.ts","sourceRoot":"","sources":["../../../../src/util/monitor-audio-level.ts"],"names":[],"mappings":"AAIA,eAAO,MAAM,iBAAiB,iDAI3B;IACD,gBAAgB,EAAE,gBAAgB,CAAC;IACnC,SAAS,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IACnC,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB,eA6CA,CAAC"}

package/dist/types/src/util/random-queue-dxn.d.ts

@@ -0,0 +1,3 @@
+import { DXN } from '@dxos/keys';
+export declare const randomQueueDxn: () => DXN;
+//# sourceMappingURL=random-queue-dxn.d.ts.map

package/dist/types/src/util/random-queue-dxn.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"random-queue-dxn.d.ts","sourceRoot":"","sources":["../../../../src/util/random-queue-dxn.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,GAAG,EAA8B,MAAM,YAAY,CAAC;AAE7D,eAAO,MAAM,cAAc,WAC6D,CAAC"}

package/dist/types/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"version":"5.7.3"}

package/package.json

@@ -0,0 +1,93 @@
+{
+  "name": "@dxos/plugin-transcription",
+  "version": "0.7.5-staging.2ff1350",
+  "description": "Voice transcription.",
+  "homepage": "https://dxos.org",
+  "bugs": "https://github.com/dxos/dxos/issues",
+  "license": "MIT",
+  "author": "DXOS.org",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/types/src/index.d.ts",
+      "browser": "./dist/lib/browser/index.mjs",
+      "node": "./dist/lib/node-esm/index.mjs"
+    },
+    "./types": {
+      "types": "./dist/types/src/types/index.d.ts",
+      "browser": "./dist/lib/browser/types/index.mjs",
+      "node": "./dist/lib/node-esm/types/index.mjs"
+    }
+  },
+  "types": "dist/types/src/index.d.ts",
+  "typesVersions": {
+    "*": {
+      "types": [
+        "dist/types/src/types/index.d.ts"
+      ]
+    }
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@preact/signals-core": "^1.6.0",
+    "date-fns": "^3.3.1",
+    "effect": "^3.12.3",
+    "extendable-media-recorder": "^9.2.19",
+    "extendable-media-recorder-wav-encoder": "^7.0.123",
+    "react-resize-detector": "^12.0.2",
+    "react-router-dom": "^6.4.0",
+    "react-use": "^17.5.1",
+    "wavefile": "^11.0.0",
+    "@dxos/app-framework": "0.7.5-staging.2ff1350",
+    "@dxos/artifact": "0.7.5-staging.2ff1350",
+    "@dxos/assistant": "0.7.5-staging.2ff1350",
+    "@dxos/async": "0.7.5-staging.2ff1350",
+    "@dxos/codec-protobuf": "0.7.5-staging.2ff1350",
+    "@dxos/context": "0.7.5-staging.2ff1350",
+    "@dxos/debug": "0.7.5-staging.2ff1350",
+    "@dxos/client": "0.7.5-staging.2ff1350",
+    "@dxos/display-name": "0.7.5-staging.2ff1350",
+    "@dxos/echo-db": "0.7.5-staging.2ff1350",
+    "@dxos/echo-schema": "0.7.5-staging.2ff1350",
+    "@dxos/invariant": "0.7.5-staging.2ff1350",
+    "@dxos/keys": "0.7.5-staging.2ff1350",
+    "@dxos/live-object": "0.7.5-staging.2ff1350",
+    "@dxos/log": "0.7.5-staging.2ff1350",
+    "@dxos/plugin-client": "0.7.5-staging.2ff1350",
+    "@dxos/plugin-graph": "0.7.5-staging.2ff1350",
+    "@dxos/plugin-markdown": "0.7.5-staging.2ff1350",
+    "@dxos/plugin-space": "0.7.5-staging.2ff1350",
+    "@dxos/protocols": "0.7.5-staging.2ff1350",
+    "@dxos/react-client": "0.7.5-staging.2ff1350",
+    "@dxos/react-edge-client": "0.7.5-staging.2ff1350",
+    "@dxos/react-ui-components": "0.7.5-staging.2ff1350",
+    "@dxos/react-ui-stack": "0.7.5-staging.2ff1350",
+    "@dxos/schema": "0.7.5-staging.2ff1350",
+    "@dxos/tracing": "0.7.5-staging.2ff1350",
+    "@dxos/util": "0.7.5-staging.2ff1350",
+    "@dxos/react-ui-sfx": "0.7.5-staging.2ff1350"
+  },
+  "devDependencies": {
+    "@types/react": "~18.2.0",
+    "react": "~18.2.0",
+    "react-dom": "~18.2.0",
+    "tailwindcss": "~3.4.1",
+    "@dxos/random": "0.7.5-staging.2ff1350",
+    "@dxos/react-ui": "0.7.5-staging.2ff1350",
+    "@dxos/storybook-utils": "0.7.5-staging.2ff1350",
+    "@dxos/react-ui-theme": "0.7.5-staging.2ff1350",
+    "@dxos/test-utils": "0.7.5-staging.2ff1350"
+  },
+  "optionalDependencies": {
+    "react": "~18.2.0",
+    "react-dom": "~18.2.0",
+    "@dxos/react-ui": "0.7.5-staging.2ff1350",
+    "@dxos/react-ui-theme": "0.7.5-staging.2ff1350"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/TranscriptionPlugin.tsx

@@ -0,0 +1,57 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { Capabilities, Events, contributes, defineModule, definePlugin } from '@dxos/app-framework';
+import { ClientCapabilities, ClientEvents } from '@dxos/plugin-client';
+
+import { AppGraphBuilder, IntentResolver, ReactSurface, TranscriptionCapabilities } from './capabilities';
+import { meta, TRANSCRIPTION_PLUGIN } from './meta';
+import translations from './translations';
+import { TranscriptType } from './types';
+
+export const TranscriptionPlugin = () =>
+  definePlugin(meta, [
+    defineModule({
+      id: `${meta.id}/module/translations`,
+      activatesOn: Events.SetupTranslations,
+      activate: () => contributes(Capabilities.Translations, translations),
+    }),
+    defineModule({
+      id: `${meta.id}/module/transcription`,
+      activatesOn: Events.Startup,
+      activate: () => contributes(TranscriptionCapabilities.Transcription, null),
+    }),
+    defineModule({
+      id: `${meta.id}/module/metadata`,
+      activatesOn: Events.SetupMetadata,
+      activate: () =>
+        contributes(Capabilities.Metadata, {
+          id: TranscriptType.typename,
+          metadata: {
+            placeholder: ['transcript title placeholder', { ns: TRANSCRIPTION_PLUGIN }],
+            icon: 'ph--subtitles--regular',
+          },
+        }),
+    }),
+    defineModule({
+      id: `${meta.id}/module/schema`,
+      activatesOn: ClientEvents.SetupSchema,
+      activate: () => [contributes(ClientCapabilities.Schema, [TranscriptType])],
+    }),
+    defineModule({
+      id: `${meta.id}/module/react-surface`,
+      activatesOn: Events.SetupReactSurface,
+      activate: ReactSurface,
+    }),
+    defineModule({
+      id: `${meta.id}/module/intent-resolver`,
+      activatesOn: Events.SetupIntentResolver,
+      activate: IntentResolver,
+    }),
+    defineModule({
+      id: `${meta.id}/module/app-graph-builder`,
+      activatesOn: Events.SetupAppGraph,
+      activate: AppGraphBuilder,
+    }),
+  ]);

package/src/capabilities/app-graph-builder.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capabilities, contributes, type PluginsContext } from '@dxos/app-framework';
+
+export default (context: PluginsContext) => contributes(Capabilities.AppGraphBuilder, []);

package/src/capabilities/capabilities.ts

@@ -0,0 +1,12 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { defineCapability } from '@dxos/app-framework';
+
+import { TRANSCRIPTION_PLUGIN } from '../meta';
+
+export namespace TranscriptionCapabilities {
+  // TODO(burdon): Currently this is just a marker; adapt useVoiceInput to use this.
+  export const Transcription = defineCapability<null>(`${TRANSCRIPTION_PLUGIN}/capability/transcription`);
+}

package/src/capabilities/index.ts

@@ -0,0 +1,11 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { lazy } from '@dxos/app-framework';
+
+export const AppGraphBuilder = lazy(() => import('./app-graph-builder'));
+export const IntentResolver = lazy(() => import('./intent-resolver'));
+export const ReactSurface = lazy(() => import('./react-surface'));
+
+export * from './capabilities';

package/src/capabilities/intent-resolver.ts

@@ -0,0 +1,24 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { Capabilities, contributes, createResolver, type PluginsContext } from '@dxos/app-framework';
+import { create } from '@dxos/live-object';
+
+import { TranscriptionAction, TranscriptType } from '../types';
+import { getTimeStr, randomQueueDxn } from '../util';
+
+export default (context: PluginsContext) =>
+  contributes(Capabilities.IntentResolver, [
+    createResolver({
+      intent: TranscriptionAction.Create,
+      resolve: ({ name }) => {
+        const transcript = create(TranscriptType, {
+          queue: randomQueueDxn().toString(), // TODO(burdon): Associate with space.
+          name: name ?? `Transcript ${getTimeStr(Date.now())}`,
+        });
+
+        return { data: { object: transcript } };
+      },
+    }),
+  ]);

package/src/capabilities/react-surface.tsx

@@ -0,0 +1,21 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import React from 'react';
+
+import { Capabilities, contributes, createSurface } from '@dxos/app-framework';
+
+import { TranscriptionContainer } from '../components';
+import { TRANSCRIPTION_PLUGIN } from '../meta';
+import { isTranscript, type TranscriptType } from '../types';
+
+export default () =>
+  contributes(Capabilities.ReactSurface, [
+    createSurface({
+      id: TRANSCRIPTION_PLUGIN,
+      role: 'article',
+      filter: (data): data is { subject: TranscriptType } => isTranscript(data.subject),
+      component: ({ data, role }) => <TranscriptionContainer transcript={data.subject} />,
+    }),
+  ]);

package/src/components/Transcript/Transcript.stories.tsx

@@ -0,0 +1,65 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import '@dxos-theme';
+
+import { type StoryObj, type Meta } from '@storybook/react';
+import React from 'react';
+
+import { faker } from '@dxos/random';
+import { ScrollContainer } from '@dxos/react-ui-components';
+import { withTheme, withLayout } from '@dxos/storybook-utils';
+
+import { Transcript } from './Transcript';
+import translations from '../../translations';
+
+const meta: Meta<typeof Transcript> = {
+  title: 'plugins/plugin-transcription/Transcript',
+  component: Transcript,
+  render: (args) => (
+    <div className='flex w-[30rem] p-2'>
+      <ScrollContainer>
+        <Transcript {...args} />
+      </ScrollContainer>
+    </div>
+  ),
+  decorators: [
+    withTheme,
+    withLayout({
+      tooltips: true,
+      fullscreen: true,
+      classNames: 'justify-center',
+    }),
+  ],
+  parameters: {
+    translations,
+  },
+};
+
+export default meta;
+
+type Story = StoryObj<typeof Transcript>;
+
+let start = new Date();
+const next = () => {
+  start = new Date(start.getTime() - Math.random() * 30_000);
+  return start;
+};
+
+const names = Array.from({ length: 3 }, () => faker.person.fullName());
+
+export const Default: Story = {
+  args: {
+    blocks: Array.from({ length: 6 + Math.floor(Math.random()) * 15 }, () => ({
+      id: faker.string.uuid(),
+      author: faker.helpers.arrayElement(names),
+      segments: Array.from({ length: 1 + Math.floor(Math.random() * 3) }, () => ({
+        started: next(),
+        text: faker.lorem.paragraph(),
+      })),
+    })).reverse(),
+  },
+};
+
+export const Empty: Story = {};

package/src/components/Transcript/Transcript.tsx

@@ -0,0 +1,69 @@
+//
+// Copyright 2023 DXOS.org
+//
+
+import { formatDistanceToNow } from 'date-fns/formatDistanceToNow';
+import React, { type FC } from 'react';
+
+import { Icon, IconButton, useTranslation } from '@dxos/react-ui';
+
+import { TRANSCRIPTION_PLUGIN } from '../../meta';
+import { type TranscriptBlock } from '../../types';
+
+// TODO(burdon): react-ui-list.
+// TODO(burdon): Actions (e.g., mark, summarize, translate, label, delete).
+
+export type TranscriptProps = {
+  blocks?: TranscriptBlock[];
+};
+
+export const Transcript: FC<TranscriptProps> = ({ blocks }) => {
+  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
+  const hoverButton = 'p-1 min-bs-1 transition-opacity duration-250 opacity-10 group-hover:opacity-100';
+
+  return (
+    <div role='list' className='flex flex-col grow gap-2 m-1'>
+      {blocks?.map((block) => (
+        <div
+          role='listitem'
+          key={block.id}
+          className='flex flex-col py-1 border border-transparent hover:border-separator rounded'
+        >
+          <div className='group flex items-center px-2'>
+            <Icon icon='ph--user--regular' />
+            <div className='px-2 text-sm text-subdued'>{block.author}</div>
+            <div className='grow' />
+            <IconButton
+              disabled
+              icon='ph--x--regular'
+              label={t('delete button')}
+              iconOnly
+              size={4}
+              classNames={hoverButton}
+            />
+          </div>
+
+          {block.segments.map((segment, i) => (
+            <div key={i} className='group flex flex-col'>
+              <div className='px-2'>{segment.text}</div>
+              <div className='flex gap-1 items-center justify-between px-2'>
+                <div className='grow' />
+                <div className='truncate text-xs text-subdued'>
+                  {formatDistanceToNow(segment.started, { addSuffix: true })}
+                </div>
+                <IconButton
+                  disabled
+                  icon='ph--bookmark-simple--regular'
+                  label={t('bookmark button')}
+                  iconOnly
+                  size={4}
+                  classNames={hoverButton}
+                />
+              </div>
+            </div>
+          ))}
+        </div>
+      ))}
+    </div>
+  );
+};

package/src/components/Transcript/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './Transcript';