@dxos/plugin-transcription 0.8.2-main.fbd8ed0 → 0.8.2-staging.4d6ad0f

package/dist/types/src/components/transcription.stories.d.ts

@@ -1,18 +0,0 @@
-import '@dxos-theme';
-import { type Meta, type StoryObj } from '@storybook/react';
-import React from 'react';
-import { type DXN } from '@dxos/keys';
-declare const Microphone: ({ entityExtraction }: {
-    entityExtraction?: boolean;
-}) => React.JSX.Element;
-declare const AudioFile: ({ queueDxn, audioUrl }: {
-    queueDxn: DXN;
-    audioUrl: string;
-    transcriptUrl: string;
-}) => React.JSX.Element;
-declare const meta: Meta<typeof AudioFile>;
-export default meta;
-export declare const Default: StoryObj<typeof Microphone>;
-export declare const EntityExtraction: StoryObj<typeof Microphone>;
-export declare const File: StoryObj<typeof AudioFile>;
-//# sourceMappingURL=transcription.stories.d.ts.map

package/dist/types/src/components/transcription.stories.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"transcription.stories.d.ts","sourceRoot":"","sources":["../../../../src/components/transcription.stories.tsx"],"names":[],"mappings":"AAIA,OAAO,aAAa,CAAC;AAErB,OAAO,EAAE,KAAK,IAAI,EAAE,KAAK,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5D,OAAO,KASN,MAAM,OAAO,CAAC;AAQf,OAAO,EAAE,KAAK,GAAG,EAAE,MAAM,YAAY,CAAC;AAmDtC,QAAA,MAAM,UAAU,GAAI,sBAAsB;IAAE,gBAAgB,CAAC,EAAE,OAAO,CAAA;CAAE,sBAiFvE,CAAC;AAEF,QAAA,MAAM,SAAS,GAAI,wBAAwB;IAAE,QAAQ,EAAE,GAAG,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAC;IAAC,aAAa,EAAE,MAAM,CAAA;CAAE,sBAgEpG,CAAC;AAEF,QAAA,MAAM,IAAI,EAAE,IAAI,CAAC,OAAO,SAAS,CA4ChC,CAAC;AAEF,eAAe,IAAI,CAAC;AAEpB,eAAO,MAAM,OAAO,EAAE,QAAQ,CAAC,OAAO,UAAU,CAK/C,CAAC;AAEF,eAAO,MAAM,gBAAgB,EAAE,QAAQ,CAAC,OAAO,UAAU,CAKxD,CAAC;AAEF,eAAO,MAAM,IAAI,EAAE,QAAQ,CAAC,OAAO,SAAS,CAQ3C,CAAC"}

package/dist/types/src/entity-extraction/entity-extraction.d.ts

@@ -1,42 +0,0 @@
-import { Schema } from 'effect';
-import type { AIServiceClient } from '@dxos/assistant';
-import { type BaseEchoObject } from '@dxos/echo-schema';
-import { DataType } from '@dxos/schema';
-type ProcessTranscriptMessageParams = {
-    message: DataType.Message;
-    aiService: AIServiceClient;
-    context: {
-        objects?: BaseEchoObject[];
-    };
-    options?: {
-        /**
-         * Timeout for the entity extraction.
-         */
-        timeout?: number;
-        /**
-         * Fallback to raw text if the entity extraction fails.
-         * Otherwise the function will throw an error.
-         * @default false
-         */
-        fallbackToRaw?: boolean;
-    };
-};
-type ProcessTranscriptMessageResult = {
-    message: DataType.Message;
-    timeElapsed: number;
-};
-declare const ReferencedQuotes: Schema.Struct<{
-    references: Schema.Array$<Schema.Struct<{
-        quote: typeof Schema.String;
-        id: typeof Schema.String;
-    }>>;
-}>;
-interface ReferencedQuotes extends Schema.Schema.Type<typeof ReferencedQuotes> {
-}
-export declare const processTranscriptMessage: (params: ProcessTranscriptMessageParams) => Promise<ProcessTranscriptMessageResult>;
-/**
- * Finds and replaces all quotes with DXNs references.
- */
-export declare const postprocessText: (text: string, quotes: ReferencedQuotes) => string;
-export {};
-//# sourceMappingURL=entity-extraction.d.ts.map

package/dist/types/src/entity-extraction/entity-extraction.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"entity-extraction.d.ts","sourceRoot":"","sources":["../../../../src/entity-extraction/entity-extraction.ts"],"names":[],"mappings":"AAIA,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAGhC,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAGvD,OAAO,EAAE,KAAK,cAAc,EAAoB,MAAM,mBAAmB,CAAC;AAG1E,OAAO,EAAE,QAAQ,EAAE,MAAM,cAAc,CAAC;AAIxC,KAAK,8BAA8B,GAAG;IACpC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC;IAC1B,SAAS,EAAE,eAAe,CAAC;IAC3B,OAAO,EAAE;QACP,OAAO,CAAC,EAAE,cAAc,EAAE,CAAC;KAC5B,CAAC;IAEF,OAAO,CAAC,EAAE;QACR;;WAEG;QACH,OAAO,CAAC,EAAE,MAAM,CAAC;QAEjB;;;;WAIG;QACH,aAAa,CAAC,EAAE,OAAO,CAAC;KACzB,CAAC;CACH,CAAC;AAEF,KAAK,8BAA8B,GAAG;IACpC,OAAO,EAAE,QAAQ,CAAC,OAAO,CAAC;IAC1B,WAAW,EAAE,MAAM,CAAC;CACrB,CAAC;AAUF,QAAA,MAAM,gBAAgB;;;;;EAUpB,CAAC;AACH,UAAU,gBAAiB,SAAQ,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,gBAAgB,CAAC;CAAG;AAGjF,eAAO,MAAM,wBAAwB,GACnC,QAAQ,8BAA8B,KACrC,OAAO,CAAC,8BAA8B,CA+DxC,CAAC;AAEF;;GAEG;AAEH,eAAO,MAAM,eAAe,GAAI,MAAM,MAAM,EAAE,QAAQ,gBAAgB,WAWrE,CAAC"}

package/dist/types/src/entity-extraction/entity-extraction.test.d.ts

@@ -1,2 +0,0 @@
-export {};
-//# sourceMappingURL=entity-extraction.test.d.ts.map

package/dist/types/src/entity-extraction/entity-extraction.test.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"entity-extraction.test.d.ts","sourceRoot":"","sources":["../../../../src/entity-extraction/entity-extraction.test.ts"],"names":[],"mappings":""}

package/dist/types/src/entity-extraction/index.d.ts

@@ -1,2 +0,0 @@
-export * from './entity-extraction';
-//# sourceMappingURL=index.d.ts.map

package/dist/types/src/entity-extraction/index.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/entity-extraction/index.ts"],"names":[],"mappings":"AAIA,cAAc,qBAAqB,CAAC"}

package/src/capabilities/app-graph-builder.ts

@@ -1,233 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Rx } from '@effect-rx/rx-react';
-import { Option, pipe, type Schema } from 'effect';
-
-import { Capabilities, contributes, createIntent, type PluginContext } from '@dxos/app-framework';
-import { AIServiceEdgeClient, type AIServiceClient } from '@dxos/assistant';
-import { AI_SERVICE_ENDPOINT } from '@dxos/assistant/testing';
-import { Filter, fullyQualifiedId, getSpace, makeRef, Query, type Space } from '@dxos/client/echo';
-import { getSchemaTypename, isInstanceOf, type BaseEchoObject } from '@dxos/echo-schema';
-import { invariant } from '@dxos/invariant';
-import { DXN } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { ClientCapabilities } from '@dxos/plugin-client';
-import { PLANK_COMPANION_TYPE, ATTENDABLE_PATH_SEPARATOR } from '@dxos/plugin-deck/types';
-import { createExtension, rxFromSignal } from '@dxos/plugin-graph';
-import { DocumentType } from '@dxos/plugin-markdown/types';
-import { MeetingCapabilities, type CallState, type MediaState } from '@dxos/plugin-meeting';
-import { MeetingType, type MeetingCallProperties } from '@dxos/plugin-meeting/types';
-import { type buf } from '@dxos/protocols/buf';
-import { type TranscriptionPayloadSchema } from '@dxos/protocols/buf/dxos/edge/calls_pb';
-import { DataType } from '@dxos/schema';
-
-import { TranscriptionCapabilities } from './capabilities';
-import { processTranscriptMessage } from '../entity-extraction';
-import { TRANSCRIPTION_PLUGIN } from '../meta';
-import { TranscriptionManager } from '../transcriber';
-import { TranscriptionAction, type TranscriptionSettingsProps, TranscriptType } from '../types';
-
-// TODO(wittjosiah): Factor out.
-// TODO(wittjosiah): Can we stop using protobuf for this?
-type TranscriptionPayload = buf.MessageInitShape<typeof TranscriptionPayloadSchema>;
-
-const getMeetingTranscript = async (
-  context: PluginContext,
-  meeting: MeetingType,
-): Promise<TranscriptType | undefined> => {
-  const { dispatchPromise: dispatch } = context.getCapability(Capabilities.IntentDispatcher);
-
-  const typename = getSchemaTypename(TranscriptType)!;
-  const existing = await meeting.artifacts[typename]?.load();
-  if (existing instanceof TranscriptType) {
-    return existing;
-  }
-
-  const space = getSpace(meeting);
-  invariant(space);
-  const { data } = await dispatch(createIntent(TranscriptionAction.Create, { spaceId: space.id }));
-  meeting.artifacts[typename] = makeRef(data!.object);
-  return data?.object;
-};
-
-// TODO(wittjosiah): Introduce a sound that plays when transcription is activated/deactivated.
-export default (context: PluginContext) =>
-  contributes(Capabilities.AppGraphBuilder, [
-    createExtension({
-      id: `${TRANSCRIPTION_PLUGIN}/meeting-transcript`,
-      actions: (node) =>
-        Rx.make((get) =>
-          pipe(
-            get(node),
-            Option.flatMap((node) => (isInstanceOf(MeetingType, node.data) ? Option.some(node.data) : Option.none())),
-            Option.map((meeting) => {
-              const state = context.getCapability(TranscriptionCapabilities.MeetingTranscriptionState);
-              const enabled = get(rxFromSignal(() => state.enabled));
-              return [
-                {
-                  id: `${fullyQualifiedId(meeting)}/action/start-stop-transcription`,
-                  data: async () => {
-                    // NOTE: We are not saving the state of the transcription manager here.
-                    // We expect the state to be updated through `onCallStateUpdated` once it is propagated through Swarm.
-                    // This is done to avoid race conditions and to not handle optimistic updates.
-                    const transcript = await getMeetingTranscript(context, meeting);
-                    invariant(transcript, 'Failed to create transcript');
-
-                    const callManager = context.getCapability(MeetingCapabilities.CallManager);
-                    callManager.setActivity(getSchemaTypename(TranscriptType)!, {
-                      queueDxn: transcript.queue.dxn.toString(),
-                      enabled: !enabled,
-                    });
-                  },
-                  properties: {
-                    label: enabled
-                      ? ['stop transcription label', { ns: TRANSCRIPTION_PLUGIN }]
-                      : ['start transcription label', { ns: TRANSCRIPTION_PLUGIN }],
-                    icon: 'ph--subtitles--regular',
-                    disposition: 'toolbar',
-                    classNames: enabled ? 'bg-callAlert' : '',
-                  },
-                },
-              ];
-            }),
-            Option.getOrElse(() => []),
-          ),
-        ),
-      connector: (node) =>
-        Rx.make((get) =>
-          pipe(
-            get(node),
-            Option.flatMap((node) => (isInstanceOf(MeetingType, node.data) ? Option.some(node.data) : Option.none())),
-            Option.map((meeting) => {
-              const client = context.getCapability(ClientCapabilities.Client);
-              const state = context.getCapability(TranscriptionCapabilities.MeetingTranscriptionState);
-              const settingsStore = get(context.capabilities(Capabilities.SettingsStore))[0];
-              const settings = get(
-                rxFromSignal(() => settingsStore?.getStore<TranscriptionSettingsProps>(TRANSCRIPTION_PLUGIN)!.value),
-              );
-
-              // TODO(dmaretskyi): Request via capability.
-              const aiClient: AIServiceClient | undefined = new AIServiceEdgeClient({
-                // TODO(burdon): Get from config.
-                endpoint: AI_SERVICE_ENDPOINT.REMOTE,
-              });
-
-              return [
-                {
-                  id: `${fullyQualifiedId(meeting)}${ATTENDABLE_PATH_SEPARATOR}${getSchemaTypename(TranscriptType)}`,
-                  type: PLANK_COMPANION_TYPE,
-                  data: get(rxFromSignal(() => meeting.artifacts[getSchemaTypename(TranscriptType)!]?.target)),
-                  properties: {
-                    label: ['transcript companion label', { ns: TRANSCRIPTION_PLUGIN }],
-                    icon: 'ph--subtitles--regular',
-                    position: 'hoist',
-                    disposition: 'hidden',
-                    schema: TranscriptType,
-                    getIntent: ({ space }: { space: Space }) =>
-                      createIntent(TranscriptionAction.Create, { spaceId: space.id }),
-
-                    // TODO(burdon): Is this the right place to inisitalize the state? Change to a capability?
-                    onJoin: async ({ meeting }: { meeting?: MeetingType }) => {
-                      const identity = client.halo.identity.get();
-                      invariant(identity);
-                      const space = getSpace(meeting);
-                      invariant(space);
-
-                      let messageEnricher;
-                      if (aiClient && settings.entityExtraction) {
-                        messageEnricher = createEntityExtractionEnricher({
-                          aiClient,
-                          // TODO(dmaretskyi): Have those be discovered from the schema graph or contributed by capabilities?
-                          //  This forced me to add a dependency on markdown plugin.
-                          //  This will be replaced with a vector search index anyway, so its not a big deal.
-                          contextTypes: [DocumentType, DataType.Person, DataType.Organization],
-                          space,
-                        });
-                      }
-
-                      // TODO(burdon): The TranscriptionManager singleton is part of the state and should just be updated here.
-                      state.transcriptionManager = await new TranscriptionManager({
-                        edgeClient: client.edge,
-                        messageEnricher,
-                      })
-                        .setIdentityDid(identity.did)
-                        .open();
-                    },
-                    onLeave: async () => {
-                      await state.transcriptionManager?.close();
-                      state.transcriptionManager = undefined;
-                      state.enabled = false;
-                    },
-                    onCallStateUpdated: async (callState: CallState) => {
-                      const typename = getSchemaTypename(TranscriptType);
-                      const transcription = typename ? callState.activities?.[typename] : undefined;
-                      if (!transcription?.payload) {
-                        return;
-                      }
-
-                      const payload: TranscriptionPayload = transcription.payload;
-                      state.enabled = !!payload.enabled;
-                      if (payload.queueDxn) {
-                        // NOTE: Must set queue before enabling transcription.
-                        state.transcriptionManager?.setQueue(DXN.parse(payload.queueDxn));
-                      }
-                      await state.transcriptionManager?.setEnabled(payload.enabled);
-                    },
-                    onMediaStateUpdated: async ([mediaState, isSpeaking]: [MediaState, boolean]) => {
-                      void state.transcriptionManager?.setAudioTrack(mediaState.audioTrack);
-                      void state.transcriptionManager?.setRecording(isSpeaking);
-                    },
-                  } satisfies MeetingCallProperties,
-                },
-              ];
-            }),
-            Option.getOrElse(() => []),
-          ),
-        ),
-    }),
-  ]);
-
-type EntityExtractionEnricherFactoryOptions = {
-  aiClient: AIServiceClient;
-  contextTypes: Schema.Schema.AnyNoContext[];
-  space: Space;
-};
-
-const createEntityExtractionEnricher = ({ aiClient, contextTypes, space }: EntityExtractionEnricherFactoryOptions) => {
-  return async (message: DataType.Message) => {
-    const { objects } = await space.db
-      .query(Query.select(Filter.or(...contextTypes.map((s) => Filter.type(s as Schema.Schema<BaseEchoObject>)))))
-      .run();
-    log.info('context', { objects });
-
-    const { message: enhancedMessage, timeElapsed } = await processTranscriptMessage({
-      aiService: aiClient,
-      message,
-      context: {
-        objects: await Promise.all(objects.map((o) => processContextObject(o))),
-      },
-      options: { timeout: ENTITY_EXTRACTOR_TIMEOUT, fallbackToRaw: true },
-    });
-
-    log.info('entity extraction time', { timeElapsed });
-    return enhancedMessage;
-  };
-};
-
-// TODO(dmaretskyi): Use Type.Any
-const processContextObject = async (object: BaseEchoObject): Promise<any> => {
-  // TODO(dmaretskyi): Documents need special processing is the content is behind a ref.
-  // TODO(dmaretskyi): Think about a way to handle this serialization with a decorator.
-  // if (isInstanceOf(DocumentType, object)) {
-  //   return {
-  //     ...object,
-  //     content: await object.content.load(),
-  //   };
-  // }
-
-  return object;
-};
-
-const ENTITY_EXTRACTOR_TIMEOUT = 25_000;

package/src/capabilities/meeting-transcription-state.ts

@@ -1,16 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { contributes } from '@dxos/app-framework';
-import { live } from '@dxos/live-object';
-
-import { TranscriptionCapabilities } from './capabilities';
-
-export default () => {
-  const state = live<TranscriptionCapabilities.MeetingTranscriptionState>({
-    enabled: false,
-  });
-
-  return contributes(TranscriptionCapabilities.MeetingTranscriptionState, state);
-};

package/src/capabilities/settings.ts

@@ -1,20 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-import { Capabilities, contributes } from '@dxos/app-framework';
-import { live } from '@dxos/live-object';
-
-import { TRANSCRIPTION_PLUGIN } from '../meta';
-import { TranscriptionSettingsSchema } from '../types';
-
-export default () => {
-  // TODO(wittjosiah): `live` currently doesn't handle schema default values.
-  const settings = live(TranscriptionSettingsSchema, { entityExtraction: true });
-
-  return contributes(Capabilities.Settings, {
-    schema: TranscriptionSettingsSchema,
-    prefix: TRANSCRIPTION_PLUGIN,
-    value: settings,
-  });
-};

package/src/components/TranscriptionSettings/TranscriptionSettings.tsx

@@ -1,26 +0,0 @@
-//
-// Copyright 2023 DXOS.org
-//
-
-import React from 'react';
-
-import { Input, useTranslation } from '@dxos/react-ui';
-import { DeprecatedFormContainer, DeprecatedFormInput } from '@dxos/react-ui-form';
-
-import { TRANSCRIPTION_PLUGIN } from '../../meta';
-import { type TranscriptionSettingsProps } from '../../types';
-
-export const TranscriptionSettings = ({ settings }: { settings: TranscriptionSettingsProps }) => {
-  const { t } = useTranslation(TRANSCRIPTION_PLUGIN);
-
-  return (
-    <DeprecatedFormContainer>
-      <DeprecatedFormInput label={t('settings entity extraction label')}>
-        <Input.Switch
-          checked={!!settings.entityExtraction}
-          onCheckedChange={(checked) => (settings.entityExtraction = checked)}
-        />
-      </DeprecatedFormInput>
-    </DeprecatedFormContainer>
-  );
-};

package/src/components/TranscriptionSettings/index.ts

@@ -1,5 +0,0 @@
-//
-// Copyright 2025 DXOS.org
-//
-
-export * from './TranscriptionSettings';