@dxos/react-ui-canvas-compute 0.7.5-labs.5f04cf6

package/src/shapes/GptRealtime.tsx

@@ -0,0 +1,168 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React, { useState } from 'react';
+
+import { S } from '@dxos/echo-schema';
+import { Icon } from '@dxos/react-ui';
+import { type ShapeComponentProps, type ShapeDef } from '@dxos/react-ui-canvas-editor';
+
+import { createFunctionAnchors } from './common';
+import { ComputeShape, createShape, type CreateShapeProps } from './defs';
+
+export const GptRealtimeShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('gpt-realtime'),
+  }),
+);
+
+export type GptRealtimeShape = S.Schema.Type<typeof GptRealtimeShape>;
+
+export type CreateGptRealtimeProps = CreateShapeProps<GptRealtimeShape>;
+
+export const createGptRealtime = (props: CreateGptRealtimeProps) =>
+  createShape<GptRealtimeShape>({ type: 'gpt-realtime', size: { width: 256, height: 256 }, ...props });
+
+export const GptRealtimeComponent = ({ shape }: ShapeComponentProps<GptRealtimeShape>) => {
+  const [isLive, setIsLive] = useState(false);
+  const [isReady, setIsReady] = useState(false);
+
+  const start = async () => {
+    setIsLive(true);
+
+    try {
+      // Create WebRTC connection
+      const peerConnection = new RTCPeerConnection();
+
+      // Handle incoming audio
+      peerConnection.ontrack = (event) => {
+        const audioElement = document.createElement('audio');
+        audioElement.srcObject = event.streams[0];
+        audioElement.autoplay = true;
+        audioElement.controls = false;
+        audioElement.style.display = 'none';
+        document.body.appendChild(audioElement);
+        setIsReady(true);
+      };
+
+      // Get microphone stream
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+
+      // Add microphone track to connection
+      stream.getTracks().forEach((track) => peerConnection.addTransceiver(track, { direction: 'sendrecv' }));
+
+      // Create offer
+      const offer = await peerConnection.createOffer();
+      await peerConnection.setLocalDescription(offer);
+
+      // Send offer to backend and get answer
+      const response = await fetch(`${AI_SERVICE_URL}/rtc-connect`, {
+        method: 'POST',
+        body: offer.sdp,
+        headers: {
+          'Content-Type': 'application/sdp',
+        },
+      });
+
+      const answer = await response.text();
+
+      // Set remote description with answer
+      await peerConnection.setRemoteDescription({
+        sdp: answer,
+        type: 'answer',
+      });
+
+      const dataChannel = peerConnection.createDataChannel('response');
+
+      const configureData = () => {
+        console.log('Configuring data channel');
+        const event = {
+          type: 'session.update',
+          session: {
+            modalities: ['text', 'audio'],
+            // Provide the tools. Note they match the keys in the `fns` object above
+            tools: [],
+          },
+        };
+        dataChannel.send(JSON.stringify(event));
+      };
+
+      dataChannel.addEventListener('open', (ev) => {
+        console.log('Opening data channel', ev);
+        configureData();
+      });
+
+      // {
+      //     "type": "response.function_call_arguments.done",
+      //     "event_id": "event_Ad2gt864G595umbCs2aF9",
+      //     "response_id": "resp_Ad2griUWUjsyeLyAVtTtt",
+      //     "item_id": "item_Ad2gsxA84w9GgEvFwW1Ex",
+      //     "output_index": 1,
+      //     "call_id": "call_PG12S5ER7l7HrvZz",
+      //     "name": "get_weather",
+      //     "arguments": "{\"location\":\"Portland, Oregon\"}"
+      // }
+
+      dataChannel.addEventListener('message', async (ev) => {
+        const msg = JSON.parse(ev.data);
+        // console.log('realtime gpt event', msg);
+        // Handle function calls
+        if (msg.type === 'response.function_call_arguments.done') {
+          // const fn = fns[msg.name];
+          // if (fn !== undefined) {
+          //   console.log(`Calling local function ${msg.name} with ${msg.arguments}`);
+          //   const args = JSON.parse(msg.arguments);
+          //   const result = await fn(args);
+          //   console.log('result', result);
+          //   // Let OpenAI know that the function has been called and share it's output
+          //   const event = {
+          //     type: 'conversation.item.create',
+          //     item: {
+          //       type: 'function_call_output',
+          //       call_id: msg.call_id, // call_id from the function_call message
+          //       output: JSON.stringify(result), // result of the function
+          //     },
+          //   };
+          //   dataChannel.send(JSON.stringify(event));
+          // }
+        }
+      });
+    } catch (error) {
+      console.error('Error in realtime session:', error);
+      throw error;
+    }
+  };
+
+  return (
+    <div className='flex w-full justify-center items-center'>
+      <Icon
+        icon={isReady ? 'ph--waveform--regular' : isLive ? 'ph--pulse--regular' : 'ph--play--regular'}
+        size={16}
+        classNames={!isLive && 'cursor-pointer'}
+        onClick={start}
+      />
+    </div>
+  );
+};
+
+export const gptRealtimeShape: ShapeDef<GptRealtimeShape> = {
+  type: 'gpt-realtime',
+  name: 'GPT Realtime',
+  icon: 'ph--pulse--regular',
+  component: GptRealtimeComponent,
+  createShape: createGptRealtime,
+  // TODO(dmaretskyi): Can we fetch the schema dynamically?
+  getAnchors: (shape) =>
+    createFunctionAnchors(
+      shape,
+      S.Struct({
+        audio: S.Any,
+      }),
+      S.Struct({}),
+    ),
+  resizable: true,
+};
+
+const AI_SERVICE_URL = 'http://localhost:8788';

package/src/shapes/Json.tsx

@@ -0,0 +1,103 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React from 'react';
+
+import { DEFAULT_INPUT, DefaultOutput, JsonTransformInput } from '@dxos/conductor';
+import { S } from '@dxos/echo-schema';
+import { type ShapeComponentProps, type ShapeDef } from '@dxos/react-ui-canvas-editor';
+import { createAnchorMap } from '@dxos/react-ui-canvas-editor';
+import { JsonFilter } from '@dxos/react-ui-syntax-highlighter';
+
+import { createFunctionAnchors, getHeight, Box } from './common';
+import { ComputeShape, createAnchorId, createShape, type CreateShapeProps } from './defs';
+import { useComputeNodeState } from '../hooks';
+
+//
+// Data
+//
+
+export const JsonShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('json'),
+  }),
+);
+
+export type JsonShape = S.Schema.Type<typeof JsonShape>;
+
+export const JsonTransformShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('json-transform'),
+  }),
+);
+
+export type JsonTransformShape = S.Schema.Type<typeof JsonTransformShape>;
+
+//
+// Component
+//
+
+export type JsonComponentProps = ShapeComponentProps<JsonShape>;
+
+export const JsonComponent = ({ shape, ...props }: JsonComponentProps) => {
+  const { runtime } = useComputeNodeState(shape);
+  const input = runtime.inputs[DEFAULT_INPUT];
+  const value = input?.type === 'executed' ? input.value : undefined;
+
+  return (
+    <Box shape={shape}>
+      <JsonFilter data={value} classNames='text-xs' />
+    </Box>
+  );
+};
+
+export type JsonTransformComponentProps = ShapeComponentProps<JsonTransformShape>;
+
+export const JsonTransformComponent = ({ shape, ...props }: JsonTransformComponentProps) => {
+  return <Box shape={shape} />;
+};
+
+//
+// Defs
+//
+
+export type CreateJsonProps = CreateShapeProps<JsonShape>;
+
+export const createJson = (props: CreateJsonProps) =>
+  createShape<JsonShape>({ type: 'json', size: { width: 256, height: 256 }, ...props });
+
+export const jsonShape: ShapeDef<JsonShape> = {
+  type: 'json',
+  name: 'JSON',
+  icon: 'ph--code--regular',
+  component: (props) => <JsonComponent {...props} />,
+  createShape: createJson,
+  getAnchors: (shape) =>
+    createAnchorMap(shape, {
+      [createAnchorId('input')]: { x: -1, y: 0 },
+      [createAnchorId('output')]: { x: 1, y: 0 },
+    }),
+  resizable: true,
+};
+
+export type CreateJsonTransformProps = CreateShapeProps<JsonTransformShape> & { expression?: string };
+
+export const createJsonTransform = (props: CreateJsonTransformProps) =>
+  createShape<JsonTransformShape>({
+    type: 'json-transform',
+    size: { width: 128, height: getHeight(JsonTransformInput) },
+    ...props,
+  });
+
+export const jsonTransformShape: ShapeDef<JsonTransformShape> = {
+  type: 'json-transform',
+  name: 'Transform',
+  icon: 'ph--shuffle-simple--regular',
+  component: (props) => <JsonTransformComponent {...props} />,
+  createShape: createJsonTransform,
+  getAnchors: (shape) => createFunctionAnchors(shape, JsonTransformInput, DefaultOutput),
+  resizable: true,
+};

package/src/shapes/Logic.tsx

@@ -0,0 +1,82 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React from 'react';
+
+import { IfElseInput, IfElseOutput, IfInput, IfOutput } from '@dxos/conductor';
+import { S } from '@dxos/echo-schema';
+import { type ShapeComponentProps, type ShapeDef } from '@dxos/react-ui-canvas-editor';
+
+import { createFunctionAnchors, FunctionBody, getHeight } from './common';
+import { ComputeShape, createShape, type CreateShapeProps } from './defs';
+
+//
+// Data
+//
+
+export const IfShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('if'),
+  }),
+);
+
+export type IfShape = S.Schema.Type<typeof IfShape>;
+
+export const IfElseShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('if-else'),
+  }),
+);
+
+export type IfElseShape = S.Schema.Type<typeof IfElseShape>;
+
+//
+// Components
+//
+
+export type IfComponentProps = ShapeComponentProps<IfShape>;
+
+export const IfComponent = ({ shape, ...props }: IfComponentProps) => {
+  return <FunctionBody shape={shape} inputSchema={IfInput} outputSchema={IfOutput} />;
+};
+
+export type IfElseComponentProps = ShapeComponentProps<IfElseShape>;
+
+export const IfElseComponent = ({ shape, ...props }: IfElseComponentProps) => {
+  return <FunctionBody shape={shape} inputSchema={IfElseInput} outputSchema={IfElseOutput} />;
+};
+
+//
+// Defs
+//
+
+export type CreateIfProps = CreateShapeProps<IfShape> & { if?: string };
+
+export const createIf = (props: CreateIfProps) =>
+  createShape<IfShape>({ type: 'if', size: { width: 192, height: getHeight(IfInput) }, ...props });
+
+export const ifShape: ShapeDef<IfShape> = {
+  type: 'if',
+  name: 'IF',
+  icon: 'ph--arrows-split--regular',
+  component: (props) => <IfComponent {...props} />,
+  createShape: createIf,
+  getAnchors: (shape) => createFunctionAnchors(shape, IfInput, IfOutput),
+};
+
+export type CreateIfElseProps = CreateShapeProps<IfShape> & { if?: string };
+
+export const createIfElse = (props: CreateIfElseProps) =>
+  createShape<IfElseShape>({ type: 'if-else', size: { width: 192, height: getHeight(IfElseInput) }, ...props });
+
+export const ifElseShape: ShapeDef<IfElseShape> = {
+  type: 'if-else',
+  name: 'IF/ELSE',
+  icon: 'ph--arrows-merge--regular',
+  component: (props) => <IfElseComponent {...props} />,
+  createShape: createIfElse,
+  getAnchors: (shape) => createFunctionAnchors(shape, IfElseInput, IfElseOutput),
+};

package/src/shapes/Queue.tsx

@@ -0,0 +1,78 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React, { Fragment } from 'react';
+
+import { DEFAULT_OUTPUT, QueueInput, QueueOutput } from '@dxos/conductor';
+import { S } from '@dxos/echo-schema';
+import { type ThemedClassName } from '@dxos/react-ui';
+import { type ShapeComponentProps, type ShapeDef } from '@dxos/react-ui-canvas-editor';
+import { mx } from '@dxos/react-ui-theme';
+
+import { createFunctionAnchors } from './common';
+import { Box, type BoxActionHandler } from './common';
+import { ComputeShape, createShape, type CreateShapeProps } from './defs';
+import { useComputeNodeState } from '../hooks';
+
+export const QueueShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('queue'),
+  }),
+);
+
+export type QueueShape = S.Schema.Type<typeof QueueShape>;
+
+export type CreateQueueProps = CreateShapeProps<QueueShape>;
+
+export const createQueue = (props: CreateQueueProps) =>
+  createShape<QueueShape>({ type: 'queue', size: { width: 256, height: 512 }, ...props });
+
+export const QueueComponent = ({ shape }: ShapeComponentProps<QueueShape>) => {
+  const { runtime } = useComputeNodeState(shape);
+  const items = runtime.outputs[DEFAULT_OUTPUT]?.type === 'executed' ? runtime.outputs[DEFAULT_OUTPUT].value : [];
+
+  const handleAction: BoxActionHandler = (action) => {
+    if (action === 'run') {
+      runtime.evalNode();
+    }
+  };
+
+  return (
+    <Box shape={shape} status={`${items.length} items`} onAction={handleAction}>
+      <div className='flex flex-col w-full overflow-y-scroll divide-y divide-separator'>
+        {[...items].map((item, i) => (
+          <QueueItem key={i} classNames='p-1 px-2' item={item} />
+        ))}
+      </div>
+    </Box>
+  );
+};
+
+export const QueueItem = ({ classNames, item }: ThemedClassName<{ item: any }>) => {
+  if (typeof item !== 'object') {
+    return <div className={mx(classNames, 'whitespace-pre-wrap')}>{item}</div>;
+  }
+
+  return (
+    <div className={mx('grid grid-cols-[80px,1fr]', classNames)}>
+      {Object.entries(item).map(([key, value]) => (
+        <Fragment key={key}>
+          <div className='p-1 text-xs text-subdued'>{key}</div>
+          <div>{typeof value === 'string' ? value : JSON.stringify(value)}</div>
+        </Fragment>
+      ))}
+    </div>
+  );
+};
+
+export const queueShape: ShapeDef<QueueShape> = {
+  type: 'queue',
+  name: 'Queue',
+  icon: 'ph--queue--regular',
+  component: QueueComponent,
+  createShape: createQueue,
+  getAnchors: (shape) => createFunctionAnchors(shape, QueueInput, QueueOutput),
+  resizable: true,
+};

package/src/shapes/RNG.tsx

@@ -0,0 +1,84 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React, { useEffect, useState } from 'react';
+
+import { DEFAULT_OUTPUT } from '@dxos/conductor';
+import { S } from '@dxos/echo-schema';
+import { Icon, type IconProps } from '@dxos/react-ui';
+import { createAnchorMap, type ShapeComponentProps, type ShapeDef } from '@dxos/react-ui-canvas-editor';
+import { mx } from '@dxos/react-ui-theme';
+
+import { ComputeShape, createAnchorId, createShape, type CreateShapeProps } from './defs';
+import { useComputeNodeState } from '../hooks';
+
+export const RandomShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('rng'),
+    min: S.optional(S.Number),
+    max: S.optional(S.Number),
+  }),
+);
+
+export type RandomShape = S.Schema.Type<typeof RandomShape>;
+
+export type CreateRandomProps = CreateShapeProps<RandomShape>;
+
+export const createRandom = (props: CreateRandomProps) =>
+  createShape<RandomShape>({ type: 'rng', size: { width: 64, height: 64 }, ...props });
+
+const icons = [
+  'ph--dice-one--regular',
+  'ph--dice-two--regular',
+  'ph--dice-three--regular',
+  'ph--dice-four--regular',
+  'ph--dice-five--regular',
+  'ph--dice-six--regular',
+];
+
+const pickIcon = () => icons[Math.floor(Math.random() * icons.length)];
+
+// TODO(burdon): Optional range.
+export const RandomComponent = ({ shape }: ShapeComponentProps<RandomShape>) => {
+  const { runtime } = useComputeNodeState(shape);
+
+  const [spin, setSpin] = useState(false);
+  const [icon, setIcon] = useState(pickIcon());
+  useEffect(() => {
+    if (!spin) {
+      return;
+    }
+
+    const i = setInterval(() => setIcon(pickIcon()), 250);
+    const t1 = setTimeout(() => clearInterval(i), 900);
+    const t2 = setTimeout(() => setSpin(false), 1_100);
+    return () => {
+      clearInterval(i);
+      clearTimeout(t1);
+      clearTimeout(t2);
+    };
+  }, [spin]);
+
+  const handleClick: IconProps['onClick'] = (ev) => {
+    ev.stopPropagation();
+    runtime.setOutput(DEFAULT_OUTPUT, Math.random());
+    setSpin(true);
+  };
+
+  return (
+    <div className='flex grow items-center justify-center'>
+      <Icon icon={icon} classNames={mx(spin && 'animate-[spin_1s]')} size={10} onClick={handleClick} />
+    </div>
+  );
+};
+
+export const randomShape: ShapeDef<RandomShape> = {
+  type: 'rng',
+  name: 'Random',
+  icon: 'ph--dice-six--regular',
+  component: RandomComponent,
+  createShape: createRandom,
+  getAnchors: (shape) => createAnchorMap(shape, { [createAnchorId('output')]: { x: 1, y: 0 } }),
+};

package/src/shapes/Scope.tsx

@@ -0,0 +1,54 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React from 'react';
+
+import { DEFAULT_INPUT } from '@dxos/conductor';
+import { S } from '@dxos/echo-schema';
+import { createAnchorMap, type ShapeComponentProps, type ShapeDef } from '@dxos/react-ui-canvas-editor';
+import { useAudioStream, Chaos, shaderPresets } from '@dxos/react-ui-sfx';
+
+import { ComputeShape, createAnchorId, createShape, type CreateShapeProps } from './defs';
+import { useComputeNodeState } from '../hooks';
+
+export const ScopeShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('scope'),
+  }),
+);
+
+export type ScopeShape = S.Schema.Type<typeof ScopeShape>;
+
+export type CreateScopeProps = CreateShapeProps<ScopeShape>;
+
+export const createScope = (props: CreateScopeProps) =>
+  createShape<ScopeShape>({
+    type: 'scope',
+    size: { width: 128, height: 128 },
+    classNames: 'rounded-full border-primary-800',
+    ...props,
+  });
+
+export const ScopeComponent = ({ shape }: ShapeComponentProps<ScopeShape>) => {
+  const { runtime } = useComputeNodeState(shape);
+  const input = runtime.inputs[DEFAULT_INPUT];
+  const active = input?.type === 'executed' ? input.value : false;
+  const { getAverage } = useAudioStream(active);
+
+  return (
+    <div className='flex w-full justify-center items-center bg-black'>
+      <Chaos active={active} getValue={getAverage} options={{ ...shaderPresets.heptapod, zoom: 1.2 }} />
+    </div>
+  );
+};
+
+export const scopeShape: ShapeDef<ScopeShape> = {
+  type: 'scope',
+  name: 'Scope',
+  icon: 'ph--waveform--regular',
+  component: ScopeComponent,
+  createShape: createScope,
+  getAnchors: (shape) => createAnchorMap(shape, { [createAnchorId('input')]: { x: -1, y: 0 } }),
+};

package/src/shapes/Surface.tsx

@@ -0,0 +1,57 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React from 'react';
+
+import { Surface } from '@dxos/app-framework';
+import { DEFAULT_INPUT } from '@dxos/conductor';
+import { S } from '@dxos/echo-schema';
+import { type ShapeComponentProps, type ShapeDef } from '@dxos/react-ui-canvas-editor';
+import { createAnchorMap } from '@dxos/react-ui-canvas-editor';
+
+import { Box, type BoxActionHandler } from './common';
+import { ComputeShape, createAnchorId, createShape, type CreateShapeProps } from './defs';
+import { useComputeNodeState } from '../hooks';
+
+export const SurfaceShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('surface'),
+  }),
+);
+
+export type SurfaceShape = S.Schema.Type<typeof SurfaceShape>;
+
+export type CreateSurfaceProps = CreateShapeProps<SurfaceShape>;
+
+export const createSurface = (props: CreateSurfaceProps) =>
+  createShape<SurfaceShape>({ type: 'surface', size: { width: 384, height: 384 }, ...props });
+
+export const SurfaceComponent = ({ shape }: ShapeComponentProps<SurfaceShape>) => {
+  const { runtime } = useComputeNodeState(shape);
+  const input = runtime.inputs[DEFAULT_INPUT];
+  const value = input?.type === 'executed' ? input.value : null;
+
+  const handleAction: BoxActionHandler = (action) => {
+    if (action === 'run') {
+      runtime.evalNode();
+    }
+  };
+
+  return (
+    <Box shape={shape} onAction={handleAction}>
+      {value !== null && <Surface role='canvas-node' data={{ value }} limit={1} />}
+    </Box>
+  );
+};
+
+export const surfaceShape: ShapeDef<SurfaceShape> = {
+  type: 'surface',
+  name: 'Surface',
+  icon: 'ph--frame-corners--regular',
+  component: SurfaceComponent,
+  createShape: createSurface,
+  getAnchors: (shape) => createAnchorMap(shape, { [createAnchorId('input')]: { x: -1, y: 0 } }),
+  resizable: true,
+};

package/src/shapes/Switch.tsx

@@ -0,0 +1,53 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import React, { useEffect, useState } from 'react';
+
+import { DEFAULT_OUTPUT } from '@dxos/conductor';
+import { S } from '@dxos/echo-schema';
+import { Input } from '@dxos/react-ui';
+import { createAnchorMap, type ShapeComponentProps, type ShapeDef } from '@dxos/react-ui-canvas-editor';
+
+import { ComputeShape, createAnchorId, createShape, type CreateShapeProps } from './defs';
+import { useComputeNodeState } from '../hooks';
+
+export const SwitchShape = S.extend(
+  ComputeShape,
+  S.Struct({
+    type: S.Literal('switch'),
+  }),
+);
+
+export type SwitchShape = S.Schema.Type<typeof SwitchShape>;
+
+export type CreateSwitchProps = CreateShapeProps<SwitchShape>;
+
+export const createSwitch = (props: CreateSwitchProps) =>
+  createShape<SwitchShape>({ type: 'switch', size: { width: 64, height: 64 }, ...props });
+
+// TODO(burdon): Should model as a constant.
+export const SwitchComponent = ({ shape }: ShapeComponentProps<SwitchShape>) => {
+  const { runtime } = useComputeNodeState(shape);
+  const [value, setValue] = useState(false);
+  useEffect(() => {
+    runtime.setOutput(DEFAULT_OUTPUT, value);
+  }, [value]);
+
+  return (
+    <div className='flex w-full justify-center items-center' onClick={(ev) => ev.stopPropagation()}>
+      <Input.Root>
+        <Input.Switch checked={value} onCheckedChange={(value) => setValue(value)} />
+      </Input.Root>
+    </div>
+  );
+};
+
+export const switchShape: ShapeDef<SwitchShape> = {
+  type: 'switch',
+  name: 'Switch',
+  icon: 'ph--toggle-left--regular',
+  component: SwitchComponent,
+  createShape: createSwitch,
+  getAnchors: (shape) => createAnchorMap(shape, { [createAnchorId('output')]: { x: 1, y: 0 } }),
+};