@dxos/react-ui-canvas-compute 0.7.5-labs.5f04cf6

package/dist/types/src/testing/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/testing/index.ts"],"names":[],"mappings":"AAIA,cAAc,YAAY,CAAC"}

package/dist/types/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"version":"5.7.2"}

package/package.json

@@ -0,0 +1,85 @@
+{
+  "name": "@dxos/react-ui-canvas-compute",
+  "version": "0.7.5-labs.5f04cf6",
+  "description": "A compute graph extension for the canvas editor component.",
+  "homepage": "https://dxos.org",
+  "bugs": "https://github.com/dxos/dxos/issues",
+  "license": "MIT",
+  "author": "DXOS.org",
+  "exports": {
+    ".": {
+      "types": "./dist/types/src/index.d.ts",
+      "browser": "./dist/lib/browser/index.mjs",
+      "node": "./dist/lib/node-esm/index.mjs"
+    }
+  },
+  "types": "dist/types/src/index.d.ts",
+  "files": [
+    "dist",
+    "src"
+  ],
+  "dependencies": {
+    "@antv/graphlib": "^2.0.4",
+    "@antv/layout": "^1.2.13",
+    "chess.js": "^1.0.0",
+    "@dxos/app-framework": "0.7.5-labs.5f04cf6",
+    "@dxos/assistant": "0.7.5-labs.5f04cf6",
+    "@dxos/context": "0.7.5-labs.5f04cf6",
+    "@dxos/artifact": "0.7.5-labs.5f04cf6",
+    "@dxos/conductor": "0.7.5-labs.5f04cf6",
+    "@dxos/debug": "0.7.5-labs.5f04cf6",
+    "@dxos/async": "0.7.5-labs.5f04cf6",
+    "@dxos/echo-schema": "0.7.5-labs.5f04cf6",
+    "@dxos/edge-client": "0.7.5-labs.5f04cf6",
+    "@dxos/graph": "0.7.5-labs.5f04cf6",
+    "@dxos/keys": "0.7.5-labs.5f04cf6",
+    "@dxos/invariant": "0.7.5-labs.5f04cf6",
+    "@dxos/functions": "0.7.5-labs.5f04cf6",
+    "@dxos/log": "0.7.5-labs.5f04cf6",
+    "@dxos/node-std": "0.7.5-labs.5f04cf6",
+    "@dxos/react-client": "0.7.5-labs.5f04cf6",
+    "@dxos/live-object": "0.7.5-labs.5f04cf6",
+    "@dxos/react-edge-client": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui-attention": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui-canvas-editor": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui-canvas": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui-form": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui-editor": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui-sfx": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui-stack": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui-syntax-highlighter": "0.7.5-labs.5f04cf6",
+    "@dxos/schema": "0.7.5-labs.5f04cf6",
+    "@dxos/util": "0.7.5-labs.5f04cf6"
+  },
+  "devDependencies": {
+    "@effect/ai": "^0.5.1",
+    "@effect/platform": "^0.72.1",
+    "@types/lodash.defaultsdeep": "^4.6.6",
+    "@types/react": "~18.2.0",
+    "@types/react-dom": "~18.2.0",
+    "effect": "^3.12.3",
+    "lodash.defaultsdeep": "^4.6.1",
+    "react": "~18.2.0",
+    "react-dom": "~18.2.0",
+    "vite": "5.4.7",
+    "@dxos/artifact-testing": "0.7.5-labs.5f04cf6",
+    "@dxos/app-framework": "0.7.5-labs.5f04cf6",
+    "@dxos/compute": "0.7.5-labs.5f04cf6",
+    "@dxos/random": "0.7.5-labs.5f04cf6",
+    "@dxos/echo-signals": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui-theme": "0.7.5-labs.5f04cf6",
+    "@dxos/storybook-utils": "0.7.5-labs.5f04cf6"
+  },
+  "peerDependencies": {
+    "@effect/schema": "^0.75.5",
+    "effect": "^3.12.3",
+    "react": "~18.2.0",
+    "react-dom": "~18.2.0",
+    "@dxos/react-ui-theme": "0.7.5-labs.5f04cf6",
+    "@dxos/react-ui": "0.7.5-labs.5f04cf6"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}

package/src/README.md

@@ -0,0 +1,47 @@
+# Compute
+
+- This folder represents a "compute plugin" for the Canvas component.
+- It defines a set of Shapes that implement a state machine.
+
+### Notes (burdon)
+
+- [x] Text/markdown view (parse cot, etc.)
+- [x] Surface view
+- [ ] Prompt with handlebar variables (dynamic schema)
+- [ ] Artifact tool (find artifact defintions AND instances); conext (e.g., current game).
+  - [ ] Chess plugin defines artifact to generate and process FEN?
+  - [ ] Deploy funciton to play game.
+- [ ] Function node (call script)
+- [ ] Group nodes (sub-graph)
+- [ ] Markdown surface node (parse COT)
+- [ ] Auto layout graphs
+
+### Notes (dmaretskyi)
+
+- Consider how different types of edges affect computations:
+  - Being able to provide values by reference
+    - e.g. GPT gets connected to a vector index that is represented by a separate node that can be queried
+    - history node can be provided by reference (as a mutable Thread object) that GPT mutates without requiring a back reference
+  - Consider separating purely functional inputs from imperative ones
+    - This is what UnrealEngine does - functional inputs trigger computations when the data changes, imperative inputs can trigger computations with the same or no data
+    - imperative inputs can represent events or streams of data
+    - functional inputs represent values
+
+- persisting compute graph and canvas in ECHO
+  - (Canvas)-[.nodes]->(CanvasNode)-[.node]->(ComputeNode)<-[.nodes]-(ComputeGraph), (Canvas)-[.computeGraph]->(ComputeGraph)
+
+  - Computations can operate purely on the ComputeGraph
+  - Computations return an ExecutionTrace that is streamable
+  - Canvas graph doesn't directly interact with the compute graph, but reads the execution trace
+
+  - ExecutionTrace
+    - has an execution trace id -- one graph can have multiple independent parallel traces
+    - series of events - each event is associated with a particular node in a particular graph
+    - events include:
+      - inputs/outputs being updated
+      - logs associated with a node or a graph
+      - node state updates (can a node have internal state separate from inputs/outputs)
+      - timestep? - to correlate events that happen during a single execution time
+
+
+

package/src/compute-layout.ts

@@ -0,0 +1,37 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { DefaultInput, DefaultOutput } from '@dxos/conductor';
+import { toEffectSchema } from '@dxos/echo-schema';
+import { type Anchor, ShapeLayout, ShapeRegistry } from '@dxos/react-ui-canvas-editor';
+
+import { type ComputeGraphController } from './graph';
+import { type ComputeShape, createFunctionAnchors } from './shapes';
+
+// TODO(burdon): Customize layout. Specialize ComputeShapeDef and registry.
+export class ComputeShapeLayout extends ShapeLayout {
+  constructor(
+    private _controller: ComputeGraphController,
+    registry: ShapeRegistry,
+  ) {
+    super(registry);
+  }
+
+  // TODO(burdon): Doesn't update.
+  override getAnchors(shape: ComputeShape): Record<string, Anchor> {
+    const shapeDef = this._registry.getShapeDef(shape.type);
+    let anchors = shapeDef?.getAnchors?.(shape) ?? {};
+    if (shape.node) {
+      const node = this._controller.graph.getNode(shape.node);
+      if (node.inputSchema || node.outputSchema) {
+        // TODO(burdon): Requires that component defined input and output types.
+        const inputSchema = node.inputSchema ? toEffectSchema(node.inputSchema) : DefaultInput;
+        const outputSchema = node.outputSchema ? toEffectSchema(node.outputSchema) : DefaultOutput;
+        anchors = createFunctionAnchors(shape, inputSchema, outputSchema);
+      }
+    }
+
+    return anchors;
+  }
+}

package/src/compute.stories.tsx

@@ -0,0 +1,362 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import '@dxos-theme';
+
+import type { Meta, StoryObj } from '@storybook/react';
+import React, { type PropsWithChildren, useEffect, useMemo, useRef, useState } from 'react';
+
+import { withPluginManager } from '@dxos/app-framework/testing';
+import { capabilities, createEdgeServices } from '@dxos/artifact-testing';
+import { type UnsubscribeCallback } from '@dxos/async';
+import { type ComputeGraphModel, ComputeNode, DefaultInput, DefaultOutput } from '@dxos/conductor';
+import { S, toEffectSchema } from '@dxos/echo-schema';
+import { withClientProvider } from '@dxos/react-client/testing';
+import { Select, Toolbar } from '@dxos/react-ui';
+import { withAttention } from '@dxos/react-ui-attention/testing';
+import {
+  type Anchor,
+  Editor,
+  type EditorController,
+  type EditorRootProps,
+  ShapeLayout,
+  ShapeRegistry,
+} from '@dxos/react-ui-canvas-editor';
+import { Container, useSelection } from '@dxos/react-ui-canvas-editor/testing';
+import { JsonFilter } from '@dxos/react-ui-syntax-highlighter';
+import { withLayout, withTheme } from '@dxos/storybook-utils';
+
+import { type ComputeGraphController } from './graph';
+import { ComputeContext, useGraphMonitor } from './hooks';
+import { computeShapes } from './registry';
+import { createFunctionAnchors, type ComputeShape } from './shapes';
+import {
+  createControlCircuit,
+  createGPTRealtimeCircuit,
+  createLogicCircuit,
+  createComputeGraphController,
+  createBasicCircuit,
+  createGptCircuit,
+  createAudioCircuit,
+  createTransformCircuit,
+  createTemplateCircuit,
+  createArtifactCircuit,
+} from './testing';
+import { ComputeShapeLayout } from './compute-layout';
+
+const FormSchema = S.omit<any, any, ['subgraph']>('subgraph')(ComputeNode);
+
+const sidebarTypes: NonNullable<RenderProps['sidebar']>[] = ['canvas', 'compute', 'controller', 'selected'] as const;
+
+// TODO(burdon): Move to async/context?
+const combine = (...cbs: UnsubscribeCallback[]) => {
+  return () => {
+    for (const cb of cbs) {
+      cb();
+    }
+  };
+};
+
+type RenderProps = EditorRootProps<ComputeShape> &
+  PropsWithChildren<{
+    init?: boolean;
+    sidebar?: 'canvas' | 'compute' | 'controller' | 'selected';
+    computeGraph?: ComputeGraphModel;
+    controller?: ComputeGraphController;
+  }>;
+
+const Render = ({
+  id = 'test',
+  children,
+  graph,
+  controller,
+  init,
+  sidebar: _sidebar,
+  registry,
+  ...props
+}: RenderProps) => {
+  const [, forceUpdate] = useState({});
+
+  const editorRef = useRef<EditorController>(null);
+
+  // Selection.
+  const [selection, selected] = useSelection(graph);
+
+  const getComputeNode = (id?: string): ComputeNode | undefined => {
+    if (id) {
+      const node = graph?.getNode(id) as ComputeShape;
+      if (node?.node) {
+        return controller?.graph.getNode(node.node);
+      }
+    }
+  };
+
+  // Sidebar.
+  const [sidebar, setSidebar] = useState(_sidebar);
+  const json = useMemo(() => {
+    switch (sidebar) {
+      case 'canvas':
+        return { graph };
+      case 'compute':
+        return { graph: controller?.graph };
+      case 'controller':
+        return { state: controller?.state };
+      case 'selected':
+        return { shape: selected, compute: getComputeNode(selected?.id) };
+    }
+  }, [graph, controller, sidebar, selected]);
+
+  // Controller.
+  useEffect(() => {
+    if (!controller || !graph) {
+      return;
+    }
+
+    void controller.open();
+    const off = combine(
+      controller.update.on(() => {
+        void editorRef.current?.update();
+        forceUpdate({});
+      }),
+
+      // TODO(burdon): Every node is called on every update.
+      controller.output.on(({ nodeId, property, value }) => {
+        if (value.type === 'not-executed') {
+          // If the node didn't execute, don't trigger.
+          return;
+        }
+
+        const edge = graph.edges.find((edge) => {
+          const source = graph.getNode(edge.source);
+          return (source as ComputeShape).node === nodeId && edge.output === property;
+        });
+
+        if (edge) {
+          void editorRef.current?.action?.({ type: 'trigger', edges: [edge] });
+        }
+      }),
+    );
+
+    return () => {
+      void controller.close();
+      off();
+    };
+  }, [graph, controller]);
+
+  // Sync monitor.
+  const graphMonitor = useGraphMonitor(controller?.graph);
+
+  // Layout.
+  const layout = useMemo(
+    () => (controller && registry ? new ComputeShapeLayout(controller, registry) : undefined),
+    [controller, registry],
+  );
+
+  if (!controller) {
+    return <div />;
+  }
+
+  return (
+    <div className='grid grid-cols-[1fr,360px] w-full h-full'>
+      <ComputeContext.Provider value={{ controller }}>
+        <Container id={id} classNames={['flex grow overflow-hidden', !sidebar && 'col-span-2']}>
+          <Editor.Root<ComputeShape>
+            ref={editorRef}
+            id={id}
+            graph={graph}
+            graphMonitor={graphMonitor}
+            layout={layout}
+            registry={registry}
+            selection={selection}
+            autoZoom
+            {...props}
+          >
+            <Editor.Canvas>{children}</Editor.Canvas>
+            <Editor.UI />
+          </Editor.Root>
+        </Container>
+      </ComputeContext.Provider>
+
+      {sidebar && (
+        <Container id='sidebar' classNames='flex flex-col h-full overflow-hidden'>
+          <Toolbar.Root classNames='p-1'>
+            <Select.Root value={sidebar} onValueChange={(value) => setSidebar(value as RenderProps['sidebar'])}>
+              <Select.TriggerButton classNames='is-full'>{sidebar}</Select.TriggerButton>
+              <Select.Portal>
+                <Select.Content>
+                  <Select.Viewport>
+                    {sidebarTypes.map((type) => (
+                      <Select.Item key={type} value={type}>
+                        {type}
+                      </Select.Item>
+                    ))}
+                  </Select.Viewport>
+                </Select.Content>
+              </Select.Portal>
+            </Select.Root>
+          </Toolbar.Root>
+
+          <div className='flex flex-col h-full overflow-hidden divide-y divider-separator'>
+            {sidebar === 'selected' && selected && (
+              <div>Form</div>
+              // <Form<ComputeNode> schema={FormSchema} values={getComputeNode(selected.id) ?? {}} Custom={{}} />
+            )}
+
+            <JsonFilter data={json} />
+          </div>
+        </Container>
+      )}
+    </div>
+  );
+};
+
+const meta: Meta<RenderProps> = {
+  title: 'ui/react-ui-canvas-compute/compute',
+  component: Editor.Root,
+  render: Render,
+  decorators: [
+    withClientProvider({ createIdentity: true, createSpace: true }),
+    withTheme,
+    withAttention,
+    withLayout({ fullscreen: true, tooltips: true }),
+    withPluginManager({ capabilities }),
+  ],
+};
+
+export default meta;
+
+type Story = StoryObj<RenderProps>;
+
+export const Default: Story = {
+  args: {
+    // debug: true,
+    showGrid: false,
+    snapToGrid: false,
+    sidebar: 'selected',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(),
+  },
+};
+
+export const Beacon: Story = {
+  args: {
+    // debug: true,
+    showGrid: false,
+    snapToGrid: false,
+    sidebar: 'selected',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(createBasicCircuit()),
+  },
+};
+
+export const Transform: Story = {
+  args: {
+    // debug: true,
+    showGrid: false,
+    snapToGrid: false,
+    sidebar: 'selected',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(createTransformCircuit()),
+  },
+};
+
+export const Logic: Story = {
+  args: {
+    // debug: true,
+    showGrid: false,
+    snapToGrid: false,
+    sidebar: 'compute',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(createLogicCircuit()),
+  },
+};
+
+export const Control: Story = {
+  args: {
+    // debug: true,
+    showGrid: false,
+    snapToGrid: false,
+    sidebar: 'compute',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(createControlCircuit()),
+  },
+};
+
+// export const Ollama: Story = {
+//   args: {
+//     // debug: true,
+//     showGrid: false,
+//     snapToGrid: false,
+//     registry: new ShapeRegistry(computeShapes),
+//     ...createComputeGraphController(createTest3(), createEdgeServices()),
+//   },
+// };
+
+export const Template: Story = {
+  args: {
+    showGrid: false,
+    snapToGrid: false,
+    // sidebar: 'controller',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(createTemplateCircuit(), createEdgeServices()),
+  },
+};
+
+export const GPT: Story = {
+  args: {
+    // debug: true,
+    showGrid: false,
+    snapToGrid: false,
+    // sidebar: 'json',
+    sidebar: 'controller',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(createGptCircuit({ history: true }), createEdgeServices()),
+  },
+};
+
+export const Plugins: Story = {
+  args: {
+    // debug: true,
+    showGrid: false,
+    snapToGrid: false,
+    // sidebar: 'json',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(
+      createGptCircuit({ history: true, image: true, artifact: true }),
+      createEdgeServices(),
+    ),
+  },
+};
+
+export const Artifact: Story = {
+  args: {
+    // debug: true,
+    showGrid: false,
+    snapToGrid: false,
+    // sidebar: 'json',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(createArtifactCircuit(), createEdgeServices()),
+  },
+};
+
+export const Audio: Story = {
+  args: {
+    // debug: true,
+    showGrid: false,
+    snapToGrid: false,
+    sidebar: 'controller',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(createAudioCircuit(), createEdgeServices()),
+  },
+};
+
+export const Voice: Story = {
+  args: {
+    showGrid: false,
+    snapToGrid: false,
+    sidebar: 'controller',
+    registry: new ShapeRegistry(computeShapes),
+    ...createComputeGraphController(createGPTRealtimeCircuit(), createEdgeServices()),
+  },
+};