@dxos/react-ui-canvas-compute 0.7.5-labs.5f04cf6

package/src/graph/controller.ts

@@ -0,0 +1,405 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type Context, Effect, Either, Exit, Layer, Scope } from 'effect';
+
+import { type ImageContentBlock } from '@dxos/artifact';
+import { Event, synchronized } from '@dxos/async';
+import {
+  isNotExecuted,
+  makeValueBag,
+  type ComputeEdge,
+  type ComputeEvent,
+  type ComputeGraphModel,
+  type ComputeMeta,
+  type ComputeNode,
+  type ComputeRequirements,
+  EventLogger,
+  type GptInput,
+  type GptOutput,
+  GptService,
+  GraphExecutor,
+  MockGpt,
+  QueueService,
+  SpaceService,
+  type ValueBag,
+} from '@dxos/conductor';
+import { Resource } from '@dxos/context';
+import type { EdgeClient, EdgeHttpClient } from '@dxos/edge-client';
+import { log } from '@dxos/log';
+import { type CanvasGraphModel } from '@dxos/react-ui-canvas-editor';
+
+import { resolveComputeNode } from './node-defs';
+import { createComputeGraph } from '../hooks';
+import { type ComputeShape } from '../shapes';
+
+// TODO(burdon): API package for conductor.
+export const InvalidStateError = Error;
+
+export type FunctionCallback<INPUT, OUTPUT> = (input: INPUT) => Promise<OUTPUT>;
+
+/**
+ * Callback to notify the controller of a scheduled update.
+ */
+export type AsyncUpdate<T> = (value: T) => void;
+
+export interface GptExecutor {
+  invoke: FunctionCallback<GptInput, GptOutput>;
+
+  // TODO(dmaretskyi): A hack to get image artifacts working. Rework into querying images from the ai-service store.
+  imageCache: Map<string, ImageContentBlock>;
+}
+
+export type RuntimeValue =
+  | {
+      type: 'not-executed'; // TODO(burdon): Different from pending?
+    }
+  | {
+      type: 'executed';
+      value: any;
+    }
+  | {
+      type: 'pending';
+    }
+  | {
+      type: 'error';
+      error: string;
+    };
+
+export type Services = {
+  gpt: Context.Tag.Service<GptService>;
+  edgeClient?: EdgeClient;
+  edgeHttpClient?: EdgeHttpClient;
+  spaceService?: Context.Tag.Service<SpaceService>;
+};
+
+type ComputeOutputEvent = {
+  nodeId: string;
+  property: string;
+  value: RuntimeValue;
+};
+
+/**
+ * Nodes that will automatically trigger the execution of the graph on startup.
+ */
+const AUTO_TRIGGER_NODES = ['chat', 'switch', 'constant'];
+
+export const createComputeGraphController = (graph: CanvasGraphModel<ComputeShape>, services?: Partial<Services>) => {
+  const computeGraph = createComputeGraph(graph);
+  const controller = new ComputeGraphController(computeGraph);
+  controller.setServices(services ?? {});
+  return { controller, graph };
+};
+
+/**
+ * Controller that manages compute graph state, execution, and service coordination.
+ */
+export class ComputeGraphController extends Resource {
+  private readonly _executor = new GraphExecutor({
+    computeNodeResolver: (node) => resolveComputeNode(node),
+  });
+
+  private _services: Partial<Services> = {};
+
+  /**
+   * Canvas force-sets outputs of those nodes.
+   */
+  private _forcedOutputs: Record<string, Record<string, unknown>> = {};
+
+  /**
+   * Runtime state of the execution graph.
+   */
+  private _runtimeStateInputs: Record<string, Record<string, RuntimeValue>> = {};
+
+  private _runtimeStateOutputs: Record<string, Record<string, RuntimeValue>> = {};
+
+  // TODO(burdon): Remove? Make state reactive?
+  public readonly update = new Event();
+
+  /** Computed result. */
+  public readonly output = new Event<ComputeOutputEvent>();
+
+  public readonly events = new Event<ComputeEvent>();
+
+  constructor(
+    /** Persistent compute graph. */
+    private readonly _graph: ComputeGraphModel,
+  ) {
+    super();
+  }
+
+  toJSON() {
+    return {
+      graph: this._graph,
+      state: {
+        inputs: this._runtimeStateInputs,
+        outputs: this._runtimeStateOutputs,
+      },
+      forcedOutputs: this._forcedOutputs,
+    };
+  }
+
+  setServices(services: Partial<Services>) {
+    Object.assign(this._services, services);
+  }
+
+  get graph() {
+    return this._graph;
+  }
+
+  get userState() {
+    return this._forcedOutputs;
+  }
+
+  get inputStates() {
+    return this._runtimeStateInputs;
+  }
+
+  get outputStates() {
+    return this._runtimeStateOutputs;
+  }
+
+  /**
+   * Inputs and outputs for all nodes.
+   */
+  get state() {
+    const ids = [...new Set([...Object.keys(this._runtimeStateInputs), ...Object.keys(this._runtimeStateOutputs)])];
+    return Object.fromEntries(
+      ids.map((id) => [
+        id,
+        {
+          node: this._graph.getNode(id),
+          input: this._runtimeStateInputs[id],
+          output: this._runtimeStateOutputs[id],
+        },
+      ]),
+    );
+  }
+
+  addNode(node: ComputeNode) {
+    this._graph.addNode(node);
+  }
+
+  addEdge(edge: ComputeEdge) {
+    this._graph.addEdge(edge);
+  }
+
+  getComputeNode(nodeId: string): ComputeNode {
+    return this._graph.getNode(nodeId);
+  }
+
+  getInputs(nodeId: string) {
+    return this._runtimeStateInputs[nodeId] ?? {};
+  }
+
+  getOutputs(nodeId: string) {
+    return this._runtimeStateOutputs[nodeId] ?? {};
+  }
+
+  setOutput(nodeId: string, property: string, value: any) {
+    this._forcedOutputs[nodeId] ??= {};
+    this._forcedOutputs[nodeId][property] = value;
+
+    queueMicrotask(async () => {
+      try {
+        await this.exec(nodeId);
+      } catch (err) {
+        log.catch(err);
+      }
+    });
+  }
+
+  async getMeta(node: ComputeNode): Promise<ComputeMeta> {
+    const { meta } = await resolveComputeNode(node);
+    return meta;
+  }
+
+  async evalNode(nodeId: string) {
+    const executor = this._executor.clone();
+    await executor.load(this._graph);
+
+    for (const [nodeId, outputs] of Object.entries(this._forcedOutputs)) {
+      executor.setOutputs(nodeId, Effect.succeed(makeValueBag(outputs)));
+    }
+
+    const services = this._createServiceLayer();
+    await Effect.runPromise(
+      Effect.gen(this, function* () {
+        const scope = yield* Scope.make();
+
+        // TODO(dmaretskyi): Code duplication.
+        const executable = yield* Effect.promise(() => resolveComputeNode(this._graph.getNode(nodeId)));
+        const computingOutputs = executable.exec != null;
+        // TODO(dmaretskyi): Check if the node has a compute function and run computeOutputs if it does.
+        const effect = (computingOutputs ? executor.computeOutputs(nodeId) : executor.computeInputs(nodeId)).pipe(
+          Effect.withSpan('runGraph'),
+          Effect.provide(services),
+          Scope.extend(scope),
+
+          Effect.flatMap(computeValueBag),
+          Effect.withSpan('test'),
+          Effect.tap((values) => {
+            for (const [key, value] of Object.entries(values)) {
+              if (computingOutputs) {
+                this._onOutputComputed(nodeId, key, value);
+              } else {
+                this._onInputComputed(nodeId, key, value);
+              }
+            }
+          }),
+        );
+
+        yield* effect;
+
+        yield* Scope.close(scope, Exit.void);
+      }),
+    );
+
+    this.update.emit();
+  }
+
+  /**
+   * Executes the graph.
+   * @param startFromNode - Node to start from, otherwise all {@link AUTO_TRIGGER_NODES} are executed.
+   */
+  @synchronized
+  async exec(startFromNode?: string) {
+    this._runtimeStateInputs = {};
+    this._runtimeStateOutputs = {};
+    const executor = this._executor.clone();
+    await executor.load(this._graph);
+
+    for (const [nodeId, outputs] of Object.entries(this._forcedOutputs)) {
+      executor.setOutputs(nodeId, Effect.succeed(makeValueBag(outputs)));
+    }
+
+    // TODO(dmaretskyi): Stop hardcoding.
+    const triggerNodes =
+      startFromNode != null
+        ? [this._graph.getNode(startFromNode)]
+        : this._graph.nodes.filter((node) => node.type != null && AUTO_TRIGGER_NODES.includes(node.type));
+    const allAffectedNodes = [...new Set(triggerNodes.flatMap((node) => executor.getAllDependantNodes(node.id)))];
+
+    const services = this._createServiceLayer();
+    await Effect.runPromise(
+      Effect.gen(this, function* () {
+        const scope = yield* Scope.make();
+
+        // TODO(burdon): Return map?
+        const tasks: Effect.Effect<unknown, any, never>[] = [];
+        for (const node of allAffectedNodes) {
+          // TODO(dmaretskyi): Code duplication.
+          const executable = yield* Effect.promise(() => resolveComputeNode(this._graph.getNode(node)));
+          const computingOutputs = executable.exec != null;
+
+          // TODO(dmaretskyi): Check if the node has a compute function and run computeOutputs if it does.
+          const effect = (computingOutputs ? executor.computeOutputs(node) : executor.computeInputs(node)).pipe(
+            Effect.withSpan('runGraph'),
+            Effect.provide(services),
+            Scope.extend(scope),
+            Effect.flatMap(computeValueBag),
+            Effect.withSpan('test'),
+            Effect.tap((values) => {
+              for (const [key, value] of Object.entries(values)) {
+                if (computingOutputs) {
+                  this._onOutputComputed(node, key, value);
+                } else {
+                  this._onInputComputed(node, key, value);
+                }
+              }
+            }),
+          );
+
+          tasks.push(effect);
+        }
+
+        //
+        yield* Effect.all(tasks);
+
+        //
+        yield* Scope.close(scope, Exit.void);
+      }),
+    );
+
+    this.update.emit();
+  }
+
+  private _createServiceLayer(): Layer.Layer<Exclude<ComputeRequirements, Scope.Scope>> {
+    const services = { ...DEFAULT_SERVICES, ...this._services };
+    const logLayer = Layer.succeed(EventLogger, this._createLogger());
+    const gptLayer = Layer.succeed(GptService, services.gpt!);
+    const queueLayer =
+      services.edgeHttpClient != null ? QueueService.fromClient(services.edgeHttpClient) : QueueService.notAvailable;
+
+    const spaceLayer =
+      services.spaceService != null ? Layer.succeed(SpaceService, services.spaceService) : SpaceService.empty;
+
+    return Layer.mergeAll(logLayer, gptLayer, queueLayer, spaceLayer);
+  }
+
+  private _createLogger(): Context.Tag.Service<EventLogger> {
+    return {
+      log: (event) => {
+        this._handleEvent(event);
+      },
+      nodeId: undefined, // Not in a context of a specific node.
+    };
+  }
+
+  private _handleEvent(event: ComputeEvent) {
+    log('handleEvent', { event });
+    switch (event.type) {
+      case 'compute-input': {
+        this._onInputComputed(event.nodeId, event.property, { type: 'executed', value: event.value });
+        break;
+      }
+
+      case 'compute-output': {
+        this._onOutputComputed(event.nodeId, event.property, { type: 'executed', value: event.value });
+        break;
+      }
+    }
+    this.events.emit(event);
+  }
+
+  private _onInputComputed(nodeId: string, property: string, value: RuntimeValue) {
+    this._runtimeStateInputs[nodeId] ??= {};
+    this._runtimeStateInputs[nodeId][property] = value;
+  }
+
+  private _onOutputComputed(nodeId: string, property: string, value: RuntimeValue) {
+    this._runtimeStateOutputs[nodeId] ??= {};
+    this._runtimeStateOutputs[nodeId][property] = value;
+
+    // TODO(burdon): Only fire if changed?
+    this.output.emit({ nodeId, property, value });
+  }
+}
+
+const DEFAULT_SERVICES: Services = {
+  gpt: new MockGpt(),
+};
+
+/**
+ * Waits for all effects in the bag to complete and returns the `RuntimeValue` for each property.
+ */
+const computeValueBag = (bag: ValueBag<any>): Effect.Effect<Record<string, RuntimeValue>, never, never> => {
+  return Effect.all(
+    Object.entries(bag.values).map(([key, eff]) =>
+      Effect.either(eff).pipe(
+        Effect.map((value) => {
+          if (Either.isLeft(value)) {
+            if (isNotExecuted(value.left)) {
+              return [key, { type: 'not-executed' }] as const;
+            } else {
+              return [key, { type: 'error', error: value.left }] as const;
+            }
+          } else {
+            return [key, { type: 'executed', value: value.right }] as const;
+          }
+        }),
+      ),
+    ),
+  ).pipe(Effect.map((entries) => Object.fromEntries(entries) as Record<string, RuntimeValue>));
+};

package/src/graph/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './controller';
+export * from './node-defs';

package/src/graph/node-defs.ts

@@ -0,0 +1,82 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import {
+  NODE_INPUT,
+  NODE_OUTPUT,
+  type ComputeNode,
+  type Executable,
+  type NodeType,
+  registry,
+  getTemplateInputSchema,
+} from '@dxos/conductor';
+import { raise } from '@dxos/debug';
+import { ObjectId, toJsonSchema } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+
+import { type ComputeShape, type ConstantShape, type TemplateShape } from '../shapes';
+
+export const resolveComputeNode = async (node: ComputeNode): Promise<Executable> => {
+  const impl = registry[node.type as NodeType];
+  invariant(impl, `Unknown node type: ${node.type}`);
+  return impl;
+};
+
+export const isValidComputeNode = (type: string): boolean => {
+  return nodeFactory[type as NodeType] !== undefined;
+};
+
+export const createComputeNode = (shape: ComputeShape): ComputeNode => {
+  const type = shape.type ?? raise(new Error('Type not specified'));
+  const factory = nodeFactory[type as NodeType] ?? raise(new Error(`Unknown shape type: ${type}`));
+  return factory(shape);
+};
+
+const nodeFactory: Record<NodeType | 'trigger', (shape: ComputeShape) => ComputeNode> = {
+  // System.
+  [NODE_INPUT]: () => createNode(NODE_INPUT),
+  [NODE_OUTPUT]: () => createNode(NODE_OUTPUT),
+
+  // Extensions.
+  ['text-to-image' as const]: () => createNode('text-to-image'), // TODO(burdon): Rename ai-impage-tool
+  ['and' as const]: () => createNode('and'),
+  ['append' as const]: () => createNode('append'),
+  ['audio' as const]: () => createNode('audio'),
+  ['beacon' as const]: () => createNode('beacon'),
+  ['chat' as const]: () => createNode('chat'),
+  ['constant' as const]: (shape) =>
+    createNode('constant', {
+      value: (shape as ConstantShape).value,
+    }),
+  ['database' as const]: () => createNode('database'),
+  ['gpt' as const]: () => createNode('gpt'),
+  ['gpt-realtime' as const]: () => createNode('gpt-realtime'),
+  ['if' as const]: () => createNode('if'),
+  ['if-else' as const]: () => createNode('if-else'),
+  ['function' as const]: () => createNode('function'),
+  ['json' as const]: () => createNode('json'),
+  ['json-transform' as const]: () => createNode('json-transform'),
+  ['not' as const]: () => createNode('not'),
+  ['or' as const]: () => createNode('or'),
+  ['queue' as const]: () => createNode('queue'),
+  ['rng' as const]: () => createNode('rng'),
+  ['reducer' as const]: () => createNode('reducer'),
+  ['scope' as const]: () => createNode('scope'),
+  ['surface' as const]: () => createNode('surface'),
+  ['switch' as const]: () => createNode('switch'),
+  ['template' as const]: (shape) => {
+    const node = createNode('template', { valueType: (shape as TemplateShape).valueType, value: shape.text });
+    node.inputSchema = toJsonSchema(getTemplateInputSchema(node));
+    return node;
+  },
+  ['text' as const]: () => createNode('text'),
+  ['thread' as const]: () => createNode('thread'),
+  ['trigger' as const]: () => createNode(NODE_INPUT),
+};
+
+const createNode = (type: string, props?: Partial<ComputeNode>): ComputeNode => ({
+  id: ObjectId.random(),
+  type,
+  ...props,
+});

package/src/hooks/compute-context.ts

@@ -0,0 +1,19 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { createContext, useContext } from 'react';
+
+import { raise } from '@dxos/debug';
+
+import type { ComputeGraphController } from '../graph';
+
+export type ComputeContextType = {
+  controller: ComputeGraphController;
+};
+
+export const ComputeContext = createContext<ComputeContextType | null>(null);
+
+export const useComputeContext = () => {
+  return useContext(ComputeContext) ?? raise(new Error('Missing ComputeContext'));
+};

package/src/hooks/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+export * from './compute-context';
+export * from './useComputeNodeState';
+export * from './useGraphMonitor';

package/src/hooks/useComputeNodeState.ts

@@ -0,0 +1,83 @@
+//
+// Copyright 2025 DXOS.org
+//
+
+import { useCallback, useEffect, useState } from 'react';
+
+import type { ComputeNode, ComputeMeta, ComputeEvent } from '@dxos/conductor';
+import { S } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+
+import { useComputeContext } from './compute-context';
+import { type RuntimeValue } from '../graph';
+import { type ComputeShape } from '../shapes';
+
+export type ComputeNodeState = {
+  node: ComputeNode;
+  meta: ComputeMeta;
+  runtime: {
+    inputs: Record<string, RuntimeValue>;
+    outputs: Record<string, RuntimeValue>;
+    setOutput: (property: string, value: any) => void;
+    evalNode: () => void;
+    subscribeToEventLog: (cb: (event: ComputeEvent) => void) => void;
+  };
+};
+
+/**
+ * Runtime state of a compute node.
+ */
+export const useComputeNodeState = (shape: ComputeShape): ComputeNodeState => {
+  const { controller } = useComputeContext();
+  invariant(controller);
+
+  const [meta, setMeta] = useState<ComputeMeta>();
+  useEffect(() => {
+    let disposed = false;
+    queueMicrotask(async () => {
+      invariant(shape.node, 'Node not specified');
+      const node = controller.getComputeNode(shape.node);
+      const meta = await controller.getMeta(node);
+      if (disposed) {
+        return;
+      }
+      setMeta(meta);
+    });
+
+    return () => {
+      disposed = true;
+    };
+  }, [shape.node]);
+
+  const evalNode = useCallback(() => {
+    return controller.evalNode(shape.node!);
+  }, [shape.node]);
+
+  const subscribeToEventLog = useCallback(
+    (cb: (event: ComputeEvent) => void) => {
+      return controller.events.on((ev) => {
+        if (ev.nodeId === shape.node) {
+          cb(ev);
+        }
+      });
+    },
+    [shape.node],
+  );
+
+  return {
+    node: controller.getComputeNode(shape.node!),
+    meta: meta ?? {
+      input: S.Struct({}),
+      output: S.Struct({}),
+    },
+    runtime: {
+      inputs: controller.getInputs(shape.node!),
+      outputs: controller.getOutputs(shape.node!),
+      setOutput: (property: string, value: any) => {
+        controller.setOutput(shape.node!, property, value);
+      },
+      evalNode,
+      subscribeToEventLog,
+    },
+  };
+};