@dxos/plugin-sheet 0.6.12-main.5cc132e → 0.6.12-main.c974201

package/src/extensions/compute.ts

@@ -0,0 +1,98 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { syntaxTree } from '@codemirror/language';
+import {
+  type EditorState,
+  type Extension,
+  type RangeSet,
+  RangeSetBuilder,
+  StateEffect,
+  StateField,
+  type Transaction,
+} from '@codemirror/state';
+import { Decoration, EditorView, ViewPlugin, WidgetType } from '@codemirror/view';
+
+import { type ComputeNode } from '../graph';
+
+const LANGUAGE_TAG = 'dx';
+
+// TODO(burdon): Create marker just for our decorator?
+const updateAllDecorations = StateEffect.define<void>();
+
+export type ComputeOptions = {};
+
+export const compute = (computeNode: ComputeNode, options: ComputeOptions = {}): Extension => {
+  const update = (state: EditorState) => {
+    const builder = new RangeSetBuilder();
+    syntaxTree(state).iterate({
+      enter: (node) => {
+        if (node.name === 'FencedCode') {
+          const cursor = state.selection.main.head;
+          if (state.readOnly || cursor < node.from || cursor > node.to) {
+            const info = node.node.getChild('CodeInfo');
+            if (info) {
+              const type = state.sliceDoc(info.from, info.to);
+              const text = node.node.getChild('CodeText');
+              if (type === LANGUAGE_TAG && text) {
+                const content = state.sliceDoc(text.from, text.to);
+                // TODO(burdon): Map unique reference onto cell; e.g., track ordered list?
+                computeNode.setValue({ col: 0, row: 0 }, content);
+                const value = computeNode.getValue({ col: 0, row: 0 });
+                builder.add(
+                  node.from,
+                  node.to,
+                  Decoration.replace({
+                    widget: new DxWidget(String(value)),
+                  }),
+                );
+              }
+            }
+          }
+        }
+      },
+    });
+
+    return builder.finish();
+  };
+
+  return [
+    // Graph subscription.
+    ViewPlugin.fromClass(
+      class {
+        private readonly _subscription: any;
+        constructor(view: EditorView) {
+          this._subscription = computeNode.graph.update.on(() => {
+            view.dispatch({
+              effects: updateAllDecorations.of(),
+            });
+          });
+        }
+
+        destroy() {
+          this._subscription();
+        }
+      },
+    ),
+
+    // Decorations.
+    StateField.define<RangeSet<any>>({
+      create: (state) => update(state),
+      update: (_: RangeSet<any>, tr: Transaction) => update(tr.state),
+      provide: (field) => EditorView.decorations.from(field),
+    }),
+  ];
+};
+
+class DxWidget extends WidgetType {
+  constructor(private readonly value: string) {
+    super();
+  }
+
+  override toDOM(view: EditorView) {
+    const div = document.createElement('div');
+    div.innerText = this.value;
+    return div;
+  }
+}

package/src/extensions/index.ts

@@ -0,0 +1,5 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './compute';

package/src/{components/ComputeGraph → graph}/async-function.ts

@@ -2,7 +2,6 @@
 // Copyright 2024 DXOS.org
 //
 
-import { CellError, ErrorType, EmptyValue, FunctionPlugin, type HyperFormula } from 'hyperformula';
 import { type InterpreterState } from 'hyperformula/typings/interpreter/InterpreterState';
 import { type InterpreterValue } from 'hyperformula/typings/interpreter/InterpreterValue';
 import { type ProcedureAst } from 'hyperformula/typings/parser';
@@ -12,6 +11,8 @@ import { debounce, type UnsubscribeCallback } from '@dxos/async';
 import { type Space } from '@dxos/client/echo';
 import { log } from '@dxos/log';
 
+import { CellError, ErrorType, EmptyValue, FunctionPlugin, type HyperFormula } from '#hyperformula';
+
 // TODO(burdon): API gateways!
 //  https://publicapis.io
 //  https://api-ninjas.com/api/cryptoprice
@@ -82,6 +83,7 @@ export class FunctionContext {
     }, this._options.recalculationDelay);
   }
 
+  // TODO(burdon): Remove?
   get space() {
     return this._space;
   }

package/src/graph/compute-graph.browser.test.ts

@@ -0,0 +1,104 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { describe, expect, test } from 'vitest';
+
+import { Trigger } from '@dxos/async';
+import { Client } from '@dxos/client';
+import { create } from '@dxos/client/echo';
+import { Context } from '@dxos/context';
+import { type S } from '@dxos/echo-schema';
+import { FunctionType } from '@dxos/plugin-script/types';
+
+import { createComputeGraphRegistry } from './compute-graph';
+import { addressFromA1Notation, createSheet } from '../defs';
+import { SheetModel } from '../model';
+import { type CellScalarValue } from '../types';
+
+// TODO(burdon): Vitest issues:
+//  - Cannot test Hyperformula
+//    - throws "Cannot convert undefined or null to object" in vitest (without browser).
+//    - throws "process.nextTick is not a function" (with browser)
+//    - throws "Buffer already defined" (if nodeExternal: true in config)
+//  - Need better docs; esp. vitest config.
+//    - NOTE: For non-browser tests, import types from x-plugin/types (otherwise will bring in react deps).
+//  - Can't add flags to our tools?
+//  - test.only / test.skip ignored?
+
+/**
+ * NOTE: Browser test required for hyperformula due to raw translation files.
+ */
+describe('compute graph', () => {
+  // TODO(burdon): Replace with builder.
+  const createModel = async (types?: S.Schema<any>[]) => {
+    const ctx = new Context();
+    const client = new Client();
+    if (types) {
+      client.addTypes(types);
+    }
+    await client.initialize();
+    await client.halo.createIdentity();
+    const space = await client.spaces.create();
+    ctx.onDispose(() => client.destroy());
+
+    const registry = createComputeGraphRegistry();
+    await registry.open(ctx);
+
+    const graph = await registry.createGraph(space);
+    await graph.open(ctx);
+
+    const sheet = createSheet({ rows: 5, columns: 5 });
+    const model = new SheetModel(graph, sheet);
+    await model.open(ctx);
+
+    // TODO(burdon): Move event propagation into graph.
+    graph.update.on(() => model.update.emit());
+
+    return { space, graph, model };
+  };
+
+  test('map functions', async () => {
+    const { space, graph } = await createModel([FunctionType]);
+
+    // Create script.
+    const fn = space.db.add(create(FunctionType, { version: 1, binding: 'TEST' }));
+    const id = graph.mapFunctionBindingToId('TEST()');
+    expect(id).to.eq(`${fn.id}()`);
+  });
+
+  test('cross-node references', async () => {
+    const { graph } = await createModel();
+
+    // Create ndoes.
+    const node1 = graph.getOrCreateNode('node-1');
+    const node2 = graph.getOrCreateNode('node-2');
+    node1.hf.setCellContents({ sheet: node1.sheetId, row: 1, col: 1 }, 100);
+    node2.hf.setCellContents({ sheet: node2.sheetId, row: 1, col: 1 }, `=${node1.sheetId}!A1`);
+    const value1 = node1.hf.getCellValue({ sheet: node1.sheetId, col: 1, row: 1 });
+    const value2 = node1.hf.getCellValue({ sheet: node2.sheetId, col: 1, row: 1 });
+    expect(value1).to.eq(value2);
+  });
+
+  test('async function', async () => {
+    const { graph, model } = await createModel();
+
+    // Triggers function.
+    model.setValue(addressFromA1Notation('A1'), '=TEST()');
+    const trigger = new Trigger<CellScalarValue>();
+    model.update.on(() => {
+      const value = model.getValue(addressFromA1Notation('A1'));
+      trigger.wake(value);
+    });
+
+    // Get initial value (i.e., null).
+    const v1 = model.getValue(addressFromA1Notation('A1'));
+    expect(v1).to.be.null;
+    expect(graph.context.info.invocations.TEST).to.eq(undefined);
+
+    // Wait until async update triggered.
+    const v2 = await trigger.wait();
+    expect(v2).not.to.be.null;
+    expect(graph.context.info.invocations.TEST).to.eq(1);
+  });
+});

package/src/graph/compute-graph.stories.tsx

@@ -0,0 +1,92 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import '@dxos-theme';
+
+import React, { useEffect, useRef, useState } from 'react';
+
+import { FunctionType } from '@dxos/plugin-script/types';
+import { create, useSpace, Filter } from '@dxos/react-client/echo';
+import { withClientProvider } from '@dxos/react-client/testing';
+import { Toolbar, Button, Input } from '@dxos/react-ui';
+import { SyntaxHighlighter } from '@dxos/react-ui-syntax-highlighter';
+import { withTheme } from '@dxos/storybook-utils';
+
+import { createSheet } from '../defs';
+import { useComputeGraph, useSheetModel } from '../hooks';
+import { withGraphDecorator } from '../testing';
+import { SheetType } from '../types';
+
+const FUNCTION_NAME = 'TEST';
+
+const Story = () => {
+  const space = useSpace();
+  const graph = useComputeGraph(space);
+  const [sheet, setSheet] = useState<SheetType>();
+  const [text, setText] = useState(`${FUNCTION_NAME}(100)`);
+  const [result, setResult] = useState<any>();
+  const model = useSheetModel(space, sheet);
+  useEffect(() => {
+    if (space) {
+      const sheet = space.db.add(createSheet());
+      setSheet(sheet);
+    }
+  }, [space]);
+
+  useEffect(() => {
+    if (space && graph) {
+      graph.update.on(() => {
+        const f1 = graph.getFunctions({ standard: true, echo: false });
+        const f2 = graph.getFunctions({ standard: false, echo: true });
+        setResult({ functions: { standard: f1.length, echo: f2.length } });
+      });
+
+      space.db.add(create(FunctionType, { version: 1, binding: FUNCTION_NAME }));
+    }
+  }, [space, graph]);
+
+  const inputRef = useRef<HTMLInputElement | null>(null);
+  const handleTest = async () => {
+    if (space && graph) {
+      const { objects } = await space.db.query(Filter.schema(FunctionType)).run();
+      const mapped = graph.mapFunctionBindingToId(text);
+      const unmapped = graph.mapFunctionBindingFromId(mapped);
+      const internal = graph.mapFormulaToNative(text);
+      setResult({ mapped, unmapped, internal, functions: objects.map((object) => object.id) });
+    }
+
+    inputRef.current?.focus();
+  };
+
+  return (
+    <div className='flex flex-col gap-2 '>
+      <Toolbar.Root>
+        <Input.Root>
+          <Input.TextInput
+            ref={inputRef}
+            placeholder='Formula'
+            value={text}
+            onChange={(ev) => setText(ev.target.value)}
+          />
+        </Input.Root>
+        <Button onClick={handleTest}>Test</Button>
+      </Toolbar.Root>
+      <SyntaxHighlighter language='json'>
+        {JSON.stringify({ space: space?.id, graph: graph?.id, sheet: sheet?.id, model: model?.id, result }, null, 2)}
+      </SyntaxHighlighter>
+    </div>
+  );
+};
+
+export default {
+  title: 'plugin-sheet/functions',
+  decorators: [
+    withClientProvider({ types: [FunctionType, SheetType], createIdentity: true, createSpace: true }),
+    withGraphDecorator,
+    withTheme,
+  ],
+  render: (args: any) => <Story {...args} />,
+};
+
+export const Default = {};

package/src/graph/compute-graph.ts

@@ -0,0 +1,290 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type FunctionPluginDefinition } from 'hyperformula';
+import { type ConfigParams } from 'hyperformula/typings/ConfigParams';
+import { type FunctionTranslationsPackage } from 'hyperformula/typings/interpreter';
+
+import { Event } from '@dxos/async';
+import { type SpaceId, type Space, Filter, fullyQualifiedId } from '@dxos/client/echo';
+import { Resource } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { FunctionType } from '@dxos/plugin-script/types';
+import { nonNullable } from '@dxos/util';
+
+import { HyperFormula } from '#hyperformula';
+import { FunctionContext, type FunctionContextOptions } from './async-function';
+import { ComputeNode } from './compute-node';
+import { EdgeFunctionPlugin, EdgeFunctionPluginTranslations } from './edge-function';
+import { defaultFunctions, type FunctionDefinition } from './function-defs';
+
+//
+// NOTE: The package.json file defines the packaged #hyperformula module.
+//
+
+// TODO(wittjosiah): Factor out.
+const OBJECT_ID_LENGTH = 60; // 33 (space id) + 26 (object id) + 1 (separator).
+
+// TODO(burdon): Change to "DX".
+const CUSTOM_FUNCTION = 'ECHO';
+
+export type ComputeGraphPlugin = {
+  plugin: FunctionPluginDefinition;
+  translations: FunctionTranslationsPackage;
+};
+
+export type ComputeGraphOptions = {
+  plugins?: ComputeGraphPlugin[];
+} & Partial<FunctionContextOptions> &
+  Partial<ConfigParams>;
+
+export const defaultOptions: ComputeGraphOptions = {
+  licenseKey: 'gpl-v3',
+  plugins: [
+    {
+      plugin: EdgeFunctionPlugin,
+      translations: EdgeFunctionPluginTranslations,
+    },
+  ],
+};
+
+/**
+ * Marker for sheets that are managed by an ECHO object.
+ */
+const PREFIX = '__';
+export const createSheetName = (id: string) => `${PREFIX}${id}`;
+export const getSheetId = (name: string): string | undefined =>
+  name.startsWith(PREFIX) ? name.slice(PREFIX.length) : undefined;
+
+/**
+ * NOTE: Async imports to decouple hyperformula deps.
+ */
+export const createComputeGraphRegistry = (options: Partial<FunctionContextOptions> = {}) => {
+  return new ComputeGraphRegistry({
+    ...defaultOptions,
+    ...options,
+  });
+};
+
+/**
+ * Manages a collection of ComputeGraph instances for each space.
+ *
+ * [ComputePlugin] => [ComputeGraphRegistry] => [ComputeGraph(Space)] => [ComputeNode(Object)]
+ */
+// TODO(burdon): Move graph into separate plugin; isolate HF deps.
+export class ComputeGraphRegistry extends Resource {
+  private readonly _registry = new Map<SpaceId, ComputeGraph>();
+
+  constructor(private readonly _options: ComputeGraphOptions = defaultOptions) {
+    super();
+    this._options.plugins?.forEach(({ plugin, translations }) => {
+      HyperFormula.registerFunctionPlugin(plugin, translations);
+    });
+  }
+
+  getGraph(spaceId: SpaceId): ComputeGraph | undefined {
+    return this._registry.get(spaceId);
+  }
+
+  async getOrCreateGraph(space: Space): Promise<ComputeGraph> {
+    let graph = this.getGraph(space.id);
+    if (!graph) {
+      log.info('create graph', { space: space.id });
+      graph = await this.createGraph(space);
+    }
+
+    return graph;
+  }
+
+  async createGraph(space: Space): Promise<ComputeGraph> {
+    invariant(!this._registry.has(space.id), `ComputeGraph already exists for space: ${space.id}`);
+    const hf = HyperFormula.buildEmpty(this._options);
+    const graph = new ComputeGraph(hf, space, this._options);
+    await graph.open(this._ctx);
+    this._registry.set(space.id, graph);
+    return graph;
+  }
+}
+
+/**
+ * Per-space compute and dependency graph.
+ * Consists of multiple ComputeNode (corresponding to a HyperFormula sheet).
+ * Manages the set of custom functions.
+ * HyperFormula manages the dependency graph.
+ */
+// TODO(burdon): Tests.
+export class ComputeGraph extends Resource {
+  public readonly id = `graph-${PublicKey.random().truncate()}`;
+
+  // Map of nodes indexed by sheet number.
+  private readonly _nodes = new Map<number, ComputeNode>();
+
+  // Cached function objects.
+  private _functions: FunctionType[] = [];
+
+  // The context is passed to all functions.
+  public readonly context = new FunctionContext(this._hf, this._space, this.refresh.bind(this), this._options);
+
+  // TODO(burdon): Typed events.
+  // TODO(burdon): Tie into HyperFormula dependency graph.
+  // TODO(burdon): Event propagation.
+  public readonly update = new Event();
+
+  constructor(
+    private readonly _hf: HyperFormula,
+    private readonly _space?: Space,
+    private readonly _options?: Partial<FunctionContextOptions>,
+  ) {
+    super();
+
+    this._hf.updateConfig({ context: this.context });
+  }
+
+  // TODO(burdon): Remove.
+  get hf() {
+    return this._hf;
+  }
+
+  refresh() {
+    log('refresh', { id: this.id });
+    this.update.emit();
+  }
+
+  getFunctions({ standard = true, echo = true }: { standard?: boolean; echo?: boolean } = {}): FunctionDefinition[] {
+    return [
+      ...(standard
+        ? this._hf
+            .getRegisteredFunctionNames()
+            .map((name) => defaultFunctions.find((fn) => fn.name === name) ?? { name })
+        : []),
+      ...(echo ? this._functions.map((fn) => ({ name: fn.binding! })) : []),
+    ];
+  }
+
+  /**
+   * Get or create cell representing a sheet.
+   */
+  // TODO(burdon): Async (open node).
+  //  The graph should be an extensible factory that plugins extend with model constructors.
+  //  This would enable on-the-fly instantiation of new models when then are referenced.
+  //  E.g., Cross-object reference would be stored as "ObjectId!A1"
+  //  The graph would then load the object and create a ComputeNode (model) of the appropriate type.
+  getOrCreateNode(name: string): ComputeNode {
+    invariant(name.length);
+    if (!this._hf.doesSheetExist(name)) {
+      log.info('created node', { space: this._space?.id, name });
+      this._hf.addSheet(name);
+      this.update.emit();
+    }
+
+    const sheetId = this._hf.getSheetId(name);
+    invariant(sheetId !== undefined);
+
+    // TODO(burdon): Chain context?
+    const node = new ComputeNode(this, sheetId);
+    this._nodes.set(sheetId, node);
+    return node;
+  }
+
+  /**
+   * Map bound value to custom function invocation.
+   * E.g., "HELLO(...args)" => "EDGE("HELLO", ...args)".
+   */
+  mapFormulaToNative(formula: string): string {
+    return (
+      formula
+        // Sheet references.
+        // https://hyperformula.handsontable.com/guide/cell-references.html#cell-references
+        .replace(/['"]?([ \w]+)['"]?!/, (_match, name) => {
+          if (name) {
+            // TODO(burdon): What if not loaded?
+            const objects = this._hf
+              .getSheetNames()
+              .map((name) => {
+                const id = getSheetId(name);
+                return id ? this._space?.db.getObjectById(id) : undefined;
+              })
+              .filter(nonNullable);
+
+            for (const obj of objects) {
+              if (obj.name === name || obj.title === name) {
+                return `${createSheetName(obj.id)}!`;
+              }
+            }
+          }
+
+          return `${name}!`;
+        })
+
+        // Functions.
+        .replace(/(\w+)\((.*)\)/g, (match, binding, args) => {
+          const fn = this._functions.find((fn) => fn.binding === binding);
+          if (!fn) {
+            return match;
+          }
+
+          if (args.trim() === '') {
+            return `${CUSTOM_FUNCTION}("${binding}")`;
+          } else {
+            return `${CUSTOM_FUNCTION}("${binding}", ${args})`;
+          }
+        })
+    );
+  }
+
+  /**
+   * Map from binding to fully qualified ECHO ID (to store).
+   * E.g., HELLO() => spaceId:objectId()
+   */
+  mapFunctionBindingToId(formula: string) {
+    return formula.replace(/(\w+)\((.*)\)/g, (match, binding, args) => {
+      if (binding === CUSTOM_FUNCTION || defaultFunctions.find((fn) => fn.name === binding)) {
+        return match;
+      }
+
+      const fn = this._functions.find((fn) => fn.binding === binding);
+      if (fn) {
+        const id = fullyQualifiedId(fn);
+        return `${id}(${args})`;
+      } else {
+        return match;
+      }
+    });
+  }
+
+  /**
+   * Map from fully qualified ECHO ID to binding (from store).
+   * E.g., spaceId:objectId() => HELLO()
+   */
+  mapFunctionBindingFromId(formula: string) {
+    return formula.replace(/(\w+):([a-zA-Z0-9]+)\((.*)\)/g, (match, spaceId, objectId, args) => {
+      const id = `${spaceId}:${objectId}`;
+      if (id.length !== OBJECT_ID_LENGTH) {
+        return match;
+      }
+
+      const fn = this._functions.find((fn) => fullyQualifiedId(fn) === id);
+      if (fn?.binding) {
+        return `${fn.binding}(${args})`;
+      } else {
+        return match;
+      }
+    });
+  }
+
+  protected override async _open() {
+    if (this._space) {
+      // Subscribe to remote function definitions.
+      const query = this._space.db.query(Filter.schema(FunctionType));
+      const unsubscribe = query.subscribe(({ objects }) => {
+        this._functions = objects.filter(({ binding }) => binding);
+        this.update.emit();
+      });
+
+      this._ctx.onDispose(unsubscribe);
+    }
+  }
+}

package/src/graph/compute-node.ts

@@ -0,0 +1,51 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Event } from '@dxos/async';
+import { Resource } from '@dxos/context';
+
+import { DetailedCellError } from '#hyperformula';
+import { type ComputeGraph } from './compute-graph';
+import { type CellAddress } from '../defs';
+import { type CellScalarValue } from '../types';
+
+/**
+ * Individual "sheet" (typically corresponds to an ECHO object).
+ */
+// TODO(burdon): Factor out common HF wrapper from from SheetModel.
+export class ComputeNode extends Resource {
+  // TODO(burdon): Chaing events.
+  public readonly update = new Event();
+
+  constructor(
+    private readonly _graph: ComputeGraph,
+    public readonly sheetId: number,
+  ) {
+    super();
+  }
+
+  // TODO(burdon): Remove?
+  get graph() {
+    return this._graph;
+  }
+
+  get hf() {
+    return this._graph.hf;
+  }
+
+  getValue(cell: CellAddress): CellScalarValue {
+    const value = this._graph.hf.getCellValue({ sheet: this.sheetId, row: cell.row, col: cell.col });
+    if (value instanceof DetailedCellError) {
+      return null;
+    }
+
+    return value;
+  }
+
+  setValue(cell: CellAddress, value: CellScalarValue) {
+    const mappedValue =
+      typeof value === 'string' && value.charAt(0) === '=' ? this._graph.mapFormulaToNative(value) : value;
+    this._graph.hf.setCellContents({ sheet: this.sheetId, row: cell.row, col: cell.col }, [[mappedValue]]);
+  }
+}

package/src/{components/ComputeGraph/custom.ts → graph/custom-function.ts}

@@ -2,18 +2,14 @@
 // Copyright 2024 DXOS.org
 //
 
-import { FunctionArgumentType } from 'hyperformula';
 import { type InterpreterState } from 'hyperformula/typings/interpreter/InterpreterState';
 import { type ProcedureAst } from 'hyperformula/typings/parser';
 
 import { getDeep } from '@dxos/util';
 
+import { FunctionArgumentType } from '#hyperformula';
 import { type AsyncFunction, FunctionPluginAsync } from './async-function';
-
-// TODO(burdon): Factor out.
-const parseNumberString = (str: string): number => {
-  return parseFloat(str.replace(/[^\d.]/g, ''));
-};
+import { parseNumberString } from './util';
 
 /**
  * https://hyperformula.handsontable.com/guide/custom-functions.html#add-a-simple-custom-function