@dxos/plugin-sheet 0.6.13 → 0.6.14-main.1366248

package/src/compute-graph/compute-graph.test.ts

@@ -0,0 +1,87 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type CellValue } from 'hyperformula';
+import { afterEach, beforeEach, describe, expect, test } from 'vitest';
+
+import { Trigger } from '@dxos/async';
+import { create, fullyQualifiedId } from '@dxos/client/echo';
+import { FunctionType } from '@dxos/plugin-script/types';
+
+import { DetailedCellError } from '#hyperformula';
+import { TestBuilder } from './testing';
+
+describe('ComputeGraph', () => {
+  let testBuilder: TestBuilder;
+  beforeEach(async () => {
+    testBuilder = new TestBuilder({ types: [FunctionType] });
+    await testBuilder.open();
+  });
+  afterEach(async () => {
+    await testBuilder.close();
+  });
+
+  test('map functions', async () => {
+    const space = await testBuilder.client.spaces.create();
+    const graph = testBuilder.registry.createGraph(space);
+    await graph.open();
+
+    // Create script.
+    const trigger = new Trigger();
+    graph.update.once(() => trigger.wake());
+    const functionObject = space.db.add(create(FunctionType, { version: 1, binding: 'TEST' }));
+    await trigger.wait();
+    const functions = graph.getFunctions({ echo: true });
+    expect(functions).to.toHaveLength(1);
+
+    const id = graph.mapFunctionBindingToId('TEST()');
+    expect(id).to.eq(`${fullyQualifiedId(functionObject)}()`);
+  });
+
+  test('cross-node references', async () => {
+    const space = await testBuilder.client.spaces.create();
+    const graph = testBuilder.registry.createGraph(space);
+
+    // Create nodes.
+    const node1 = await graph.getOrCreateNode('node-1').open();
+    const node2 = await graph.getOrCreateNode('node-2').open();
+    expect(graph.hf.getSheetNames()).to.toHaveLength(2);
+
+    {
+      node1.graph.hf.setCellContents({ sheet: node1.sheetId, row: 0, col: 0 }, [[100, 200, 300, '=SUM(A1:C1)']]);
+      node2.graph.hf.setCellContents({ sheet: node2.sheetId, row: 0, col: 0 }, "='node-1'!D1");
+      const value1 = node1.graph.hf.getCellValue({ sheet: node1.sheetId, col: 3, row: 0 });
+      const value2 = node2.graph.hf.getCellValue({ sheet: node2.sheetId, col: 0, row: 0 });
+      expect(value1).to.eq(value2);
+    }
+
+    // Get updated event.
+    const trigger = new Trigger<CellValue>();
+    node2.update.on(({ change }) => {
+      const value = node2.graph.hf.getCellValue({ sheet: node2.sheetId, col: 0, row: 0 });
+      expect(value).to.eq(change?.newValue);
+      trigger.wake(value);
+    });
+
+    {
+      node1.graph.hf.setCellContents({ sheet: node1.sheetId, row: 0, col: 0 }, 400);
+      const value1 = node1.graph.hf.getCellValue({ sheet: node1.sheetId, col: 3, row: 0 });
+      const value2 = await trigger.wait();
+      expect(value1).to.eq(value2);
+    }
+  });
+
+  // TODO(burdon): Dynamically load node/model based on dependencies.
+  //  - Create dependency then close model.
+  test('dynamic loading', async () => {
+    const space = await testBuilder.client.spaces.create();
+    const graph = testBuilder.registry.createGraph(space);
+
+    const node1 = await graph.getOrCreateNode('node-1').open();
+    node1.graph.hf.setCellContents({ sheet: node1.sheetId, row: 0, col: 0 }, "='node-2'!A1");
+    const value1 = node1.graph.hf.getCellValue({ sheet: node1.sheetId, col: 0, row: 0 });
+    expect(value1).to.be.instanceof(DetailedCellError);
+    expect((value1 as DetailedCellError).type).to.eq('REF');
+  });
+});

package/src/compute-graph/compute-graph.ts

@@ -0,0 +1,260 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type Listeners } from 'hyperformula/typings/Emitter';
+
+import { Event } from '@dxos/async';
+import { type Space, Filter, fullyQualifiedId } from '@dxos/client/echo';
+import { FQ_ID_LENGTH } from '@dxos/client/echo';
+import { Resource } from '@dxos/context';
+import { getTypename } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { FunctionType } from '@dxos/plugin-script/types';
+import { nonNullable } from '@dxos/util';
+
+import { ExportedCellChange, type HyperFormula } from '#hyperformula';
+import { ComputeNode } from './compute-node';
+import {
+  defaultFunctions,
+  FunctionContext,
+  type FunctionContextOptions,
+  type FunctionDefinition,
+  EDGE_FUNCTION_NAME,
+} from './functions';
+
+// TODO(burdon): Factor out compute-graph.
+
+const UNKNOWN_BINDING = '__UNKNOWN__';
+
+// TODO(burdon): Factory.
+// export type ComputeNodeGenerator = <T>(obj: T) => ComputeNode;
+
+type ObjectRef = { type: string; id: string };
+
+/**
+ * Marker for sheets that are managed by an ECHO object.
+ * Sheet ID: `dxos.org/type/SheetType@1234`
+ */
+export const createSheetName = ({ type, id }: ObjectRef) => `${type}@${id}`;
+export const parseSheetName = (name: string): Partial<ObjectRef> => {
+  const [type, id] = name.split('@');
+  return id ? { type, id } : { id: type };
+};
+
+export type ComputeGraphEvent = 'functionsUpdated' | 'valuesUpdated';
+
+/**
+ * Per-space compute and dependency graph.
+ * Consists of multiple ComputeNode (corresponding to a HyperFormula sheet).
+ * Manages the set of custom functions.
+ * HyperFormula manages the dependency graph.
+ */
+export class ComputeGraph extends Resource {
+  public readonly id = `graph-${PublicKey.random().truncate()}`;
+
+  // Map of nodes indexed by sheet number.
+  private readonly _nodes = new Map<number, ComputeNode>();
+
+  // Cached function objects.
+  private _remoteFunctions: FunctionType[] = [];
+
+  public readonly update = new Event<{ type: ComputeGraphEvent }>();
+
+  // The context is passed to all functions.
+  public readonly context: FunctionContext;
+
+  constructor(
+    private readonly _hf: HyperFormula,
+    private readonly _space?: Space,
+    private readonly _options?: Partial<FunctionContextOptions>,
+  ) {
+    super();
+
+    const contextOptions = {
+      ...this._options,
+      onUpdate: (update) => {
+        this._options?.onUpdate?.(update);
+        this.update.emit({ type: 'valuesUpdated' });
+      },
+    } satisfies Partial<FunctionContextOptions>;
+    this.context = new FunctionContext(this._hf, this._space, contextOptions);
+    this._hf.updateConfig({ context: this.context });
+
+    // TODO(burdon): If debounce then aggregate changes.
+    const onValuesUpdate: Listeners['valuesUpdated'] = (changes) => {
+      for (const change of changes) {
+        if (change instanceof ExportedCellChange) {
+          const { sheet } = change;
+          const node = this._nodes.get(sheet);
+          if (node) {
+            node.update.emit({ type: 'valuesUpdated', change });
+          }
+        }
+      }
+    };
+
+    this._hf.on('valuesUpdated', onValuesUpdate);
+    this._ctx.onDispose(() => this._hf.off('valuesUpdated', onValuesUpdate));
+  }
+
+  get hf() {
+    return this._hf;
+  }
+
+  getFunctions(
+    { standard, echo }: { standard?: boolean; echo?: boolean } = { standard: true, echo: true },
+  ): FunctionDefinition[] {
+    return [
+      ...(standard
+        ? this._hf
+            .getRegisteredFunctionNames()
+            .map((name) => defaultFunctions.find((fn) => fn.name === name) ?? { name })
+        : []),
+      ...(echo ? this._remoteFunctions.map((fn) => ({ name: fn.binding! })) : []),
+    ];
+  }
+
+  /**
+   * Get or create cell representing a sheet.
+   */
+  // TODO(burdon): Async (open node).
+  //  The graph should be an extensible factory that plugins extend with model constructors.
+  //  This would enable on-the-fly instantiation of new models when then are referenced.
+  //  E.g., Cross-object reference would be stored as "ObjectId!A1"
+  //  The graph would then load the object and create a ComputeNode (model) of the appropriate type.
+  getOrCreateNode(name: string): ComputeNode {
+    invariant(name.length);
+    if (!this._hf.doesSheetExist(name)) {
+      log('created node', { space: this._space?.id, sheet: name });
+      this._hf.addSheet(name);
+    }
+
+    const sheetId = this._hf.getSheetId(name);
+    invariant(sheetId !== undefined);
+
+    const node = new ComputeNode(this, sheetId);
+    this._nodes.set(sheetId, node);
+    return node;
+  }
+
+  /**
+   * Map bound value to custom function invocation.
+   * E.g., "HELLO(...args)" => "DX("HELLO", ...args)".
+   */
+  mapFormulaToNative(formula: string): string {
+    return (
+      formula
+        //
+        // Map cross-sheet references by name onto sheet stored by ECHO object/model.
+        // Example: "Test Sheet"!A0 => "dxos.org/type/SheetType@1234"!A0
+        // https://hyperformula.handsontable.com/guide/cell-references.html#cell-references
+        //
+        .replace(/['"]?([ \w]+)['"]?!/, (_match, name) => {
+          if (name) {
+            // TODO(burdon): Cache map.
+            const objects = this._hf
+              .getSheetNames()
+              .map((name) => {
+                const { type, id } = parseSheetName(name);
+                return type && id ? this._space?.db.getObjectById(id) : undefined;
+              })
+              .filter(nonNullable);
+
+            for (const obj of objects) {
+              if (obj.name === name) {
+                const type = getTypename(obj)!;
+                // NOTE: Names must be single quoted.
+                return `'${createSheetName({ type, id: obj.id })}'!`;
+              }
+            }
+          }
+
+          return `${name}!`;
+        })
+
+        //
+        // Map remote function references (i.e., to remote DX function invocation).
+        //
+        .replace(/(\w+)\((.*)\)/g, (match, binding, args) => {
+          const fn = this._remoteFunctions.find((fn) => fn.binding === binding);
+          if (!fn) {
+            return match;
+          }
+
+          if (args.trim() === '') {
+            return `${EDGE_FUNCTION_NAME}("${binding}")`;
+          } else {
+            return `${EDGE_FUNCTION_NAME}("${binding}", ${args})`;
+          }
+        })
+    );
+  }
+
+  /**
+   * Map from binding to fully qualified ECHO ID (to store).
+   * E.g., HELLO() => spaceId:objectId()
+   */
+  mapFunctionBindingToId(formula: string) {
+    return formula.replace(/(\w+)\((.*)\)/g, (match, binding, args) => {
+      if (binding === EDGE_FUNCTION_NAME || defaultFunctions.find((fn) => fn.name === binding)) {
+        return match;
+      }
+
+      const fn = this._remoteFunctions.find((fn) => fn.binding === binding);
+      if (fn) {
+        const id = fullyQualifiedId(fn);
+        return `${id}(${args})`;
+      } else {
+        return match;
+      }
+    });
+  }
+
+  /**
+   * Map from fully qualified ECHO ID to binding (from store).
+   * E.g., spaceId:objectId() => HELLO()
+   */
+  mapFunctionBindingFromId(formula: string) {
+    const binding = formula.replace(/(\w+):([a-zA-Z0-9]+)\((.*)\)/g, (match, spaceId, objectId, args) => {
+      const id = `${spaceId}:${objectId}`;
+      if (id.length !== FQ_ID_LENGTH) {
+        return match;
+      }
+
+      const fn = this._remoteFunctions.find((fn) => fullyQualifiedId(fn) === id);
+      if (fn?.binding) {
+        return `${fn.binding}(${args})`;
+      } else {
+        return UNKNOWN_BINDING;
+      }
+    });
+
+    if (binding.startsWith(`=${UNKNOWN_BINDING}`)) {
+      return undefined;
+    } else {
+      return binding;
+    }
+  }
+
+  protected override async _open() {
+    if (this._space) {
+      // Subscribe to remote function definitions.
+      const query = this._space.db.query(Filter.schema(FunctionType));
+      const unsubscribe = query.subscribe(({ objects }) => {
+        this._remoteFunctions = objects.filter(({ binding }) => binding);
+        this.update.emit({ type: 'functionsUpdated' });
+      });
+
+      this._ctx.onDispose(unsubscribe);
+    }
+  }
+
+  protected override async _close() {
+    for (const node of this._nodes.values()) {
+      await node.close();
+    }
+  }
+}

package/src/compute-graph/compute-node.ts

@@ -0,0 +1,62 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type Listeners } from 'hyperformula/typings/Emitter';
+import { type ExportedCellChange } from 'hyperformula/typings/Exporter';
+
+import { Event } from '@dxos/async';
+import { Resource } from '@dxos/context';
+
+import { DetailedCellError } from '#hyperformula';
+import { type ComputeGraph } from './compute-graph';
+import { type CellAddress, isFormula } from '../defs';
+import { type CellScalarValue } from '../types';
+
+export type ComputeNodeEvent = {
+  type: keyof Listeners;
+  change?: ExportedCellChange;
+};
+
+/**
+ * Individual "sheet" (typically corresponds to an ECHO object).
+ */
+// TODO(burdon): Factor out common HF wrapper from from SheetModel.
+export class ComputeNode extends Resource {
+  public readonly update = new Event<ComputeNodeEvent>();
+
+  constructor(
+    private readonly _graph: ComputeGraph,
+    public readonly sheetId: number,
+  ) {
+    super();
+  }
+
+  get graph() {
+    return this._graph;
+  }
+
+  clear() {
+    this._graph.hf.clearSheet(this.sheetId);
+  }
+
+  getValue(cell: CellAddress): CellScalarValue {
+    const value = this._graph.hf.getCellValue({ sheet: this.sheetId, row: cell.row, col: cell.col });
+    if (value instanceof DetailedCellError) {
+      return null;
+    }
+
+    return value;
+  }
+
+  setValue(cell: CellAddress, value: CellScalarValue) {
+    const mappedValue = isFormula(value) ? this._graph.mapFormulaToNative(value) : value;
+    this._graph.hf.setCellContents({ sheet: this.sheetId, row: cell.row, col: cell.col }, [[mappedValue]]);
+  }
+
+  // TODO(burdon): Load data into sheet.
+  protected override async _open() {
+    // const unsubscribe = this._graph.update.on(this.update.emit);
+    // this._ctx.onDispose(unsubscribe);
+  }
+}

package/src/{components/ComputeGraph → compute-graph/functions}/async-function.ts

@@ -2,7 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
-import { CellError, ErrorType, EmptyValue, FunctionPlugin, type HyperFormula } from 'hyperformula';
+import { type SimpleCellAddress } from 'hyperformula/typings/Cell';
 import { type InterpreterState } from 'hyperformula/typings/interpreter/InterpreterState';
 import { type InterpreterValue } from 'hyperformula/typings/interpreter/InterpreterValue';
 import { type ProcedureAst } from 'hyperformula/typings/parser';
@@ -12,28 +12,35 @@ import { debounce, type UnsubscribeCallback } from '@dxos/async';
 import { type Space } from '@dxos/client/echo';
 import { log } from '@dxos/log';
 
-// TODO(burdon): API gateways!
+import { CellError, ErrorType, EmptyValue, FunctionPlugin, type HyperFormula } from '#hyperformula';
+
+// TODO(burdon): Create API gateways:
 //  https://publicapis.io
 //  https://api-ninjas.com/api/cryptoprice
 //  https://developers.google.com/apis-explorer
 //  https://publicapis.io/coin-desk-api
 
-// TODO(burdon): Create wrapper.
 export type AsyncFunction = (...args: any) => Promise<InterpreterValue>;
 
+export type FunctionUpdateEvent = {
+  name: string;
+  cell: SimpleCellAddress;
+};
+
 export type FunctionOptions = {
   ttl?: number;
 };
 
 export type FunctionContextOptions = {
   defaultTtl: number;
-  recalculationDelay: number;
+  debounceDelay: number;
   remoteFunctionUrl: string;
+  onUpdate?: (update: FunctionUpdateEvent) => void;
 };
 
 export const defaultFunctionContextOptions: FunctionContextOptions = {
   defaultTtl: 5_000,
-  recalculationDelay: 200,
+  debounceDelay: 200,
   remoteFunctionUrl: 'https://edge.dxos.workers.dev/functions', // TODO(burdon): Config.
 };
 
@@ -65,21 +72,22 @@ export class FunctionContext {
   private _invocations: Record<string, number> = {};
 
   private readonly _options: FunctionContextOptions;
-  private readonly _onUpdate: () => void;
+
+  // Debounced update handler.
+  private readonly _onUpdate: (update: FunctionUpdateEvent) => void;
 
   constructor(
     private readonly _hf: HyperFormula,
     private readonly _space: Space | undefined,
-    onUpdate: (context: FunctionContext) => void,
     _options?: Partial<FunctionContextOptions>,
   ) {
     this._options = defaultsDeep(_options ?? {}, defaultFunctionContextOptions);
-    this._onUpdate = debounce(() => {
-      // TODO(burdon): Better way to trigger recalculation?
-      //  NOTE: rebuildAndRecalculate resets the undo history.
+    this._onUpdate = debounce((update) => {
+      log('update', update);
+      // TODO(burdon): Better way to trigger recalculation? (NOTE: rebuildAndRecalculate resets the undo history.)
       this._hf.resumeEvaluation();
-      onUpdate(this);
-    }, this._options.recalculationDelay);
+      this._options.onUpdate?.(update);
+    }, this._options.debounceDelay);
   }
 
   get space() {
@@ -134,7 +142,7 @@ export class FunctionContext {
           const value = await cb(...args);
           this._cache.set(invocationKey, { value, ts: Date.now() });
           log('set', { cell, value });
-          this._onUpdate();
+          this._onUpdate({ name, cell });
         } catch (err) {
           // TODO(burdon): Show error to user.
           log.warn('failed', { cell, err });
@@ -153,12 +161,15 @@ export class FunctionContext {
 /**
  * Base class for async functions.
  */
-export class FunctionPluginAsync extends FunctionPlugin {
+export class AsyncFunctionPlugin extends FunctionPlugin {
   get context() {
     return this.config.context as FunctionContext;
   }
 
-  runAsyncFunction(ast: ProcedureAst, state: InterpreterState, cb: AsyncFunction, options?: FunctionOptions) {
+  /**
+   * Immediately returns cached value then runs the async function.
+   */
+  protected runAsyncFunction(ast: ProcedureAst, state: InterpreterState, cb: AsyncFunction, options?: FunctionOptions) {
     const { procedureName } = ast;
     const metadata = this.metadata(procedureName);
     return this.runFunction(ast.args, state, metadata, (...args: any) => {

package/src/{components/ComputeGraph → compute-graph/functions}/edge-function.ts

@@ -3,7 +3,6 @@
 //
 
 import { effect } from '@preact/signals-core';
-import { CellError, ErrorType, FunctionArgumentType } from 'hyperformula';
 import { type InterpreterState } from 'hyperformula/typings/interpreter/InterpreterState';
 import { type ProcedureAst } from 'hyperformula/typings/parser';
 
@@ -13,17 +12,20 @@ import { getUserFunctionUrlInMetadata } from '@dxos/plugin-script/edge';
 import { FunctionType } from '@dxos/plugin-script/types';
 import { nonNullable } from '@dxos/util';
 
-import { type AsyncFunction, FunctionPluginAsync } from './async-function';
+import { CellError, ErrorType, FunctionArgumentType } from '#hyperformula';
+import { type AsyncFunction, AsyncFunctionPlugin } from './async-function';
 
-const EDGE_FUNCTION_TTL = 10_000;
+export const EDGE_FUNCTION_NAME = 'DX';
+
+const FUNCTION_TTL = 10_000;
 
 /**
- * A hyperformula function plugin for calling EDGE functions.
+ * A hyperformula function plugin for calling remote (EDGE) functions.
  *
  * https://hyperformula.handsontable.com/guide/custom-functions.html#add-a-simple-custom-function
  */
-export class EdgeFunctionPlugin extends FunctionPluginAsync {
-  edge(ast: ProcedureAst, state: InterpreterState) {
+export class EdgeFunctionPlugin extends AsyncFunctionPlugin {
+  dx(ast: ProcedureAst, state: InterpreterState) {
     const handler =
       (subscribe = false): AsyncFunction =>
       async (binding: string, ...args: any) => {
@@ -42,39 +44,41 @@ export class EdgeFunctionPlugin extends FunctionPluginAsync {
 
         if (subscribe) {
           const unsubscribe = effect(() => {
-            log.info('function changed', { fn });
+            log('function changed', { fn });
             const _ = fn?.version;
 
             // TODO(wittjosiah): `ttl` should be 0 to force a recalculation when a new version is deployed.
             //  This needs a ttl to prevent a binding change from causing the function not to be found.
-            this.runAsyncFunction(ast, state, handler(false), { ttl: EDGE_FUNCTION_TTL });
+            this.runAsyncFunction(ast, state, handler(false), { ttl: FUNCTION_TTL });
           });
 
           this.context.createSubscription(ast.procedureName, unsubscribe);
         }
 
         const path = getUserFunctionUrlInMetadata(getMeta(fn));
-        const result = await fetch(`${this.context.remoteFunctionUrl}${path}`, {
+        const response = await fetch(`${this.context.remoteFunctionUrl}${path}`, {
           method: 'POST',
           headers: { 'Content-Type': 'application/json' },
           body: JSON.stringify({ args: args.filter(nonNullable) }),
         });
-        return await result.text();
+        const result = await response.text();
+        log('function executed', { result });
+
+        return result;
       };
 
-    return this.runAsyncFunction(ast, state, handler(true), { ttl: EDGE_FUNCTION_TTL });
+    return this.runAsyncFunction(ast, state, handler(true), { ttl: FUNCTION_TTL });
   }
 }
 
 EdgeFunctionPlugin.implementedFunctions = {
-  EDGE: {
-    method: 'edge',
+  [EDGE_FUNCTION_NAME]: {
+    method: 'dx',
     parameters: [
       // Binding
       { argumentType: FunctionArgumentType.STRING },
 
-      // Remote function arguments (currently supporting up to 9).
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
+      // Remote function arguments (currently supporting up to 8).
       { argumentType: FunctionArgumentType.ANY, optionalArg: true },
       { argumentType: FunctionArgumentType.ANY, optionalArg: true },
       { argumentType: FunctionArgumentType.ANY, optionalArg: true },
@@ -90,9 +94,9 @@ EdgeFunctionPlugin.implementedFunctions = {
 
 export const EdgeFunctionPluginTranslations = {
   enGB: {
-    EDGE: 'EDGE',
+    [EDGE_FUNCTION_NAME]: 'Remote function',
   },
   enUS: {
-    EDGE: 'EDGE',
+    [EDGE_FUNCTION_NAME]: 'Remote function',
   },
 };

package/src/compute-graph/functions/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './async-function';
+export * from './edge-function';
+export * from './function-defs';

package/src/compute-graph/hyperformula.test.ts

@@ -0,0 +1,14 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { describe, test, expect } from 'vitest';
+
+import { HyperFormula } from '#hyperformula';
+
+describe('hyperformula', () => {
+  test('sanity test', async () => {
+    const hf = HyperFormula.buildEmpty({ licenseKey: 'gpl-v3' });
+    expect(hf).to.exist;
+  });
+});

package/src/compute-graph/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './compute-graph';
+export * from './compute-graph-registry';
+export * from './compute-node';
+export * from './functions';

package/src/compute-graph/testing/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './test-builder';
+export * from './test-plugin';