@dxos/plugin-sheet 0.6.12 → 0.6.13-main.548ca8d

package/src/graph/compute-graph.ts

@@ -0,0 +1,242 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type Listeners } from 'hyperformula/typings/Emitter';
+
+import { Event } from '@dxos/async';
+import { type Space, Filter, fullyQualifiedId } from '@dxos/client/echo';
+import { Resource } from '@dxos/context';
+import { getTypename } from '@dxos/echo-schema';
+import { invariant } from '@dxos/invariant';
+import { PublicKey } from '@dxos/keys';
+import { log } from '@dxos/log';
+import { FunctionType } from '@dxos/plugin-script/types';
+import { nonNullable } from '@dxos/util';
+
+import { ExportedCellChange, type HyperFormula } from '#hyperformula';
+import { ComputeNode } from './compute-node';
+import {
+  defaultFunctions,
+  FunctionContext,
+  type FunctionContextOptions,
+  type FunctionDefinition,
+  EDGE_FUNCTION_NAME,
+} from './functions';
+
+// TODO(wittjosiah): Factor out.
+const OBJECT_ID_LENGTH = 60; // 33 (space id) + 1 (separator) + 26 (object id).
+
+// TODO(burdon): Factory.
+// export type ComputeNodeGenerator = <T>(obj: T) => ComputeNode;
+
+type ObjectRef = { type: string; id: string };
+
+/**
+ * Marker for sheets that are managed by an ECHO object.
+ * Sheet ID: `dxos.org/type/SheetType@1234`
+ */
+export const createSheetName = ({ type, id }: ObjectRef) => `${type}@${id}`;
+export const parseSheetName = (name: string): Partial<ObjectRef> => {
+  const [type, id] = name.split('@');
+  return id ? { type, id } : { id: type };
+};
+
+export type ComputeGraphEvent = 'functionsUpdated';
+
+/**
+ * Per-space compute and dependency graph.
+ * Consists of multiple ComputeNode (corresponding to a HyperFormula sheet).
+ * Manages the set of custom functions.
+ * HyperFormula manages the dependency graph.
+ */
+export class ComputeGraph extends Resource {
+  public readonly id = `graph-${PublicKey.random().truncate()}`;
+
+  // Map of nodes indexed by sheet number.
+  private readonly _nodes = new Map<number, ComputeNode>();
+
+  // Cached function objects.
+  private _remoteFunctions: FunctionType[] = [];
+
+  public readonly update = new Event<{ type: ComputeGraphEvent }>();
+
+  // The context is passed to all functions.
+  public readonly context = new FunctionContext(this._hf, this._space, this._options);
+
+  constructor(
+    private readonly _hf: HyperFormula,
+    private readonly _space?: Space,
+    private readonly _options?: Partial<FunctionContextOptions>,
+  ) {
+    super();
+    this._hf.updateConfig({ context: this.context });
+    // TODO(burdon): If debounce then aggregate changes.
+    const onValuesUpdate: Listeners['valuesUpdated'] = (changes) => {
+      for (const change of changes) {
+        if (change instanceof ExportedCellChange) {
+          const { sheet } = change;
+          const node = this._nodes.get(sheet);
+          if (node) {
+            node.update.emit({ type: 'valuesUpdated', change });
+          }
+        }
+      }
+    };
+
+    this._hf.on('valuesUpdated', onValuesUpdate);
+    this._ctx.onDispose(() => this._hf.off('valuesUpdated', onValuesUpdate));
+  }
+
+  get hf() {
+    return this._hf;
+  }
+
+  getFunctions(
+    { standard, echo }: { standard?: boolean; echo?: boolean } = { standard: true, echo: true },
+  ): FunctionDefinition[] {
+    return [
+      ...(standard
+        ? this._hf
+            .getRegisteredFunctionNames()
+            .map((name) => defaultFunctions.find((fn) => fn.name === name) ?? { name })
+        : []),
+      ...(echo ? this._remoteFunctions.map((fn) => ({ name: fn.binding! })) : []),
+    ];
+  }
+
+  /**
+   * Get or create cell representing a sheet.
+   */
+  // TODO(burdon): Async (open node).
+  //  The graph should be an extensible factory that plugins extend with model constructors.
+  //  This would enable on-the-fly instantiation of new models when then are referenced.
+  //  E.g., Cross-object reference would be stored as "ObjectId!A1"
+  //  The graph would then load the object and create a ComputeNode (model) of the appropriate type.
+  getOrCreateNode(name: string): ComputeNode {
+    invariant(name.length);
+    if (!this._hf.doesSheetExist(name)) {
+      log.info('created node', { space: this._space?.id, sheet: name });
+      this._hf.addSheet(name);
+    }
+
+    const sheetId = this._hf.getSheetId(name);
+    invariant(sheetId !== undefined);
+
+    const node = new ComputeNode(this, sheetId);
+    this._nodes.set(sheetId, node);
+    return node;
+  }
+
+  /**
+   * Map bound value to custom function invocation.
+   * E.g., "HELLO(...args)" => "EDGE("HELLO", ...args)".
+   */
+  mapFormulaToNative(formula: string): string {
+    return (
+      formula
+        //
+        // Map cross-sheet references by name onto sheet stored by ECHO object/model.
+        // Example: "Test Sheet"!A0 => "dxos.org/type/SheetType@1234"!A0
+        // https://hyperformula.handsontable.com/guide/cell-references.html#cell-references
+        //
+        .replace(/['"]?([ \w]+)['"]?!/, (_match, name) => {
+          if (name) {
+            // TODO(burdon): Cache map.
+            const objects = this._hf
+              .getSheetNames()
+              .map((name) => {
+                const { type, id } = parseSheetName(name);
+                return type && id ? this._space?.db.getObjectById(id) : undefined;
+              })
+              .filter(nonNullable);
+
+            for (const obj of objects) {
+              if (obj.name === name) {
+                const type = getTypename(obj)!;
+                // NOTE: Names must be single quoted.
+                return `'${createSheetName({ type, id: obj.id })}'!`;
+              }
+            }
+          }
+
+          return `${name}!`;
+        })
+
+        //
+        // Map remote function references (i.e., to remote DX function invocation).
+        //
+        .replace(/(\w+)\((.*)\)/g, (match, binding, args) => {
+          const fn = this._remoteFunctions.find((fn) => fn.binding === binding);
+          if (!fn) {
+            return match;
+          }
+
+          if (args.trim() === '') {
+            return `${EDGE_FUNCTION_NAME}("${binding}")`;
+          } else {
+            return `${EDGE_FUNCTION_NAME}("${binding}", ${args})`;
+          }
+        })
+    );
+  }
+
+  /**
+   * Map from binding to fully qualified ECHO ID (to store).
+   * E.g., HELLO() => spaceId:objectId()
+   */
+  mapFunctionBindingToId(formula: string) {
+    return formula.replace(/(\w+)\((.*)\)/g, (match, binding, args) => {
+      if (binding === EDGE_FUNCTION_NAME || defaultFunctions.find((fn) => fn.name === binding)) {
+        return match;
+      }
+
+      const fn = this._remoteFunctions.find((fn) => fn.binding === binding);
+      if (fn) {
+        const id = fullyQualifiedId(fn);
+        return `${id}(${args})`;
+      } else {
+        return match;
+      }
+    });
+  }
+
+  /**
+   * Map from fully qualified ECHO ID to binding (from store).
+   * E.g., spaceId:objectId() => HELLO()
+   */
+  mapFunctionBindingFromId(formula: string) {
+    return formula.replace(/(\w+):([a-zA-Z0-9]+)\((.*)\)/g, (match, spaceId, objectId, args) => {
+      const id = `${spaceId}:${objectId}`;
+      if (id.length !== OBJECT_ID_LENGTH) {
+        return match;
+      }
+
+      const fn = this._remoteFunctions.find((fn) => fullyQualifiedId(fn) === id);
+      if (fn?.binding) {
+        return `${fn.binding}(${args})`;
+      } else {
+        return match;
+      }
+    });
+  }
+
+  protected override async _open() {
+    if (this._space) {
+      // Subscribe to remote function definitions.
+      const query = this._space.db.query(Filter.schema(FunctionType));
+      const unsubscribe = query.subscribe(({ objects }) => {
+        this._remoteFunctions = objects.filter(({ binding }) => binding);
+        this.update.emit({ type: 'functionsUpdated' });
+      });
+
+      this._ctx.onDispose(unsubscribe);
+    }
+  }
+
+  protected override async _close() {
+    for (const node of this._nodes.values()) {
+      await node.close();
+    }
+  }
+}

package/src/graph/compute-node.ts

@@ -0,0 +1,63 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { type Listeners } from 'hyperformula/typings/Emitter';
+import { type ExportedCellChange } from 'hyperformula/typings/Exporter';
+
+import { Event } from '@dxos/async';
+import { Resource } from '@dxos/context';
+
+import { DetailedCellError } from '#hyperformula';
+import { type ComputeGraph } from './compute-graph';
+import { type CellAddress } from '../defs';
+import { type CellScalarValue } from '../types';
+
+export type ComputeNodeEvent = {
+  type: keyof Listeners;
+  change?: ExportedCellChange;
+};
+
+/**
+ * Individual "sheet" (typically corresponds to an ECHO object).
+ */
+// TODO(burdon): Factor out common HF wrapper from from SheetModel.
+export class ComputeNode extends Resource {
+  public readonly update = new Event<ComputeNodeEvent>();
+
+  constructor(
+    private readonly _graph: ComputeGraph,
+    public readonly sheetId: number,
+  ) {
+    super();
+  }
+
+  get graph() {
+    return this._graph;
+  }
+
+  clear() {
+    this._graph.hf.clearSheet(this.sheetId);
+  }
+
+  getValue(cell: CellAddress): CellScalarValue {
+    const value = this._graph.hf.getCellValue({ sheet: this.sheetId, row: cell.row, col: cell.col });
+    if (value instanceof DetailedCellError) {
+      return null;
+    }
+
+    return value;
+  }
+
+  setValue(cell: CellAddress, value: CellScalarValue) {
+    const mappedValue =
+      typeof value === 'string' && value.charAt(0) === '=' ? this._graph.mapFormulaToNative(value) : value;
+    this._graph.hf.setCellContents({ sheet: this.sheetId, row: cell.row, col: cell.col }, [[mappedValue]]);
+  }
+
+  // TODO(burdon): Load data into sheet.
+  protected override async _open() {
+    // const unsubscribe = this._graph.update.on(this.update.emit);
+    // this._ctx.onDispose(unsubscribe);
+  }
+}

package/src/{components/ComputeGraph → graph/functions}/async-function.ts

@@ -2,7 +2,7 @@
 // Copyright 2024 DXOS.org
 //
 
-import { CellError, ErrorType, EmptyValue, FunctionPlugin, type HyperFormula } from 'hyperformula';
+import { type SimpleCellAddress } from 'hyperformula/typings/Cell';
 import { type InterpreterState } from 'hyperformula/typings/interpreter/InterpreterState';
 import { type InterpreterValue } from 'hyperformula/typings/interpreter/InterpreterValue';
 import { type ProcedureAst } from 'hyperformula/typings/parser';
@@ -12,28 +12,35 @@ import { debounce, type UnsubscribeCallback } from '@dxos/async';
 import { type Space } from '@dxos/client/echo';
 import { log } from '@dxos/log';
 
-// TODO(burdon): API gateways!
+import { CellError, ErrorType, EmptyValue, FunctionPlugin, type HyperFormula } from '#hyperformula';
+
+// TODO(burdon): Create API gateways:
 //  https://publicapis.io
 //  https://api-ninjas.com/api/cryptoprice
 //  https://developers.google.com/apis-explorer
 //  https://publicapis.io/coin-desk-api
 
-// TODO(burdon): Create wrapper.
 export type AsyncFunction = (...args: any) => Promise<InterpreterValue>;
 
+export type FunctionUpdateEvent = {
+  name: string;
+  cell: SimpleCellAddress;
+};
+
 export type FunctionOptions = {
   ttl?: number;
 };
 
 export type FunctionContextOptions = {
   defaultTtl: number;
-  recalculationDelay: number;
+  debounceDelay: number;
   remoteFunctionUrl: string;
+  onUpdate?: (update: FunctionUpdateEvent) => void;
 };
 
 export const defaultFunctionContextOptions: FunctionContextOptions = {
   defaultTtl: 5_000,
-  recalculationDelay: 200,
+  debounceDelay: 200,
   remoteFunctionUrl: 'https://edge.dxos.workers.dev/functions', // TODO(burdon): Config.
 };
 
@@ -65,21 +72,21 @@ export class FunctionContext {
   private _invocations: Record<string, number> = {};
 
   private readonly _options: FunctionContextOptions;
-  private readonly _onUpdate: () => void;
+
+  // Debounced update handler.
+  private readonly _onUpdate: (update: FunctionUpdateEvent) => void;
 
   constructor(
     private readonly _hf: HyperFormula,
     private readonly _space: Space | undefined,
-    onUpdate: (context: FunctionContext) => void,
     _options?: Partial<FunctionContextOptions>,
   ) {
     this._options = defaultsDeep(_options ?? {}, defaultFunctionContextOptions);
-    this._onUpdate = debounce(() => {
-      // TODO(burdon): Better way to trigger recalculation?
-      //  NOTE: rebuildAndRecalculate resets the undo history.
+    this._onUpdate = debounce((update) => {
+      // TODO(burdon): Better way to trigger recalculation? (NOTE: rebuildAndRecalculate resets the undo history.)
       this._hf.resumeEvaluation();
-      onUpdate(this);
-    }, this._options.recalculationDelay);
+      this._options.onUpdate?.(update);
+    }, this._options.debounceDelay);
   }
 
   get space() {
@@ -134,7 +141,7 @@ export class FunctionContext {
           const value = await cb(...args);
           this._cache.set(invocationKey, { value, ts: Date.now() });
           log('set', { cell, value });
-          this._onUpdate();
+          this._onUpdate({ name, cell });
         } catch (err) {
           // TODO(burdon): Show error to user.
           log.warn('failed', { cell, err });
@@ -153,12 +160,15 @@ export class FunctionContext {
 /**
  * Base class for async functions.
  */
-export class FunctionPluginAsync extends FunctionPlugin {
+export class AsyncFunctionPlugin extends FunctionPlugin {
   get context() {
     return this.config.context as FunctionContext;
   }
 
-  runAsyncFunction(ast: ProcedureAst, state: InterpreterState, cb: AsyncFunction, options?: FunctionOptions) {
+  /**
+   * Immediately returns cached value then runs the async function.
+   */
+  protected runAsyncFunction(ast: ProcedureAst, state: InterpreterState, cb: AsyncFunction, options?: FunctionOptions) {
     const { procedureName } = ast;
     const metadata = this.metadata(procedureName);
     return this.runFunction(ast.args, state, metadata, (...args: any) => {

package/src/{components/ComputeGraph → graph/functions}/edge-function.ts

@@ -3,7 +3,6 @@
 //
 
 import { effect } from '@preact/signals-core';
-import { CellError, ErrorType, FunctionArgumentType } from 'hyperformula';
 import { type InterpreterState } from 'hyperformula/typings/interpreter/InterpreterState';
 import { type ProcedureAst } from 'hyperformula/typings/parser';
 
@@ -13,17 +12,20 @@ import { getUserFunctionUrlInMetadata } from '@dxos/plugin-script/edge';
 import { FunctionType } from '@dxos/plugin-script/types';
 import { nonNullable } from '@dxos/util';
 
-import { type AsyncFunction, FunctionPluginAsync } from './async-function';
+import { CellError, ErrorType, FunctionArgumentType } from '#hyperformula';
+import { type AsyncFunction, AsyncFunctionPlugin } from './async-function';
 
-const EDGE_FUNCTION_TTL = 10_000;
+export const EDGE_FUNCTION_NAME = 'DX';
+
+const FUNCTION_TTL = 10_000;
 
 /**
- * A hyperformula function plugin for calling EDGE functions.
+ * A hyperformula function plugin for calling remote (EDGE) functions.
  *
  * https://hyperformula.handsontable.com/guide/custom-functions.html#add-a-simple-custom-function
  */
-export class EdgeFunctionPlugin extends FunctionPluginAsync {
-  edge(ast: ProcedureAst, state: InterpreterState) {
+export class EdgeFunctionPlugin extends AsyncFunctionPlugin {
+  dx(ast: ProcedureAst, state: InterpreterState) {
     const handler =
       (subscribe = false): AsyncFunction =>
       async (binding: string, ...args: any) => {
@@ -47,7 +49,7 @@ export class EdgeFunctionPlugin extends FunctionPluginAsync {
 
             // TODO(wittjosiah): `ttl` should be 0 to force a recalculation when a new version is deployed.
             //  This needs a ttl to prevent a binding change from causing the function not to be found.
-            this.runAsyncFunction(ast, state, handler(false), { ttl: EDGE_FUNCTION_TTL });
+            this.runAsyncFunction(ast, state, handler(false), { ttl: FUNCTION_TTL });
           });
 
           this.context.createSubscription(ast.procedureName, unsubscribe);
@@ -59,22 +61,22 @@ export class EdgeFunctionPlugin extends FunctionPluginAsync {
           headers: { 'Content-Type': 'application/json' },
           body: JSON.stringify({ args: args.filter(nonNullable) }),
         });
+
         return await result.text();
       };
 
-    return this.runAsyncFunction(ast, state, handler(true), { ttl: EDGE_FUNCTION_TTL });
+    return this.runAsyncFunction(ast, state, handler(true), { ttl: FUNCTION_TTL });
   }
 }
 
 EdgeFunctionPlugin.implementedFunctions = {
-  EDGE: {
-    method: 'edge',
+  [EDGE_FUNCTION_NAME]: {
+    method: 'dx',
     parameters: [
       // Binding
       { argumentType: FunctionArgumentType.STRING },
 
-      // Remote function arguments (currently supporting up to 9).
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
+      // Remote function arguments (currently supporting up to 8).
       { argumentType: FunctionArgumentType.ANY, optionalArg: true },
       { argumentType: FunctionArgumentType.ANY, optionalArg: true },
       { argumentType: FunctionArgumentType.ANY, optionalArg: true },
@@ -90,9 +92,9 @@ EdgeFunctionPlugin.implementedFunctions = {
 
 export const EdgeFunctionPluginTranslations = {
   enGB: {
-    EDGE: 'EDGE',
+    [EDGE_FUNCTION_NAME]: 'Remote function',
   },
   enUS: {
-    EDGE: 'EDGE',
+    [EDGE_FUNCTION_NAME]: 'Remote function',
   },
 };

package/src/graph/functions/index.ts

@@ -0,0 +1,7 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './async-function';
+export * from './edge-function';
+export * from './function-defs';

package/src/graph/hyperformula.test.ts

@@ -0,0 +1,14 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { describe, test, expect } from 'vitest';
+
+import { HyperFormula } from '#hyperformula';
+
+describe('hyperformula', () => {
+  test('sanity test', async () => {
+    const hf = HyperFormula.buildEmpty({ licenseKey: 'gpl-v3' });
+    expect(hf).to.exist;
+  });
+});

package/src/graph/index.ts

@@ -0,0 +1,8 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './compute-graph';
+export * from './compute-graph-registry';
+export * from './compute-node';
+export * from './functions';

package/src/graph/testing/index.ts

@@ -0,0 +1,6 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+export * from './test-builder';
+export * from './test-plugin';

package/src/graph/testing/test-builder.ts

@@ -0,0 +1,54 @@
+//
+// Copyright 2024 DXOS.org
+//
+
+import { Client, type ClientOptions } from '@dxos/client';
+import { type Context, Resource } from '@dxos/context';
+import { invariant } from '@dxos/invariant';
+
+import { type ComputeGraphOptions, ComputeGraphRegistry } from '../compute-graph-registry';
+
+export type TestBuilderOptions = ClientOptions & ComputeGraphOptions;
+
+// TODO(burdon): Reconcile with @dxos/client/testing.
+export class TestBuilder extends Resource {
+  private _client?: Client;
+  private _registry?: ComputeGraphRegistry;
+
+  constructor(private readonly _options: TestBuilderOptions = {}) {
+    super();
+  }
+
+  get ctx(): Context {
+    return this._ctx;
+  }
+
+  get client(): Client {
+    invariant(this._client);
+    return this._client;
+  }
+
+  get registry(): ComputeGraphRegistry {
+    invariant(this._registry);
+    return this._registry;
+  }
+
+  override async _open() {
+    const client = new Client(this._options);
+    await client.initialize();
+    await client.halo.createIdentity();
+    this._client = client;
+    this._ctx.onDispose(async () => {
+      await client.destroy();
+      this._client = undefined;
+    });
+
+    const registry = new ComputeGraphRegistry(this._options);
+    await registry.open();
+    this._registry = registry;
+    this._ctx.onDispose(async () => {
+      await registry.close();
+      this._registry = undefined;
+    });
+  }
+}

package/src/{components/ComputeGraph/custom.ts → graph/testing/test-plugin.ts}

@@ -2,24 +2,36 @@
 // Copyright 2024 DXOS.org
 //
 
-import { FunctionArgumentType } from 'hyperformula';
 import { type InterpreterState } from 'hyperformula/typings/interpreter/InterpreterState';
 import { type ProcedureAst } from 'hyperformula/typings/parser';
 
 import { getDeep } from '@dxos/util';
 
-import { type AsyncFunction, FunctionPluginAsync } from './async-function';
-
-// TODO(burdon): Factor out.
-const parseNumberString = (str: string): number => {
-  return parseFloat(str.replace(/[^\d.]/g, ''));
-};
+import { FunctionArgumentType } from '#hyperformula';
+import { type ComputeGraphPlugin } from '../compute-graph-registry';
+import { type AsyncFunction, AsyncFunctionPlugin } from '../functions';
+import { parseNumberString } from '../util';
 
 /**
+ * Testing functions run locally (not run via EDGE).
  * https://hyperformula.handsontable.com/guide/custom-functions.html#add-a-simple-custom-function
  */
-export class CustomPlugin extends FunctionPluginAsync {
+export class TestPlugin extends AsyncFunctionPlugin {
+  /**
+   * Simple local function returns input value.
+   */
   test(ast: ProcedureAst, state: InterpreterState) {
+    const handler: AsyncFunction = async (_value) => {
+      return _value;
+    };
+
+    return this.runAsyncFunction(ast, state, handler);
+  }
+
+  /**
+   * Simple local function returns random number.
+   */
+  random(ast: ProcedureAst, state: InterpreterState) {
     const handler: AsyncFunction = async () => {
       return Math.random();
     };
@@ -27,6 +39,9 @@ export class CustomPlugin extends FunctionPluginAsync {
     return this.runAsyncFunction(ast, state, handler);
   }
 
+  /**
+   * Async HTTP function.
+   */
   crypto(ast: ProcedureAst, state: InterpreterState) {
     const handler: AsyncFunction = async (_currency) => {
       const currency = (_currency || 'USD').toUpperCase();
@@ -44,9 +59,15 @@ export class CustomPlugin extends FunctionPluginAsync {
   }
 }
 
-CustomPlugin.implementedFunctions = {
+TestPlugin.implementedFunctions = {
   TEST: {
     method: 'test',
+    parameters: [{ argumentType: FunctionArgumentType.NUMBER, optionalArg: false }],
+    isVolatile: true,
+  },
+
+  RANDOM: {
+    method: 'random',
     parameters: [],
     isVolatile: true,
   },
@@ -58,13 +79,22 @@ CustomPlugin.implementedFunctions = {
   },
 };
 
-export const CustomPluginTranslations = {
+export const TestPluginTranslations = {
   enGB: {
-    TEST: 'TEST',
-    CRYPTO: 'CRYPTO',
+    TEST: 'Returns input value',
+    RANDOM: 'Random number',
+    CRYPTO: 'Crypto token value',
   },
   enUS: {
-    TEST: 'TEST',
-    CRYPTO: 'CRYPTO',
+    TEST: 'Returns input value',
+    RANDOM: 'Random number',
+    CRYPTO: 'Crypto token value',
   },
 };
+
+export const testFunctionPlugins: ComputeGraphPlugin[] = [
+  {
+    plugin: TestPlugin,
+    translations: TestPluginTranslations,
+  },
+];