@dxos/compute 0.8.4-main.fcfe5033a5 → 0.8.4-staging.60fe92afc8

package/src/compute-graph.ts

@@ -1,265 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import type * as ManagedRuntime from 'effect/ManagedRuntime';
-
-import { Event } from '@dxos/async';
-import { Filter, type Space } from '@dxos/client/echo';
-import { Resource } from '@dxos/context';
-import { Obj } from '@dxos/echo';
-import { invariant } from '@dxos/invariant';
-import { PublicKey } from '@dxos/keys';
-import { log } from '@dxos/log';
-import { Operation } from '@dxos/operation';
-import { isNonNullable } from '@dxos/util';
-import type { Listeners } from '@dxos/vendor-hyperformula';
-import { ExportedCellChange, type HyperFormula } from '@dxos/vendor-hyperformula';
-
-import { ComputeNode } from './compute-node';
-import {
-  EDGE_FUNCTION_NAME,
-  FunctionContext,
-  type FunctionContextOptions,
-  type FunctionDefinition,
-  defaultFunctions,
-} from './functions';
-
-// TODO(burdon): Factor out compute-graph.
-
-const UNKNOWN_BINDING = '__UNKNOWN__';
-
-// TODO(burdon): Factory.
-// export type ComputeNodeGenerator = <T>(obj: T) => ComputeNode;
-
-type ObjectRef = { type: string; id: string };
-
-/**
- * Marker for sheets that are managed by an ECHO object.
- * Sheet ID: `org.dxos.type.sheet-type@1234`
- */
-export const createSheetName = ({ type, id }: ObjectRef) => `${type}@${id}`;
-export const parseSheetName = (name: string): Partial<ObjectRef> => {
-  const [type, id] = name.split('@');
-  return id ? { type, id } : { id: type };
-};
-
-export type ComputeGraphEvent = 'functionsUpdated' | 'valuesUpdated';
-
-/**
- * Per-space compute and dependency graph.
- * Consists of multiple ComputeNode (corresponding to a HyperFormula sheet).
- * Manages the set of custom functions.
- * HyperFormula manages the dependency graph.
- */
-export class ComputeGraph extends Resource {
-  public readonly id = `graph-${PublicKey.random().truncate()}`;
-
-  // Map of nodes indexed by sheet number.
-  private readonly _nodes = new Map<number, ComputeNode>();
-
-  // Cached function objects.
-  private _remoteFunctions: Operation.PersistentOperation[] = [];
-
-  public readonly update = new Event<{ type: ComputeGraphEvent }>();
-
-  // The context is passed to all functions.
-  public readonly context: FunctionContext;
-
-  constructor(
-    private readonly _hf: HyperFormula,
-    private readonly _runtime: ManagedRuntime.ManagedRuntime<Operation.Service, never>,
-    private readonly _space?: Space,
-    private readonly _options?: Partial<FunctionContextOptions>,
-  ) {
-    super();
-
-    this.context = new FunctionContext(this._hf, this._runtime, this._space, {
-      ...this._options,
-      onUpdate: (update) => {
-        this._options?.onUpdate?.(update);
-        this.update.emit({ type: 'valuesUpdated' });
-      },
-    });
-    this._hf.updateConfig({ context: this.context });
-
-    // TODO(burdon): If debounce then aggregate changes.
-    const onValuesUpdate: Listeners['valuesUpdated'] = (changes) => {
-      for (const change of changes) {
-        if (change instanceof ExportedCellChange) {
-          const { sheet } = change;
-          const node = this._nodes.get(sheet);
-          if (node) {
-            node.update.emit({ type: 'valuesUpdated', change });
-          }
-        }
-      }
-    };
-
-    this._hf.on('valuesUpdated', onValuesUpdate);
-    this._ctx.onDispose(() => this._hf.off('valuesUpdated', onValuesUpdate));
-  }
-
-  get hf() {
-    return this._hf;
-  }
-
-  getFunctions(
-    { standard, echo }: { standard?: boolean; echo?: boolean } = {
-      standard: true,
-      echo: true,
-    },
-  ): FunctionDefinition[] {
-    return [
-      ...(standard
-        ? this._hf
-            .getRegisteredFunctionNames()
-            .map((name) => defaultFunctions.find((fn) => fn.name === name) ?? { name })
-        : []),
-      ...(echo ? this._remoteFunctions.map((fn) => ({ name: fn.binding! })) : []),
-    ];
-  }
-
-  /**
-   * Get or create cell representing a sheet.
-   */
-  // TODO(burdon): Async (open node).
-  //  The graph should be an extensible factory that plugins extend with model constructors.
-  //  This would enable on-the-fly instantiation of new models when then are referenced.
-  //  E.g., Cross-object reference would be stored as "ObjectId!A1"
-  //  The graph would then load the object and create a ComputeNode (model) of the appropriate type.
-  getOrCreateNode(name: string): ComputeNode {
-    invariant(name.length);
-    if (!this._hf.doesSheetExist(name)) {
-      log('created node', { space: this._space?.id, sheet: name });
-      this._hf.addSheet(name);
-    }
-
-    const sheetId = this._hf.getSheetId(name);
-    invariant(sheetId !== undefined);
-
-    const node = new ComputeNode(this, sheetId);
-    this._nodes.set(sheetId, node);
-    return node;
-  }
-
-  /**
-   * Map bound value to custom function invocation.
-   * E.g., "HELLO(...args)" => "DX("HELLO", ...args)".
-   */
-  mapFormulaToNative(formula: string): string {
-    return (
-      formula
-        //
-        // Map cross-sheet references by name onto sheet stored by ECHO object/model.
-        // Example: "Test Sheet"!A0 => "org.dxos.type.sheet-type@1234"!A0
-        // https://hyperformula.handsontable.com/guide/cell-references.html#cell-references
-        //
-        .replace(/['"]?([ \w]+)['"]?!/, (_match, name) => {
-          if (name) {
-            // TODO(burdon): Cache map.
-            const objects = this._hf
-              .getSheetNames()
-              .map((name) => {
-                const { type, id } = parseSheetName(name);
-                return type && id ? this._space?.db.getObjectById(id) : undefined;
-              })
-              .filter(isNonNullable);
-
-            for (const obj of objects) {
-              if (obj.name === name) {
-                const type = Obj.getTypename(obj)!;
-                // NOTE: Names must be single quoted.
-                return `'${createSheetName({ type, id: obj.id })}'!`;
-              }
-            }
-          }
-
-          return `${name}!`;
-        })
-
-        //
-        // Map remote function references (i.e., to remote DX function invocation).
-        //
-        .replace(/(\w+)\((.*)\)/g, (match, binding, args) => {
-          const fn = this._remoteFunctions.find((fn) => fn.binding === binding);
-          if (!fn) {
-            return match;
-          }
-
-          if (args.trim() === '') {
-            return `${EDGE_FUNCTION_NAME}("${binding}")`;
-          } else {
-            return `${EDGE_FUNCTION_NAME}("${binding}", ${args})`;
-          }
-        })
-    );
-  }
-
-  /**
-   * Map from binding to fully qualified ECHO DXN (to store).
-   * E.g., HELLO() => dxn:echo:spaceId:objectId()
-   */
-  mapFunctionBindingToId(formula: string): string {
-    return formula.replace(/(\w+)\((.*)\)/g, (match, binding, args) => {
-      if (binding === EDGE_FUNCTION_NAME || defaultFunctions.find((fn) => fn.name === binding)) {
-        return match;
-      }
-
-      const fn = this._remoteFunctions.find((fn) => fn.binding === binding);
-      if (fn) {
-        const id = Obj.getDXN(fn).toString();
-        return `${id}(${args})`;
-      } else {
-        return match;
-      }
-    });
-  }
-
-  /**
-   * Map from fully qualified ECHO DXN to binding (from store).
-   * E.g., dxn:echo:spaceId:objectId() => HELLO()
-   */
-  mapFunctionBindingFromId(formula: string): string | undefined {
-    const binding = formula.replace(
-      /dxn:([^:]+):([^:(]+):([a-zA-Z0-9]+)\((.*)\)/g,
-      (_match, kind, spaceTag, objectId, args) => {
-        const dxn = `dxn:${kind}:${spaceTag}:${objectId}`;
-        const fn = this._remoteFunctions.find((fn) => Obj.getDXN(fn).toString() === dxn);
-        if (fn?.binding) {
-          return `${fn.binding}(${args})`;
-        } else {
-          return UNKNOWN_BINDING;
-        }
-      },
-    );
-
-    if (binding.startsWith(`=${UNKNOWN_BINDING}`)) {
-      return undefined;
-    } else {
-      return binding;
-    }
-  }
-
-  protected override async _open(): Promise<void> {
-    if (this._space) {
-      // Subscribe to remote function definitions.
-      const query = this._space.db.query(Filter.type(Operation.PersistentOperation));
-      const unsubscribe = query.subscribe(
-        () => {
-          this._remoteFunctions = query.results.filter((fn) => fn.binding);
-          this.update.emit({ type: 'functionsUpdated' });
-        },
-        { fire: true },
-      );
-
-      this._ctx.onDispose(unsubscribe);
-    }
-  }
-
-  protected override async _close(): Promise<void> {
-    for (const node of this._nodes.values()) {
-      await node.close();
-    }
-  }
-}

package/src/compute-node.ts

@@ -1,59 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import { Event } from '@dxos/async';
-import { Resource } from '@dxos/context';
-import { type ExportedCellChange, type Listeners } from '@dxos/vendor-hyperformula';
-import { DetailedCellError } from '@dxos/vendor-hyperformula';
-
-import { type ComputeGraph } from './compute-graph';
-import { type CellAddress, type CellScalarValue, isFormula } from './types';
-
-export type ComputeNodeEvent = {
-  type: keyof Listeners;
-  change?: ExportedCellChange;
-};
-
-/**
- * Individual "sheet" (typically corresponds to an ECHO object).
- */
-// TODO(burdon): Factor out common HF wrapper from from SheetModel.
-export class ComputeNode extends Resource {
-  public readonly update = new Event<ComputeNodeEvent>();
-
-  constructor(
-    private readonly _graph: ComputeGraph,
-    public readonly sheetId: number,
-  ) {
-    super();
-  }
-
-  get graph() {
-    return this._graph;
-  }
-
-  clear(): void {
-    this._graph.hf.clearSheet(this.sheetId);
-  }
-
-  getValue(cell: CellAddress): CellScalarValue {
-    const value = this._graph.hf.getCellValue({ sheet: this.sheetId, row: cell.row, col: cell.col });
-    if (value instanceof DetailedCellError) {
-      return null;
-    }
-
-    return value;
-  }
-
-  setValue(cell: CellAddress, value: CellScalarValue): void {
-    const mappedValue = isFormula(value) ? this._graph.mapFormulaToNative(value) : value;
-    this._graph.hf.setCellContents({ sheet: this.sheetId, row: cell.row, col: cell.col }, [[mappedValue]]);
-  }
-
-  // TODO(burdon): Load data into sheet.
-  protected override async _open(): Promise<void> {
-    // const unsubscribe = this._graph.update.on(this.update.emit);
-    // this._ctx.onDispose(unsubscribe);
-  }
-}

package/src/functions/async-function.ts

@@ -1,184 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import type * as ManagedRuntime from 'effect/ManagedRuntime';
-import defaultsDeep from 'lodash.defaultsdeep';
-
-import { type CleanupFn, debounce } from '@dxos/async';
-import type { Space } from '@dxos/client/echo';
-import { log } from '@dxos/log';
-import type { Operation } from '@dxos/operation';
-import type { RawInterpreterValue, SimpleCellAddress } from '@dxos/vendor-hyperformula';
-import type { InterpreterState } from '@dxos/vendor-hyperformula';
-import type { InterpreterValue } from '@dxos/vendor-hyperformula';
-import type { ProcedureAst } from '@dxos/vendor-hyperformula';
-import { CellError, EmptyValue, ErrorType, FunctionPlugin, type HyperFormula } from '@dxos/vendor-hyperformula';
-
-// TODO(burdon): Create API gateways:
-//  https://publicapis.io
-//  https://api-ninjas.com/api/cryptoprice
-//  https://developers.google.com/apis-explorer
-//  https://publicapis.io/coin-desk-api
-
-export type AsyncFunction = (...args: any) => Promise<InterpreterValue>;
-
-export type FunctionUpdateEvent = {
-  name: string;
-  cell: SimpleCellAddress;
-};
-
-export type FunctionOptions = {
-  ttl?: number;
-};
-
-export type FunctionContextOptions = {
-  defaultTtl: number;
-  debounceDelay: number;
-  onUpdate?: (update: FunctionUpdateEvent) => void;
-};
-
-export const defaultFunctionContextOptions: Pick<FunctionContextOptions, 'defaultTtl' | 'debounceDelay'> = {
-  defaultTtl: 5_000,
-  debounceDelay: 200,
-};
-
-/**
- * The context singleton for the model is passed into custom functions.
- *
- * HyperFormula does not support async functions.
- * - https://hyperformula.handsontable.com/guide/custom-functions.html
- * - https://hyperformula.handsontable.com/guide/known-limitations.html#known-limitations
- * - https://github.com/handsontable/hyperformula/issues/892
- */
-export class FunctionContext {
-  // Mangle name with params.
-  static createInvocationKey(name: string, ...args: any): string {
-    return JSON.stringify({ name, ...args });
-  }
-
-  // TODO(wittjosiah): Persist cached values.
-  // Cached values for cell.
-  private readonly _cache = new Map<string, { value: InterpreterValue; ts: number }>();
-
-  // Active requests.
-  private readonly _pending = new Map<string, number>();
-
-  // Query subscriptions.
-  private readonly _subscriptions = new Map<string, CleanupFn>();
-
-  // Invocation count.
-  private _invocations: Record<string, number> = {};
-
-  private readonly _options: FunctionContextOptions;
-
-  // Debounced update handler.
-  private readonly _onUpdate: (update: FunctionUpdateEvent) => void;
-
-  constructor(
-    private readonly _hf: HyperFormula,
-    private readonly _runtime: ManagedRuntime.ManagedRuntime<Operation.Service, never>,
-    private readonly _space: Space | undefined,
-    _options?: Partial<FunctionContextOptions>,
-  ) {
-    this._options = defaultsDeep(_options ?? {}, defaultFunctionContextOptions);
-    this._onUpdate = debounce((update) => {
-      log('update', update);
-      // TODO(burdon): Better way to trigger recalculation? (NOTE: rebuildAndRecalculate resets the undo history.)
-      this._hf.resumeEvaluation();
-      this._options.onUpdate?.(update);
-    }, this._options.debounceDelay);
-  }
-
-  get space() {
-    return this._space;
-  }
-
-  get info() {
-    return { cache: this._cache.size, invocations: this._invocations };
-  }
-
-  get runtime() {
-    return this._runtime;
-  }
-
-  flush(): void {
-    this._cache.clear();
-    this._invocations = {};
-    this._subscriptions.forEach((unsubscribe) => unsubscribe());
-    this._subscriptions.clear();
-  }
-
-  createSubscription(name: string, unsubscribe: CleanupFn): void {
-    this._subscriptions.get(name)?.();
-    this._subscriptions.set(name, unsubscribe);
-  }
-
-  /**
-   * Exec the function if TTL has expired.
-   * Return the cached value.
-   */
-  invokeFunction(
-    name: string,
-    state: InterpreterState,
-    args: any[],
-    cb: AsyncFunction,
-    options?: FunctionOptions,
-  ): InterpreterValue | undefined {
-    const ttl = options?.ttl ?? this._options.defaultTtl;
-
-    const { formulaAddress: cell } = state;
-    const invocationKey = FunctionContext.createInvocationKey(name, ...args);
-    const { value = undefined, ts = 0 } = this._cache.get(invocationKey) ?? {};
-
-    const now = Date.now();
-    const delta = now - ts;
-    if ((!ts || delta > ttl) && !this._pending.has(invocationKey)) {
-      this._pending.set(invocationKey, now);
-      setTimeout(async () => {
-        this._invocations[name] = (this._invocations[name] ?? 0) + 1;
-        try {
-          // Exec function.
-          const value = await cb(...args);
-          this._cache.set(invocationKey, { value, ts: Date.now() });
-          log('set', { cell, value });
-          this._onUpdate({ name, cell });
-        } catch (err) {
-          // TODO(burdon): Show error to user.
-          log.warn('failed', { cell, err });
-          this._cache.set(invocationKey, { value: new CellError(ErrorType.ERROR, 'Function failed.'), ts: Date.now() });
-        } finally {
-          this._pending.delete(invocationKey);
-        }
-      });
-    }
-
-    log.info('invoke', { cell, name, args, cache: value });
-    return value;
-  }
-}
-
-/**
- * Base class for async functions.
- */
-export class AsyncFunctionPlugin extends FunctionPlugin {
-  get context() {
-    return this.config.context as FunctionContext;
-  }
-
-  /**
-   * Immediately returns cached value then runs the async function.
-   */
-  protected runAsyncFunction(
-    ast: ProcedureAst,
-    state: InterpreterState,
-    cb: AsyncFunction,
-    options?: FunctionOptions,
-  ): RawInterpreterValue {
-    const { procedureName } = ast;
-    const metadata = this.metadata(procedureName);
-    return this.runFunction(ast.args, state, metadata, (...args: any) => {
-      return this.context.invokeFunction(procedureName, state, args, cb, options) ?? EmptyValue;
-    });
-  }
-}

package/src/functions/edge-function.ts

@@ -1,108 +0,0 @@
-//
-// Copyright 2024 DXOS.org
-//
-
-import * as Effect from 'effect/Effect';
-import * as SchemaAST from 'effect/SchemaAST';
-
-import { Filter } from '@dxos/client/echo';
-import { JsonSchema, Obj } from '@dxos/echo';
-import { log } from '@dxos/log';
-import { Operation } from '@dxos/operation';
-import { isNonNullable } from '@dxos/util';
-import { type ProcedureAst } from '@dxos/vendor-hyperformula';
-import { type InterpreterState } from '@dxos/vendor-hyperformula';
-import { CellError, ErrorType, FunctionArgumentType } from '@dxos/vendor-hyperformula';
-
-import { type AsyncFunction, AsyncFunctionPlugin } from './async-function';
-
-export const EDGE_FUNCTION_NAME = 'DX';
-
-const FUNCTION_TTL = 10_000;
-
-/**
- * A hyperformula function plugin for calling remote (EDGE) functions.
- * https://hyperformula.handsontable.com/guide/custom-functions.html#add-a-simple-custom-function
- */
-export class EdgeFunctionPlugin extends AsyncFunctionPlugin {
-  dx(ast: ProcedureAst, state: InterpreterState) {
-    const handler =
-      (subscribe = false): AsyncFunction =>
-      async (binding: string, ...args: any) => {
-        const space = this.context.space;
-        if (!space) {
-          return new CellError(ErrorType.REF, 'Missing space');
-        }
-
-        const [fn] = await space.db.query(Filter.type(Operation.PersistentOperation, { binding })).run();
-        if (!fn) {
-          log.info('Function not found', { binding });
-          return new CellError(ErrorType.REF, 'Function not found');
-        }
-
-        if (subscribe) {
-          const unsubscribe = Obj.subscribe(fn, () => {
-            log('function changed', { fn });
-            // TODO(wittjosiah): `ttl` should be 0 to force a recalculation when a new version is deployed.
-            //  This needs a ttl to prevent a binding change from causing the function not to be found.
-            this.runAsyncFunction(ast, state, handler(false), {
-              ttl: FUNCTION_TTL,
-            });
-          });
-
-          this.context.createSubscription(ast.procedureName, unsubscribe);
-        }
-
-        const runtime = this.context.runtime;
-        const functionDef = Operation.deserialize(fn);
-        // If input schema exists, construct an object from args using the schema props order, otherwise pass { args }.
-        let input: any;
-        if (fn.inputSchema) {
-          const schema = JsonSchema.toEffectSchema(fn.inputSchema);
-          const props = SchemaAST.getPropertySignatures(schema.ast);
-          input = {} as any;
-          props.forEach(({ name }, index) => {
-            input[name.toString()] = args[index];
-          });
-        } else {
-          input = { args: args.filter(isNonNullable) };
-        }
-        const result = runtime.runPromise(Operation.invoke(functionDef, input).pipe(Effect.orDie));
-        return result as any;
-      };
-
-    return this.runAsyncFunction(ast, state, handler(true), {
-      ttl: FUNCTION_TTL,
-    });
-  }
-}
-
-EdgeFunctionPlugin.implementedFunctions = {
-  [EDGE_FUNCTION_NAME]: {
-    method: 'dx',
-    parameters: [
-      // Binding
-      { argumentType: FunctionArgumentType.STRING },
-
-      // Remote function arguments (currently supporting up to 8).
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
-      { argumentType: FunctionArgumentType.ANY, optionalArg: true },
-    ],
-    isVolatile: true,
-  },
-};
-
-export const EdgeFunctionPluginTranslations = {
-  enGB: {
-    [EDGE_FUNCTION_NAME]: 'Remote function',
-  },
-  enUS: {
-    [EDGE_FUNCTION_NAME]: 'Remote function',
-  },
-};