@finos/legend-cube-builder 0.0.3 → 0.0.5

package/src/stores/CubeExtendTyping.ts

@@ -0,0 +1,251 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  colSpec,
+  colSpecArray,
+  EmitRole,
+  type Extend,
+  type ExtendTyping,
+  func,
+  getExtendSignature,
+  hashText,
+  type IR,
+  type JsonObject,
+  lambda,
+  lambdaJson,
+  type ModelContext,
+  type NodeRegistry,
+  originOf,
+  printIR,
+  type Query,
+  QueryEmitter,
+  type Schema,
+  stableJsonText,
+} from '@finos/legend-cube';
+import {
+  CubeEngineError,
+  CubeEngineErrorKind,
+  type NodeId,
+} from '../graph-manager/CubeEngine.js';
+
+// Typing an Extend's columns with the engine (PLAN §11.7): its input's
+// relation, as Cube types any node, then one extend per column. The whole
+// chain gives every type; each shorter chain, typed with it, tells which
+// column the engine fails on, since a stored lambda has no locations.
+
+/** An engine message without the compiler's ` - Context:[…]` trail, which repeats what it was doing */
+export const withoutEngineContext = (message: string): string =>
+  message.replace(/ - Context:\[[^\n]*/gu, '');
+
+/** A column to type: its name, and its lambda, located in its text or not */
+export interface CubeExtendTypingColumn {
+  readonly name: string;
+  readonly lambda: JsonObject;
+}
+
+/** The key of the lambda that types the first `count` columns */
+const keyOf = (extendId: string, count: number): NodeId =>
+  `${extendId}#${count}`;
+
+/**
+ * The lambdas that type an Extend's columns, by key: the input's relation
+ * then the first k columns, for every k. A located lambda keeps its
+ * locations, so an error points into its text; a stored one is marked with
+ * the Extend, so an error lands on it. The input must be valid.
+ */
+export const buildCubeExtendTypingLambdas = (
+  query: Query,
+  extend: Extend,
+  columns: readonly CubeExtendTypingColumn[],
+  registry: NodeRegistry,
+  located: boolean,
+): Map<NodeId, IR> => {
+  const [inputId] = query.getInputIds(extend.id);
+  if (inputId === undefined) {
+    throw new Error(`Extend "${extend.id}" has no input to type it on`);
+  }
+  const input = new QueryEmitter(query, registry).emitRelation(inputId);
+  const lambdas = new Map<NodeId, IR>();
+  columns.reduce<IR>((relation, column, index) => {
+    const extended = func('extend', [
+      relation,
+      colSpecArray([
+        colSpec(
+          column.name,
+          lambdaJson(
+            column.lambda,
+            located ? undefined : originOf(extend.id, EmitRole.EXPRESSION),
+          ),
+        ),
+      ]),
+    ]);
+    lambdas.set(keyOf(extend.id, index + 1), lambda([], [extended]));
+    return extended;
+  }, input);
+  return lambdas;
+};
+
+/** Each model's digest, by the object, so a large model is digested once */
+const MODEL_DIGESTS = new WeakMap<ModelContext, string>();
+
+const digestModel = (model: ModelContext): string => {
+  let digest = MODEL_DIGESTS.get(model);
+  if (digest === undefined) {
+    digest = hashText(stableJsonText(model));
+    MODEL_DIGESTS.set(model, digest);
+  }
+  return digest;
+};
+
+/**
+ * What the engine is given for a node's input when it types an Extend or
+ * checks a Filter by Expression, digested (PLAN §11.7, §11.11): the input's
+ * relation as emitted, and the model. A typing or a check records it
+ * (`upstream`), since the engine sees more than Cube's schema does;
+ * `undefined` when the input can't be emitted, e.g. it is invalid.
+ */
+export const getCubeExtendUpstream = (
+  emitter: QueryEmitter,
+  extendId: string,
+  model: ModelContext,
+): string | undefined => {
+  const [inputId] = emitter.query.getInputIds(extendId);
+  if (inputId === undefined || !emitter.canEmit(inputId)) {
+    return undefined;
+  }
+  return hashText(
+    `${digestModel(model)}\n${printIR(emitter.emitRelation(inputId))}`,
+  );
+};
+
+/** The digest of what the engine was given for the input, which the typing recorded */
+export const getCubeExtendTypingUpstream = (
+  typing: ExtendTyping,
+): string | undefined =>
+  typing.kind === 'unresolved' ? undefined : typing.upstream;
+
+const upstreamOf = getCubeExtendTypingUpstream;
+
+/**
+ * Whether the Extend's typing gives its types for this input: typed, for
+ * this schema and these columns, and for this input as the engine is given
+ * it, or recorded without it, as when loaded
+ */
+export const isCubeExtendTypingCurrent = (
+  extend: Extend,
+  input: Schema,
+  upstream: string | undefined,
+): boolean => {
+  const { typing } = extend;
+  const recorded = upstreamOf(typing);
+  return (
+    typing.kind === 'typed' &&
+    extend.currentTyping(input) === typing &&
+    (recorded === undefined || recorded === upstream)
+  );
+};
+
+/** What typing an Extend gave: the typing to store, and the error the engine gave, if it failed */
+export interface CubeExtendTypingResult {
+  readonly typing: ExtendTyping;
+  readonly error?: CubeEngineError | undefined;
+}
+
+/**
+ * The typing the engine's answers give, for this input and these columns:
+ * every column's type, from the whole chain, else the first column whose
+ * chain fails, with the engine's error (a network failure on none)
+ */
+export const readCubeExtendTyping = (
+  answers: ReadonlyMap<NodeId, Schema | CubeEngineError>,
+  extend: Extend,
+  input: Schema,
+  columns: readonly CubeExtendTypingColumn[],
+  upstream: string | undefined,
+): CubeExtendTypingResult => {
+  const signature = getExtendSignature(input, extend.columns);
+  const whole = answers.get(keyOf(extend.id, columns.length));
+  if (whole !== undefined && !(whole instanceof CubeEngineError)) {
+    return {
+      typing: {
+        kind: 'typed',
+        signature,
+        types: whole.columns
+          .slice(-columns.length)
+          .map((column) => column.type),
+        upstream,
+      },
+    };
+  }
+  const failing = columns.findIndex(
+    (_, index) =>
+      answers.get(keyOf(extend.id, index + 1)) instanceof CubeEngineError,
+  );
+  const error =
+    (answers.get(keyOf(extend.id, failing + 1)) as
+      | CubeEngineError
+      | undefined) ??
+    new CubeEngineError(
+      CubeEngineErrorKind.NETWORK,
+      'The engine gave no type for these columns',
+      extend.id,
+    );
+  return {
+    typing: {
+      kind: 'failed',
+      signature,
+      message: withoutEngineContext(error.detail),
+      column:
+        failing < 0 || error.kind === CubeEngineErrorKind.NETWORK
+          ? undefined
+          : failing,
+      upstream,
+    },
+    error,
+  };
+};
+
+/**
+ * Whether two typings say the same: the same kind, signature and input as
+ * the engine was given it, and the same types, or the same failure
+ */
+export const isSameCubeExtendTyping = (
+  a: ExtendTyping,
+  b: ExtendTyping,
+): boolean => {
+  if (upstreamOf(a) !== upstreamOf(b)) {
+    return false;
+  }
+  if (a.kind === 'typed' && b.kind === 'typed') {
+    return (
+      a.signature === b.signature &&
+      a.types.length === b.types.length &&
+      a.types.every((type, index) => {
+        const other = b.types[index];
+        return other !== undefined && type.equals(other);
+      })
+    );
+  }
+  if (a.kind === 'failed' && b.kind === 'failed') {
+    return (
+      a.signature === b.signature &&
+      a.message === b.message &&
+      a.column === b.column
+    );
+  }
+  return a.kind === 'unresolved' && b.kind === 'unresolved';
+};

package/src/stores/CubeFunctionRecheck.ts

@@ -0,0 +1,97 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  emitFunctionSource,
+  type EnteredValue,
+  type FunctionSource,
+  getLiteralKinds,
+  type IR,
+  lambda,
+  type LiteralValue,
+  type ModelContext,
+  type Schema,
+} from '@finos/legend-cube';
+import {
+  CubeEngineError,
+  CubeEngineErrorKind,
+  type CubeEngine,
+  type NodeId,
+} from '../graph-manager/CubeEngine.js';
+
+/** A value of each kind, which only types the call: a function's columns don't depend on its values */
+const PLACEHOLDERS: Readonly<Record<LiteralValue['kind'], LiteralValue>> = {
+  string: { kind: 'string', value: '' },
+  boolean: { kind: 'boolean', value: false },
+  integer: { kind: 'integer', value: '0' },
+  float: { kind: 'float', value: '0' },
+  decimal: { kind: 'decimal', value: '0' },
+  strictDate: { kind: 'strictDate', value: '2000-01-01' },
+  dateTime: { kind: 'dateTime', value: '2000-01-01T00:00:00' },
+  enum: { kind: 'enum', value: '' },
+};
+
+/**
+ * The lambda that types a function source, `{| sales::ordersAbove(…)}`: its
+ * values where they are valid, else a placeholder of the parameter's type, or
+ * `[]` for an empty optional one
+ */
+export const buildCubeFunctionTypingLambda = (source: FunctionSource): IR => {
+  const values: EnteredValue[] = source.parameters.map((parameter, index) => {
+    const value = source.values[index];
+    if (value !== undefined && value.kind !== 'invalid') {
+      return value;
+    }
+    if (value === undefined && parameter.optional) {
+      return undefined;
+    }
+    const [kind] = getLiteralKinds(parameter.type);
+    return kind ? PLACEHOLDERS[kind] : undefined;
+  });
+  return lambda([], [emitFunctionSource(source.withValues(values))]);
+};
+
+/**
+ * Types a cube's functions again with the engine (PLAN §6.5), in one call:
+ * their columns as the engine types the call. One answer per source,
+ * failures included; never rejects.
+ */
+export const recheckCubeFunctionSources = async (
+  engine: CubeEngine,
+  model: ModelContext,
+  sources: readonly FunctionSource[],
+): Promise<Map<NodeId, Schema | CubeEngineError>> => {
+  try {
+    return await engine.typeLambdas(
+      model,
+      new Map(
+        sources.map((source) => [
+          source.id,
+          buildCubeFunctionTypingLambda(source),
+        ]),
+      ),
+    );
+  } catch (error) {
+    const failure =
+      error instanceof CubeEngineError
+        ? error
+        : new CubeEngineError(
+            CubeEngineErrorKind.NETWORK,
+            error instanceof Error ? error.message : String(error),
+          );
+    return new Map(sources.map((source) => [source.id, failure]));
+  }
+};

package/src/stores/CubeGridFetcher.ts

@@ -0,0 +1,302 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  buildSchemasAndValidity,
+  checkDrilldown,
+  type ColumnDirection,
+  deriveDrilldownLevel,
+  type DrilldownCell,
+  type DrilldownLevel,
+  printIR,
+  QueryEmitter,
+  SortDirection,
+} from '@finos/legend-cube';
+import { NULL_CELL_TEXT } from '../__lib__/LegendCubeLabels.js';
+import {
+  CubeEngineError,
+  CubeEngineErrorKind,
+  type CubeResultValue,
+} from '../graph-manager/CubeEngine.js';
+import type { CubeEditorState } from './CubeEditorState.js';
+import type { CubeExecutionResult } from './CubeExecutionState.js';
+import { type CubeQueryRunResult, runCubeQuery } from './CubeQueryRun.js';
+
+/** A row of a level, by the run's column positions */
+export interface CubeGridRow {
+  /** Each run column's value, `undefined` where the level shows none (a group row's column with no aggregation) */
+  readonly cells: readonly (CubeResultValue | undefined)[];
+  /** On a group row, how many of the run's rows the group holds */
+  readonly count?: number | undefined;
+}
+
+/** The rows of one level of the grid: the run's rows, or groups of them */
+export interface CubeGridLevelRows {
+  readonly level: DrilldownLevel;
+  readonly rows: readonly CubeGridRow[];
+  /** The engine returned more than the run's row limit */
+  readonly limited: boolean;
+}
+
+/** An answer for rows the grid no longer shows: a newer run, or the grid purged its rows; it is dropped */
+export class CubeGridFetchDropped extends Error {
+  constructor() {
+    super('The rows were asked for by a grid that no longer shows them');
+    this.name = 'CubeGridFetchDropped';
+  }
+}
+
+/** The most rows the cache holds across its levels; the oldest levels go first (PLAN §11.10) */
+export const MAX_CUBE_GRID_CACHED_ROWS = 200_000;
+
+export const CUBE_GRID_CONTEXT_CHANGED =
+  "The cube's model, runtime or warehouse changed since this run: execute again to see more of its rows.";
+
+interface CachedLevel {
+  readonly promise: Promise<CubeGridLevelRows>;
+  /** Its rows, once answered; 0 until then */
+  rows: number;
+}
+
+/** How a fetch is shown in the run stats, e.g. `Rows of SHIP_COUNTRY "France"` */
+const describeFetch = (
+  level: DrilldownLevel,
+  keys: readonly DrilldownCell[],
+  levels: readonly string[],
+  sorts: readonly ColumnDirection[],
+): string => {
+  const of = keys
+    .map(
+      (key, index) =>
+        `${levels[index]} ${key === null ? NULL_CELL_TEXT : JSON.stringify(String(key))}`,
+    )
+    .join(', ');
+  const sorted = sorts
+    .map(
+      ({ column, direction }) =>
+        `${column} ${direction === SortDirection.DESC ? 'descending' : 'ascending'}`,
+    )
+    .join(', ');
+  return [
+    level.isLeaf ? 'Rows' : `Groups by ${level.groupColumns.at(-1)}`,
+    of ? ` of ${of}` : '',
+    sorted ? `, sorted by ${sorted}` : '',
+  ].join('');
+};
+
+/**
+ * Fetches the levels of the grid's server-side mode (PLAN §11.10): each a
+ * query derived from the run on screen, never the live cube, with the
+ * drill-down the cube has now and the grid's sort, run as Execute runs
+ * (`runCubeQuery`), with the run's model, runtime and row limit. Answers are
+ * cached by the derived query, the oldest dropped past
+ * `MAX_CUBE_GRID_CACHED_ROWS` rows. `purge()` drops them all, aborts the
+ * fetches in flight and drops their answers, as a new run does.
+ */
+export class CubeGridFetcher {
+  readonly editorState: CubeEditorState;
+  /** The most rows the cache holds */
+  readonly maxCachedRows: number;
+  /** Bumped by each purge: an answer of an older generation is dropped */
+  private generation = 0;
+  private resultId: number | undefined;
+  /** In the order last used, the oldest first */
+  private readonly cache = new Map<string, CachedLevel>();
+  private readonly controllers = new Set<AbortController>();
+
+  constructor(
+    editorState: CubeEditorState,
+    maxCachedRows = MAX_CUBE_GRID_CACHED_ROWS,
+  ) {
+    this.editorState = editorState;
+    this.maxCachedRows = maxCachedRows;
+  }
+
+  /** How many rows the cache holds */
+  get cachedRows(): number {
+    return Array.from(this.cache.values()).reduce(
+      (total, cached) => total + cached.rows,
+      0,
+    );
+  }
+
+  /** Drops every cached level, and aborts the fetches in flight, whose answers are then dropped */
+  purge(): void {
+    this.generation += 1;
+    this.cache.clear();
+    this.controllers.forEach((controller) => controller.abort());
+    this.controllers.clear();
+  }
+
+  /**
+   * The rows of the level under the expanded groups' keys (outermost first;
+   * none for the top level), sorted by the grid. Rejects with a
+   * `CubeEngineError`, or a `CubeGridFetchDropped` when the answer is for rows
+   * the grid no longer shows.
+   */
+  fetchLevel(
+    result: CubeExecutionResult,
+    keys: readonly DrilldownCell[],
+    sorts: readonly ColumnDirection[],
+  ): Promise<CubeGridLevelRows> {
+    if (result.id !== this.resultId) {
+      this.purge();
+      this.resultId = result.id;
+    }
+    const { editorState } = this;
+    const model = result.context?.model;
+    const runtime = result.context?.runtime;
+    const captureId = result.query.selected;
+    if (!model || !runtime || captureId === undefined) {
+      return Promise.reject(
+        new Error('A run has a model, a runtime and a node'),
+      );
+    }
+    // the warehouse is read from the page when the query runs, so a fetch
+    // under another one would run the run's query elsewhere
+    if (
+      result.context !== editorState.document.context ||
+      result.warehouse !== editorState.dataProductRuntime.effectiveWarehouse
+    ) {
+      return Promise.reject(
+        new CubeEngineError(
+          CubeEngineErrorKind.EXECUTION,
+          CUBE_GRID_CONTEXT_CHANGED,
+          captureId,
+        ),
+      );
+    }
+    let level: DrilldownLevel;
+    const drilldown = checkDrilldown(
+      editorState.document.meta.drilldown ?? { columns: [], aggregations: [] },
+      result.schema,
+    );
+    try {
+      level = deriveDrilldownLevel({
+        query: result.query,
+        captureId,
+        schema: result.schema,
+        drilldown,
+        keys,
+        sorts,
+      });
+    } catch (error) {
+      return Promise.reject(
+        new CubeEngineError(
+          CubeEngineErrorKind.EXECUTION,
+          error instanceof Error ? error.message : String(error),
+          captureId,
+        ),
+      );
+    }
+    const runId = level.query.selected as string;
+    // Cube's own checks first, so the engine never gets a query Cube knows is wrong
+    const { schemas, validity } = buildSchemasAndValidity(
+      level.query,
+      editorState.registry.queryRules,
+    );
+    const problem = (validity.get(runId) ?? [])[0];
+    const schema = schemas.get(runId);
+    if (problem !== undefined || !schema) {
+      return Promise.reject(
+        new CubeEngineError(
+          CubeEngineErrorKind.EXECUTION,
+          `These rows can't be fetched: ${problem ?? 'the query has no columns'}`,
+          captureId,
+        ),
+      );
+    }
+    // the whole run, so a sort the row order keeps is in it
+    const key = printIR(
+      new QueryEmitter(level.query, editorState.registry).emitExecutionLambda({
+        rowLimit: result.rowLimit,
+        runtime,
+      }),
+    );
+    const cached = this.cache.get(key);
+    if (cached) {
+      // last used now
+      this.cache.delete(key);
+      this.cache.set(key, cached);
+      return cached.promise;
+    }
+    const generation = this.generation;
+    const controller = new AbortController();
+    this.controllers.add(controller);
+    const entry: CachedLevel = {
+      rows: 0,
+      promise: runCubeQuery(editorState, {
+        query: level.query,
+        model,
+        runtime,
+        schema,
+        rowLimit: result.rowLimit,
+        abortController: controller,
+        label: describeFetch(level, keys, drilldown.columns, sorts),
+      })
+        .then((run: CubeQueryRunResult): CubeGridLevelRows => {
+          if (generation !== this.generation) {
+            throw new CubeGridFetchDropped();
+          }
+          const positions = result.schema.columns.map(({ name }) => {
+            const output = level.columns.get(name);
+            return output === undefined
+              ? -1
+              : schema.columns.findIndex((column) => column.name === output);
+          });
+          const countAt = level.countColumn
+            ? schema.columns.findIndex(
+                (column) => column.name === level.countColumn,
+              )
+            : -1;
+          const rows = run.rows.map(
+            (row): CubeGridRow => ({
+              cells: positions.map((position) =>
+                position === -1 ? undefined : (row[position] ?? null),
+              ),
+              ...(countAt === -1 ? {} : { count: Number(row[countAt]) }),
+            }),
+          );
+          entry.rows = rows.length;
+          this.evict(key);
+          return { level, rows, limited: run.limited };
+        })
+        .catch((error: unknown) => {
+          if (this.cache.get(key) === entry) {
+            this.cache.delete(key);
+          }
+          throw generation !== this.generation
+            ? new CubeGridFetchDropped()
+            : error;
+        })
+        .finally(() => this.controllers.delete(controller)),
+    };
+    this.cache.set(key, entry);
+    return entry.promise;
+  }
+
+  /** Drops the least recently used levels until the cache holds at most its rows, never the one just answered */
+  private evict(keep: string): void {
+    for (const [key] of this.cache) {
+      if (this.cachedRows <= this.maxCachedRows) {
+        return;
+      }
+      if (key !== keep) {
+        this.cache.delete(key);
+      }
+    }
+  }
+}