@finos/legend-cube-builder 0.0.4 → 0.0.6

package/src/stores/CubeEngineChecks.ts

@@ -0,0 +1,240 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  buildSchemasAndValidity,
+  ERR_CHECKING,
+  ERR_TYPING,
+  ExpressionFilter,
+  Extend,
+  type IR,
+  isTypingError,
+  type JsonObject,
+  type ModelContext,
+  type NodeRegistry,
+  type Query,
+  QueryEmitter,
+  type QueryNode,
+  type QueryRule,
+  type Schema,
+} from '@finos/legend-cube';
+import {
+  getConditionRecheckWarning,
+  getExtendRetypeWarning,
+} from '../__lib__/LegendCubeLabels.js';
+import {
+  type CubeEngineError,
+  CubeEngineErrorKind,
+  type NodeId,
+} from '../graph-manager/CubeEngine.js';
+import {
+  buildCubeConditionCheckLambdas,
+  getCubeConditionCheckUpstream,
+  isCubeConditionCheckCurrent,
+  isSameCubeConditionCheck,
+  readCubeConditionCheck,
+} from './CubeExpressionFilterCheck.js';
+import {
+  buildCubeExtendTypingLambdas,
+  getCubeExtendTypingUpstream,
+  getCubeExtendUpstream,
+  isCubeExtendTypingCurrent,
+  isSameCubeExtendTyping,
+  readCubeExtendTyping,
+} from './CubeExtendTyping.js';
+
+// The nodes the engine checks in the background, outside the undo history:
+// an Extend, whose columns it types (PLAN §11.7), and a Filter by Expression,
+// whose condition it checks (PLAN §11.11). One path serves both, so what
+// M6.11's review fixed for Extends holds for filters too.
+
+/** A node the engine checks: an Extend or a Filter by Expression */
+export type CubeEngineCheckedNode = Extend | ExpressionFilter;
+
+export const isCubeEngineCheckedNode = (
+  node: QueryNode,
+): node is CubeEngineCheckedNode =>
+  node instanceof Extend || node instanceof ExpressionFilter;
+
+/** The digest of what the engine was given for the node's input, which its last check recorded */
+export const getCubeEngineCheckUpstream = (
+  node: CubeEngineCheckedNode,
+): string | undefined =>
+  node instanceof Extend
+    ? getCubeExtendTypingUpstream(node.typing)
+    : getCubeConditionCheckUpstream(node.check);
+
+/** What the engine's answers make of a node: the node to store, when they say anything new */
+export interface CubeEngineCheckOutcome {
+  /** The node with its new check, or `undefined` when the answers say what it says already */
+  readonly node: CubeEngineCheckedNode | undefined;
+  /** The engine's error, when the check failed */
+  readonly error?: CubeEngineError | undefined;
+  /**
+   * When the engine couldn't be reached and the node's last check still
+   * holds for this input, the warning that keeps it, as a source keeps its
+   * saved columns
+   */
+  readonly keptWarning?: string | undefined;
+}
+
+/** The lambdas that check one node, and how to read the engine's answers to them */
+export interface CubeEngineCheckRequest {
+  readonly node: CubeEngineCheckedNode;
+  readonly lambdas: ReadonlyMap<NodeId, IR>;
+  read(
+    answers: ReadonlyMap<NodeId, Schema | CubeEngineError>,
+  ): CubeEngineCheckOutcome;
+}
+
+/** What to make of a node's new check: kept with a warning, offline, when the last one holds; else stored if new */
+const outcomeOf = (
+  node: CubeEngineCheckedNode,
+  error: CubeEngineError | undefined,
+  stillCurrent: () => boolean,
+  isSame: boolean,
+  withCheck: () => CubeEngineCheckedNode,
+  warn: (firstLine: string) => string,
+): CubeEngineCheckOutcome =>
+  error?.kind === CubeEngineErrorKind.NETWORK && stillCurrent()
+    ? { node: undefined, error, keptWarning: warn(error.firstLine) }
+    : { node: isSame ? undefined : withCheck(), error };
+
+/**
+ * The request that checks a node for the input it has now, its lambdas
+ * marked with the node; `undefined` when it has nothing to send yet: an
+ * Extend column without a lambda, or a condition with a problem Cube finds
+ * itself. The input must be valid.
+ */
+export const buildCubeEngineCheckRequest = (
+  query: Query,
+  node: CubeEngineCheckedNode,
+  input: Schema,
+  upstream: string | undefined,
+  registry: NodeRegistry,
+): CubeEngineCheckRequest | undefined => {
+  if (node instanceof Extend) {
+    if (node.columns.some(({ lambda }) => lambda === undefined)) {
+      return undefined;
+    }
+    const columns = node.columns.map(({ name, lambda }) => ({
+      name,
+      lambda: lambda as JsonObject,
+    }));
+    return {
+      node,
+      lambdas: buildCubeExtendTypingLambdas(
+        query,
+        node,
+        columns,
+        registry,
+        false,
+      ),
+      read: (answers) => {
+        const { typing, error } = readCubeExtendTyping(
+          answers,
+          node,
+          input,
+          columns,
+          upstream,
+        );
+        return outcomeOf(
+          node,
+          error,
+          () => isCubeExtendTypingCurrent(node, input, upstream),
+          isSameCubeExtendTyping(typing, node.typing),
+          () => node.withTyping(typing),
+          getExtendRetypeWarning,
+        );
+      },
+    };
+  }
+  const { lambda } = node;
+  if (lambda === undefined || node.getConditionProblem() !== undefined) {
+    return undefined;
+  }
+  return {
+    node,
+    lambdas: buildCubeConditionCheckLambdas(
+      query,
+      node,
+      lambda,
+      input,
+      registry,
+      false,
+    ),
+    read: (answers) => {
+      const { check, error } = readCubeConditionCheck(
+        answers,
+        node,
+        input,
+        upstream,
+      );
+      return outcomeOf(
+        node,
+        error,
+        () => isCubeConditionCheckCurrent(node, input, upstream),
+        isSameCubeConditionCheck(check, node.check),
+        () => node.withCheck(check),
+        getConditionRecheckWarning,
+      );
+    },
+  };
+};
+
+/**
+ * A query rule for the editor (PLAN §11.7, §11.11): a node whose check
+ * recorded another input than the engine would be given now waits to be
+ * checked again (`ERR_TYPING` for an Extend, `ERR_CHECKING` for a filter),
+ * e.g. after an Extend above it gains `->toOne()`, which Cube's schema
+ * doesn't show. A check that recorded nothing, as one loaded, is used as it
+ * is; a node that already waits, or whose input is invalid, is left to its
+ * own checks.
+ */
+export const createStaleCubeEngineCheckRule =
+  (
+    registry: NodeRegistry,
+    modelOf: () => ModelContext | undefined,
+  ): QueryRule =>
+  (query) => {
+    const stale = new Map<string, readonly string[]>();
+    const model = modelOf();
+    const recorded = query.nodes.filter(
+      (node): node is CubeEngineCheckedNode =>
+        isCubeEngineCheckedNode(node) &&
+        getCubeEngineCheckUpstream(node) !== undefined,
+    );
+    if (!model || !recorded.length) {
+      return stale;
+    }
+    const { validity } = buildSchemasAndValidity(query, registry.queryRules);
+    const emitter = new QueryEmitter(query, registry);
+    recorded.forEach((node) => {
+      if ((validity.get(node.id) ?? []).some(isTypingError)) {
+        return;
+      }
+      const upstream = getCubeExtendUpstream(emitter, node.id, model);
+      if (
+        upstream !== undefined &&
+        upstream !== getCubeEngineCheckUpstream(node)
+      ) {
+        stale.set(node.id, [
+          node instanceof Extend ? ERR_TYPING : ERR_CHECKING,
+        ]);
+      }
+    });
+    return stale;
+  };

package/src/stores/CubeExamples.ts

@@ -15,7 +15,11 @@
  */
 
 import {
+  Explode,
+  Extend,
+  type JsonObject,
   AggregationFunction,
+  Pivot,
   ColumnComparisonFilter,
   CompositeFilter,
   CompositeFilterOperator,
@@ -55,6 +59,13 @@ import {
   CUBE_TRADES_RUNTIME,
   CUBE_TRADES_SCHEMA,
 } from './fixtures/CubeTradesModel.js';
+import {
+  CUBE_SALES_DATABASE,
+  CUBE_SALES_MODEL,
+  CUBE_SALES_ORDERS_TABLE,
+  CUBE_SALES_RUNTIME,
+  CUBE_SALES_SCHEMA,
+} from './fixtures/CubeSalesDuckDbModel.js';
 
 // The example cubes the source dialog opens (PLAN §6.9): two over each
 // bundled model. Each is built here rather than saved, so it has no schema
@@ -120,9 +131,21 @@ const TRADES: CubeExampleDataset = {
   schema: CUBE_TRADES_SCHEMA,
 };
 
+const SALES: CubeExampleDataset = {
+  id: 'sales',
+  name: 'Sales (DuckDB)',
+  description:
+    'Made-up sales of tea and coffee by country and year, and orders as JSON documents, on DuckDB, which runs Pivot and reads JSON',
+  tables: ['sales', 'orders'],
+  model: CUBE_SALES_MODEL,
+  runtime: CUBE_SALES_RUNTIME,
+  database: CUBE_SALES_DATABASE,
+  schema: CUBE_SALES_SCHEMA,
+};
+
 /** The datasets, in the order the examples show them */
 export const CUBE_EXAMPLE_DATASETS: readonly CubeExampleDataset[] =
-  Object.freeze([NORTHWIND, SPORTS, TRADES]);
+  Object.freeze([NORTHWIND, SPORTS, TRADES, SALES]);
 
 /**
  * A cube over the model: the nodes in order, each after the one before it
@@ -169,6 +192,77 @@ const equal = (column: string, value: string): ColumnComparisonFilter =>
     value,
   });
 
+/** An order's document read as JSON: Parse as JSON's column, as the engine parses its text (PLAN §11.20) */
+const ORDER_DOCUMENT_COLUMN = {
+  name: 'document_json',
+  code: 'x | $x.document->toOne()->fromJson()',
+  lambda: {
+    _type: 'lambda',
+    body: [
+      {
+        _type: 'func',
+        function: 'fromJson',
+        parameters: [
+          {
+            _type: 'func',
+            function: 'toOne',
+            parameters: [
+              {
+                _type: 'property',
+                parameters: [{ _type: 'var', name: 'x' }],
+                property: 'document',
+              },
+            ],
+          },
+        ],
+      },
+    ],
+    parameters: [{ _type: 'var', name: 'x' }],
+  },
+};
+
+/** An order line's field as an Extend column, `x | $x.item->get('<key>')->to(@<type>)`, as the engine parses it */
+const orderLineColumn = (
+  key: string,
+  type: string,
+): { name: string; code: string; lambda: JsonObject } => ({
+  name: key,
+  code: `x | $x.item->get('${key}')->to(@${type})`,
+  lambda: {
+    _type: 'lambda',
+    body: [
+      {
+        _type: 'func',
+        function: 'to',
+        parameters: [
+          {
+            _type: 'func',
+            function: 'get',
+            parameters: [
+              {
+                _type: 'property',
+                parameters: [{ _type: 'var', name: 'x' }],
+                property: 'item',
+              },
+              { _type: 'string', value: key },
+            ],
+          },
+          {
+            _type: 'genericTypeInstance',
+            genericType: {
+              multiplicityArguments: [],
+              rawType: { _type: 'packageableType', fullPath: type },
+              typeArguments: [],
+              typeVariableValues: [],
+            },
+          },
+        ],
+      },
+    ],
+    parameters: [{ _type: 'var', name: 'x' }],
+  },
+});
+
 export const CUBE_EXAMPLES: readonly CubeExample[] = Object.freeze([
   example(
     NORTHWIND,
@@ -454,4 +548,97 @@ export const CUBE_EXAMPLES: readonly CubeExample[] = Object.freeze([
       ]),
     }),
   ),
+  example(
+    SALES,
+    'sales-by-year',
+    'Sales by country and year',
+    'Pivots the sales by year: a row per country, a column per year summing the amounts',
+    (table) => ({
+      nodes: [
+        table('relational101', '"sales"'),
+        new Pivot(
+          'pivot101',
+          ['country'],
+          'year',
+          ['2022', '2023', '2024'].map((value) => ({
+            kind: 'integer' as const,
+            value,
+          })),
+          { column: 'amount', function: AggregationFunction.SUM },
+        ),
+        new Sort('sort101', [
+          { column: 'country', direction: SortDirection.ASC },
+        ]),
+      ],
+      connections: chain(['relational101', 'pivot101', 'sort101']),
+    }),
+  ),
+  example(
+    SALES,
+    'sales-by-product',
+    'Sales count by country and product',
+    'Pivots the sales by product: a row per country, a column per product counting its sales',
+    (table) => ({
+      nodes: [
+        table('relational101', '"sales"'),
+        new Pivot(
+          'pivot101',
+          ['country'],
+          'product',
+          ['Coffee', 'Tea'].map((value) => ({
+            kind: 'string' as const,
+            value,
+          })),
+          { column: undefined, function: AggregationFunction.COUNT_ROWS },
+        ),
+        new Sort('sort101', [
+          { column: 'country', direction: SortDirection.ASC },
+        ]),
+      ],
+      connections: chain(['relational101', 'pivot101', 'sort101']),
+    }),
+  ),
+  example(
+    SALES,
+    'sales-orders-json',
+    'Orders from JSON',
+    "Reads each order's JSON document as a Variant column: explore its fields from the column's menu",
+    (table) => ({
+      nodes: [
+        table('relational101', CUBE_SALES_ORDERS_TABLE),
+        new Extend('extend101', [ORDER_DOCUMENT_COLUMN]),
+      ],
+      connections: chain(['relational101', 'extend101']),
+    }),
+  ),
+  example(
+    SALES,
+    'sales-order-lines-json',
+    'Order lines from JSON',
+    "One row per order line, from the array in each order's JSON document, with its product and quantity",
+    (table) => ({
+      nodes: [
+        table('relational101', CUBE_SALES_ORDERS_TABLE),
+        new Extend('extend101', [ORDER_DOCUMENT_COLUMN]),
+        new Explode('explode101', 'document_json', ['items'], 'item'),
+        new Extend('extend102', [
+          orderLineColumn('product', 'String'),
+          orderLineColumn('qty', 'Integer'),
+        ]),
+        // a lateral join keeps no order: the lines by order, then product
+        new Sort('sort101', [
+          { column: 'id', direction: SortDirection.ASC },
+          { column: 'product', direction: SortDirection.ASC },
+          { column: 'qty', direction: SortDirection.ASC },
+        ]),
+      ],
+      connections: chain([
+        'relational101',
+        'extend101',
+        'explode101',
+        'extend102',
+        'sort101',
+      ]),
+    }),
+  ),
 ]);

package/src/stores/CubeExamplesState.ts

@@ -74,7 +74,7 @@ export class CubeExamplesState {
   /**
    * Starts a new cube on the dataset's model, in place of the cube (one undo
    * step), and opens the source dialog's Sample Data tab on it to add a first
-   * table
+   * table, with a way back to the examples
    */
   startOnDataset(dataset: CubeExampleDataset): void {
     this.close();
@@ -84,7 +84,9 @@ export class CubeExamplesState {
       }),
       false,
     );
-    this.editorState.sourcePicker.open(CubeSourcePickerTabKey.MODEL);
+    this.editorState.sourcePicker.openFromExamples(
+      CubeSourcePickerTabKey.MODEL,
+    );
   }
 
   /** Opens the example in place of the cube, then runs it */
@@ -92,7 +94,15 @@ export class CubeExamplesState {
     this.close();
     const document = example.createDocument();
     this.editorState.importDocument(document, false);
-    yield when(() => !this.editorState.isResolvingSources);
+    // its tables typed, and its expressions checked by the engine, as an
+    // example's Extend parsing JSON is (PLAN §11.20)
+    yield when(
+      () =>
+        !this.editorState.isResolvingSources &&
+        !this.editorState.document.query.nodes.some((node) =>
+          this.editorState.isTypingNode(node),
+        ),
+    );
     const { execution } = this.editorState;
     // not if the user has moved on, e.g. undone the opening: typing the
     // tables replaces the document but keeps its context, which is this

package/src/stores/CubeExecutionState.ts

@@ -17,26 +17,19 @@
 import {
   type CubeContext,
   isSchemasError,
-  needsDatabaseType,
   type Query,
   type Schema,
 } from '@finos/legend-cube';
 import { assertErrorThrown, type GeneratorFn } from '@finos/legend-shared';
-import {
-  action,
-  computed,
-  flow,
-  flowResult,
-  makeObservable,
-  observable,
-} from 'mobx';
+import { action, computed, flow, makeObservable, observable } from 'mobx';
 import {
   CubeEngineError,
   CubeEngineErrorKind,
-  type CubeResult,
   type CubeResultValue,
 } from '../graph-manager/CubeEngine.js';
+import { CUBE_RUN_LABEL } from '../__lib__/LegendCubeLabels.js';
 import type { CubeEditorState } from './CubeEditorState.js';
+import { type CubeQueryRunResult, runCubeQuery } from './CubeQueryRun.js';
 
 /** What the grid shows of a run: the rows within the limit, under the schema the run was made with */
 export interface CubeExecutionResult {
@@ -157,7 +150,9 @@ export class CubeExecutionState {
     }
     const captureId = query.selected;
     if (captureId === undefined) {
-      reasons.push('Select the node to run.');
+      reasons.push(
+        'Pick the step to run: Run this step, from its menu or editor.',
+      );
     } else if (
       !emitter.canEmit(captureId) ||
       // as the editor sees it: an Extend typed for another input waits
@@ -176,7 +171,7 @@ export class CubeExecutionState {
     if (this.isRunning || !this.canExecute) {
       return;
     }
-    const { document, emitter, analysis, rowLimit, host } = this.editorState;
+    const { document, analysis, rowLimit } = this.editorState;
     const { query, context } = document;
     const warehouse = this.editorState.dataProductRuntime.effectiveWarehouse;
     // checked by canExecute
@@ -192,64 +187,33 @@ export class CubeExecutionState {
     this.runController = controller;
     this.error = undefined;
     this.editorState.clearHostIssues();
+    // an export's failure is of the run it exported
+    this.editorState.export.clearError();
     try {
-      // a Drop, Slice, Limit or Distinct is written for the database it runs on
-      // (PLAN §11.4), which the model's outline gives: loaded once, waited
-      // for only then; the query, model and runtime are the ones captured
-      let databaseType: string | undefined;
-      if (needsDatabaseType(query, captureId)) {
-        yield flowResult(this.editorState.loadModelOutline(model));
-        if (this.runController !== controller || controller.signal.aborted) {
-          return;
-        }
-        databaseType = this.editorState.getRunDatabaseType(
-          query,
-          captureId,
-          model,
-          runtime,
-        );
-      }
-      const lambda = emitter.emitExecutionLambda({
-        rowLimit,
+      // the query, model and runtime are the ones captured
+      const run = (yield runCubeQuery(this.editorState, {
+        query,
+        model,
         runtime,
-        databaseType,
-      });
-      const response = (yield host.engine.execute(model, lambda, {
+        schema,
+        rowLimit,
         abortController: controller,
-      })) as CubeResult;
+        label: CUBE_RUN_LABEL.EXECUTE,
+      })) as CubeQueryRunResult;
       if (this.runController !== controller) {
         return;
       }
-      // rows are read by position, so the engine's columns must be the query's
-      if (response.columns.length !== schema.columns.length) {
-        throw new CubeEngineError(
-          CubeEngineErrorKind.EXECUTION,
-          `The engine returned ${response.columns.length} columns, but the query has ${schema.columns.length}`,
-          captureId,
-        );
-      }
-      const misplaced = schema.columns.findIndex(
-        (column, position) => response.columns[position] !== column.name,
-      );
-      if (misplaced !== -1) {
-        throw new CubeEngineError(
-          CubeEngineErrorKind.EXECUTION,
-          `The engine returned the column "${response.columns[misplaced]}" at position ${misplaced + 1}, but the query has "${schema.columns[misplaced]?.name}" there`,
-          captureId,
-        );
-      }
-      const limited = response.rows.length > rowLimit;
       this.result = {
         id: ++this.runCount,
         query,
         context,
         warehouse,
         schema,
-        rows: limited ? response.rows.slice(0, rowLimit) : response.rows,
+        rows: run.rows,
         rowLimit,
-        limited,
-        sql: response.sql,
-        durationMs: response.durationMs,
+        limited: run.limited,
+        sql: run.sql,
+        durationMs: run.durationMs,
       };
     } catch (error) {
       // a stopped or replaced run shows nothing
@@ -301,6 +265,11 @@ export class CubeExecutionState {
   /** Stops any run and forgets the last one's rows and error, e.g. when another cube is opened */
   reset(): void {
     this.stop();
+    this.editorState.gridFetcher.purge();
+    this.editorState.export.stop();
+    // an export's failure is of the run it exported (review of M7)
+    this.editorState.export.clearError();
+    this.editorState.grid.closeSettings();
     this.result = undefined;
     this.error = undefined;
   }