@finos/legend-cube-builder 0.0.4 → 0.0.5

package/src/stores/CubeEditorState.ts

@@ -21,10 +21,11 @@ import {
   createNodeRegistry,
   CubeDocument,
   diffSchemas,
-  Extend,
+  dependsOnDatabaseType,
   findLostSortOrders,
+  getDatabaseProblemOfType,
+  getDatabaseProblems,
   isTypingError,
-  type JsonObject,
   MESSAGE_SORT_COLUMNS_CUT,
   MESSAGE_SORT_COLUMNS_DROPPED,
   MESSAGE_SORT_ORDER_LOST,
@@ -38,7 +39,13 @@ import {
   rereadQueryFilterValues,
   type Schema,
   type SchemaInferenceResult,
+  ServiceSource,
   type SourceDefinition,
+  foldColumnName,
+  FunctionSource,
+  copyQueryNode,
+  type CubeContext,
+  SourceNode,
 } from '@finos/legend-cube';
 import type { CommandRegistrar } from '@finos/legend-application';
 import type { GeneratorFn } from '@finos/legend-shared';
@@ -58,7 +65,6 @@ import {
   CUBE_EDITOR_CLOSED_REASON,
   DEFAULT_ROW_LIMIT,
   getSchemaDriftWarning,
-  getExtendRetypeWarning,
   getSourceRecheckWarning,
   LEGEND_CUBE_USER_DATA_KEY,
   MAX_UNDO_STEPS,
@@ -75,31 +81,49 @@ import {
 } from '../__lib__/LegendCubeIngestLabels.js';
 import { isCubeDataProductModel } from '../graph-manager/CubeDataProduct.js';
 import { isCubeIngestModel } from '../graph-manager/CubeIngest.js';
+import { isCubeServiceModel } from '../graph-manager/CubeService.js';
 import {
   CubeEngineError,
   CubeEngineErrorKind,
   type CubeModelOutline,
   type NodeId,
 } from '../graph-manager/CubeEngine.js';
+import {
+  CUBE_LAKEHOUSE_RUNTIME_KIND,
+  getLakehouseDatabaseType,
+} from '../graph-manager/CubeLakehouseRuntime.js';
 import { getDatabaseType } from '../graph-manager/CubeModelOutlineHelper.js';
 import { recheckCubeDataProductSources } from './CubeDataProductRecheck.js';
 import { recheckCubeIngestSources } from './CubeIngestRecheck.js';
+import { recheckCubeServiceSources } from './CubeServiceRecheck.js';
+import {
+  CUBE_FUNCTION_MESSAGE,
+  getFunctionDriftWarning,
+  getFunctionRecheckWarning,
+} from '../__lib__/LegendCubeFunctionLabels.js';
+import { recheckCubeFunctionSources } from './CubeFunctionRecheck.js';
 import { CubeDataProductRuntimeState } from './CubeDataProductRuntimeState.js';
 import { CubeExamplesState } from './CubeExamplesState.js';
+import { CubeExportState } from './CubeExportState.js';
+import { CubeGridFetcher } from './CubeGridFetcher.js';
+import { CubeGridState } from './CubeGridState.js';
+import { CubeDevSettingsState } from './CubeDevSettingsState.js';
+import { CubeRunStatsState } from './CubeRunStatsState.js';
 import { CubeExecutionState } from './CubeExecutionState.js';
+import { CubeExpressionFilterDraft } from './editors/CubeExpressionFilterDraft.js';
 import { CubeExtendDraft } from './editors/CubeExtendDraft.js';
 import {
-  buildCubeExtendTypingLambdas,
-  createStaleCubeExtendTypingRule,
-  getCubeExtendTypingUpstream,
-  getCubeExtendUpstream,
-  isCubeExtendTypingCurrent,
-  isSameCubeExtendTyping,
-  readCubeExtendTyping,
-} from './CubeExtendTyping.js';
+  buildCubeEngineCheckRequest,
+  type CubeEngineCheckedNode,
+  createStaleCubeEngineCheckRule,
+  getCubeEngineCheckUpstream,
+  isCubeEngineCheckedNode,
+} from './CubeEngineChecks.js';
+import { getCubeExtendUpstream } from './CubeExtendTyping.js';
 import { type CubeHost, loadCubeModelOutline } from './CubeHost.js';
 import { CubeNodeEditorState } from './CubeNodeEditorState.js';
 import { CubeShowPureState } from './CubeShowPureState.js';
+import { CubeLambdaImportState } from './CubeLambdaImportState.js';
 import { CubeSourcePickerState } from './CubeSourcePickerState.js';
 import { CubeSpecTransferState } from './CubeSpecTransferState.js';
 
@@ -128,11 +152,13 @@ const isFocusInNodeEditor = (): boolean =>
     `[data-testid="${LEGEND_CUBE_TEST_ID.NODE_EDITOR}"]`,
   ) !== null && document.activeElement !== null;
 
-/** A source Cube types again to see drift: a table, or a data product's access point */
+/** A source Cube types again to see drift: a table, a function, an access point, an ingest data set or a service */
 type RecheckedSource =
   | RelationalTableSource
+  | FunctionSource
   | DataProductAccessPointSource
-  | IngestDatasetSource;
+  | IngestDatasetSource
+  | ServiceSource;
 
 /**
  * The state of one Cube page (PLAN §7.8). The document is immutable: every
@@ -144,20 +170,59 @@ export class CubeEditorState implements CommandRegistrar {
   readonly host: CubeHost;
   /** One registry for inference, emission and the saved spec */
   readonly registry: NodeRegistry;
-  /** The editor's own query rule: an Extend typed for another input than its own waits to be typed again (PLAN §11.7) */
-  private readonly staleExtendTyping: QueryRule;
+  /**
+   * The editor's own query rule: an Extend typed, or a Filter by Expression
+   * checked, for another input than its own waits to be checked again (PLAN
+   * §11.7, §11.11)
+   */
+  private readonly staleEngineChecks: QueryRule;
+  /**
+   * The editor's query rule for the database a node runs on: `%` or `_`
+   * given to a pattern on DuckDB (PLAN §11.11), once the model's outline
+   * tells the database
+   */
+  private readonly databaseRefusals: QueryRule;
   readonly execution: CubeExecutionState;
+  /** The latest runs, fetches and exports (spec §12.6) */
+  readonly runStats: CubeRunStatsState;
+  /** The Dev Settings, at the top left (PLAN §11.16) */
+  readonly devSettings: CubeDevSettingsState;
+  /** The grid's server-side levels (PLAN §11.10) */
+  readonly gridFetcher: CubeGridFetcher;
+  /** The grid's mode and its server-side levels (PLAN §11.10) */
+  readonly grid: CubeGridState;
+  /** The results' export (spec §12.5) */
+  readonly export: CubeExportState;
   /** Where a lakehouse cube runs, a data product's or an ingest one's: its class and warehouse */
   readonly dataProductRuntime: CubeDataProductRuntimeState;
   readonly sourcePicker: CubeSourcePickerState;
   readonly specTransfer: CubeSpecTransferState;
   readonly examples: CubeExamplesState;
   readonly showPure: CubeShowPureState;
+  /** Import Pure: the lambda the page opened on, read into the cube (PLAN §11.18) */
+  readonly lambdaImport: CubeLambdaImportState;
   readonly nodeEditor: CubeNodeEditorState;
 
   document: CubeDocument;
   /** Earlier documents, oldest first */
   history: readonly CubeDocument[] = [];
+  /**
+   * The step copied last, as it was, with the cube's context then, which a
+   * pasted source needs; kept for the page, across cubes
+   */
+  copiedNode:
+    | { readonly node: QueryNode; readonly context: CubeContext | undefined }
+    | undefined;
+  /**
+   * The changes undone, the latest last, to redo; a new change clears them.
+   * Each keeps whether it was read-only and whether undoing it left another
+   * cube (an import), so a redo restores both
+   */
+  redoStack: readonly {
+    readonly document: CubeDocument;
+    readonly readOnly: boolean;
+    readonly opensAnotherCube: boolean;
+  }[] = [];
   /** Engine errors by node id: shown with the node's own errors, never part of inference */
   hostIssues: ReadonlyMap<string, CubeHostIssue> = new Map();
   /**
@@ -166,18 +231,31 @@ export class CubeEditorState implements CommandRegistrar {
    */
   warnings: ReadonlyMap<number, readonly string[]> = new Map();
   /**
-   * The outline of each model the cube used, loaded when an editor needs it,
-   * e.g. for the Join's 'type unknown' warning; by reference, never observed
-   * deeply
+   * The outline of each model the cube used, by the model's JSON
+   * (`modelKey`): loaded when the cube reads a table, or an editor or a run
+   * needs it, e.g. for the Join's 'type unknown' warning or the database a
+   * node runs on (PLAN §11.12); by reference, never observed deeply
    */
-  private modelOutlines: ReadonlyMap<ModelContext, CubeModelOutline> =
-    new Map();
+  private modelOutlines: ReadonlyMap<string, CubeModelOutline> = new Map();
+  /** Each model's key in `modelOutlines`, its JSON, worked out once per model object */
+  private readonly modelKeys = new WeakMap<ModelContext, string>();
+  /** The outlines on their way, by model key, so a second load waits for the first */
+  private readonly loadingOutlines = new Map<
+    string,
+    Promise<CubeModelOutline>
+  >();
   /** Sources sent to the engine to be typed again, until it answers */
   private pendingSources: ReadonlySet<QueryNode> = new Set();
-  /** Extends sent to the engine to be typed, until it answers (PLAN §11.7) */
-  private typingExtends: ReadonlySet<QueryNode> = new Set();
-  /** Types the Extends that wait for it, whenever one does (PLAN §11.7 Q4) */
-  private readonly retyping: IReactionDisposer;
+  /** Extends and Filters by Expression sent to the engine to be checked, until it answers (PLAN §11.7, §11.11) */
+  private checkingNodes: ReadonlySet<QueryNode> = new Set();
+  /** Checks the nodes that wait for the engine, whenever one does (PLAN §11.7 Q4) */
+  private readonly rechecking: IReactionDisposer;
+  /**
+   * Loads the model's outline once the cube reads a table, so the database it
+   * runs on is known before any run: some nodes are refused there, or written
+   * another way (PLAN §11.4, §11.11, §11.12)
+   */
+  private readonly outlineLoading: IReactionDisposer;
   /** The rows a run returns; kept per user, never in the cube */
   rowLimit: number;
   /** The palette shows icons only; kept per user, never in the cube */
@@ -195,18 +273,21 @@ export class CubeEditorState implements CommandRegistrar {
   constructor(host: CubeHost, document = new CubeDocument()) {
     makeObservable<
       CubeEditorState,
-      'pendingSources' | 'modelOutlines' | 'typingExtends'
+      'pendingSources' | 'modelOutlines' | 'checkingNodes'
     >(this, {
       pendingSources: observable.ref,
-      typingExtends: observable.ref,
-      extendsToType: computed,
-      isTypingExtends: computed,
-      retypeExtends: flow,
+      checkingNodes: observable.ref,
+      nodesToCheck: computed,
+      isCheckingExpressions: computed,
+      checkExpressions: flow,
       modelOutlines: observable.ref,
       modelOutline: computed,
       loadModelOutline: flow,
       document: observable.ref,
       history: observable.ref,
+      redoStack: observable.ref,
+      canRedo: computed,
+      redo: action,
       hostIssues: observable.ref,
       warnings: observable.ref,
       derivedWarnings: computed,
@@ -220,10 +301,14 @@ export class CubeEditorState implements CommandRegistrar {
       hasEditsToApply: computed,
       executeEdited: action,
       undoEdited: action,
+      redoEdited: action,
       isDialogOpen: computed,
       applyDocument: action,
       undo: action,
       importDocument: action,
+      openDocument: action,
+      startNewCube: action,
+      isEmptyCube: computed,
       reresolveSources: flow,
       refreshSource: flow,
       applyQuery: action,
@@ -233,19 +318,34 @@ export class CubeEditorState implements CommandRegistrar {
       addConfiguredNode: action,
       dropNode: action,
       removeNode: action,
+      copiedNode: observable.ref,
+      canPaste: computed,
+      copyNode: action,
+      pasteNode: action,
+      duplicateNode: action,
       swapInputs: action,
       setShowGraph: action,
       setPaletteCollapsed: action,
       setHostIssue: action,
       clearHostIssues: action,
       setRowLimit: action,
+      setColumnWidth: action,
     });
     this.host = host;
     this.registry = createNodeRegistry();
-    this.staleExtendTyping = createStaleCubeExtendTypingRule(
+    this.staleEngineChecks = createStaleCubeEngineCheckRule(
       this.registry,
       () => this.document.context?.model,
     );
+    this.databaseRefusals = (query) =>
+      new Map(
+        query.nodes
+          .map((node): [string, string[]] => [
+            node.id,
+            this.getNodeDatabaseProblems(query, node),
+          ])
+          .filter(([, problems]) => problems.length > 0),
+      );
     this.document = document;
     const storedLimit = host.applicationStore.userDataService.getNumericValue(
       LEGEND_CUBE_USER_DATA_KEY.ROW_LIMIT,
@@ -258,17 +358,40 @@ export class CubeEditorState implements CommandRegistrar {
         LEGEND_CUBE_USER_DATA_KEY.PALETTE_COLLAPSED,
       ) ?? false;
     this.execution = new CubeExecutionState(this);
+    this.runStats = new CubeRunStatsState();
+    this.devSettings = new CubeDevSettingsState(host);
+    this.gridFetcher = new CubeGridFetcher(this);
+    this.grid = new CubeGridState(this);
+    this.export = new CubeExportState(this);
     this.dataProductRuntime = new CubeDataProductRuntimeState(this);
     this.sourcePicker = new CubeSourcePickerState(this);
     this.specTransfer = new CubeSpecTransferState(this);
     this.examples = new CubeExamplesState(this);
     this.showPure = new CubeShowPureState(this);
+    this.lambdaImport = new CubeLambdaImportState(this);
     this.nodeEditor = new CubeNodeEditorState(this);
-    this.retyping = reaction(
-      () => this.extendsToType,
+    this.rechecking = reaction(
+      () => this.nodesToCheck,
       (waiting) => {
         if (waiting.length) {
-          flowResult(this.retypeExtends(waiting)).catch(
+          flowResult(this.checkExpressions(waiting)).catch(
+            this.host.applicationStore.alertUnhandledError,
+          );
+        }
+      },
+      { fireImmediately: true },
+    );
+    // the model, while the cube reads a table: another model loads too
+    this.outlineLoading = reaction(
+      () =>
+        this.document.query.nodes.some(
+          (node) => node instanceof RelationalTableSource,
+        )
+          ? this.document.context?.model
+          : undefined,
+      (model) => {
+        if (model !== undefined) {
+          flowResult(this.loadModelOutline(model)).catch(
             this.host.applicationStore.alertUnhandledError,
           );
         }
@@ -279,13 +402,15 @@ export class CubeEditorState implements CommandRegistrar {
 
   /**
    * Each node's schema and errors, query-level rules included, as the
-   * emitter sees them, and an Extend typed for another input than its own
-   * waiting (`ERR_TYPING`), which only the editor can tell
+   * emitter sees them; an Extend or a Filter by Expression checked for
+   * another input than its own waiting, and a filter its database would
+   * misread refused, which only the editor can tell
    */
   get analysis(): SchemaInferenceResult {
     return buildSchemasAndValidity(this.document.query, [
       ...this.registry.queryRules,
-      this.staleExtendTyping,
+      this.staleEngineChecks,
+      this.databaseRefusals,
     ]);
   }
 
@@ -306,47 +431,50 @@ export class CubeEditorState implements CommandRegistrar {
   }
 
   /**
-   * The Extends of the cube shown that wait for the engine to type their
-   * columns, their input valid, and that aren't being typed (PLAN §11.7):
-   * new expressions, or an input that changed
+   * The Extends and Filters by Expression of the cube shown that wait for
+   * the engine, their input valid, and that aren't being checked (PLAN §11.7,
+   * §11.11): new expressions, or an input that changed
    */
-  get extendsToType(): readonly Extend[] {
+  get nodesToCheck(): readonly CubeEngineCheckedNode[] {
     const { validity } = this.analysis;
     return this.document.query.nodes.filter(
-      (node): node is Extend =>
-        node instanceof Extend &&
-        !this.typingExtends.has(node) &&
+      (node): node is CubeEngineCheckedNode =>
+        isCubeEngineCheckedNode(node) &&
+        !this.checkingNodes.has(node) &&
         (validity.get(node.id) ?? []).some(isTypingError),
     );
   }
 
-  /** The cube shown has Extends the engine is typing */
-  get isTypingExtends(): boolean {
+  /** The cube shown has expressions the engine is checking: an Extend's columns, a filter's condition */
+  get isCheckingExpressions(): boolean {
     return this.document.query.nodes.some((node) =>
-      this.typingExtends.has(node),
+      this.checkingNodes.has(node),
     );
   }
 
-  /** The node waits for the engine to type it, or is being typed: an Extend (PLAN §11.7) */
+  /** The node waits for the engine to check it, or is being checked: an Extend or a Filter by Expression (PLAN §11.7, §11.11) */
   isTypingNode(node: QueryNode): boolean {
     return (
-      this.typingExtends.has(node) ||
+      this.checkingNodes.has(node) ||
       (this.analysis.validity.get(node.id) ?? []).some(isTypingError)
     );
   }
 
   /**
-   * Types Extends' columns with the engine, in one call, outside the undo
-   * history (PLAN §11.7 Q4): each for the input it has now, its columns
-   * after its input's relation, as typing types any node. The typing goes in
-   * place of the very nodes sent, in the cube shown and the undo snapshots
-   * that give the engine the same input; one the user changed meanwhile is
-   * left to type again. Each typing records the input as the engine was given
-   * it (`upstream`). A typing the engine couldn't give names the column it
-   * failed on; when the engine can't be reached, a typing still current for
-   * the input is kept, with a warning, as a source's saved columns are.
+   * Checks expressions with the engine, in one call, outside the undo history
+   * (PLAN §11.7 Q4, §11.11): an Extend's columns typed, a Filter by
+   * Expression's condition checked, each for the input it has now, after its
+   * input's relation, as typing types any node. The check goes in place of
+   * the very nodes sent, in the cube shown and the undo snapshots that give
+   * the engine the same input; one the user changed meanwhile is left to
+   * check again. Each check records the input as the engine was given it
+   * (`upstream`). A check the engine couldn't give says why; when the engine
+   * can't be reached, a check still current for the input is kept, with a
+   * warning, as a source's saved columns are.
    */
-  *retypeExtends(nodes: readonly Extend[]): GeneratorFn<void> {
+  *checkExpressions(
+    nodes: readonly CubeEngineCheckedNode[],
+  ): GeneratorFn<void> {
     const { context, query } = this.document;
     if (!context) {
       return;
@@ -358,37 +486,25 @@ export class CubeEditorState implements CommandRegistrar {
       const input = inputId === undefined ? undefined : schemas.get(inputId);
       if (
         query.getNode(node.id) !== node ||
-        this.typingExtends.has(node) ||
-        !input ||
-        node.columns.some(({ lambda }) => lambda === undefined)
+        this.checkingNodes.has(node) ||
+        !input
       ) {
         return [];
       }
-      const columns = node.columns.map(({ name, lambda }) => ({
-        name,
-        lambda: lambda as JsonObject,
-      }));
-      return [
-        {
-          node,
-          input,
-          columns,
-          upstream: getCubeExtendUpstream(emitter, node.id, context.model),
-          lambdas: buildCubeExtendTypingLambdas(
-            query,
-            node,
-            columns,
-            this.registry,
-            false,
-          ),
-        },
-      ];
+      const request = buildCubeEngineCheckRequest(
+        query,
+        node,
+        input,
+        getCubeExtendUpstream(emitter, node.id, context.model),
+        this.registry,
+      );
+      return request ? [request] : [];
     });
     if (!requests.length) {
       return;
     }
-    const sent = requests.map(({ node }) => node);
-    this.typingExtends = new Set([...this.typingExtends, ...sent]);
+    const sent: readonly QueryNode[] = requests.map(({ node }) => node);
+    this.checkingNodes = new Set([...this.checkingNodes, ...sent]);
     let answers: ReadonlyMap<NodeId, Schema | CubeEngineError>;
     try {
       answers = (yield this.host.engine.typeLambdas(
@@ -409,41 +525,31 @@ export class CubeEditorState implements CommandRegistrar {
         ),
       );
     } finally {
-      this.typingExtends = new Set(
-        [...this.typingExtends].filter(
-          (node) => !sent.includes(node as Extend),
-        ),
+      this.checkingNodes = new Set(
+        [...this.checkingNodes].filter((node) => !sent.includes(node)),
       );
     }
-    // a typing that comes back the same leaves its node, and the query, alone
+    // a check that comes back the same leaves its node, and the query, alone
     const warnings = new Map(this.warnings);
     const replacements = new Map<QueryNode, QueryNode>(
-      requests.flatMap(({ node, input, columns, upstream }) => {
-        const { typing, error } = readCubeExtendTyping(
-          answers,
-          node,
-          input,
-          columns,
-          upstream,
-        );
-        if (
-          error?.kind === CubeEngineErrorKind.NETWORK &&
-          isCubeExtendTypingCurrent(node, input, upstream)
-        ) {
-          warnings.set(node.key, [getExtendRetypeWarning(error.firstLine)]);
+      requests.flatMap((request) => {
+        const { node } = request;
+        const outcome = request.read(answers);
+        if (outcome.keptWarning !== undefined) {
+          warnings.set(node.key, [outcome.keptWarning]);
           return [];
         }
         warnings.delete(node.key);
-        return isSameCubeExtendTyping(typing, node.typing)
-          ? []
-          : [[node, node.withTyping(typing)] as const];
+        return outcome.node ? [[node, outcome.node] as const] : [];
       }),
     );
     this.warnings = warnings;
-    // an undo snapshot whose input differs keeps the typing it had
+    // an undo snapshot whose input differs keeps the check it had
     const emitters = new Map<Query, QueryEmitter>([[query, emitter]]);
     this.replaceOutsideHistory(replacements, (snapshot, resolved) => {
-      const upstream = getCubeExtendTypingUpstream((resolved as Extend).typing);
+      const upstream = getCubeEngineCheckUpstream(
+        resolved as CubeEngineCheckedNode,
+      );
       if (upstream === undefined) {
         return true;
       }
@@ -525,17 +631,31 @@ export class CubeEditorState implements CommandRegistrar {
     return model === undefined ? undefined : this.getModelOutline(model);
   }
 
+  /**
+   * A model's key for its outline: its JSON, so a cube imported on the same
+   * model, a new object with the same content, shares the outline loaded
+   */
+  private modelKey(model: ModelContext): string {
+    let key = this.modelKeys.get(model);
+    if (key === undefined) {
+      key = JSON.stringify(model);
+      this.modelKeys.set(model, key);
+    }
+    return key;
+  }
+
   /** The outline of a model, once loaded */
   getModelOutline(model: ModelContext): CubeModelOutline | undefined {
-    return this.modelOutlines.get(model);
+    return this.modelOutlines.get(this.modelKey(model));
   }
 
   /**
    * The database type a run of the query up to the node needs, with the
-   * model's runtime (PLAN §11.4), from the model's outline once it is loaded
-   * (`loadModelOutline`): the type of the runtime's connections to the
-   * databases the node reads. None without an outline, which writes every
-   * operation the native way.
+   * model's runtime (PLAN §11.4): for a data product or ingest cube, the
+   * database its lakehouse runtime routes to (§11.12); else, from the model's
+   * outline once it is loaded (`loadModelOutline`), the type of the runtime's
+   * connections to the databases the node reads. None without an outline,
+   * which writes every operation the native way.
    */
   getRunDatabaseType(
     query: Query,
@@ -543,6 +663,9 @@ export class CubeEditorState implements CommandRegistrar {
     model: ModelContext,
     runtime: string,
   ): string | undefined {
+    if (isCubeDataProductModel(model) || isCubeIngestModel(model)) {
+      return getLakehouseDatabaseType(CUBE_LAKEHOUSE_RUNTIME_KIND);
+    }
     const outline = this.getModelOutline(model);
     if (!outline) {
       return undefined;
@@ -552,6 +675,11 @@ export class CubeEditorState implements CommandRegistrar {
       const node = query.getNode(id);
       if (node instanceof RelationalTableSource) {
         databases.add(node.database);
+      } else if (node instanceof FunctionSource) {
+        // a function reads what Cube can't see: any database of the runtime
+        outline.runtimes
+          .find(({ path }) => path === runtime)
+          ?.connections?.forEach(({ storePath }) => databases.add(storePath));
       }
       query
         .getInputIds(id)
@@ -562,32 +690,85 @@ export class CubeEditorState implements CommandRegistrar {
   }
 
   /**
-   * Loads the outline of a model, the cube's by default, once per model. It
-   * only adds warnings and database types, so a model that fails to load
-   * shows none and runs every operation the native way.
+   * A node's problems on the database the query up to it runs on, with the
+   * cube's runtime (`getRunDatabaseType`): a Pivot where it can't run (PLAN
+   * §11.12); a filter giving `%` or `_` to a pattern on DuckDB, which would
+   * match them as wildcards (§11.11). None when the database isn't known.
+   */
+  getNodeDatabaseProblems(query: Query, node: QueryNode): string[] {
+    const { context } = this.document;
+    const runtime = context?.runtime;
+    if (!context || runtime === undefined || !dependsOnDatabaseType(node)) {
+      return [];
+    }
+    return getDatabaseProblems(
+      node,
+      this.getRunDatabaseType(query, node.id, context.model, runtime),
+    );
+  }
+
+  /**
+   * Why a node of the type can't be added after the node, on the database the
+   * query up to that node runs on (PLAN §11.12): a Pivot where it can't run.
+   * `undefined` when it can, or the database isn't known.
+   */
+  getAddDatabaseProblem(
+    type: string,
+    afterId: string | undefined,
+  ): string | undefined {
+    const { context, query } = this.document;
+    const runtime = context?.runtime;
+    if (!context || runtime === undefined || afterId === undefined) {
+      return undefined;
+    }
+    return getDatabaseProblemOfType(
+      type,
+      this.getRunDatabaseType(query, afterId, context.model, runtime),
+    );
+  }
+
+  /**
+   * Loads the outline of a model, the cube's by default, once per model: a
+   * load already on its way is waited for, not started again, e.g. a run's
+   * while the cube's opening load is still out. It only adds warnings,
+   * database types and a database's refusals (PLAN §11.11, §11.12), so a
+   * model that fails to load shows none and runs every operation the native
+   * way.
    */
   *loadModelOutline(
     model: ModelContext | undefined = this.document.context?.model,
   ): GeneratorFn<void> {
-    if (model === undefined || this.modelOutlines.has(model)) {
+    if (model === undefined) {
       return;
     }
+    const key = this.modelKey(model);
+    if (this.modelOutlines.has(key)) {
+      return;
+    }
+    let loading = this.loadingOutlines.get(key);
+    if (!loading) {
+      loading = loadCubeModelOutline(this.host, model);
+      this.loadingOutlines.set(key, loading);
+    }
     try {
-      const outline = (yield loadCubeModelOutline(
-        this.host,
-        model,
-      )) as CubeModelOutline;
-      this.modelOutlines = new Map([...this.modelOutlines, [model, outline]]);
+      const outline = (yield loading) as CubeModelOutline;
+      if (!this.modelOutlines.has(key)) {
+        this.modelOutlines = new Map([...this.modelOutlines, [key, outline]]);
+      }
     } catch {
       // no outline, no warning: nothing is blocked on it
+    } finally {
+      this.loadingOutlines.delete(key);
     }
   }
 
-  /** One of Cube's dialogs is open: the source picker, Import or Export, Show Pure */
+  /** One of Cube's dialogs is open: the source picker, Examples, the group rows' settings, Import or Export, Show Pure */
   get isDialogOpen(): boolean {
     return (
       this.sourcePicker.isOpen ||
       this.examples.isOpen ||
+      this.grid.isSettingsShown ||
+      this.grid.formatColumn !== undefined ||
       this.specTransfer.mode !== undefined ||
       this.showPure.isOpen
     );
@@ -617,7 +798,7 @@ export class CubeEditorState implements CommandRegistrar {
           isFocusInNodeEditor(),
         ),
     });
-    // F10 in the Extend editor (PLAN §11.7)
+    // F10 in the Extend and Filter by Expression editors (PLAN §11.7, §11.11)
     commandService.registerCommand({
       key: LEGEND_CUBE_COMMAND_KEY.VALIDATE_EXPRESSIONS,
       trigger: () => {
@@ -625,19 +806,36 @@ export class CubeEditorState implements CommandRegistrar {
         return (
           !this.isDialogOpen &&
           !this.readOnly &&
-          draft instanceof CubeExtendDraft &&
+          (draft instanceof CubeExtendDraft ||
+            draft instanceof CubeExpressionFilterDraft) &&
           !draft.validating
         );
       },
       action: () => {
         const { draft } = this.nodeEditor;
-        if (draft instanceof CubeExtendDraft) {
+        if (
+          draft instanceof CubeExtendDraft ||
+          draft instanceof CubeExpressionFilterDraft
+        ) {
           flowResult(draft.validate()).catch(
             this.host.applicationStore.alertUnhandledError,
           );
         }
       },
     });
+    commandService.registerCommand({
+      key: LEGEND_CUBE_COMMAND_KEY.REDO,
+      trigger: () =>
+        !this.isDialogOpen &&
+        !this.nodeEditor.isHeld &&
+        this.canRedo &&
+        !isTypingText(),
+      action: () =>
+        this.nodeEditor.runFromKeyboard(
+          () => this.redoEdited(),
+          isFocusInNodeEditor(),
+        ),
+    });
     commandService.registerCommand({
       key: LEGEND_CUBE_COMMAND_KEY.UNDO,
       trigger: () =>
@@ -694,6 +892,10 @@ export class CubeEditorState implements CommandRegistrar {
     return this.history.length > 0 && !this.readOnly;
   }
 
+  get canRedo(): boolean {
+    return this.redoStack.length > 0;
+  }
+
   /** The one way to change the cube; the document before goes to the undo history */
   applyDocument(next: CubeDocument): void {
     if (next === this.document) {
@@ -709,6 +911,41 @@ export class CubeEditorState implements CommandRegistrar {
    * without applying its edits, and drops the last run's rows and errors,
    * which belong to the cube before. Never runs the cube.
    */
+  /** Whether the cube is a new one: no name, no context, no node */
+  get isEmptyCube(): boolean {
+    const { document } = this;
+    return (
+      document.name === undefined &&
+      document.context === undefined &&
+      document.query.nodes.length === 0
+    );
+  }
+
+  /**
+   * Replaces the cube with a new, empty one, as an import does (one undo
+   * step, so Undo brings the cube back); a cube already new is left as it is
+   */
+  startNewCube(): void {
+    if (!this.isEmptyCube) {
+      this.importDocument(new CubeDocument(), false);
+    }
+  }
+
+  /**
+   * Opens the page on a cube read for it, as `initialDocument` opens one:
+   * no undo step, as there was no cube before it (Import Pure's
+   * `initialLambda`, PLAN §11.18). Its sources come typed; its expressions
+   * are checked as any are.
+   */
+  openDocument(next: CubeDocument): void {
+    this.execution.reset();
+    this.sourcePicker.close();
+    this.nodeEditor.discard(CUBE_EDITOR_CLOSED_REASON.CUBE_REPLACED);
+    this.hostIssues = new Map();
+    this.document = next;
+    this.readOnly = false;
+  }
+
   importDocument(next: CubeDocument, readOnly: boolean): void {
     this.pushHistory();
     this.importedOver.add(this.document);
@@ -723,14 +960,10 @@ export class CubeEditorState implements CommandRegistrar {
     flowResult(this.reresolveSources()).catch(
       this.host.applicationStore.alertUnhandledError,
     );
-    // typed once on load too, as sources are checked again: the model may
-    // have changed the types of its functions (PLAN §11.7)
+    // checked once on load too, as sources are checked again: the model may
+    // have changed the types of its functions (PLAN §11.7, §11.11)
     flowResult(
-      this.retypeExtends(
-        next.query.nodes.filter(
-          (node): node is Extend => node instanceof Extend,
-        ),
-      ),
+      this.checkExpressions(next.query.nodes.filter(isCubeEngineCheckedNode)),
     ).catch(this.host.applicationStore.alertUnhandledError);
   }
 
@@ -784,10 +1017,27 @@ export class CubeEditorState implements CommandRegistrar {
             node instanceof IngestDatasetSource,
         )
       : [];
+    // a function on another kind of cube is the no-mix rule's to flag
+    const functions =
+      isCubeDataProductModel(context.model) ||
+      isCubeIngestModel(context.model) ||
+      isCubeServiceModel(context.model)
+        ? []
+        : kept.filter(
+            (node): node is FunctionSource => node instanceof FunctionSource,
+          );
+    // services are typed in one place, still to come (PLAN §6.4)
+    const services = isCubeServiceModel(context.model)
+      ? kept.filter(
+          (node): node is ServiceSource => node instanceof ServiceSource,
+        )
+      : [];
     const sources: readonly RecheckedSource[] = [
       ...tables,
+      ...functions,
       ...accessPoints,
       ...dataSets,
+      ...services,
     ];
     if (!sources.length) {
       return;
@@ -795,53 +1045,72 @@ export class CubeEditorState implements CommandRegistrar {
     this.pendingSources = new Set([...this.pendingSources, ...sources]);
     let answers: ReadonlyMap<string, Schema | CubeEngineError>;
     try {
-      const [tableAnswers, accessPointAnswers, dataSetAnswers] =
-        (yield Promise.all([
-          tables.length
-            ? this.host.engine
-                .resolveSchemas(
-                  context.model,
-                  new Map(
-                    tables.map((source) => [
-                      source.id,
-                      [source.database, source.schema, source.table],
-                    ]),
-                  ),
-                )
-                .catch((error: unknown) => {
-                  const failure =
-                    error instanceof CubeEngineError
-                      ? error
-                      : new CubeEngineError(
-                          CubeEngineErrorKind.NETWORK,
-                          error instanceof Error
-                            ? error.message
-                            : String(error),
-                        );
-                  return new Map(tables.map((source) => [source.id, failure]));
-                })
-            : new Map(),
-          accessPoints.length
-            ? recheckCubeDataProductSources(
-                this.host.dataProductCatalog,
+      const [
+        tableAnswers,
+        accessPointAnswers,
+        dataSetAnswers,
+        serviceAnswers,
+        functionAnswers,
+      ] = (yield Promise.all([
+        tables.length
+          ? this.host.engine
+              .resolveSchemas(
                 context.model,
-                accessPoints,
-                fresh,
-              )
-            : new Map(),
-          dataSets.length
-            ? recheckCubeIngestSources(
-                this.host.ingestCatalog,
-                context.model,
-                dataSets,
-                fresh,
+                new Map(
+                  tables.map((source) => [
+                    source.id,
+                    [source.database, source.schema, source.table],
+                  ]),
+                ),
               )
-            : new Map(),
-        ])) as ReadonlyMap<string, Schema | CubeEngineError>[];
+              .catch((error: unknown) => {
+                const failure =
+                  error instanceof CubeEngineError
+                    ? error
+                    : new CubeEngineError(
+                        CubeEngineErrorKind.NETWORK,
+                        error instanceof Error ? error.message : String(error),
+                      );
+                return new Map(tables.map((source) => [source.id, failure]));
+              })
+          : new Map(),
+        accessPoints.length
+          ? recheckCubeDataProductSources(
+              this.host.dataProductCatalog,
+              context.model,
+              accessPoints,
+              fresh,
+            )
+          : new Map(),
+        dataSets.length
+          ? recheckCubeIngestSources(
+              this.host.ingestCatalog,
+              context.model,
+              dataSets,
+              fresh,
+            )
+          : new Map(),
+        services.length
+          ? recheckCubeServiceSources(
+              this.host.serviceCatalog,
+              context.model,
+              services,
+            )
+          : new Map(),
+        functions.length
+          ? recheckCubeFunctionSources(
+              this.host.engine,
+              context.model,
+              functions,
+            )
+          : new Map(),
+      ])) as ReadonlyMap<string, Schema | CubeEngineError>[];
       answers = new Map([
         ...(tableAnswers ?? []),
         ...(accessPointAnswers ?? []),
         ...(dataSetAnswers ?? []),
+        ...(serviceAnswers ?? []),
+        ...(functionAnswers ?? []),
       ]);
     } finally {
       this.pendingSources = new Set(
@@ -856,6 +1125,7 @@ export class CubeEditorState implements CommandRegistrar {
     sources.forEach((source) => {
       const isAccessPoint = source instanceof DataProductAccessPointSource;
       const isDataSet = source instanceof IngestDatasetSource;
+      const isFunction = source instanceof FunctionSource;
       const answer =
         answers.get(source.id) ??
         new CubeEngineError(
@@ -864,7 +1134,9 @@ export class CubeEditorState implements CommandRegistrar {
             ? CUBE_DATA_PRODUCT_RECHECK_MESSAGE.NO_ANSWER
             : isDataSet
               ? CUBE_INGEST_RECHECK_MESSAGE.NO_ANSWER
-              : 'The engine gave no schema for this table',
+              : isFunction
+                ? CUBE_FUNCTION_MESSAGE.NO_ANSWER
+                : 'The engine gave no schema for this table',
           source.id,
         );
       const saved =
@@ -878,7 +1150,9 @@ export class CubeEditorState implements CommandRegistrar {
               ? getAccessPointRecheckWarning(answer.firstLine)
               : isDataSet
                 ? getDataSetRecheckWarning(answer.firstLine)
-                : getSourceRecheckWarning(answer.firstLine),
+                : isFunction
+                  ? getFunctionRecheckWarning(answer.firstLine)
+                  : getSourceRecheckWarning(answer.firstLine),
           ]);
         } else if (
           source.resolution.kind !== 'failed' ||
@@ -907,7 +1181,9 @@ export class CubeEditorState implements CommandRegistrar {
             ? getAccessPointDriftWarning(diff)
             : isDataSet
               ? getDataSetDriftWarning(diff)
-              : getSchemaDriftWarning(diff),
+              : isFunction
+                ? getFunctionDriftWarning(diff)
+                : getSchemaDriftWarning(diff),
         ]);
       }
       replacements.set(source, resolved);
@@ -964,6 +1240,11 @@ export class CubeEditorState implements CommandRegistrar {
       return next;
     };
     this.history = this.history.map(check);
+    // a change undone meanwhile is redone with the checked nodes too
+    this.redoStack = this.redoStack.map((entry) => ({
+      ...entry,
+      document: check(entry.document),
+    }));
     const next = check(this.document);
     if (next !== this.document) {
       this.replaceDocument(next);
@@ -981,7 +1262,9 @@ export class CubeEditorState implements CommandRegistrar {
     if (
       source instanceof RelationalTableSource ||
       source instanceof DataProductAccessPointSource ||
-      source instanceof IngestDatasetSource
+      source instanceof IngestDatasetSource ||
+      source instanceof ServiceSource ||
+      source instanceof FunctionSource
     ) {
       // a Refresh reads the deployed artifact or definition again
       yield flowResult(this.reresolveSources([source], true));
@@ -1003,25 +1286,91 @@ export class CubeEditorState implements CommandRegistrar {
       return;
     }
     this.history = this.history.slice(0, -1);
+    this.redoStack = [
+      ...this.redoStack,
+      {
+        document: this.document,
+        readOnly: this.readOnly,
+        opensAnotherCube: this.importedOver.has(previous),
+      },
+    ];
     this.readOnly = this.readOnlyDocuments.has(previous);
     if (this.importedOver.has(previous)) {
       // the cube before an import is another cube, as the import was
       this.sourcePicker.close();
       this.nodeEditor.discard(CUBE_EDITOR_CLOSED_REASON.CUBE_REPLACED);
     }
-    const { query } = this.document;
+    const { query, meta } = this.document;
+    const restored = previous.withQuery(
+      previous.query === query ? query : previous.query.clone(),
+    );
+    // widths take no undo step, so the same cube keeps the ones set since
+    // (review of M7); the cube before an import keeps its own
+    this.replaceDocument(
+      this.importedOver.has(previous)
+        ? restored
+        : restored.withMeta({
+            ...restored.meta,
+            presentation: {
+              ...restored.meta.presentation,
+              columnWidths: meta.presentation.columnWidths,
+            },
+          }),
+    );
+  }
+
+  /**
+   * Brings back the last change undone, as Ctrl+Shift+Z does: the cube shown
+   * goes back to the undo history, and a change that opened another cube
+   * (an import, New) closes the editor and the source dialog again
+   */
+  redo(): void {
+    const next = this.redoStack.at(-1);
+    if (!next) {
+      return;
+    }
+    this.redoStack = this.redoStack.slice(0, -1);
+    this.pushHistory(true);
+    if (next.opensAnotherCube) {
+      this.importedOver.add(this.history.at(-1) as CubeDocument);
+      this.sourcePicker.close();
+      this.nodeEditor.discard(CUBE_EDITOR_CLOSED_REASON.CUBE_REPLACED);
+    }
+    this.readOnly = next.readOnly;
+    const { query, meta } = this.document;
+    const restored = next.document.withQuery(
+      next.document.query === query ? query : next.document.query.clone(),
+    );
+    // widths take no undo step: the same cube keeps the ones set since
     this.replaceDocument(
-      previous.withQuery(
-        previous.query === query ? query : previous.query.clone(),
-      ),
+      next.opensAnotherCube
+        ? restored
+        : restored.withMeta({
+            ...restored.meta,
+            presentation: {
+              ...restored.meta.presentation,
+              columnWidths: meta.presentation.columnWidths,
+            },
+          }),
     );
   }
 
-  private pushHistory(): void {
+  /** Redo, as its button and Ctrl+Shift+Z do: finishes the node editor first */
+  redoEdited(): void {
+    if (this.nodeEditor.finishApplied() && this.canRedo) {
+      this.redo();
+    }
+  }
+
+  /** Keeps the cube shown for undo; a new change, not a redo, drops what was undone */
+  private pushHistory(keepRedo = false): void {
     if (this.readOnly) {
       this.readOnlyDocuments.add(this.document);
     }
     this.history = [...this.history, this.document].slice(-MAX_UNDO_STEPS);
+    if (!keepRedo) {
+      this.redoStack = [];
+    }
   }
 
   /**
@@ -1063,17 +1412,43 @@ export class CubeEditorState implements CommandRegistrar {
   }
 
   /**
-   * The kinds of source the host serves: data products and ingest data sets
-   * only with their catalogs. The cube's kind may still disable one
-   * (`canAddNode`)
+   * The kinds of source the host serves: data products, ingest data sets and
+   * services only with their catalogs, tables and functions only with a tab
+   * that reads them (Studio offers ingest data sets alone). The cube's kind
+   * may still disable one (`canAddNode`)
    */
   get offeredSources(): readonly SourceDefinition[] {
-    return this.registry.sources.filter(
-      (definition) =>
-        (definition.type !== DataProductAccessPointSource.TYPE ||
-          this.sourcePicker.dataProductTab.isAvailable) &&
-        (definition.type !== IngestDatasetSource.TYPE ||
-          this.sourcePicker.ingestTab.isAvailable),
+    const {
+      modelTab,
+      projectTab,
+      projectFunctionTab,
+      directTab,
+      dataProductTab,
+      ingestTab,
+      serviceTab,
+    } = this.sourcePicker;
+    const isServed = (type: string): boolean => {
+      switch (type) {
+        case DataProductAccessPointSource.TYPE:
+          return dataProductTab.isAvailable;
+        case IngestDatasetSource.TYPE:
+          return ingestTab.isAvailable;
+        case ServiceSource.TYPE:
+          return serviceTab.isAvailable;
+        case FunctionSource.TYPE:
+          return modelTab.isAvailable || projectFunctionTab.isAvailable;
+        case RelationalTableSource.TYPE:
+          return (
+            modelTab.isAvailable ||
+            projectTab.isAvailable ||
+            directTab.isAvailable
+          );
+        default:
+          return true;
+      }
+    };
+    return this.registry.sources.filter((definition) =>
+      isServed(definition.type),
     );
   }
 
@@ -1108,19 +1483,19 @@ export class CubeEditorState implements CommandRegistrar {
    * after `afterId`; a source opens the source picker, which adds it once
    * the engine has typed it. Does nothing `canAddNode` refuses.
    */
-  addNode(type: string, afterId?: string): void {
+  addNode(type: string, afterId?: string): string | undefined {
     const definition = this.registry.get(type);
     if (!definition || !this.canAddNode(type, afterId)) {
-      return;
+      return undefined;
     }
     if (definition.kind === 'source') {
-      this.sourcePicker.open(this.sourcePicker.tabForSourceType(type)?.key);
-      return;
+      this.sourcePicker.openForSourceType(type);
+      return undefined;
     }
     const { query } = this.document;
-    this.applyQuery(
-      query.add(definition.create(query.generateId(type)), afterId),
-    );
+    const id = query.generateId(type);
+    this.applyQuery(query.add(definition.create(id), afterId));
+    return id;
   }
 
   /**
@@ -1184,6 +1559,95 @@ export class CubeEditorState implements CommandRegistrar {
     );
   }
 
+  /** Whether a step can be copied: a known type's, so its copy reads back */
+  canCopyNode(nodeId: string): boolean {
+    const node = this.document.query.getNode(nodeId);
+    return node !== undefined && this.registry.get(node.type) !== undefined;
+  }
+
+  /** Copies a step, as Cmd/Ctrl+C on it does, for Paste */
+  copyNode(nodeId: string): void {
+    const node = this.document.query.getNode(nodeId);
+    if (node && this.canCopyNode(nodeId)) {
+      this.copiedNode = { node, context: this.document.context };
+    }
+  }
+
+  /**
+   * Whether the step copied can be pasted here: a transform always; a source
+   * only into a cube of the same model and runtime, or an empty cube, which
+   * takes them, as a first table does
+   */
+  get canPaste(): boolean {
+    const copied = this.copiedNode;
+    return (
+      copied !== undefined &&
+      !this.readOnly &&
+      this.canTakeCopy(copied.node, copied.context)
+    );
+  }
+
+  /** Pastes the step copied: after the step given when it takes an input, else on its own; one undo step */
+  pasteNode(afterId?: string): void {
+    const copied = this.copiedNode;
+    if (copied && this.canPaste) {
+      this.addCopy(copied.node, copied.context, afterId);
+    }
+  }
+
+  canDuplicateNode(nodeId: string): boolean {
+    return !this.readOnly && this.canCopyNode(nodeId);
+  }
+
+  /** A copy of the step after it, as Cmd/Ctrl+D does, leaving the step copied for Paste as it was */
+  duplicateNode(nodeId: string): void {
+    const node = this.document.query.getNode(nodeId);
+    if (node && this.canDuplicateNode(nodeId)) {
+      this.addCopy(node, this.document.context, nodeId);
+    }
+  }
+
+  private canTakeCopy(
+    node: QueryNode,
+    context: CubeContext | undefined,
+  ): boolean {
+    if (!(node instanceof SourceNode)) {
+      return true;
+    }
+    const { document } = this;
+    return (
+      document.query.isEmpty ||
+      JSON.stringify(document.context) === JSON.stringify(context)
+    );
+  }
+
+  private addCopy(
+    node: QueryNode,
+    context: CubeContext | undefined,
+    afterId?: string,
+  ): void {
+    const { document } = this;
+    const copy = copyQueryNode(
+      node,
+      document.query.generateId(node.type),
+      this.registry,
+    );
+    if (!copy) {
+      return;
+    }
+    const after =
+      afterId !== undefined && document.query.canAdd(copy, afterId)
+        ? afterId
+        : undefined;
+    const query = document.query.add(copy, after);
+    // a source pasted into an empty cube brings its model and runtime
+    this.applyDocument(
+      copy instanceof SourceNode && document.query.isEmpty && context
+        ? document.withContext(context).withQuery(query)
+        : document.withQuery(query),
+    );
+  }
+
   canSwapInputs(nodeId: string): boolean {
     return !this.readOnly && this.document.query.canSwapInputs(nodeId);
   }
@@ -1211,6 +1675,31 @@ export class CubeEditorState implements CommandRegistrar {
     }
   }
 
+  /**
+   * Keeps a grid column's width, which the cube saves (PLAN §11.10): by the
+   * column's name, in any case. Presentation the user sets by dragging, so
+   * no undo step, and the rows stay current.
+   */
+  setColumnWidth(column: string, width: number): void {
+    if (!Number.isFinite(width) || width <= 0) {
+      return;
+    }
+    const { meta } = this.document;
+    const { presentation } = meta;
+    const others = presentation.columnWidths.filter(
+      (each) => foldColumnName(each.column) !== foldColumnName(column),
+    );
+    this.replaceDocument(
+      this.document.withMeta({
+        ...meta,
+        presentation: {
+          ...presentation,
+          columnWidths: [...others, { column, width: Math.round(width) }],
+        },
+      }),
+    );
+  }
+
   setHostIssue(nodeId: string, error: CubeEngineError): void {
     this.hostIssues = new Map([
       ...this.hostIssues,
@@ -1248,7 +1737,8 @@ export class CubeEditorState implements CommandRegistrar {
 
   /** Stops any run; call when the page closes */
   dispose(): void {
-    this.retyping();
+    this.rechecking();
+    this.outlineLoading();
     this.execution.stop();
     this.nodeEditor.dispose();
   }