@finos/legend-cube-builder 0.0.4 → 0.0.6

package/lib/stores/CubeEditorState.js

@@ -13,27 +13,43 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { DataProductAccessPointSource, IngestDatasetSource, buildSchemasAndValidity, createNodeRegistry, CubeDocument, diffSchemas, Extend, findLostSortOrders, isTypingError, MESSAGE_SORT_COLUMNS_CUT, MESSAGE_SORT_COLUMNS_DROPPED, MESSAGE_SORT_ORDER_LOST, QueryEmitter, RelationalTableSource, rereadQueryFilterValues, } from '@finos/legend-cube';
+import { DataProductAccessPointSource, IngestDatasetSource, buildSchemasAndValidity, createNodeRegistry, CubeDocument, diffSchemas, dependsOnDatabaseType, findLostSortOrders, getDatabaseProblemOfType, getDatabaseProblems, isTypingError, MESSAGE_SORT_COLUMNS_CUT, MESSAGE_SORT_COLUMNS_DROPPED, MESSAGE_SORT_ORDER_LOST, QueryEmitter, RelationalTableSource, rereadQueryFilterValues, ServiceSource, foldColumnName, FunctionSource, copyQueryNode, SourceNode, } from '@finos/legend-cube';
 import { action, computed, flow, flowResult, makeObservable, observable, reaction, } from 'mobx';
 import { LEGEND_CUBE_COMMAND_KEY } from '../__lib__/LegendCubeCommand.js';
 import { LEGEND_CUBE_TEST_ID } from '../__lib__/LegendCubeTesting.js';
-import { CUBE_EDITOR_CLOSED_REASON, DEFAULT_ROW_LIMIT, getSchemaDriftWarning, getExtendRetypeWarning, getSourceRecheckWarning, LEGEND_CUBE_USER_DATA_KEY, MAX_UNDO_STEPS, } from '../__lib__/LegendCubeLabels.js';
+import { CUBE_EDITOR_CLOSED_REASON, DEFAULT_ROW_LIMIT, getSchemaDriftWarning, getSourceRecheckWarning, LEGEND_CUBE_USER_DATA_KEY, MAX_UNDO_STEPS, } from '../__lib__/LegendCubeLabels.js';
 import { CUBE_DATA_PRODUCT_RECHECK_MESSAGE, getAccessPointDriftWarning, getAccessPointRecheckWarning, } from '../__lib__/LegendCubeDataProductLabels.js';
 import { CUBE_INGEST_RECHECK_MESSAGE, getDataSetDriftWarning, getDataSetRecheckWarning, } from '../__lib__/LegendCubeIngestLabels.js';
 import { isCubeDataProductModel } from '../graph-manager/CubeDataProduct.js';
 import { isCubeIngestModel } from '../graph-manager/CubeIngest.js';
+import { isCubeServiceModel } from '../graph-manager/CubeService.js';
 import { CubeEngineError, CubeEngineErrorKind, } from '../graph-manager/CubeEngine.js';
+import { CUBE_LAKEHOUSE_RUNTIME_KIND, getLakehouseDatabaseType, } from '../graph-manager/CubeLakehouseRuntime.js';
 import { getDatabaseType } from '../graph-manager/CubeModelOutlineHelper.js';
 import { recheckCubeDataProductSources } from './CubeDataProductRecheck.js';
 import { recheckCubeIngestSources } from './CubeIngestRecheck.js';
+import { recheckCubeServiceSources } from './CubeServiceRecheck.js';
+import { CUBE_FUNCTION_MESSAGE, getFunctionDriftWarning, getFunctionRecheckWarning, } from '../__lib__/LegendCubeFunctionLabels.js';
+import { recheckCubeFunctionSources } from './CubeFunctionRecheck.js';
 import { CubeDataProductRuntimeState } from './CubeDataProductRuntimeState.js';
 import { CubeExamplesState } from './CubeExamplesState.js';
+import { CubeTourState } from './CubeTourState.js';
+import { CubeExportState } from './CubeExportState.js';
+import { CubeGridFetcher } from './CubeGridFetcher.js';
+import { CubeGridState } from './CubeGridState.js';
+import { CubeDevSettingsState } from './CubeDevSettingsState.js';
+import { CubeRunStatsState } from './CubeRunStatsState.js';
 import { CubeExecutionState } from './CubeExecutionState.js';
+import { CubeExpressionFilterDraft } from './editors/CubeExpressionFilterDraft.js';
 import { CubeExtendDraft } from './editors/CubeExtendDraft.js';
-import { buildCubeExtendTypingLambdas, createStaleCubeExtendTypingRule, getCubeExtendTypingUpstream, getCubeExtendUpstream, isCubeExtendTypingCurrent, isSameCubeExtendTyping, readCubeExtendTyping, } from './CubeExtendTyping.js';
+import { buildCubeEngineCheckRequest, createStaleCubeEngineCheckRule, getCubeEngineCheckUpstream, isCubeEngineCheckedNode, } from './CubeEngineChecks.js';
+import { getCubeExtendUpstream } from './CubeExtendTyping.js';
 import { loadCubeModelOutline } from './CubeHost.js';
 import { CubeNodeEditorState } from './CubeNodeEditorState.js';
 import { CubeShowPureState } from './CubeShowPureState.js';
+import { CubeJsonViewerState } from './variant/CubeJsonViewerState.js';
+import { CubeJsonExplorerState } from './variant/CubeJsonExplorerState.js';
+import { CubeLambdaImportState } from './CubeLambdaImportState.js';
 import { CubeSourcePickerState } from './CubeSourcePickerState.js';
 import { CubeSpecTransferState } from './CubeSpecTransferState.js';
 const isValidRowLimit = (value) => value !== undefined && Number.isSafeInteger(value) && value >= 1;
@@ -56,19 +72,58 @@ export class CubeEditorState {
     host;
     /** One registry for inference, emission and the saved spec */
     registry;
-    /** The editor's own query rule: an Extend typed for another input than its own waits to be typed again (PLAN §11.7) */
-    staleExtendTyping;
+    /**
+     * The editor's own query rule: an Extend typed, or a Filter by Expression
+     * checked, for another input than its own waits to be checked again (PLAN
+     * §11.7, §11.11)
+     */
+    staleEngineChecks;
+    /**
+     * The editor's query rule for the database a node runs on: `%` or `_`
+     * given to a pattern on DuckDB (PLAN §11.11), once the model's outline
+     * tells the database
+     */
+    databaseRefusals;
     execution;
+    /** The latest runs, fetches and exports (spec §12.6) */
+    runStats;
+    /** The Dev Settings, at the top left (PLAN §11.16) */
+    devSettings;
+    /** The grid's server-side levels (PLAN §11.10) */
+    gridFetcher;
+    /** The grid's mode and its server-side levels (PLAN §11.10) */
+    grid;
+    /** The results' export (spec §12.5) */
+    export;
     /** Where a lakehouse cube runs, a data product's or an ingest one's: its class and warehouse */
     dataProductRuntime;
     sourcePicker;
     specTransfer;
     examples;
     showPure;
+    /** The editor's guided tour (PLAN §11.21) */
+    tour;
+    /** Open JSON: a Variant cell's document as a tree (PLAN §11.20) */
+    jsonViewer = new CubeJsonViewerState();
+    /** Explore JSON: a Variant column's fields, and the columns they can become (PLAN §11.20) */
+    jsonExplorer;
+    /** Import Pure: the lambda the page opened on, read into the cube (PLAN §11.18) */
+    lambdaImport;
     nodeEditor;
     document;
     /** Earlier documents, oldest first */
     history = [];
+    /**
+     * The step copied last, as it was, with the cube's context then, which a
+     * pasted source needs; kept for the page, across cubes
+     */
+    copiedNode;
+    /**
+     * The changes undone, the latest last, to redo; a new change clears them.
+     * Each keeps whether it was read-only and whether undoing it left another
+     * cube (an import), so a redo restores both
+     */
+    redoStack = [];
     /** Engine errors by node id: shown with the node's own errors, never part of inference */
     hostIssues = new Map();
     /**
@@ -77,17 +132,28 @@ export class CubeEditorState {
      */
     warnings = new Map();
     /**
-     * The outline of each model the cube used, loaded when an editor needs it,
-     * e.g. for the Join's 'type unknown' warning; by reference, never observed
-     * deeply
+     * The outline of each model the cube used, by the model's JSON
+     * (`modelKey`): loaded when the cube reads a table, or an editor or a run
+     * needs it, e.g. for the Join's 'type unknown' warning or the database a
+     * node runs on (PLAN §11.12); by reference, never observed deeply
      */
     modelOutlines = new Map();
+    /** Each model's key in `modelOutlines`, its JSON, worked out once per model object */
+    modelKeys = new WeakMap();
+    /** The outlines on their way, by model key, so a second load waits for the first */
+    loadingOutlines = new Map();
     /** Sources sent to the engine to be typed again, until it answers */
     pendingSources = new Set();
-    /** Extends sent to the engine to be typed, until it answers (PLAN §11.7) */
-    typingExtends = new Set();
-    /** Types the Extends that wait for it, whenever one does (PLAN §11.7 Q4) */
-    retyping;
+    /** Extends and Filters by Expression sent to the engine to be checked, until it answers (PLAN §11.7, §11.11) */
+    checkingNodes = new Set();
+    /** Checks the nodes that wait for the engine, whenever one does (PLAN §11.7 Q4) */
+    rechecking;
+    /**
+     * Loads the model's outline once the cube reads a table, so the database it
+     * runs on is known before any run: some nodes are refused there, or written
+     * another way (PLAN §11.4, §11.11, §11.12)
+     */
+    outlineLoading;
     /** The rows a run returns; kept per user, never in the cube */
     rowLimit;
     /** The palette shows icons only; kept per user, never in the cube */
@@ -104,15 +170,18 @@ export class CubeEditorState {
     constructor(host, document = new CubeDocument()) {
         makeObservable(this, {
             pendingSources: observable.ref,
-            typingExtends: observable.ref,
-            extendsToType: computed,
-            isTypingExtends: computed,
-            retypeExtends: flow,
+            checkingNodes: observable.ref,
+            nodesToCheck: computed,
+            isCheckingExpressions: computed,
+            checkExpressions: flow,
             modelOutlines: observable.ref,
             modelOutline: computed,
             loadModelOutline: flow,
             document: observable.ref,
             history: observable.ref,
+            redoStack: observable.ref,
+            canRedo: computed,
+            redo: action,
             hostIssues: observable.ref,
             warnings: observable.ref,
             derivedWarnings: computed,
@@ -126,10 +195,14 @@ export class CubeEditorState {
             hasEditsToApply: computed,
             executeEdited: action,
             undoEdited: action,
+            redoEdited: action,
             isDialogOpen: computed,
             applyDocument: action,
             undo: action,
             importDocument: action,
+            openDocument: action,
+            startNewCube: action,
+            isEmptyCube: computed,
             reresolveSources: flow,
             refreshSource: flow,
             applyQuery: action,
@@ -137,18 +210,31 @@ export class CubeEditorState {
             connect: action,
             addNode: action,
             addConfiguredNode: action,
+            addConfiguredNodes: action,
             dropNode: action,
             removeNode: action,
+            copiedNode: observable.ref,
+            canPaste: computed,
+            copyNode: action,
+            pasteNode: action,
+            duplicateNode: action,
             swapInputs: action,
             setShowGraph: action,
             setPaletteCollapsed: action,
             setHostIssue: action,
             clearHostIssues: action,
             setRowLimit: action,
+            setColumnWidth: action,
         });
         this.host = host;
         this.registry = createNodeRegistry();
-        this.staleExtendTyping = createStaleCubeExtendTypingRule(this.registry, () => this.document.context?.model);
+        this.staleEngineChecks = createStaleCubeEngineCheckRule(this.registry, () => this.document.context?.model);
+        this.databaseRefusals = (query) => new Map(query.nodes
+            .map((node) => [
+            node.id,
+            this.getNodeDatabaseProblems(query, node),
+        ])
+            .filter(([, problems]) => problems.length > 0));
         this.document = document;
         const storedLimit = host.applicationStore.userDataService.getNumericValue(LEGEND_CUBE_USER_DATA_KEY.ROW_LIMIT);
         this.rowLimit = isValidRowLimit(storedLimit)
@@ -157,27 +243,45 @@ export class CubeEditorState {
         this.isPaletteCollapsed =
             host.applicationStore.userDataService.getBooleanValue(LEGEND_CUBE_USER_DATA_KEY.PALETTE_COLLAPSED) ?? false;
         this.execution = new CubeExecutionState(this);
+        this.runStats = new CubeRunStatsState();
+        this.devSettings = new CubeDevSettingsState(host);
+        this.gridFetcher = new CubeGridFetcher(this);
+        this.grid = new CubeGridState(this);
+        this.export = new CubeExportState(this);
         this.dataProductRuntime = new CubeDataProductRuntimeState(this);
         this.sourcePicker = new CubeSourcePickerState(this);
         this.specTransfer = new CubeSpecTransferState(this);
         this.examples = new CubeExamplesState(this);
         this.showPure = new CubeShowPureState(this);
+        this.tour = new CubeTourState(this);
+        this.jsonExplorer = new CubeJsonExplorerState(this);
+        this.lambdaImport = new CubeLambdaImportState(this);
         this.nodeEditor = new CubeNodeEditorState(this);
-        this.retyping = reaction(() => this.extendsToType, (waiting) => {
+        this.rechecking = reaction(() => this.nodesToCheck, (waiting) => {
             if (waiting.length) {
-                flowResult(this.retypeExtends(waiting)).catch(this.host.applicationStore.alertUnhandledError);
+                flowResult(this.checkExpressions(waiting)).catch(this.host.applicationStore.alertUnhandledError);
+            }
+        }, { fireImmediately: true });
+        // the model, while the cube reads a table: another model loads too
+        this.outlineLoading = reaction(() => this.document.query.nodes.some((node) => node instanceof RelationalTableSource)
+            ? this.document.context?.model
+            : undefined, (model) => {
+            if (model !== undefined) {
+                flowResult(this.loadModelOutline(model)).catch(this.host.applicationStore.alertUnhandledError);
             }
         }, { fireImmediately: true });
     }
     /**
      * Each node's schema and errors, query-level rules included, as the
-     * emitter sees them, and an Extend typed for another input than its own
-     * waiting (`ERR_TYPING`), which only the editor can tell
+     * emitter sees them; an Extend or a Filter by Expression checked for
+     * another input than its own waiting, and a filter its database would
+     * misread refused, which only the editor can tell
      */
     get analysis() {
         return buildSchemasAndValidity(this.document.query, [
             ...this.registry.queryRules,
-            this.staleExtendTyping,
+            this.staleEngineChecks,
+            this.databaseRefusals,
         ]);
     }
     get emitter() {
@@ -192,37 +296,38 @@ export class CubeEditorState {
         return this.pendingSources.has(node);
     }
     /**
-     * The Extends of the cube shown that wait for the engine to type their
-     * columns, their input valid, and that aren't being typed (PLAN §11.7):
-     * new expressions, or an input that changed
+     * The Extends and Filters by Expression of the cube shown that wait for
+     * the engine, their input valid, and that aren't being checked (PLAN §11.7,
+     * §11.11): new expressions, or an input that changed
      */
-    get extendsToType() {
+    get nodesToCheck() {
         const { validity } = this.analysis;
-        return this.document.query.nodes.filter((node) => node instanceof Extend &&
-            !this.typingExtends.has(node) &&
+        return this.document.query.nodes.filter((node) => isCubeEngineCheckedNode(node) &&
+            !this.checkingNodes.has(node) &&
             (validity.get(node.id) ?? []).some(isTypingError));
     }
-    /** The cube shown has Extends the engine is typing */
-    get isTypingExtends() {
-        return this.document.query.nodes.some((node) => this.typingExtends.has(node));
+    /** The cube shown has expressions the engine is checking: an Extend's columns, a filter's condition */
+    get isCheckingExpressions() {
+        return this.document.query.nodes.some((node) => this.checkingNodes.has(node));
     }
-    /** The node waits for the engine to type it, or is being typed: an Extend (PLAN §11.7) */
+    /** The node waits for the engine to check it, or is being checked: an Extend or a Filter by Expression (PLAN §11.7, §11.11) */
     isTypingNode(node) {
-        return (this.typingExtends.has(node) ||
+        return (this.checkingNodes.has(node) ||
             (this.analysis.validity.get(node.id) ?? []).some(isTypingError));
     }
     /**
-     * Types Extends' columns with the engine, in one call, outside the undo
-     * history (PLAN §11.7 Q4): each for the input it has now, its columns
-     * after its input's relation, as typing types any node. The typing goes in
-     * place of the very nodes sent, in the cube shown and the undo snapshots
-     * that give the engine the same input; one the user changed meanwhile is
-     * left to type again. Each typing records the input as the engine was given
-     * it (`upstream`). A typing the engine couldn't give names the column it
-     * failed on; when the engine can't be reached, a typing still current for
-     * the input is kept, with a warning, as a source's saved columns are.
+     * Checks expressions with the engine, in one call, outside the undo history
+     * (PLAN §11.7 Q4, §11.11): an Extend's columns typed, a Filter by
+     * Expression's condition checked, each for the input it has now, after its
+     * input's relation, as typing types any node. The check goes in place of
+     * the very nodes sent, in the cube shown and the undo snapshots that give
+     * the engine the same input; one the user changed meanwhile is left to
+     * check again. Each check records the input as the engine was given it
+     * (`upstream`). A check the engine couldn't give says why; when the engine
+     * can't be reached, a check still current for the input is kept, with a
+     * warning, as a source's saved columns are.
      */
-    *retypeExtends(nodes) {
+    *checkExpressions(nodes) {
         const { context, query } = this.document;
         if (!context) {
             return;
@@ -233,30 +338,18 @@ export class CubeEditorState {
             const [inputId] = query.getInputIds(node.id);
             const input = inputId === undefined ? undefined : schemas.get(inputId);
             if (query.getNode(node.id) !== node ||
-                this.typingExtends.has(node) ||
-                !input ||
-                node.columns.some(({ lambda }) => lambda === undefined)) {
+                this.checkingNodes.has(node) ||
+                !input) {
                 return [];
             }
-            const columns = node.columns.map(({ name, lambda }) => ({
-                name,
-                lambda: lambda,
-            }));
-            return [
-                {
-                    node,
-                    input,
-                    columns,
-                    upstream: getCubeExtendUpstream(emitter, node.id, context.model),
-                    lambdas: buildCubeExtendTypingLambdas(query, node, columns, this.registry, false),
-                },
-            ];
+            const request = buildCubeEngineCheckRequest(query, node, input, getCubeExtendUpstream(emitter, node.id, context.model), this.registry);
+            return request ? [request] : [];
         });
         if (!requests.length) {
             return;
         }
         const sent = requests.map(({ node }) => node);
-        this.typingExtends = new Set([...this.typingExtends, ...sent]);
+        this.checkingNodes = new Set([...this.checkingNodes, ...sent]);
         let answers;
         try {
             answers = (yield this.host.engine.typeLambdas(context.model, new Map(requests.flatMap(({ lambdas }) => [...lambdas]))));
@@ -268,27 +361,25 @@ export class CubeEditorState {
             answers = new Map(requests.flatMap(({ lambdas }) => [...lambdas.keys()].map((key) => [key, failure])));
         }
         finally {
-            this.typingExtends = new Set([...this.typingExtends].filter((node) => !sent.includes(node)));
+            this.checkingNodes = new Set([...this.checkingNodes].filter((node) => !sent.includes(node)));
         }
-        // a typing that comes back the same leaves its node, and the query, alone
+        // a check that comes back the same leaves its node, and the query, alone
         const warnings = new Map(this.warnings);
-        const replacements = new Map(requests.flatMap(({ node, input, columns, upstream }) => {
-            const { typing, error } = readCubeExtendTyping(answers, node, input, columns, upstream);
-            if (error?.kind === CubeEngineErrorKind.NETWORK &&
-                isCubeExtendTypingCurrent(node, input, upstream)) {
-                warnings.set(node.key, [getExtendRetypeWarning(error.firstLine)]);
+        const replacements = new Map(requests.flatMap((request) => {
+            const { node } = request;
+            const outcome = request.read(answers);
+            if (outcome.keptWarning !== undefined) {
+                warnings.set(node.key, [outcome.keptWarning]);
                 return [];
             }
             warnings.delete(node.key);
-            return isSameCubeExtendTyping(typing, node.typing)
-                ? []
-                : [[node, node.withTyping(typing)]];
+            return outcome.node ? [[node, outcome.node]] : [];
         }));
         this.warnings = warnings;
-        // an undo snapshot whose input differs keeps the typing it had
+        // an undo snapshot whose input differs keeps the check it had
         const emitters = new Map([[query, emitter]]);
         this.replaceOutsideHistory(replacements, (snapshot, resolved) => {
-            const upstream = getCubeExtendTypingUpstream(resolved.typing);
+            const upstream = getCubeEngineCheckUpstream(resolved);
             if (upstream === undefined) {
                 return true;
             }
@@ -355,18 +446,34 @@ export class CubeEditorState {
         const model = this.document.context?.model;
         return model === undefined ? undefined : this.getModelOutline(model);
     }
+    /**
+     * A model's key for its outline: its JSON, so a cube imported on the same
+     * model, a new object with the same content, shares the outline loaded
+     */
+    modelKey(model) {
+        let key = this.modelKeys.get(model);
+        if (key === undefined) {
+            key = JSON.stringify(model);
+            this.modelKeys.set(model, key);
+        }
+        return key;
+    }
     /** The outline of a model, once loaded */
     getModelOutline(model) {
-        return this.modelOutlines.get(model);
+        return this.modelOutlines.get(this.modelKey(model));
     }
     /**
      * The database type a run of the query up to the node needs, with the
-     * model's runtime (PLAN §11.4), from the model's outline once it is loaded
-     * (`loadModelOutline`): the type of the runtime's connections to the
-     * databases the node reads. None without an outline, which writes every
-     * operation the native way.
+     * model's runtime (PLAN §11.4): for a data product or ingest cube, the
+     * database its lakehouse runtime routes to (§11.12); else, from the model's
+     * outline once it is loaded (`loadModelOutline`), the type of the runtime's
+     * connections to the databases the node reads. None without an outline,
+     * which writes every operation the native way.
      */
     getRunDatabaseType(query, nodeId, model, runtime) {
+        if (isCubeDataProductModel(model) || isCubeIngestModel(model)) {
+            return getLakehouseDatabaseType(CUBE_LAKEHOUSE_RUNTIME_KIND);
+        }
         const outline = this.getModelOutline(model);
         if (!outline) {
             return undefined;
@@ -377,6 +484,12 @@ export class CubeEditorState {
             if (node instanceof RelationalTableSource) {
                 databases.add(node.database);
             }
+            else if (node instanceof FunctionSource) {
+                // a function reads what Cube can't see: any database of the runtime
+                outline.runtimes
+                    .find(({ path }) => path === runtime)
+                    ?.connections?.forEach(({ storePath }) => databases.add(storePath));
+            }
             query
                 .getInputIds(id)
                 .forEach((inputId) => inputId !== undefined && visit(inputId));
@@ -385,28 +498,75 @@ export class CubeEditorState {
         return getDatabaseType(outline, runtime, [...databases]);
     }
     /**
-     * Loads the outline of a model, the cube's by default, once per model. It
-     * only adds warnings and database types, so a model that fails to load
-     * shows none and runs every operation the native way.
+     * A node's problems on the database the query up to it runs on, with the
+     * cube's runtime (`getRunDatabaseType`): a Pivot where it can't run (PLAN
+     * §11.12); a filter giving `%` or `_` to a pattern on DuckDB, which would
+     * match them as wildcards (§11.11). None when the database isn't known.
+     */
+    getNodeDatabaseProblems(query, node) {
+        const { context } = this.document;
+        const runtime = context?.runtime;
+        if (!context || runtime === undefined || !dependsOnDatabaseType(node)) {
+            return [];
+        }
+        return getDatabaseProblems(node, this.getRunDatabaseType(query, node.id, context.model, runtime));
+    }
+    /**
+     * Why a node of the type can't be added after the node, on the database the
+     * query up to that node runs on (PLAN §11.12): a Pivot where it can't run.
+     * `undefined` when it can, or the database isn't known.
+     */
+    getAddDatabaseProblem(type, afterId) {
+        const { context, query } = this.document;
+        const runtime = context?.runtime;
+        if (!context || runtime === undefined || afterId === undefined) {
+            return undefined;
+        }
+        return getDatabaseProblemOfType(type, this.getRunDatabaseType(query, afterId, context.model, runtime));
+    }
+    /**
+     * Loads the outline of a model, the cube's by default, once per model: a
+     * load already on its way is waited for, not started again, e.g. a run's
+     * while the cube's opening load is still out. It only adds warnings,
+     * database types and a database's refusals (PLAN §11.11, §11.12), so a
+     * model that fails to load shows none and runs every operation the native
+     * way.
      */
     *loadModelOutline(model = this.document.context?.model) {
-        if (model === undefined || this.modelOutlines.has(model)) {
+        if (model === undefined) {
+            return;
+        }
+        const key = this.modelKey(model);
+        if (this.modelOutlines.has(key)) {
             return;
         }
+        let loading = this.loadingOutlines.get(key);
+        if (!loading) {
+            loading = loadCubeModelOutline(this.host, model);
+            this.loadingOutlines.set(key, loading);
+        }
         try {
-            const outline = (yield loadCubeModelOutline(this.host, model));
-            this.modelOutlines = new Map([...this.modelOutlines, [model, outline]]);
+            const outline = (yield loading);
+            if (!this.modelOutlines.has(key)) {
+                this.modelOutlines = new Map([...this.modelOutlines, [key, outline]]);
+            }
         }
         catch {
             // no outline, no warning: nothing is blocked on it
         }
+        finally {
+            this.loadingOutlines.delete(key);
+        }
     }
-    /** One of Cube's dialogs is open: the source picker, Import or Export, Show Pure */
+    /** One of Cube's dialogs is open: the source picker, Examples, the group rows' settings, Import or Export, Show Pure, or the tour */
     get isDialogOpen() {
         return (this.sourcePicker.isOpen ||
             this.examples.isOpen ||
+            this.grid.isSettingsShown ||
+            this.grid.formatColumn !== undefined ||
             this.specTransfer.mode !== undefined ||
-            this.showPure.isOpen);
+            this.showPure.isOpen ||
+            this.tour.isOpen);
     }
     /**
      * The page's keyboard shortcuts, while it is open (spec §17.12). Each does
@@ -427,23 +587,33 @@ export class CubeEditorState {
                 (this.execution.canExecute || this.hasEditsToApply),
             action: () => this.nodeEditor.runFromKeyboard(() => this.executeEdited(), isFocusInNodeEditor()),
         });
-        // F10 in the Extend editor (PLAN §11.7)
+        // F10 in the Extend and Filter by Expression editors (PLAN §11.7, §11.11)
         commandService.registerCommand({
             key: LEGEND_CUBE_COMMAND_KEY.VALIDATE_EXPRESSIONS,
             trigger: () => {
                 const { draft } = this.nodeEditor;
                 return (!this.isDialogOpen &&
                     !this.readOnly &&
-                    draft instanceof CubeExtendDraft &&
+                    (draft instanceof CubeExtendDraft ||
+                        draft instanceof CubeExpressionFilterDraft) &&
                     !draft.validating);
             },
             action: () => {
                 const { draft } = this.nodeEditor;
-                if (draft instanceof CubeExtendDraft) {
+                if (draft instanceof CubeExtendDraft ||
+                    draft instanceof CubeExpressionFilterDraft) {
                     flowResult(draft.validate()).catch(this.host.applicationStore.alertUnhandledError);
                 }
             },
         });
+        commandService.registerCommand({
+            key: LEGEND_CUBE_COMMAND_KEY.REDO,
+            trigger: () => !this.isDialogOpen &&
+                !this.nodeEditor.isHeld &&
+                this.canRedo &&
+                !isTypingText(),
+            action: () => this.nodeEditor.runFromKeyboard(() => this.redoEdited(), isFocusInNodeEditor()),
+        });
         commandService.registerCommand({
             key: LEGEND_CUBE_COMMAND_KEY.UNDO,
             trigger: () => !this.isDialogOpen &&
@@ -485,6 +655,9 @@ export class CubeEditorState {
     get canUndo() {
         return this.history.length > 0 && !this.readOnly;
     }
+    get canRedo() {
+        return this.redoStack.length > 0;
+    }
     /** The one way to change the cube; the document before goes to the undo history */
     applyDocument(next) {
         if (next === this.document) {
@@ -499,6 +672,36 @@ export class CubeEditorState {
      * without applying its edits, and drops the last run's rows and errors,
      * which belong to the cube before. Never runs the cube.
      */
+    /** Whether the cube is a new one: no name, no context, no node */
+    get isEmptyCube() {
+        const { document } = this;
+        return (document.name === undefined &&
+            document.context === undefined &&
+            document.query.nodes.length === 0);
+    }
+    /**
+     * Replaces the cube with a new, empty one, as an import does (one undo
+     * step, so Undo brings the cube back); a cube already new is left as it is
+     */
+    startNewCube() {
+        if (!this.isEmptyCube) {
+            this.importDocument(new CubeDocument(), false);
+        }
+    }
+    /**
+     * Opens the page on a cube read for it, as `initialDocument` opens one:
+     * no undo step, as there was no cube before it (Import Pure's
+     * `initialLambda`, PLAN §11.18). Its sources come typed; its expressions
+     * are checked as any are.
+     */
+    openDocument(next) {
+        this.execution.reset();
+        this.sourcePicker.close();
+        this.nodeEditor.discard(CUBE_EDITOR_CLOSED_REASON.CUBE_REPLACED);
+        this.hostIssues = new Map();
+        this.document = next;
+        this.readOnly = false;
+    }
     importDocument(next, readOnly) {
         this.pushHistory();
         this.importedOver.add(this.document);
@@ -511,9 +714,9 @@ export class CubeEditorState {
         this.document = next;
         this.readOnly = readOnly;
         flowResult(this.reresolveSources()).catch(this.host.applicationStore.alertUnhandledError);
-        // typed once on load too, as sources are checked again: the model may
-        // have changed the types of its functions (PLAN §11.7)
-        flowResult(this.retypeExtends(next.query.nodes.filter((node) => node instanceof Extend))).catch(this.host.applicationStore.alertUnhandledError);
+        // checked once on load too, as sources are checked again: the model may
+        // have changed the types of its functions (PLAN §11.7, §11.11)
+        flowResult(this.checkExpressions(next.query.nodes.filter(isCubeEngineCheckedNode))).catch(this.host.applicationStore.alertUnhandledError);
     }
     /**
      * Types the cube's sources again, outside the undo history (PLAN §10.3,
@@ -551,10 +754,22 @@ export class CubeEditorState {
         const dataSets = isCubeIngestModel(context.model)
             ? kept.filter((node) => node instanceof IngestDatasetSource)
             : [];
+        // a function on another kind of cube is the no-mix rule's to flag
+        const functions = isCubeDataProductModel(context.model) ||
+            isCubeIngestModel(context.model) ||
+            isCubeServiceModel(context.model)
+            ? []
+            : kept.filter((node) => node instanceof FunctionSource);
+        // services are typed in one place, still to come (PLAN §6.4)
+        const services = isCubeServiceModel(context.model)
+            ? kept.filter((node) => node instanceof ServiceSource)
+            : [];
         const sources = [
             ...tables,
+            ...functions,
             ...accessPoints,
             ...dataSets,
+            ...services,
         ];
         if (!sources.length) {
             return;
@@ -562,7 +777,7 @@ export class CubeEditorState {
         this.pendingSources = new Set([...this.pendingSources, ...sources]);
         let answers;
         try {
-            const [tableAnswers, accessPointAnswers, dataSetAnswers] = (yield Promise.all([
+            const [tableAnswers, accessPointAnswers, dataSetAnswers, serviceAnswers, functionAnswers,] = (yield Promise.all([
                 tables.length
                     ? this.host.engine
                         .resolveSchemas(context.model, new Map(tables.map((source) => [
@@ -572,9 +787,7 @@ export class CubeEditorState {
                         .catch((error) => {
                         const failure = error instanceof CubeEngineError
                             ? error
-                            : new CubeEngineError(CubeEngineErrorKind.NETWORK, error instanceof Error
-                                ? error.message
-                                : String(error));
+                            : new CubeEngineError(CubeEngineErrorKind.NETWORK, error instanceof Error ? error.message : String(error));
                         return new Map(tables.map((source) => [source.id, failure]));
                     })
                     : new Map(),
@@ -584,11 +797,19 @@ export class CubeEditorState {
                 dataSets.length
                     ? recheckCubeIngestSources(this.host.ingestCatalog, context.model, dataSets, fresh)
                     : new Map(),
+                services.length
+                    ? recheckCubeServiceSources(this.host.serviceCatalog, context.model, services)
+                    : new Map(),
+                functions.length
+                    ? recheckCubeFunctionSources(this.host.engine, context.model, functions)
+                    : new Map(),
             ]));
             answers = new Map([
                 ...(tableAnswers ?? []),
                 ...(accessPointAnswers ?? []),
                 ...(dataSetAnswers ?? []),
+                ...(serviceAnswers ?? []),
+                ...(functionAnswers ?? []),
             ]);
         }
         finally {
@@ -600,12 +821,15 @@ export class CubeEditorState {
         sources.forEach((source) => {
             const isAccessPoint = source instanceof DataProductAccessPointSource;
             const isDataSet = source instanceof IngestDatasetSource;
+            const isFunction = source instanceof FunctionSource;
             const answer = answers.get(source.id) ??
                 new CubeEngineError(CubeEngineErrorKind.COMPILE, isAccessPoint
                     ? CUBE_DATA_PRODUCT_RECHECK_MESSAGE.NO_ANSWER
                     : isDataSet
                         ? CUBE_INGEST_RECHECK_MESSAGE.NO_ANSWER
-                        : 'The engine gave no schema for this table', source.id);
+                        : isFunction
+                            ? CUBE_FUNCTION_MESSAGE.NO_ANSWER
+                            : 'The engine gave no schema for this table', source.id);
             const saved = source.resolution.kind === 'resolved'
                 ? source.resolution.schema
                 : undefined;
@@ -616,7 +840,9 @@ export class CubeEditorState {
                             ? getAccessPointRecheckWarning(answer.firstLine)
                             : isDataSet
                                 ? getDataSetRecheckWarning(answer.firstLine)
-                                : getSourceRecheckWarning(answer.firstLine),
+                                : isFunction
+                                    ? getFunctionRecheckWarning(answer.firstLine)
+                                    : getSourceRecheckWarning(answer.firstLine),
                     ]);
                 }
                 else if (source.resolution.kind !== 'failed' ||
@@ -641,7 +867,9 @@ export class CubeEditorState {
                         ? getAccessPointDriftWarning(diff)
                         : isDataSet
                             ? getDataSetDriftWarning(diff)
-                            : getSchemaDriftWarning(diff),
+                            : isFunction
+                                ? getFunctionDriftWarning(diff)
+                                : getSchemaDriftWarning(diff),
                 ]);
             }
             replacements.set(source, resolved);
@@ -692,6 +920,11 @@ export class CubeEditorState {
             return next;
         };
         this.history = this.history.map(check);
+        // a change undone meanwhile is redone with the checked nodes too
+        this.redoStack = this.redoStack.map((entry) => ({
+            ...entry,
+            document: check(entry.document),
+        }));
         const next = check(this.document);
         if (next !== this.document) {
             this.replaceDocument(next);
@@ -707,7 +940,9 @@ export class CubeEditorState {
         const source = this.document.query.getNode(nodeId);
         if (source instanceof RelationalTableSource ||
             source instanceof DataProductAccessPointSource ||
-            source instanceof IngestDatasetSource) {
+            source instanceof IngestDatasetSource ||
+            source instanceof ServiceSource ||
+            source instanceof FunctionSource) {
             // a Refresh reads the deployed artifact or definition again
             yield flowResult(this.reresolveSources([source], true));
         }
@@ -727,20 +962,80 @@ export class CubeEditorState {
             return;
         }
         this.history = this.history.slice(0, -1);
+        this.redoStack = [
+            ...this.redoStack,
+            {
+                document: this.document,
+                readOnly: this.readOnly,
+                opensAnotherCube: this.importedOver.has(previous),
+            },
+        ];
         this.readOnly = this.readOnlyDocuments.has(previous);
         if (this.importedOver.has(previous)) {
             // the cube before an import is another cube, as the import was
             this.sourcePicker.close();
             this.nodeEditor.discard(CUBE_EDITOR_CLOSED_REASON.CUBE_REPLACED);
         }
-        const { query } = this.document;
-        this.replaceDocument(previous.withQuery(previous.query === query ? query : previous.query.clone()));
+        const { query, meta } = this.document;
+        const restored = previous.withQuery(previous.query === query ? query : previous.query.clone());
+        // widths take no undo step, so the same cube keeps the ones set since
+        // (review of M7); the cube before an import keeps its own
+        this.replaceDocument(this.importedOver.has(previous)
+            ? restored
+            : restored.withMeta({
+                ...restored.meta,
+                presentation: {
+                    ...restored.meta.presentation,
+                    columnWidths: meta.presentation.columnWidths,
+                },
+            }));
     }
-    pushHistory() {
+    /**
+     * Brings back the last change undone, as Ctrl+Shift+Z does: the cube shown
+     * goes back to the undo history, and a change that opened another cube
+     * (an import, New) closes the editor and the source dialog again
+     */
+    redo() {
+        const next = this.redoStack.at(-1);
+        if (!next) {
+            return;
+        }
+        this.redoStack = this.redoStack.slice(0, -1);
+        this.pushHistory(true);
+        if (next.opensAnotherCube) {
+            this.importedOver.add(this.history.at(-1));
+            this.sourcePicker.close();
+            this.nodeEditor.discard(CUBE_EDITOR_CLOSED_REASON.CUBE_REPLACED);
+        }
+        this.readOnly = next.readOnly;
+        const { query, meta } = this.document;
+        const restored = next.document.withQuery(next.document.query === query ? query : next.document.query.clone());
+        // widths take no undo step: the same cube keeps the ones set since
+        this.replaceDocument(next.opensAnotherCube
+            ? restored
+            : restored.withMeta({
+                ...restored.meta,
+                presentation: {
+                    ...restored.meta.presentation,
+                    columnWidths: meta.presentation.columnWidths,
+                },
+            }));
+    }
+    /** Redo, as its button and Ctrl+Shift+Z do: finishes the node editor first */
+    redoEdited() {
+        if (this.nodeEditor.finishApplied() && this.canRedo) {
+            this.redo();
+        }
+    }
+    /** Keeps the cube shown for undo; a new change, not a redo, drops what was undone */
+    pushHistory(keepRedo = false) {
         if (this.readOnly) {
             this.readOnlyDocuments.add(this.document);
         }
         this.history = [...this.history, this.document].slice(-MAX_UNDO_STEPS);
+        if (!keepRedo) {
+            this.redoStack = [];
+        }
     }
     /**
      * Engine errors belong to the query and the context they came from, so
@@ -775,15 +1070,32 @@ export class CubeEditorState {
         }
     }
     /**
-     * The kinds of source the host serves: data products and ingest data sets
-     * only with their catalogs. The cube's kind may still disable one
-     * (`canAddNode`)
+     * The kinds of source the host serves: data products, ingest data sets and
+     * services only with their catalogs, tables and functions only with a tab
+     * that reads them (Studio offers ingest data sets alone). The cube's kind
+     * may still disable one (`canAddNode`)
      */
     get offeredSources() {
-        return this.registry.sources.filter((definition) => (definition.type !== DataProductAccessPointSource.TYPE ||
-            this.sourcePicker.dataProductTab.isAvailable) &&
-            (definition.type !== IngestDatasetSource.TYPE ||
-                this.sourcePicker.ingestTab.isAvailable));
+        const { modelTab, projectTab, projectFunctionTab, directTab, dataProductTab, ingestTab, serviceTab, } = this.sourcePicker;
+        const isServed = (type) => {
+            switch (type) {
+                case DataProductAccessPointSource.TYPE:
+                    return dataProductTab.isAvailable;
+                case IngestDatasetSource.TYPE:
+                    return ingestTab.isAvailable;
+                case ServiceSource.TYPE:
+                    return serviceTab.isAvailable;
+                case FunctionSource.TYPE:
+                    return modelTab.isAvailable || projectFunctionTab.isAvailable;
+                case RelationalTableSource.TYPE:
+                    return (modelTab.isAvailable ||
+                        projectTab.isAvailable ||
+                        directTab.isAvailable);
+                default:
+                    return true;
+            }
+        };
+        return this.registry.sources.filter((definition) => isServed(definition.type));
     }
     /**
      * Whether a node of the type can be added: a transform, unconnected or
@@ -816,14 +1128,16 @@ export class CubeEditorState {
     addNode(type, afterId) {
         const definition = this.registry.get(type);
         if (!definition || !this.canAddNode(type, afterId)) {
-            return;
+            return undefined;
         }
         if (definition.kind === 'source') {
-            this.sourcePicker.open(this.sourcePicker.tabForSourceType(type)?.key);
-            return;
+            this.sourcePicker.openForSourceType(type);
+            return undefined;
         }
         const { query } = this.document;
-        this.applyQuery(query.add(definition.create(query.generateId(type)), afterId));
+        const id = query.generateId(type);
+        this.applyQuery(query.add(definition.create(id), afterId));
+        return id;
     }
     /**
      * Adds a node made elsewhere, with its settings, as the grid's quick
@@ -832,11 +1146,28 @@ export class CubeEditorState {
      * read-only cube, or when the query can't take it.
      */
     addConfiguredNode(node, afterId) {
-        const { query } = this.document;
-        if (this.readOnly || !query.canAdd(node, afterId)) {
+        this.addConfiguredNodes([node], afterId);
+    }
+    /**
+     * Adds nodes made elsewhere one after another, the first spliced in after
+     * `afterId`, as one undo step, the last then running when that one did
+     * (Explore JSON's Extend then Explode Array). Does nothing in a read-only
+     * cube, or when the query can't take one of them.
+     */
+    addConfiguredNodes(nodes, afterId) {
+        if (this.readOnly || !nodes.length) {
             return;
         }
-        this.applyQuery(query.add(node, afterId));
+        let query = this.document.query;
+        let after = afterId;
+        for (const node of nodes) {
+            if (!query.canAdd(node, after)) {
+                return;
+            }
+            query = query.add(node, after);
+            after = node.id;
+        }
+        this.applyQuery(query);
     }
     /** Whether dropping a node on another does anything: connect it, or else move it after it */
     canDropNode(nodeId, targetId) {
@@ -875,6 +1206,69 @@ export class CubeEditorState {
             ? this.document.withContext(undefined).withQuery(query)
             : this.document.withQuery(query));
     }
+    /** Whether a step can be copied: a known type's, so its copy reads back */
+    canCopyNode(nodeId) {
+        const node = this.document.query.getNode(nodeId);
+        return node !== undefined && this.registry.get(node.type) !== undefined;
+    }
+    /** Copies a step, as Cmd/Ctrl+C on it does, for Paste */
+    copyNode(nodeId) {
+        const node = this.document.query.getNode(nodeId);
+        if (node && this.canCopyNode(nodeId)) {
+            this.copiedNode = { node, context: this.document.context };
+        }
+    }
+    /**
+     * Whether the step copied can be pasted here: a transform always; a source
+     * only into a cube of the same model and runtime, or an empty cube, which
+     * takes them, as a first table does
+     */
+    get canPaste() {
+        const copied = this.copiedNode;
+        return (copied !== undefined &&
+            !this.readOnly &&
+            this.canTakeCopy(copied.node, copied.context));
+    }
+    /** Pastes the step copied: after the step given when it takes an input, else on its own; one undo step */
+    pasteNode(afterId) {
+        const copied = this.copiedNode;
+        if (copied && this.canPaste) {
+            this.addCopy(copied.node, copied.context, afterId);
+        }
+    }
+    canDuplicateNode(nodeId) {
+        return !this.readOnly && this.canCopyNode(nodeId);
+    }
+    /** A copy of the step after it, as Cmd/Ctrl+D does, leaving the step copied for Paste as it was */
+    duplicateNode(nodeId) {
+        const node = this.document.query.getNode(nodeId);
+        if (node && this.canDuplicateNode(nodeId)) {
+            this.addCopy(node, this.document.context, nodeId);
+        }
+    }
+    canTakeCopy(node, context) {
+        if (!(node instanceof SourceNode)) {
+            return true;
+        }
+        const { document } = this;
+        return (document.query.isEmpty ||
+            JSON.stringify(document.context) === JSON.stringify(context));
+    }
+    addCopy(node, context, afterId) {
+        const { document } = this;
+        const copy = copyQueryNode(node, document.query.generateId(node.type), this.registry);
+        if (!copy) {
+            return;
+        }
+        const after = afterId !== undefined && document.query.canAdd(copy, afterId)
+            ? afterId
+            : undefined;
+        const query = document.query.add(copy, after);
+        // a source pasted into an empty cube brings its model and runtime
+        this.applyDocument(copy instanceof SourceNode && document.query.isEmpty && context
+            ? document.withContext(context).withQuery(query)
+            : document.withQuery(query));
+    }
     canSwapInputs(nodeId) {
         return !this.readOnly && this.document.query.canSwapInputs(nodeId);
     }
@@ -897,6 +1291,26 @@ export class CubeEditorState {
             }));
         }
     }
+    /**
+     * Keeps a grid column's width, which the cube saves (PLAN §11.10): by the
+     * column's name, in any case. Presentation the user sets by dragging, so
+     * no undo step, and the rows stay current.
+     */
+    setColumnWidth(column, width) {
+        if (!Number.isFinite(width) || width <= 0) {
+            return;
+        }
+        const { meta } = this.document;
+        const { presentation } = meta;
+        const others = presentation.columnWidths.filter((each) => foldColumnName(each.column) !== foldColumnName(column));
+        this.replaceDocument(this.document.withMeta({
+            ...meta,
+            presentation: {
+                ...presentation,
+                columnWidths: [...others, { column, width: Math.round(width) }],
+            },
+        }));
+    }
     setHostIssue(nodeId, error) {
         this.hostIssues = new Map([
             ...this.hostIssues,
@@ -924,7 +1338,8 @@ export class CubeEditorState {
     }
     /** Stops any run; call when the page closes */
     dispose() {
-        this.retyping();
+        this.rechecking();
+        this.outlineLoading();
         this.execution.stop();
         this.nodeEditor.dispose();
     }