@finos/legend-cube-builder 0.0.3 → 0.0.5

package/src/stores/CubeHost.ts

@@ -15,7 +15,24 @@
  */
 
 import type { GenericLegendApplicationStore } from '@finos/legend-application';
-import type { CubeEngine } from '../graph-manager/CubeEngine.js';
+import type { ModelContext } from '@finos/legend-cube';
+import type { CubeConnectionExplorer } from '../graph-manager/CubeConnectionExplorer.js';
+import type { CubeDataProductCatalog } from '../graph-manager/CubeDataProductCatalog.js';
+import type { CubeEngineClientSettings } from '../graph-manager/CubeEngineClientSettings.js';
+import type { CubeIngestCatalog } from '../graph-manager/CubeIngestCatalog.js';
+import {
+  CubeEngineError,
+  CubeEngineErrorKind,
+  type CubeEngine,
+  type CubeModelOutline,
+} from '../graph-manager/CubeEngine.js';
+import {
+  checkCubeProjectModel,
+  getCubeProjectCoordinates,
+  isCubeProjectModel,
+} from '../graph-manager/CubeProject.js';
+import type { CubeProjectCatalog } from '../graph-manager/CubeProjectCatalog.js';
+import type { CubeServiceCatalog } from '../graph-manager/CubeServiceCatalog.js';
 import type { LocalModelCatalog } from './LocalModelCatalog.js';
 
 /**
@@ -29,4 +46,105 @@ export interface CubeHost {
   readonly applicationStore: GenericLegendApplicationStore;
   readonly engine: CubeEngine;
   readonly modelCatalog: LocalModelCatalog;
+  /**
+   * Reads the databases behind direct connections (PLAN §6.8); without one,
+   * the source picker offers no database connection
+   */
+  readonly connectionExplorer?: CubeConnectionExplorer | undefined;
+  /**
+   * The deployed data products (PLAN §6.8); without one, the source dialog
+   * offers no data products
+   */
+  readonly dataProductCatalog?: CubeDataProductCatalog | undefined;
+  /**
+   * The deployed ingest definitions (PLAN §6.7); without one, the source
+   * dialog offers no ingest data sets
+   */
+  readonly ingestCatalog?: CubeIngestCatalog | undefined;
+  /**
+   * The published projects in the host's depot (PLAN §6.3); without one, the
+   * source dialog offers no projects
+   */
+  readonly projectCatalog?: CubeProjectCatalog | undefined;
+  /**
+   * The deployed services (PLAN §6.4), in progress; without one, the source
+   * dialog offers no services
+   */
+  readonly serviceCatalog?: CubeServiceCatalog | undefined;
+  /**
+   * The engine URL and request compression its engine clients share, which
+   * Dev Settings changes (PLAN §11.16); without them, Dev Settings can't
+   * change either
+   */
+  readonly engineClientSettings?: CubeEngineClientSettings | undefined;
+  /**
+   * Whether the page offers Cube's sample data: the source dialog's Sample
+   * Data tab and the Examples. Offered unless the host says no, as Studio
+   * does, where a cube reads only the project's ingest definitions
+   */
+  readonly offersSampleData?: boolean | undefined;
+  /**
+   * The one project whose own sources the page reads, e.g. the one Studio
+   * edits: the source dialog lists its ingest definitions and data products
+   * by name and path, with no class, deployment or search across the
+   * platform, and says which lakehouse runtime its runs use
+   */
+  readonly project?: CubeHostProject | undefined;
 }
+
+/** The lakehouse runtime a project's runs use, as the source dialog shows it */
+export interface CubeHostProjectRuntime {
+  /** e.g. `demo-dev, from demo::ingest::LakehouseRuntime` */
+  readonly label: string;
+  /** The warehouse a new cube starts on, when the runtime names one */
+  readonly warehouse?: string | undefined;
+}
+
+/** The one project a host's page reads the sources of (`CubeHost.project`) */
+export interface CubeHostProject {
+  /** Its lakehouse runtime; fails with a `CubeEngineError` saying why when it has none */
+  resolveRuntime(): Promise<CubeHostProjectRuntime>;
+}
+
+/** Whether the host's page reads one project's own sources, as Studio's does */
+export const isCubeProjectHost = (host: CubeHost): boolean =>
+  host.project !== undefined;
+
+/** Whether the host offers Cube's sample data and examples */
+export const isCubeSampleDataOffered = (host: CubeHost): boolean =>
+  host.offersSampleData !== false;
+
+/** Why the host can't list a project model's Databases */
+const NO_PROJECT_CATALOG =
+  "This page can't read published projects: its host has no depot";
+
+/**
+ * A model's databases and runtimes, wherever its kind keeps them: a
+ * project's in the host's depot (PLAN §6.3), any other through the model
+ * catalog's engine
+ */
+export const loadCubeModelOutline = (
+  host: CubeHost,
+  model: ModelContext,
+): Promise<CubeModelOutline> => {
+  if (!isCubeProjectModel(model)) {
+    return host.modelCatalog.loadOutline(model);
+  }
+  const project = getCubeProjectCoordinates(model);
+  if (!project) {
+    return Promise.reject(
+      new CubeEngineError(
+        CubeEngineErrorKind.UNSUPPORTED_MODEL,
+        checkCubeProjectModel(model).join('\n'),
+      ),
+    );
+  }
+  return host.projectCatalog
+    ? host.projectCatalog.loadOutline(project)
+    : Promise.reject(
+        new CubeEngineError(
+          CubeEngineErrorKind.UNSUPPORTED_MODEL,
+          NO_PROJECT_CATALOG,
+        ),
+      );
+};

package/src/stores/CubeIngestRecheck.ts

@@ -0,0 +1,92 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type {
+  IngestDatasetSource,
+  ModelContext,
+  Schema,
+} from '@finos/legend-cube';
+import { CUBE_INGEST_RECHECK_MESSAGE } from '../__lib__/LegendCubeIngestLabels.js';
+import {
+  CubeEngineError,
+  CubeEngineErrorKind,
+  type NodeId,
+} from '../graph-manager/CubeEngine.js';
+import {
+  checkCubeIngestModel,
+  getCubeIngestSettings,
+} from '../graph-manager/CubeIngest.js';
+import type { CubeIngestCatalog } from '../graph-manager/CubeIngestCatalog.js';
+
+/**
+ * Reads an ingest cube's data sets again (PLAN §6.7): their columns as their
+ * deployed definitions declare them, through the catalog, with no engine
+ * call; `fresh` reads the definitions again rather than what this page visit
+ * read. One answer per source, failures included; never rejects.
+ */
+export const recheckCubeIngestSources = async (
+  catalog: CubeIngestCatalog | undefined,
+  model: ModelContext,
+  sources: readonly IngestDatasetSource[],
+  fresh = false,
+): Promise<Map<NodeId, Schema | CubeEngineError>> => {
+  const failAll = (
+    kind: CubeEngineErrorKind,
+    detail: string,
+  ): Map<NodeId, Schema | CubeEngineError> =>
+    new Map(
+      sources.map((source) => [
+        source.id,
+        new CubeEngineError(kind, detail, source.id),
+      ]),
+    );
+  if (!catalog) {
+    return failAll(
+      CubeEngineErrorKind.UNSUPPORTED_MODEL,
+      CUBE_INGEST_RECHECK_MESSAGE.NO_CATALOG,
+    );
+  }
+  const settings = getCubeIngestSettings(model);
+  if (!settings) {
+    return failAll(
+      CubeEngineErrorKind.UNSUPPORTED_MODEL,
+      checkCubeIngestModel(model).join('\n'),
+    );
+  }
+  try {
+    return await catalog.resolveSchemas(
+      settings.environmentType,
+      new Map(
+        sources.map((source) => [
+          source.id,
+          {
+            ingestDefinitionUrn: source.ingestDefinitionUrn,
+            ingestDefinition: source.ingestDefinition,
+            dataSet: source.dataSet,
+          },
+        ]),
+      ),
+      { fresh },
+    );
+  } catch (error) {
+    return error instanceof CubeEngineError
+      ? failAll(error.kind, error.detail)
+      : failAll(
+          CubeEngineErrorKind.NETWORK,
+          error instanceof Error ? error.message : String(error),
+        );
+  }
+};

package/src/stores/CubeLambdaImportState.ts

@@ -0,0 +1,434 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  CubeDocument,
+  CubeLambdaReadError,
+  ExpressionFilter,
+  Extend,
+  findCubeLambdaExtends,
+  findCubeLambdaSources,
+  isSameCubeIR,
+  lambda,
+  QueryEmitter,
+  readCubeQuery,
+  splitCubeLambda,
+  Schema,
+  type CubeContext,
+  type IR,
+  type JsonObject,
+  type Query,
+} from '@finos/legend-cube';
+import {
+  assertErrorThrown,
+  parseLosslessJSON,
+  type GeneratorFn,
+  type PlainObject,
+} from '@finos/legend-shared';
+import { action, flow, makeObservable, observable, when } from 'mobx';
+import { CubeEngineError } from '../graph-manager/CubeEngine.js';
+import {
+  CUBE_EXPRESSION_CODEC,
+  readCubeLambdaJson,
+} from '../graph-manager/protocol/pure/CubeEngineBuilder.js';
+import type { CubeEditorState } from './CubeEditorState.js';
+import { bindCubeLambdaSources } from './CubeLambdaSources.js';
+
+// Import Pure (PLAN §11.18): a lambda read into a cube, its sources bound by
+// the host, each step checked by writing it again. A host can open the page
+// on one (`CubeEditor`'s `initialLambda`), e.g. Studio's Edit MatView. Once
+// the cube's expressions are checked, it is written again and compared with
+// the lambda, so the page can say when saving would write it another way.
+
+/** A lambda a host opens the page on: its protocol JSON, and what to call it */
+export interface CubeInitialLambda {
+  readonly lambda: PlainObject;
+  /** What the page calls it, e.g. `materialized view TRADES_VIEW` */
+  readonly label: string;
+}
+
+/** Whether Cube writes the cube it read back as the lambda it read */
+export enum CubeLambdaFidelity {
+  /** The same Pure, so saving it changes nothing */
+  EXACT = 'exact',
+  /** Pure that reads the same rows, written Cube's way */
+  REWRITTEN = 'rewritten',
+  /**
+   * Not known: a step isn't valid, an expression couldn't be checked, or a
+   * step read with unknown columns is written another way
+   */
+  UNCHECKED = 'unchecked',
+}
+
+export const CUBE_LAMBDA_IMPORT_MESSAGE = {
+  READING: (label: string): string => `Reading ${label}…`,
+  CANT_READ: (label: string): string => `Cube can't read ${label}`,
+  NOT_NEW: 'Cube opens a lambda on a new cube only.',
+  REWRITTEN: (label: string): string =>
+    `Cube writes ${label} another way, which reads the same rows: what Cube shows and saves is its own Pure.`,
+  UNCHECKED: (label: string): string =>
+    `Cube couldn't check that its steps read the same rows as ${label}: review them.`,
+  GRID_LIMIT:
+    "The limit beside from() is read as a Limit step: it may be the grid's row limit of the run the Pure was copied from.",
+  /** What the banner calls a lambda imported from the Import Pure dialog */
+  IMPORTED: 'the Pure imported',
+  NOT_JSON: (reason: string): string => `Can't read the JSON: ${reason}`,
+};
+
+/** The source id the engine's grammar gives what Import Pure parses, so its errors name it */
+export const CUBE_IMPORT_PURE_SOURCE_ID = 'cube:import-pure';
+
+export interface CubeLambdaImportError {
+  readonly message: string;
+  readonly detail?: string | undefined;
+}
+
+const toImportError = (error: unknown): CubeLambdaImportError => {
+  if (error instanceof CubeEngineError) {
+    return {
+      message: error.firstLine,
+      detail: error.detail !== error.firstLine ? error.detail : undefined,
+    };
+  }
+  assertErrorThrown(error);
+  return { message: error.message };
+};
+
+/** A lambda read into a cube: the cube, and what to compare it with once checked */
+export interface CubeLambdaReading {
+  readonly document: CubeDocument;
+  /** The lambda as read, which the cube is compared with */
+  readonly lambda: IR;
+  /** Whether it was a run's lambda, with `from()`, rather than a matview's relation */
+  readonly isRun: boolean;
+  /**
+   * The steps read without being checked, as the columns before them weren't
+   * known: read as Cube writes them, so the cube can't be vouched for unless
+   * it is written back the same
+   */
+  readonly unchecked: readonly string[];
+  readonly notes: readonly string[];
+}
+
+/** The engine's grammar of an expression's lambda, as the editor shows it */
+const codeOf = async (
+  editorState: CubeEditorState,
+  expression: JsonObject,
+): Promise<string> =>
+  (
+    await editorState.host.engine.renderPure(
+      CUBE_EXPRESSION_CODEC.read(expression),
+    )
+  ).trim();
+
+/** Each Filter by Expression's and Extend column's code, the engine's grammar of its lambda */
+const withExpressionCode = async (
+  editorState: CubeEditorState,
+  query: Query,
+): Promise<Query> => {
+  let next = query;
+  for (const node of query.nodes) {
+    if (node instanceof ExpressionFilter && node.lambda && !node.code) {
+      next = next.replace(
+        node.withCondition(await codeOf(editorState, node.lambda), node.lambda),
+      );
+    } else if (node instanceof Extend) {
+      const columns = [];
+      for (const column of node.columns) {
+        columns.push(
+          column.lambda && !column.code
+            ? { ...column, code: await codeOf(editorState, column.lambda) }
+            : column,
+        );
+      }
+      next = next.replace(node.withColumns(columns));
+    }
+  }
+  return next;
+};
+
+/**
+ * The schema of each chain of extends of the lambda, typed by the engine in
+ * one call before the steps are read, so the steps after an Extend can be
+ * checked; none when the engine can't type them, as then the editor will
+ * say why
+ */
+const typeExtends = async (
+  editorState: CubeEditorState,
+  context: CubeContext,
+  chains: readonly { readonly read: IR; readonly typed: IR }[],
+): Promise<Map<IR, Schema>> => {
+  const schemas = new Map<IR, Schema>();
+  if (!chains.length) {
+    return schemas;
+  }
+  try {
+    const typed = await editorState.host.engine.typeLambdas(
+      context.model,
+      new Map(
+        chains.map(({ typed: relation }, index) => [
+          `extend${index}`,
+          lambda([], [relation]),
+        ]),
+      ),
+    );
+    chains.forEach(({ read }, index) => {
+      const schema = typed.get(`extend${index}`);
+      if (schema instanceof Schema) {
+        schemas.set(read, schema);
+      }
+    });
+  } catch {
+    // typed again by the editor, which says why it can't
+  }
+  return schemas;
+};
+
+/**
+ * A lambda's protocol JSON read into a cube on the editor's host: its sources
+ * bound, its steps read, its expressions given their code. Throws a
+ * `CubeLambdaReadError`, or the engine's error, naming what can't be read.
+ */
+export const readCubeLambdaDocument = async (
+  editorState: CubeEditorState,
+  json: unknown,
+): Promise<CubeLambdaReading> => {
+  const read = readCubeLambdaJson(json);
+  const parts = splitCubeLambda(read);
+  const binding = await bindCubeLambdaSources(
+    editorState,
+    findCubeLambdaSources(parts),
+  );
+  const { query, unchecked } = readCubeQuery(parts, {
+    bindSource: binding.bindSource,
+    expressions: CUBE_EXPRESSION_CODEC,
+    relationSchemas: await typeExtends(
+      editorState,
+      binding.context,
+      findCubeLambdaExtends(parts),
+    ),
+    registry: editorState.registry,
+  });
+  return {
+    document: new CubeDocument({
+      context: {
+        ...binding.context,
+        ...(parts.runtime !== undefined ? { runtime: parts.runtime } : {}),
+      },
+      query: await withExpressionCode(editorState, query),
+    }),
+    lambda: read,
+    isRun: parts.runtime !== undefined,
+    unchecked,
+    notes: parts.limitBesideFrom ? [CUBE_LAMBDA_IMPORT_MESSAGE.GRID_LIMIT] : [],
+  };
+};
+
+/**
+ * Whether the cube, written again as the lambda was written (a matview's
+ * relation, or a run with no row limit), is the lambda it was read from.
+ * Written another way, it reads the same rows only when every step was
+ * checked as it was read: a step read with unknown columns, e.g. a select
+ * taken for the one a Join writes, may not.
+ */
+export const getCubeLambdaFidelity = (
+  editorState: CubeEditorState,
+  reading: Pick<CubeLambdaReading, 'lambda' | 'isRun' | 'unchecked'>,
+): CubeLambdaFidelity => {
+  const { query, context } = editorState.document;
+  const selected = query.selected;
+  const emitter = new QueryEmitter(query, editorState.registry);
+  if (selected === undefined || !emitter.canEmit(selected)) {
+    return CubeLambdaFidelity.UNCHECKED;
+  }
+  const written =
+    reading.isRun && context?.runtime !== undefined
+      ? emitter.emitExecutionLambda({ runtime: context.runtime })
+      : lambda([], [emitter.emitRelation(selected, { withRowOrder: true })]);
+  if (
+    isSameCubeIR(written, reading.lambda, {
+      readExpression: CUBE_EXPRESSION_CODEC.read,
+    })
+  ) {
+    return CubeLambdaFidelity.EXACT;
+  }
+  return reading.unchecked.length
+    ? CubeLambdaFidelity.UNCHECKED
+    : CubeLambdaFidelity.REWRITTEN;
+};
+
+/**
+ * Opens the page on a host's lambda (`CubeEditor`'s `initialLambda`): read
+ * into a cube, opened as the first one, with no undo step; a lambda that
+ * can't be read leaves the cube empty and says why. Once the cube's
+ * expressions are checked, whether Cube writes it back the same way.
+ */
+export class CubeLambdaImportState {
+  readonly editorState: CubeEditorState;
+  /** What the page calls the lambda it opened on */
+  label: string | undefined;
+  isReading = false;
+  error: CubeLambdaImportError | undefined;
+  /** Whether Cube writes the cube back as the lambda; none until checked */
+  fidelity: CubeLambdaFidelity | undefined;
+  notes: readonly string[] = [];
+  /** Import Pure, from the Cube menu: the dialog, its text, why its last import failed */
+  isDialogOpen = false;
+  text = '';
+  isImporting = false;
+  dialogError: CubeLambdaImportError | undefined;
+
+  constructor(editorState: CubeEditorState) {
+    makeObservable(this, {
+      label: observable,
+      isReading: observable,
+      error: observable.ref,
+      fidelity: observable,
+      notes: observable.ref,
+      isDialogOpen: observable,
+      text: observable,
+      isImporting: observable,
+      dialogError: observable.ref,
+      open: flow,
+      dismiss: action,
+      openDialog: action,
+      closeDialog: action,
+      setText: action,
+      importText: flow,
+    });
+    this.editorState = editorState;
+  }
+
+  *open(initial: CubeInitialLambda): GeneratorFn<void> {
+    const { editorState } = this;
+    this.label = initial.label;
+    this.isReading = true;
+    this.error = undefined;
+    this.fidelity = undefined;
+    try {
+      const reading = (yield readCubeLambdaDocument(
+        editorState,
+        initial.lambda,
+      )) as CubeLambdaReading;
+      // the user may have started a cube while it was read
+      if (!editorState.isEmptyCube) {
+        throw new CubeLambdaReadError(CUBE_LAMBDA_IMPORT_MESSAGE.NOT_NEW);
+      }
+      editorState.openDocument(reading.document);
+      this.notes = reading.notes;
+      this.isReading = false;
+      // checked as the editor checks any expression: then compared
+      yield when(
+        () =>
+          !editorState.isResolvingSources &&
+          !editorState.isCheckingExpressions &&
+          !editorState.nodesToCheck.length,
+      );
+      this.fidelity = getCubeLambdaFidelity(editorState, reading);
+    } catch (error) {
+      this.error = toImportError(error);
+    } finally {
+      this.isReading = false;
+    }
+  }
+
+  openDialog(): void {
+    this.isDialogOpen = true;
+    this.dialogError = undefined;
+  }
+
+  closeDialog(): void {
+    this.isDialogOpen = false;
+    this.isImporting = false;
+  }
+
+  setText(text: string): void {
+    this.text = text;
+    this.dialogError = undefined;
+  }
+
+  /**
+   * Import Pure (PLAN §11.18): the dialog's text, Pure or the engine's JSON
+   * of a lambda, read into a cube that replaces this one, as one undo step;
+   * then, once checked, whether Cube writes it back the same. A lambda that
+   * can't be read leaves the cube as it was, and says why in the dialog.
+   */
+  *importText(): GeneratorFn<void> {
+    const { editorState } = this;
+    const text = this.text.trim();
+    if (!text || this.isImporting) {
+      return;
+    }
+    this.isImporting = true;
+    this.dialogError = undefined;
+    try {
+      let json: unknown;
+      if (text.startsWith('{')) {
+        try {
+          json = parseLosslessJSON(text);
+        } catch (error) {
+          assertErrorThrown(error);
+          throw new CubeLambdaReadError(
+            CUBE_LAMBDA_IMPORT_MESSAGE.NOT_JSON(error.message),
+          );
+        }
+      } else {
+        json = (
+          (yield editorState.host.engine.parseExpression(
+            text,
+            CUBE_IMPORT_PURE_SOURCE_ID,
+          )) as Awaited<
+            ReturnType<typeof editorState.host.engine.parseExpression>
+          >
+        ).lambda;
+      }
+      const reading = (yield readCubeLambdaDocument(
+        editorState,
+        json,
+      )) as CubeLambdaReading;
+      if (!this.isDialogOpen) {
+        // closed while it was read
+        return;
+      }
+      editorState.importDocument(reading.document, false);
+      this.isDialogOpen = false;
+      this.text = '';
+      this.label = CUBE_LAMBDA_IMPORT_MESSAGE.IMPORTED;
+      this.error = undefined;
+      this.fidelity = undefined;
+      this.notes = reading.notes;
+      this.isImporting = false;
+      yield when(
+        () =>
+          !editorState.isResolvingSources &&
+          !editorState.isCheckingExpressions &&
+          !editorState.nodesToCheck.length,
+      );
+      this.fidelity = getCubeLambdaFidelity(editorState, reading);
+    } catch (error) {
+      this.dialogError = toImportError(error);
+    } finally {
+      this.isImporting = false;
+    }
+  }
+
+  /** Closes the banner */
+  dismiss(): void {
+    this.error = undefined;
+    this.fidelity = undefined;
+    this.notes = [];
+  }
+}