@finos/legend-application-studio 28.21.51 → 28.21.53

package/src/stores/editor/embedded-cube/LegendStudioCubeIngestCatalog.ts

@@ -0,0 +1,235 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Schema } from '@finos/legend-cube';
+import {
+  CubeDataProductEnvironmentType,
+  CubeEngineError,
+  CubeEngineErrorKind,
+  CubeIngestDefinitionCandidate,
+  CubeIngestEnvironment,
+  CubeIngestProducer,
+  formatCubeIngestUrn,
+  parseCubeIngestUrn,
+  readCubeIngestDataSets,
+  type CubeIngestCatalog,
+  type CubeIngestDataSet,
+  type CubeIngestDataSetLocation,
+  type CubeIngestDefinitionList,
+  type CubeIngestDefinitionSource,
+  type NodeId,
+} from '@finos/legend-cube-builder';
+import type { IngestDefinition } from '@finos/legend-graph';
+import type { PlainObject } from '@finos/legend-shared';
+import type { EditorStore } from '../EditorStore.js';
+import type { LegendStudioCubeWorkspace } from './LegendStudioCubeWorkspace.js';
+
+// Legend Cube in Studio: a cube reads the ingest definitions of the project
+// being edited, as the workspace holds them, and runs on the project's
+// lakehouse runtime, as Studio's own queries of an ingest definition do. The
+// Ingest tab lists every definition of the project, under one producer that
+// stands for the project, each named by the URN it would have if this
+// project deployed it.
+
+/** The one class a Studio cube reads the project's definitions as, so the Ingest tab offers no other */
+const STUDIO_CUBE_INGEST_CLASS = CubeDataProductEnvironmentType.PRODUCTION;
+
+/** The one producer deployment the tab lists: the project itself, whatever its definitions' deployments */
+export const LEGEND_STUDIO_CUBE_PROJECT_PRODUCER = '0';
+
+export const LEGEND_STUDIO_CUBE_MESSAGE = {
+  GONE: (definition: string): string =>
+    `The project has no ingest definition ${definition} any more.`,
+  UNREADABLE: (definition: string): string =>
+    `Cube can't read the data sets of ingest definition ${definition}.`,
+  NO_DATA_SET: (dataSet: string, definition: string): string =>
+    `Ingest definition ${definition} has no data set "${dataSet}".`,
+};
+
+/**
+ * The project's own ingest definitions, for the Ingest tab and for the
+ * engine, which types and runs on them as the workspace holds them now
+ */
+export class LegendStudioCubeIngestCatalog
+  implements CubeIngestCatalog, CubeIngestDefinitionSource
+{
+  readonly environmentTypes = [STUDIO_CUBE_INGEST_CLASS];
+  readonly editorStore: EditorStore;
+  readonly workspace: LegendStudioCubeWorkspace;
+
+  constructor(editorStore: EditorStore, workspace: LegendStudioCubeWorkspace) {
+    this.editorStore = editorStore;
+    this.workspace = workspace;
+  }
+
+  /** The project's own definitions, not its dependencies' */
+  private get definitions(): IngestDefinition[] {
+    return this.editorStore.graphManagerState.graph.ownIngests;
+  }
+
+  /** The URN a cube names the project's definition by */
+  urnOf(definition: IngestDefinition): string {
+    return formatCubeIngestUrn({
+      environmentType: STUDIO_CUBE_INGEST_CLASS,
+      ...this.workspace.project,
+      definition: definition.path,
+    });
+  }
+
+  private definitionOf(urn: string): IngestDefinition {
+    const definition = this.definitions.find(
+      (each) => this.urnOf(each) === urn,
+    );
+    if (!definition) {
+      throw new CubeEngineError(
+        CubeEngineErrorKind.COMPILE,
+        LEGEND_STUDIO_CUBE_MESSAGE.GONE(
+          parseCubeIngestUrn(urn)?.definition ?? urn,
+        ),
+      );
+    }
+    return definition;
+  }
+
+  /** The project's lakehouse runtime, which a run names, with its warehouse */
+  async resolveEnvironment(
+    environmentType: CubeDataProductEnvironmentType,
+  ): Promise<CubeIngestEnvironment> {
+    const runtime = await this.workspace.resolveRuntime();
+    return new CubeIngestEnvironment({
+      environmentType,
+      name: runtime.label,
+      runtimeEnvironment: runtime.environment,
+      warehouse: runtime.warehouse,
+    });
+  }
+
+  async getRuntimeEnvironment(): Promise<string> {
+    return (await this.workspace.resolveRuntime()).environment;
+  }
+
+  /** One producer, the project, so the tab lists every definition at once */
+  async listProducers(): Promise<readonly CubeIngestProducer[]> {
+    const { groupId, artifactId } = this.workspace.project;
+    return [
+      new CubeIngestProducer({
+        deploymentId: LEGEND_STUDIO_CUBE_PROJECT_PRODUCER,
+        urn: `${groupId}:${artifactId}`,
+      }),
+    ];
+  }
+
+  /** Every definition of the project, whatever its app directory deployment */
+  async listDefinitions(): Promise<CubeIngestDefinitionList> {
+    return {
+      candidates: this.definitions.map(
+        (each) =>
+          new CubeIngestDefinitionCandidate({
+            urn: this.urnOf(each),
+            definition: each.path,
+            ...this.workspace.project,
+          }),
+      ),
+      droppedCount: 0,
+    };
+  }
+
+  /** The workspace's elements, which a typing's or a run's model holds */
+  getModelElements(): Promise<readonly PlainObject[]> {
+    return this.workspace.getModelElements();
+  }
+
+  /** The definition's element, as the workspace holds it now */
+  async getDefinitionElement(urn: string): Promise<PlainObject> {
+    return this.editorStore.graphManagerState.graphManager.elementToEntity(
+      this.definitionOf(urn),
+      { pruneSourceInformation: true },
+    ).content;
+  }
+
+  async describe(urn: string): Promise<readonly CubeIngestDataSet[]> {
+    const element = await this.getDefinitionElement(urn);
+    try {
+      return readCubeIngestDataSets(element);
+    } catch {
+      throw new CubeEngineError(
+        CubeEngineErrorKind.COMPILE,
+        LEGEND_STUDIO_CUBE_MESSAGE.UNREADABLE(this.definitionOf(urn).path),
+      );
+    }
+  }
+
+  async resolveSchemas(
+    environmentType: CubeDataProductEnvironmentType,
+    sources: ReadonlyMap<NodeId, CubeIngestDataSetLocation>,
+  ): Promise<Map<NodeId, Schema | CubeEngineError>> {
+    const described = new Map<
+      string,
+      readonly CubeIngestDataSet[] | CubeEngineError
+    >();
+    for (const { ingestDefinitionUrn } of sources.values()) {
+      if (!described.has(ingestDefinitionUrn)) {
+        try {
+          described.set(
+            ingestDefinitionUrn,
+            await this.describe(ingestDefinitionUrn),
+          );
+        } catch (error) {
+          described.set(
+            ingestDefinitionUrn,
+            error instanceof CubeEngineError
+              ? error
+              : new CubeEngineError(
+                  CubeEngineErrorKind.COMPILE,
+                  error instanceof Error ? error.message : String(error),
+                ),
+          );
+        }
+      }
+    }
+    return new Map(
+      [...sources].map(([nodeId, source]) => {
+        const read = described.get(source.ingestDefinitionUrn);
+        if (read instanceof CubeEngineError || read === undefined) {
+          return [
+            nodeId,
+            new CubeEngineError(
+              read?.kind ?? CubeEngineErrorKind.COMPILE,
+              read?.detail ??
+                LEGEND_STUDIO_CUBE_MESSAGE.GONE(source.ingestDefinition),
+              nodeId,
+            ),
+          ];
+        }
+        const dataSet = read.find((each) => each.name === source.dataSet);
+        return [
+          nodeId,
+          dataSet?.schema && dataSet.isPickable
+            ? dataSet.schema
+            : new CubeEngineError(
+                CubeEngineErrorKind.COMPILE,
+                dataSet?.disabledReason ??
+                  LEGEND_STUDIO_CUBE_MESSAGE.NO_DATA_SET(
+                    source.dataSet,
+                    source.ingestDefinition,
+                  ),
+                nodeId,
+              ),
+        ];
+      }),
+    );
+  }
+}

package/src/stores/editor/embedded-cube/LegendStudioCubeWorkspace.ts

@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  CubeEngineError,
+  CubeEngineErrorKind,
+  type CubeDataProductModelSource,
+  type CubeHostProject,
+  type CubeHostProjectRuntime,
+  type CubeLakehouseEnvironment,
+} from '@finos/legend-cube-builder';
+import {
+  LakehouseBaseRuntime,
+  LakehouseRuntime,
+  ModelStore,
+  PrimitiveType,
+  type PackageableElement,
+  type PackageableRuntime,
+} from '@finos/legend-graph';
+import type { PlainObject } from '@finos/legend-shared';
+import type { EditorStore } from '../EditorStore.js';
+
+// The project Studio edits, as Legend Cube reads it: its coordinates, the
+// lakehouse runtime its runs use, and the model its typing and runs read,
+// the workspace as it is now. A cube in Studio reads only the project's own
+// ingest definitions and data products, which aren't deployed: they are
+// typed and run on the workspace, as Studio's own queries of them are.
+
+/** The project's coordinates when Studio has no project configuration, which Cube's sources still name */
+const UNKNOWN_PROJECT = { groupId: 'studio', artifactId: 'workspace' };
+
+/** The version a cube of the workspace's data products names: the workspace, never a release */
+export const LEGEND_STUDIO_CUBE_WORKSPACE_VERSION = 'workspace';
+
+export const LEGEND_STUDIO_CUBE_WORKSPACE_MESSAGE = {
+  NO_RUNTIME:
+    "Add a lakehouse runtime to the project: a cube runs the project's ingest data sets and data products on it, as Studio's own queries do.",
+  NO_ENVIRONMENT: (runtime: string): string =>
+    `The project's lakehouse runtime ${runtime} names no environment.`,
+};
+
+/** The lakehouse runtime a run uses, read from the project */
+export interface LegendStudioCubeRuntime extends CubeHostProjectRuntime {
+  readonly path: string;
+  readonly environment: string;
+}
+
+export class LegendStudioCubeWorkspace
+  implements
+    CubeHostProject,
+    CubeLakehouseEnvironment,
+    CubeDataProductModelSource
+{
+  readonly editorStore: EditorStore;
+
+  constructor(editorStore: EditorStore) {
+    this.editorStore = editorStore;
+  }
+
+  /** The project's coordinates, which the URNs and models of Cube's sources carry */
+  get project(): { groupId: string; artifactId: string } {
+    const configuration =
+      this.editorStore.projectConfigurationEditorState.projectConfiguration;
+    return configuration
+      ? {
+          groupId: configuration.groupId,
+          artifactId: configuration.artifactId,
+        }
+      : UNKNOWN_PROJECT;
+  }
+
+  /**
+   * The lakehouse runtime a run uses: the project's first, as Studio's own
+   * queries of an ingest definition or a data product pick
+   * (`getCompatibleRuntimesFromAccessorOwner`, `LakehouseDataProductExecutionState`)
+   */
+  get lakehouseRuntime(): PackageableRuntime | undefined {
+    return this.editorStore.graphManagerState.usableRuntimes.find(
+      (runtime) => runtime.runtimeValue instanceof LakehouseBaseRuntime,
+    );
+  }
+
+  async resolveRuntime(): Promise<LegendStudioCubeRuntime> {
+    const runtime = this.lakehouseRuntime;
+    if (!runtime) {
+      throw new CubeEngineError(
+        CubeEngineErrorKind.UNSUPPORTED_MODEL,
+        LEGEND_STUDIO_CUBE_WORKSPACE_MESSAGE.NO_RUNTIME,
+      );
+    }
+    const value = runtime.runtimeValue as LakehouseBaseRuntime;
+    if (!value.environment) {
+      throw new CubeEngineError(
+        CubeEngineErrorKind.UNSUPPORTED_MODEL,
+        LEGEND_STUDIO_CUBE_WORKSPACE_MESSAGE.NO_ENVIRONMENT(runtime.path),
+      );
+    }
+    return {
+      path: runtime.path,
+      environment: value.environment,
+      label: `${value.environment}, from ${runtime.path}`,
+      warehouse:
+        value instanceof LakehouseRuntime ? value.warehouse : undefined,
+    };
+  }
+
+  /** The environment a data product run's lakehouse runtime names: the project runtime's */
+  async resolveEnvironment(): Promise<string> {
+    return (await this.resolveRuntime()).environment;
+  }
+
+  /**
+   * The elements a cube's typing and runs read, as protocol JSON: the
+   * workspace's own and its dependencies', as Studio's own queries send
+   * them, so a definition's materialized views and a data product's access
+   * points find what they read
+   */
+  async getModelElements(): Promise<readonly PlainObject[]> {
+    const { graph, graphManager } = this.editorStore.graphManagerState;
+    return [...graph.allOwnElements, ...graph.dependencyManager.allOwnElements]
+      .filter(
+        (element: PackageableElement) =>
+          !(element instanceof PrimitiveType) &&
+          !(element instanceof ModelStore),
+      )
+      .map(
+        (element) =>
+          graphManager.elementToEntity(element, {
+            pruneSourceInformation: true,
+          }).content,
+      );
+  }
+
+  /** The model a cube of the workspace's data products reads them from: the workspace */
+  async getProjectModel(): Promise<PlainObject> {
+    return { _type: 'data', elements: await this.getModelElements() };
+  }
+}

package/src/stores/graph-modifier/DSL_Ingest_GraphModifierHelper.ts

@@ -0,0 +1,214 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { RawLambda, type IngestDefinition } from '@finos/legend-graph';
+import { isObject, isString, type PlainObject } from '@finos/legend-shared';
+import { action } from 'mobx';
+
+/** The source of a data set whose rows a function computes: a materialized view */
+const MATERIALIZED_VIEW_SOURCE_TYPE = 'FunctionSource';
+
+/**
+ * The lists the engine gives every data set it parses, and reads: a data set
+ * Studio writes holds each, empty when it has none
+ */
+const DATA_SET_LIST_KEYS = [
+  'ingestPartitionColumns',
+  'preprocessors',
+  'storageLayoutClusterColumns',
+  'storageLayoutPartitionColumns',
+] as const;
+
+/** The ingest definition's data sets, as its content holds them */
+export const getIngestDataSetNames = (ingest: IngestDefinition): string[] =>
+  (Array.isArray(ingest.content.datasets) ? ingest.content.datasets : [])
+    .map((dataSet: unknown) =>
+      isObject(dataSet) ? (dataSet as PlainObject).name : undefined,
+    )
+    .filter(isString);
+
+/** What a materialized view's data set says besides its name and its query */
+export interface IngestMaterializedViewOptions {
+  /** The columns that identify a row, in the order of the view's columns; none for no key */
+  readonly primaryKey: readonly string[];
+  /** The classification of its data, e.g. `DP10`, as a data product's access points take one */
+  readonly sensitivity: string;
+}
+
+/** A materialized view of an ingest definition, as its content holds it */
+export interface IngestMaterializedView {
+  readonly name: string;
+  /** Its query, as protocol JSON */
+  readonly lambda: PlainObject;
+  readonly primaryKey: readonly string[];
+  /** The classification of its data; none for a view without one */
+  readonly sensitivity: string | undefined;
+}
+
+/** The materialized views of the ingest definition, as its content holds them */
+export const getIngestMaterializedViews = (
+  ingest: IngestDefinition,
+): IngestMaterializedView[] =>
+  (Array.isArray(ingest.content.datasets) ? ingest.content.datasets : [])
+    .filter(isObject)
+    .flatMap((dataSet) => {
+      const { name, source, primaryKey, privacyClassification } =
+        dataSet as PlainObject;
+      const lambda = isObject(source)
+        ? (source as PlainObject).function
+        : undefined;
+      const sensitivity = isObject(privacyClassification)
+        ? (privacyClassification as PlainObject).sensitivity
+        : undefined;
+      return isString(name) &&
+        (source as PlainObject | undefined)?._type ===
+          MATERIALIZED_VIEW_SOURCE_TYPE &&
+        isObject(lambda)
+        ? [
+            {
+              name,
+              lambda: lambda as PlainObject,
+              primaryKey: Array.isArray(primaryKey)
+                ? primaryKey.filter(isString)
+                : [],
+              sensitivity: isString(sensitivity) ? sensitivity : undefined,
+            },
+          ]
+        : [];
+    });
+
+/**
+ * Adds a materialized view to the ingest definition: a data set whose rows
+ * the lambda (protocol JSON) computes, with its primary key and the
+ * classification of its data. It holds every key the engine gives a data
+ * set it parses, its lists empty, as the engine reads them all: a view
+ * without a `privacyClassification` failed with a NullPointerException.
+ * The content is replaced, so the change is seen, and the view is listed
+ * for Lineage and View SQL as one read from the definition would be
+ */
+export const ingestDefinition_addMaterializedView = action(
+  (
+    ingest: IngestDefinition,
+    name: string,
+    lambda: PlainObject,
+    options: IngestMaterializedViewOptions,
+  ): void => {
+    const dataSets = Array.isArray(ingest.content.datasets)
+      ? (ingest.content.datasets as unknown[])
+      : [];
+    ingest.content = {
+      ...ingest.content,
+      datasets: [
+        ...dataSets,
+        {
+          name,
+          primaryKey: [...options.primaryKey],
+          privacyClassification: { sensitivity: options.sensitivity },
+          source: { _type: MATERIALIZED_VIEW_SOURCE_TYPE, function: lambda },
+          ...Object.fromEntries(DATA_SET_LIST_KEYS.map((key) => [key, []])),
+        },
+      ],
+    };
+    ingest.TEMPORARY_MATVIEW_FUNCTION_DATA_SETS = [
+      ...(ingest.TEMPORARY_MATVIEW_FUNCTION_DATA_SETS ?? []),
+      {
+        name,
+        source: {
+          function: new RawLambda(
+            Array.isArray(lambda.parameters) ? lambda.parameters : [],
+            lambda.body as object | undefined,
+          ),
+        },
+      },
+    ];
+  },
+);
+
+/**
+ * Replaces a materialized view's lambda (protocol JSON) in place: its name,
+ * its place among the data sets and its other keys stay, as does its place
+ * among the views listed for Lineage and View SQL. A primary key and a
+ * classification given replace the view's, and the lists the engine reads
+ * that the view lacks, e.g. as Studio wrote a view before it wrote them, are
+ * added empty
+ */
+export const ingestDefinition_updateMaterializedView = action(
+  (
+    ingest: IngestDefinition,
+    name: string,
+    lambda: PlainObject,
+    options?: IngestMaterializedViewOptions,
+  ): void => {
+    const dataSets = Array.isArray(ingest.content.datasets)
+      ? (ingest.content.datasets as unknown[])
+      : [];
+    ingest.content = {
+      ...ingest.content,
+      datasets: dataSets.map((dataSet) => {
+        if (!isObject(dataSet)) {
+          return dataSet;
+        }
+        const {
+          name: dataSetName,
+          source,
+          privacyClassification,
+        } = dataSet as PlainObject;
+        if (
+          dataSetName !== name ||
+          !isObject(source) ||
+          (source as PlainObject)._type !== MATERIALIZED_VIEW_SOURCE_TYPE
+        ) {
+          return dataSet;
+        }
+        return {
+          ...dataSet,
+          ...(options
+            ? {
+                primaryKey: [...options.primaryKey],
+                privacyClassification: {
+                  ...(isObject(privacyClassification)
+                    ? (privacyClassification as PlainObject)
+                    : {}),
+                  sensitivity: options.sensitivity,
+                },
+              }
+            : {}),
+          source: { ...source, function: lambda },
+          ...Object.fromEntries(
+            DATA_SET_LIST_KEYS.filter(
+              (key) => !Array.isArray((dataSet as PlainObject)[key]),
+            ).map((key) => [key, []]),
+          ),
+        };
+      }),
+    };
+    ingest.TEMPORARY_MATVIEW_FUNCTION_DATA_SETS =
+      ingest.TEMPORARY_MATVIEW_FUNCTION_DATA_SETS?.map((view) =>
+        view.name === name
+          ? {
+              ...view,
+              source: {
+                ...view.source,
+                function: new RawLambda(
+                  Array.isArray(lambda.parameters) ? lambda.parameters : [],
+                  lambda.body as object | undefined,
+                ),
+              },
+            }
+          : view,
+      );
+  },
+);

package/tsconfig.json

@@ -187,6 +187,11 @@
     "./src/stores/editor/editor-state/entity-diff-editor-state/EntityDiffViewState.ts",
     "./src/stores/editor/editor-state/project-configuration-editor-state/ProjectConfigurationEditorState.ts",
     "./src/stores/editor/editor-state/project-configuration-editor-state/ProjectDependencyEditorState.ts",
+    "./src/stores/editor/embedded-cube/EmbeddedCubeState.ts",
+    "./src/stores/editor/embedded-cube/LegendStudioCubeDataProductCatalog.ts",
+    "./src/stores/editor/embedded-cube/LegendStudioCubeHost.ts",
+    "./src/stores/editor/embedded-cube/LegendStudioCubeIngestCatalog.ts",
+    "./src/stores/editor/embedded-cube/LegendStudioCubeWorkspace.ts",
     "./src/stores/editor/panel-group/DevToolPanelState.ts",
     "./src/stores/editor/panel-group/StudioSQLPlaygroundPanelState.ts",
     "./src/stores/editor/sidebar-state/BulkServiceRegistrationState.ts",
@@ -221,6 +226,7 @@
     "./src/stores/graph-modifier/DSL_ExternalFormat_GraphModifierHelper.ts",
     "./src/stores/graph-modifier/DSL_FunctionActivator_GraphModifierHelper.ts",
     "./src/stores/graph-modifier/DSL_Generation_GraphModifierHelper.ts",
+    "./src/stores/graph-modifier/DSL_Ingest_GraphModifierHelper.ts",
     "./src/stores/graph-modifier/DSL_LakehouseRuntime_GraphModifierHelper.ts",
     "./src/stores/graph-modifier/DSL_Mapping_GraphModifierHelper.ts",
     "./src/stores/graph-modifier/DSL_Service_GraphModifierHelper.ts",
@@ -360,6 +366,8 @@
     "./src/components/editor/editor-group/uml-editor/StereotypeSelector.tsx",
     "./src/components/editor/editor-group/uml-editor/TaggedValueEditor.tsx",
     "./src/components/editor/editor-group/uml-editor/UMLEditor.tsx",
+    "./src/components/editor/embedded-cube/EmbeddedCube.tsx",
+    "./src/components/editor/embedded-cube/EmbeddedCubeEditor.tsx",
     "./src/components/editor/panel-group/ConsolePanel.tsx",
     "./src/components/editor/panel-group/DevToolPanel.tsx",
     "./src/components/editor/panel-group/PanelGroup.tsx",