@finos/legend-cube-builder 0.0.4 → 0.0.6

package/src/stores/LegendCubeHost.ts

@@ -0,0 +1,243 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { GenericLegendApplicationStore } from '@finos/legend-application';
+import { DepotServerClient } from '@finos/legend-server-depot';
+import {
+  LakehouseContractServerClient,
+  LakehouseIngestServerClient,
+  LakehousePlatformServerClient,
+} from '@finos/legend-server-lakehouse';
+import { MarketplaceServerClient } from '@finos/legend-server-marketplace';
+import { EXTERNAL_APPLICATION_NAVIGATION__generateCubeMarketplaceDataProductUrl } from '../__lib__/LegendCubeNavigation.js';
+import type { CubeConnectionExplorer } from '../graph-manager/CubeConnectionExplorer.js';
+import { CubeDataProductEnvironmentType } from '../graph-manager/CubeDataProduct.js';
+import type { CubeDataProductCatalog } from '../graph-manager/CubeDataProductCatalog.js';
+import type { CubeEngine } from '../graph-manager/CubeEngine.js';
+import { CubeEngineClientSettings } from '../graph-manager/CubeEngineClientSettings.js';
+import type { CubeIngestCatalog } from '../graph-manager/CubeIngestCatalog.js';
+import type { CubeProjectCatalog } from '../graph-manager/CubeProjectCatalog.js';
+import type { CubeServiceCatalog } from '../graph-manager/CubeServiceCatalog.js';
+import {
+  buildCubeConnectionExplorer,
+  buildCubeDataProductCatalog,
+  buildCubeEngine,
+  buildCubeIngestCatalog,
+  buildCubeLakehouseEnvironment,
+  buildCubeProjectCatalog,
+  buildCubeServiceCatalog,
+  type CubeEngineConfig,
+  type CubeLakehouseServices,
+} from '../graph-manager/protocol/pure/CubeEngineBuilder.js';
+import { getCubeRememberedWarehouse } from './CubeDataProductWarehouse.js';
+import type { CubeHost } from './CubeHost.js';
+import { LocalModelCatalog } from './LocalModelCatalog.js';
+
+/**
+ * The servers of a Legend deployment hosting the Cube page, as its config
+ * names them (PLAN §11.18): the engine, and each of the others only where the
+ * deployment has one. Legend Query and Legend Analytics each read theirs from
+ * their own config.
+ */
+export interface LegendCubeHostConfig {
+  readonly engine: CubeEngineConfig;
+  /** Published projects (PLAN §6.3) and, with a lakehouse, the data products' models */
+  readonly depotServerUrl?: string | undefined;
+  /** The lakehouse's contracts: data products need it, and a depot (PLAN §6.8) */
+  readonly lakehouseContractUrl?: string | undefined;
+  /** The lakehouse platform, which names the ingest servers: ingest data sets need it (PLAN §6.7) */
+  readonly lakehousePlatformUrl?: string | undefined;
+  /** The marketplace server, whose search the data product tab uses */
+  readonly marketplaceServerUrl?: string | undefined;
+  /** Legend Marketplace, which a production data product links to */
+  readonly marketplaceApplicationUrl?: string | undefined;
+  /** Legend Marketplace's production-parallel deployment, which a production-parallel data product links to */
+  readonly marketplaceProductionParallelUrl?: string | undefined;
+  /** The stereotype marking access point groups open to everyone, as Studio and Marketplace configure it */
+  readonly enterpriseStereotype?: CubeLakehouseServices['enterpriseStereotype'];
+  /** The lakehouse environment the host remembers for the viewer */
+  readonly getPreferredEnvironment?: (() => string | undefined) | undefined;
+}
+
+/**
+ * What Cube needs to read and run data products (PLAN §6.8), with the
+ * deployment's lakehouse and depot: none without both, so a deployment
+ * without them offers no data products. With a marketplace server too, its
+ * search is used; with the lakehouse platform, ingest data sets are offered.
+ */
+export const buildLegendCubeLakehouseServices = (
+  applicationStore: GenericLegendApplicationStore,
+  config: LegendCubeHostConfig,
+): CubeLakehouseServices | undefined => {
+  const { tracerService } = applicationStore;
+  if (!config.lakehouseContractUrl || !config.depotServerUrl) {
+    return undefined;
+  }
+  const contractServerClient = new LakehouseContractServerClient({
+    baseUrl: config.lakehouseContractUrl,
+  });
+  contractServerClient.setTracerService(tracerService);
+  const depotServerClient = new DepotServerClient({
+    serverUrl: config.depotServerUrl,
+  });
+  depotServerClient.setTracerService(tracerService);
+  let marketplaceServerClient: MarketplaceServerClient | undefined;
+  if (config.marketplaceServerUrl) {
+    marketplaceServerClient = new MarketplaceServerClient({
+      serverUrl: config.marketplaceServerUrl,
+      // the client only stores it, never reading it, and Cube calls no subscription route
+      subscriptionUrl: '',
+    });
+    marketplaceServerClient.setTracerService(tracerService);
+  }
+  // ingest data sets need the platform, which names the ingest servers; the
+  // ingest client has no server of its own: each call names one
+  let platformServerClient: LakehousePlatformServerClient | undefined;
+  let ingestServerClient: LakehouseIngestServerClient | undefined;
+  if (config.lakehousePlatformUrl) {
+    platformServerClient = new LakehousePlatformServerClient(
+      config.lakehousePlatformUrl,
+    );
+    platformServerClient.setTracerService(tracerService);
+    ingestServerClient = new LakehouseIngestServerClient(undefined);
+    ingestServerClient.setTracerService(tracerService);
+  }
+  return {
+    platformServerClient,
+    ingestServerClient,
+    contractServerClient,
+    depotServerClient,
+    marketplaceServerClient,
+    getAccessToken: () => applicationStore.getAccessToken(),
+    getCurrentUser: () => applicationStore.identityService.currentUser,
+    getPreferredEnvironment: config.getPreferredEnvironment,
+    enterpriseStereotype: config.enterpriseStereotype,
+    // the marketplace of the deployment's class, where its deployment is
+    getMarketplaceLink: (target) => {
+      const marketplaceUrl =
+        target.environmentType ===
+        CubeDataProductEnvironmentType.PRODUCTION_PARALLEL
+          ? config.marketplaceProductionParallelUrl
+          : config.marketplaceApplicationUrl;
+      return marketplaceUrl
+        ? EXTERNAL_APPLICATION_NAVIGATION__generateCubeMarketplaceDataProductUrl(
+            marketplaceUrl,
+            target.dataProductId,
+            target.deploymentId,
+            target.accessPointGroup,
+          )
+        : undefined;
+    },
+  };
+};
+
+/** The published projects in the deployment's depot, or none without a depot */
+export const buildLegendCubeProjectCatalog = (
+  applicationStore: GenericLegendApplicationStore,
+  config: LegendCubeHostConfig,
+): CubeProjectCatalog | undefined => {
+  if (!config.depotServerUrl) {
+    return undefined;
+  }
+  const depotServerClient = new DepotServerClient({
+    serverUrl: config.depotServerUrl,
+  });
+  depotServerClient.setTracerService(applicationStore.tracerService);
+  return buildCubeProjectCatalog(depotServerClient);
+};
+
+/** What tests give the host in place of what it would build */
+export interface LegendCubeHostParts {
+  readonly engine?: CubeEngine | undefined;
+  readonly connectionExplorer?: CubeConnectionExplorer | undefined;
+  readonly projectCatalog?: CubeProjectCatalog | undefined;
+}
+
+/**
+ * A Legend deployment as the host of the Cube page (PLAN §3.5, §11.18): the
+ * application's store, the engine and the bundled models, direct database
+ * connections read through the engine, the services in its registry, and
+ * the deployment's projects, data products and ingest data sets where it has
+ * the servers for them. Legend Query and Legend Analytics each build one per
+ * visit to the page, from their own config.
+ */
+export class LegendCubeHost implements CubeHost {
+  readonly applicationStore: GenericLegendApplicationStore;
+  readonly engine: CubeEngine;
+  readonly modelCatalog: LocalModelCatalog;
+  readonly connectionExplorer: CubeConnectionExplorer;
+  readonly dataProductCatalog: CubeDataProductCatalog | undefined;
+  readonly ingestCatalog: CubeIngestCatalog | undefined;
+  readonly projectCatalog: CubeProjectCatalog | undefined;
+  readonly serviceCatalog: CubeServiceCatalog;
+  /** The engine URL and compression Cube's engine clients share, which Dev Settings changes */
+  readonly engineClientSettings: CubeEngineClientSettings;
+
+  constructor(
+    applicationStore: GenericLegendApplicationStore,
+    config: LegendCubeHostConfig,
+    parts?: LegendCubeHostParts,
+  ) {
+    this.applicationStore = applicationStore;
+    const { tracerService } = applicationStore;
+    this.engineClientSettings = new CubeEngineClientSettings(config.engine);
+    const lakehouse = buildLegendCubeLakehouseServices(
+      applicationStore,
+      config,
+    );
+    this.ingestCatalog = lakehouse
+      ? buildCubeIngestCatalog(
+          config.engine,
+          tracerService,
+          lakehouse,
+          this.engineClientSettings,
+        )
+      : undefined;
+    this.engine =
+      parts?.engine ??
+      buildCubeEngine(config.engine, tracerService, {
+        ingestCatalog: this.ingestCatalog,
+        lakehouseEnvironment: lakehouse
+          ? buildCubeLakehouseEnvironment(lakehouse)
+          : undefined,
+        getRememberedWarehouse: () =>
+          getCubeRememberedWarehouse(applicationStore.userDataService),
+        clientSettings: this.engineClientSettings,
+      });
+    this.dataProductCatalog = lakehouse
+      ? buildCubeDataProductCatalog(lakehouse)
+      : undefined;
+    this.modelCatalog = new LocalModelCatalog(this.engine);
+    // its own client, configured as the engine's
+    this.connectionExplorer =
+      parts?.connectionExplorer ??
+      buildCubeConnectionExplorer(
+        config.engine,
+        tracerService,
+        this.engineClientSettings,
+      );
+    // it reads nothing until the source dialog's Project tab opens
+    this.projectCatalog =
+      parts?.projectCatalog ??
+      buildLegendCubeProjectCatalog(applicationStore, config);
+    // in progress (PLAN §6.4); it reads nothing until the Service tab opens
+    this.serviceCatalog = buildCubeServiceCatalog(
+      config.engine,
+      tracerService,
+      this.engineClientSettings,
+    );
+  }
+}

package/src/stores/LocalModelCatalog.ts

@@ -24,7 +24,10 @@ import {
   CUBE_NORTHWIND_SAMPLE_MODEL,
 } from './fixtures/CubeNorthwindModel.js';
 import { CUBE_SPORTS_MODEL } from './fixtures/CubeSportsModel.js';
-import { CUBE_TRADES_MODEL } from './fixtures/CubeTradesModel.js';
+import {
+  CUBE_TRADES_MODEL,
+  CUBE_TRADES_MODEL_BEFORE_FUNCTIONS,
+} from './fixtures/CubeTradesModel.js';
 
 // The models of the slice (PLAN §6.2.3): Pure text bundled with Cube, or
 // pasted by the user. Picking one gives the model context the cube saves, the
@@ -55,6 +58,8 @@ export const BUNDLED_MODELS: readonly BundledModel[] = Object.freeze([
     id: 'cube-sample-trades',
     label: 'Trades',
     model: CUBE_TRADES_MODEL,
+    // before its functions (PLAN §6.5)
+    earlierModels: [CUBE_TRADES_MODEL_BEFORE_FUNCTIONS],
   },
 ]);
 

package/src/stores/editors/CubeExplodeDraft.ts

@@ -0,0 +1,272 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  type Explode,
+  getVariantDatabaseProblem,
+  type Schema,
+} from '@finos/legend-cube';
+import { assertErrorThrown, type GeneratorFn } from '@finos/legend-shared';
+import { action, computed, flow, makeObservable, observable } from 'mobx';
+import { CubeEngineError } from '../../graph-manager/CubeEngine.js';
+import type { CubeEditorState } from '../CubeEditorState.js';
+import { CubeQueryRunAborted } from '../CubeQueryRun.js';
+import {
+  type CubeJsonField,
+  getCubeJsonElementName,
+  getCubeJsonFields,
+  getFreeCubeColumnName,
+} from '../variant/CubeJsonOffers.js';
+import { readCubeJsonSample } from '../variant/CubeJsonSampling.js';
+import type { CubeJsonSample } from '../variant/CubeJsonShape.js';
+import { isSameList } from './CubeAggregationRows.js';
+import { CubeNodeDraft } from './CubeNodeDraft.js';
+
+/** An array a sample of the column holds, to explode */
+export interface CubeExplodeArray {
+  /** The keys from the column to it, empty for the column itself */
+  readonly path: readonly string[];
+  /** What it held, e.g. `list of object` */
+  readonly description: string;
+  /** Its share, 0 to 1, of the places it could be */
+  readonly presence: number;
+}
+
+/** The arrays of a column's fields, in tree order */
+const arraysOf = (field: CubeJsonField): CubeExplodeArray[] => [
+  ...(field.offers.some((offer) => offer.explode !== undefined)
+    ? [
+        {
+          path: field.path,
+          description: field.description,
+          presence: field.presence,
+        },
+      ]
+    : []),
+  ...field.children.flatMap(arraysOf),
+];
+
+/**
+ * The Explode Array editor's draft (PLAN §11.20): the Variant column, the
+ * keys to its array and the new column's name. The arrays to pick from come
+ * from a sample of the column's documents, read after the node's input as
+ * Explore JSON reads them. Until a name is typed, the name follows the
+ * column and keys (`<last key>_element`, free in the input); picking another
+ * column drops the keys and the sample, which were the other column's.
+ */
+export class CubeExplodeDraft extends CubeNodeDraft<Explode> {
+  readonly editorState: CubeEditorState;
+  column: string;
+  path: readonly string[];
+  name: string;
+  /** What a sample of the column held, once read */
+  sample:
+    | { readonly column: string; readonly sample: CubeJsonSample }
+    | undefined;
+  isSampling = false;
+  /** Why the sample couldn't be read */
+  sampleError: string | undefined;
+  /** The name follows the column and keys until one is typed */
+  private namedByDefault: boolean;
+  private controller: AbortController | undefined;
+  /** Anything was changed; until then, `build()` gives the original back */
+  private touched = false;
+
+  constructor(original: Explode, editorState: CubeEditorState) {
+    super(original);
+    makeObservable<CubeExplodeDraft, 'touched' | 'namedByDefault'>(this, {
+      column: observable,
+      path: observable.ref,
+      name: observable,
+      sample: observable.ref,
+      isSampling: observable,
+      sampleError: observable,
+      namedByDefault: observable,
+      touched: observable,
+      arrays: computed,
+      sampleProblem: computed,
+      setColumn: action,
+      setKey: action,
+      addKey: action,
+      removeKey: action,
+      pickArray: action,
+      setName: action,
+      stopSample: action,
+      readSample: flow,
+    });
+    this.editorState = editorState;
+    this.column = original.column;
+    this.path = original.path;
+    this.name = original.name;
+    // a new node's, or one still named as Explore JSON names it
+    this.namedByDefault =
+      original.name === '' ||
+      original.name === this.defaultName(original.column, original.path);
+  }
+
+  /** The element's default name for a column and keys: `<last key>_element`, free in the input */
+  private defaultName(column: string, path: readonly string[]): string {
+    return column
+      ? getFreeCubeColumnName(
+          getCubeJsonElementName(column, path),
+          this.input?.schema.names() ?? [],
+        )
+      : '';
+  }
+
+  /**
+   * Why Find arrays can't read the column: the database the node's input
+   * runs on has no JSON functions; `undefined` when it can, or isn't known
+   */
+  get sampleProblem(): string | undefined {
+    const { editorState, input } = this;
+    const { context, query } = editorState.document;
+    return input === undefined || context?.runtime === undefined
+      ? undefined
+      : getVariantDatabaseProblem(
+          editorState.getRunDatabaseType(
+            query,
+            input.id,
+            context.model,
+            context.runtime,
+          ),
+        );
+  }
+
+  /** The node's input: its id and schema, once known */
+  private get input(): { id: string; schema: Schema } | undefined {
+    const [id] = this.editorState.document.query.getInputIds(this.original.id);
+    const schema =
+      id === undefined ? undefined : this.editorState.analysis.schemas.get(id);
+    return id === undefined || !schema ? undefined : { id, schema };
+  }
+
+  /** The arrays the sample found, if it was of the picked column */
+  get arrays(): CubeExplodeArray[] | undefined {
+    const { sample, column } = this;
+    return sample && sample.column === column
+      ? arraysOf(getCubeJsonFields(column, sample.sample))
+      : undefined;
+  }
+
+  private edited(): void {
+    this.touched = true;
+    if (this.namedByDefault) {
+      this.name = this.defaultName(this.column, this.path);
+    }
+  }
+
+  /** Picks the column; another column's keys and sample are dropped */
+  setColumn(column: string): void {
+    if (column === this.column) {
+      return;
+    }
+    this.stopSample();
+    this.column = column;
+    this.path = [];
+    this.sample = undefined;
+    this.sampleError = undefined;
+    this.edited();
+  }
+
+  setKey(index: number, key: string): void {
+    this.path = this.path.map((each, at) => (at === index ? key : each));
+    this.edited();
+  }
+
+  addKey(): void {
+    this.path = [...this.path, ''];
+    this.edited();
+  }
+
+  removeKey(index: number): void {
+    this.path = this.path.filter((_, at) => at !== index);
+    this.edited();
+  }
+
+  /** Takes the keys of an array the sample found */
+  pickArray(path: readonly string[]): void {
+    this.path = path;
+    this.edited();
+  }
+
+  /** Types the name: it no longer follows the column and keys */
+  setName(name: string): void {
+    this.name = name;
+    this.namedByDefault = false;
+    this.touched = true;
+  }
+
+  stopSample(): void {
+    this.controller?.abort();
+    this.controller = undefined;
+    this.isSampling = false;
+  }
+
+  /**
+   * Reads the picked column's first values after the node's input, as
+   * Explore JSON does (listed in Runs, can be stopped), for the arrays they
+   * hold; picking another column abandons it
+   */
+  *readSample(): GeneratorFn<void> {
+    const { editorState, column, input } = this;
+    const picked = column ? input?.schema.lookup(column) : undefined;
+    if (this.isSampling || !input || !picked || this.sampleProblem) {
+      return;
+    }
+    const controller = new AbortController();
+    this.controller = controller;
+    this.isSampling = true;
+    this.sampleError = undefined;
+    try {
+      const sample = (yield readCubeJsonSample(editorState, {
+        document: editorState.document,
+        afterId: input.id,
+        column: picked,
+        everyRow: false,
+        abortController: controller,
+      })) as CubeJsonSample | undefined;
+      if (this.controller === controller && sample) {
+        this.sample = { column, sample };
+      }
+    } catch (error) {
+      if (
+        this.controller !== controller ||
+        error instanceof CubeQueryRunAborted
+      ) {
+        return;
+      }
+      assertErrorThrown(error);
+      this.sampleError =
+        error instanceof CubeEngineError ? error.firstLine : error.message;
+    } finally {
+      if (this.controller === controller) {
+        this.controller = undefined;
+        this.isSampling = false;
+      }
+    }
+  }
+
+  build(): Explode {
+    const { original, column, path, name } = this;
+    return !this.touched ||
+      (column === original.column &&
+        isSameList(path, original.path) &&
+        name === original.name)
+      ? original
+      : original.with({ column, path, name });
+  }
+}