@finos/legend-cube-builder 0.0.4 → 0.0.6

package/src/graph-manager/CubeIngestCatalog.ts

@@ -15,6 +15,7 @@
  */
 
 import type { Schema } from '@finos/legend-cube';
+import type { PlainObject } from '@finos/legend-shared';
 import type { CubeDataProductEnvironmentType } from './CubeDataProduct.js';
 import type { CubeEngineError, NodeId } from './CubeEngine.js';
 
@@ -56,15 +57,22 @@ export class CubeIngestEnvironment {
   readonly name: string;
   /** The environment a run's lakehouse runtime names */
   readonly runtimeEnvironment: string;
+  /**
+   * The warehouse a new cube starts with, when the environment has one, e.g.
+   * the runtime of a Studio project; else the one the viewer last picked
+   */
+  readonly warehouse: string | undefined;
 
   constructor(fields: {
     environmentType: CubeDataProductEnvironmentType;
     name: string;
     runtimeEnvironment: string;
+    warehouse?: string | undefined;
   }) {
     this.environmentType = fields.environmentType;
     this.name = fields.name;
     this.runtimeEnvironment = fields.runtimeEnvironment;
+    this.warehouse = fields.warehouse;
   }
 }
 
@@ -167,3 +175,28 @@ export interface CubeIngestCatalog {
     options?: { readonly fresh?: boolean | undefined },
   ): Promise<Map<NodeId, Schema | CubeEngineError>>;
 }
+
+/**
+ * Where an ingest cube's engine reads the definitions it types and runs on,
+ * as protocol JSON, by URN, and the environment a run's lakehouse runtime
+ * names: the ingest servers for the deployed ones, or the host's own, e.g.
+ * Studio's workspace. Calls fail with a `CubeEngineError`
+ */
+export interface CubeIngestDefinitionSource {
+  /** A definition's element, as protocol JSON, by its URN, for the cube's class */
+  getDefinitionElement(
+    urn: string,
+    environmentType: CubeDataProductEnvironmentType,
+  ): Promise<PlainObject>;
+  /** The environment a run's lakehouse runtime names, for the cube's class */
+  getRuntimeEnvironment(
+    environmentType: CubeDataProductEnvironmentType,
+  ): Promise<string>;
+  /**
+   * Every element a typing's or a run's model holds, as protocol JSON, when
+   * the host has more than the definitions, e.g. Studio's whole workspace,
+   * whose definitions' materialized views read other elements; the
+   * definitions alone otherwise
+   */
+  getModelElements?(): Promise<readonly PlainObject[]>;
+}

package/src/graph-manager/CubeLakehouseRuntime.ts

@@ -0,0 +1,43 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The lakehouse runtimes, by the kind the engine's protocol names them
+ * (legend-graph's subtypes of the base lakehouse runtime): a Databricks one
+ * will be another (PLAN §11.12)
+ */
+export enum CubeLakehouseRuntimeKind {
+  LAKEHOUSE = 'LakehouseRuntime',
+  SINGLE_STORE = 'LakehouseSingleStoreRuntime',
+}
+
+/** The runtime Cube writes for a data product or ingest cube (PLAN §6.8) */
+export const CUBE_LAKEHOUSE_RUNTIME_KIND = CubeLakehouseRuntimeKind.LAKEHOUSE;
+
+/**
+ * The engine's database type each lakehouse runtime routes to (PLAN §11.12):
+ * every `LakehouseRuntime` to Snowflake, for now (user, 2026-10-11); a
+ * SingleStore runtime to SingleStore, the engine's `MemSQL`. The one place to
+ * change when the routing changes or a runtime is added.
+ */
+const LAKEHOUSE_DATABASE_TYPES: ReadonlyMap<string, string> = new Map([
+  [CubeLakehouseRuntimeKind.LAKEHOUSE, 'Snowflake'],
+  [CubeLakehouseRuntimeKind.SINGLE_STORE, 'MemSQL'],
+]);
+
+/** The database type a lakehouse runtime of the kind runs on; none for a kind Cube doesn't know */
+export const getLakehouseDatabaseType = (kind: string): string | undefined =>
+  LAKEHOUSE_DATABASE_TYPES.get(kind);

package/src/graph-manager/CubeProjectCatalog.ts

@@ -32,9 +32,9 @@ export interface CubeProjectCatalog {
   /** The project's released versions, newest first: never a SNAPSHOT or an alias */
   listVersions(project: CubeProjectSummary): Promise<readonly string[]>;
   /**
-   * The project's own Databases at the version, and the runtimes of the
-   * project and its dependencies; a dependency's Databases are not offered
-   * (user, 2026-10-09)
+   * The project's own Databases and functions at the version, and the
+   * runtimes of the project and its dependencies; a dependency's Databases
+   * and functions are not offered (user, 2026-10-09, 2026-10-12)
    */
   loadOutline(project: CubeProjectCoordinates): Promise<CubeModelOutline>;
 }

package/src/graph-manager/CubeService.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { ModelContext } from '@finos/legend-cube';
+
+// A cube on services (PLAN §6.4), in progress: its model only marks it as
+// one, with no runtime. How a service is read, and what it runs on, are
+// still to come
+
+export const CUBE_SERVICE_MODEL_TYPE = 'cubeService';
+
+export const createCubeServiceModel = (): ModelContext =>
+  ({ _type: CUBE_SERVICE_MODEL_TYPE }) as ModelContext;
+
+export const isCubeServiceModel = (model: ModelContext): boolean =>
+  model._type === CUBE_SERVICE_MODEL_TYPE;

package/src/graph-manager/CubeServiceCatalog.ts

@@ -0,0 +1,35 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// The deployed services a cube can pick (PLAN §6.4): the port the source
+// dialog's Service tab reads, over the engine's service registry
+
+/** A deployed service, as the registry lists it */
+export interface CubeServiceSummary {
+  /** The service's name, its element path */
+  readonly name: string;
+  readonly title?: string | undefined;
+  /** Its URL pattern, e.g. `/trades/byDesk`, which the engine's SQL reads it by */
+  readonly pattern: string;
+  readonly documentation: string;
+  /** Who owns it: its owners, or its deployment as `Deployment <id>`; none when the registry doesn't say */
+  readonly owners: readonly string[];
+}
+
+export interface CubeServiceCatalog {
+  /** Every deployed service, by title, then name */
+  listServices(): Promise<readonly CubeServiceSummary[]>;
+}

package/src/graph-manager/protocol/pure/CubeEngineBuilder.ts

@@ -22,21 +22,42 @@ import type {
   LakehousePlatformServerClient,
 } from '@finos/legend-server-lakehouse';
 import type { MarketplaceServerClient } from '@finos/legend-server-marketplace';
-import type { TracerService } from '@finos/legend-shared';
+import type { CubeExpressionCodec, IR, JsonObject } from '@finos/legend-cube';
+import {
+  parseLosslessJSON,
+  type PlainObject,
+  stringifyLosslessJSON,
+  type TracerService,
+} from '@finos/legend-shared';
 import type { CubeConnectionExplorer } from '../../CubeConnectionExplorer.js';
 import type {
   CubeDataProductCatalog,
+  CubeDataProductModelSource,
   CubeMarketplaceLinkTarget,
 } from '../../CubeDataProductCatalog.js';
 import type { CubeEngine } from '../../CubeEngine.js';
-import type { CubeIngestCatalog } from '../../CubeIngestCatalog.js';
+import type { CubeEngineClientSettings } from '../../CubeEngineClientSettings.js';
+import type {
+  CubeIngestCatalog,
+  CubeIngestDataSet,
+  CubeIngestDefinitionSource,
+} from '../../CubeIngestCatalog.js';
 import type { CubeLakehouseEnvironment } from '../../CubeLakehouseEnvironment.js';
 import type { CubeProjectCatalog } from '../../CubeProjectCatalog.js';
+import type { CubeServiceCatalog } from '../../CubeServiceCatalog.js';
 import { V1_CubeLakehouseEnvironmentResolver } from './v1/V1_CubeLakehouseEnvironmentResolver.js';
 import { V1_LegendCubeConnectionExplorer } from './v1/V1_LegendCubeConnectionExplorer.js';
 import { V1_LegendCubeDataProductCatalog } from './v1/V1_LegendCubeDataProductCatalog.js';
 import { V1_LegendCubeIngestCatalog } from './v1/V1_LegendCubeIngestCatalog.js';
 import { V1_LegendCubeProjectCatalog } from './v1/V1_LegendCubeProjectCatalog.js';
+import { V1_LegendCubeServiceCatalog } from './v1/V1_LegendCubeServiceCatalog.js';
+import { V1_readCubeIngestDataSets } from './v1/V1_CubeIngestDefinition.js';
+import { V1_serializeCubeLambda } from './v1/V1_CubeLambdaSerializer.js';
+import {
+  V1_readCubeLambda,
+  V1_readCubeValueSpecification,
+} from './v1/V1_CubeLambdaReader.js';
+import { V1_readCubeExpression } from './v1/V1_CubeExpression.js';
 import {
   type V1_CubeEngineConfig,
   V1_LegendCubeEngine,
@@ -59,16 +80,32 @@ export const buildCubeEngine = (
     getRememberedWarehouse?: (() => string | undefined) | undefined;
     /** Ingest cubes run only with one, built by `buildCubeIngestCatalog` */
     ingestCatalog?: CubeIngestCatalog | undefined;
+    /** Dev Settings' engine URL and compression, shared by the host's engine clients (PLAN §11.16) */
+    clientSettings?: CubeEngineClientSettings | undefined;
+    /**
+     * The definitions ingest cubes read, for a host whose catalog isn't
+     * built by `buildCubeIngestCatalog`, e.g. Studio's workspace
+     */
+    ingestDefinitionSource?: CubeIngestDefinitionSource | undefined;
+    /**
+     * The model data product cubes read their project from, for a host whose
+     * data products aren't deployed, e.g. Studio's workspace; the project's
+     * deployed version otherwise
+     */
+    dataProductModelSource?: CubeDataProductModelSource | undefined;
   },
 ): CubeEngine =>
   new V1_LegendCubeEngine(config, tracerService, {
+    clientSettings: options?.clientSettings,
     lakehouseEnvironment: options?.lakehouseEnvironment,
     getRememberedWarehouse: options?.getRememberedWarehouse,
+    dataProductModels: options?.dataProductModelSource,
     // the catalog reads the definitions the engine types and runs on
     ingestDefinitions:
-      options?.ingestCatalog instanceof V1_LegendCubeIngestCatalog
+      options?.ingestDefinitionSource ??
+      (options?.ingestCatalog instanceof V1_LegendCubeIngestCatalog
         ? options.ingestCatalog
-        : undefined,
+        : undefined),
   });
 
 /**
@@ -78,8 +115,9 @@ export const buildCubeEngine = (
 export const buildCubeConnectionExplorer = (
   config: CubeEngineConfig,
   tracerService: TracerService,
+  clientSettings?: CubeEngineClientSettings,
 ): CubeConnectionExplorer =>
-  new V1_LegendCubeConnectionExplorer(config, tracerService);
+  new V1_LegendCubeConnectionExplorer(config, tracerService, clientSettings);
 
 /** What a host gives Cube to read its deployed data products (PLAN §6.8) */
 export interface CubeLakehouseServices {
@@ -134,6 +172,23 @@ export const buildCubeProjectCatalog = (
   depotServerClient: DepotServerClient,
 ): CubeProjectCatalog => new V1_LegendCubeProjectCatalog(depotServerClient);
 
+/**
+ * The services in the engine's registry (PLAN §6.4), read through their own
+ * client, configured as the engine's. The registry isn't in every engine: a
+ * host whose engine has none offers no catalog.
+ */
+export const buildCubeServiceCatalog = (
+  config: CubeEngineConfig,
+  tracerService: TracerService,
+  clientSettings?: CubeEngineClientSettings,
+): CubeServiceCatalog => {
+  const engineClient = new V1_EngineServerClient(config);
+  // every call throws without one
+  engineClient.setTracerService(tracerService);
+  clientSettings?.register(engineClient);
+  return new V1_LegendCubeServiceCatalog(() => engineClient.getServicesInfo());
+};
+
 /** The viewer's lakehouse environment, which the engine needs to run data product cubes */
 export const buildCubeLakehouseEnvironment = (
   services: CubeLakehouseServices,
@@ -154,6 +209,7 @@ export const buildCubeIngestCatalog = (
   config: CubeEngineConfig,
   tracerService: TracerService,
   services: CubeLakehouseServices,
+  clientSettings?: CubeEngineClientSettings,
 ): CubeIngestCatalog | undefined => {
   const { platformServerClient, ingestServerClient } = services;
   if (!platformServerClient || !ingestServerClient) {
@@ -162,6 +218,7 @@ export const buildCubeIngestCatalog = (
   const engineClient = new V1_EngineServerClient(config);
   // every call throws without one
   engineClient.setTracerService(tracerService);
+  clientSettings?.register(engineClient);
   const environment = new V1_CubeLakehouseEnvironmentResolver(
     services.contractServerClient,
     services.getAccessToken,
@@ -176,3 +233,56 @@ export const buildCubeIngestCatalog = (
     services.getAccessToken,
   );
 };
+
+/** The JSON without any `sourceInformation`, at any depth */
+const withoutSourceInformation = (json: unknown): unknown =>
+  Array.isArray(json)
+    ? json.map(withoutSourceInformation)
+    : json !== null && typeof json === 'object'
+      ? Object.fromEntries(
+          Object.entries(json)
+            .filter(([key]) => key !== 'sourceInformation')
+            .map(([key, value]) => [key, withoutSourceInformation(value)]),
+        )
+      : json;
+
+/**
+ * A lambda Cube emitted, as protocol JSON for a host to keep, e.g. in a
+ * Studio ingest definition: plain JSON numbers, and none of the source
+ * information Cube stamps on a lambda to place the engine's errors
+ */
+export const serializeCubeLambdaForHost = (lambda: IR): PlainObject =>
+  withoutSourceInformation(
+    JSON.parse(stringifyLosslessJSON(V1_serializeCubeLambda(lambda))),
+  ) as PlainObject;
+
+/**
+ * The data sets of an ingest definition's content, its protocol JSON, in
+ * order, typed from what it declares; for a host that lists definitions of
+ * its own, e.g. Studio's workspace. Throws when the content can't be read.
+ */
+export const readCubeIngestDataSets = (
+  content: PlainObject,
+): CubeIngestDataSet[] => V1_readCubeIngestDataSets(content);
+
+/**
+ * The IR of a lambda's protocol JSON, Cube's or the engine's grammar's, for
+ * Import Pure (PLAN §11.18): a host's matview, or a pasted lambda. Throws a
+ * `CubeLambdaReadError` naming what Cube can't read.
+ */
+export const readCubeLambdaJson = (json: unknown): IR =>
+  V1_readCubeLambda(json);
+
+/**
+ * How Import Pure stores an expression it reads into a step, and reads one a
+ * step stores (PLAN §11.18): as the Extend and Filter by Expression editors
+ * store the engine's JSON, with no source information and number literals as
+ * their digit strings
+ */
+export const CUBE_EXPRESSION_CODEC: CubeExpressionCodec = {
+  write: (lambda: IR): JsonObject =>
+    V1_readCubeExpression(
+      parseLosslessJSON(stringifyLosslessJSON(V1_serializeCubeLambda(lambda))),
+    ).lambda,
+  read: (json: JsonObject): IR => V1_readCubeValueSpecification(json),
+};

package/src/graph-manager/protocol/pure/v1/V1_CubeDataProductModel.ts

@@ -20,6 +20,7 @@ import {
   CUBE_DATA_PRODUCT_RUNTIME_PATH,
   type CubeDataProductProject,
 } from '../../../CubeDataProduct.js';
+import { CUBE_LAKEHOUSE_RUNTIME_KIND } from '../../../CubeLakehouseRuntime.js';
 
 // The models a data product cube's engine calls run on (PLAN §6.8), built for
 // each call and never saved: the project at its saved version, and for a run
@@ -62,24 +63,29 @@ export const V1_hasCubeDataProductAccessor = (ir: IR): boolean => {
   return visit(ir);
 };
 
-/** The model typing runs on: the project alone, with no runtime */
+/**
+ * The model typing runs on: the project alone, with no runtime; the host's
+ * model of it when it has one, e.g. Studio's workspace
+ */
 export const V1_buildCubeDataProductTypingContext = (
   project: CubeDataProductProject,
-): PlainObject => pointerOf(project);
+  projectModel?: PlainObject | undefined,
+): PlainObject => projectModel ?? pointerOf(project);
 
 /**
- * The model a run runs on: the project, then a model holding only the
- * lakehouse runtime at Cube's fixed path, with the viewer's environment and
- * the warehouse, always given
+ * The model a run runs on: the project (the host's model of it, when it has
+ * one), then a model holding only the lakehouse runtime at Cube's fixed path,
+ * with the viewer's environment and the warehouse, always given
  */
 export const V1_buildCubeDataProductExecutionContext = (
   project: CubeDataProductProject,
   environment: string,
   warehouse: string,
+  projectModel?: PlainObject | undefined,
 ): PlainObject => ({
   _type: 'combination',
   contexts: [
-    pointerOf(project),
+    projectModel ?? pointerOf(project),
     {
       _type: 'data',
       elements: [
@@ -87,7 +93,7 @@ export const V1_buildCubeDataProductExecutionContext = (
           _type: 'runtime',
           ...splitPath(CUBE_DATA_PRODUCT_RUNTIME_PATH),
           runtimeValue: {
-            _type: 'LakehouseRuntime',
+            _type: CUBE_LAKEHOUSE_RUNTIME_KIND,
             connectionStores: [],
             connections: [],
             mappings: [],