@finos/legend-cube-builder 0.0.3 → 0.0.4

package/src/graph-manager/protocol/pure/v1/V1_CubeIngestModel.ts

@@ -0,0 +1,183 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { IR } from '@finos/legend-cube';
+import type { PlainObject } from '@finos/legend-shared';
+import type { CubeDataProductEnvironmentType } from '../../../CubeDataProduct.js';
+import {
+  CUBE_INGEST_RUNTIME_PATH,
+  type CubeIngestSettings,
+  parseCubeIngestUrn,
+} from '../../../CubeIngest.js';
+
+// The models an ingest cube's engine calls run on (PLAN §6.7), built for
+// each call and never saved, as Data Cube builds its producer source's: the
+// definitions the call reads, as the ingest server serves them, and for a
+// run a lakehouse runtime at Cube's fixed path
+
+/**
+ * The deployed definitions an ingest cube reads, from the host's ingest
+ * servers: what the engine needs to type and run them
+ */
+export interface V1_CubeIngestDefinitionSource {
+  /** A definition's element, as protocol JSON, by its URN, for the cube's class */
+  getDefinitionElement(
+    urn: string,
+    environmentType: CubeDataProductEnvironmentType,
+  ): Promise<PlainObject>;
+  /** The environment a run's lakehouse runtime names, for the cube's class */
+  getRuntimeEnvironment(
+    environmentType: CubeDataProductEnvironmentType,
+  ): Promise<string>;
+}
+
+/** An ingest data set a lambda reads */
+export interface V1_CubeIngestAccessor {
+  readonly definition: string;
+  readonly dataSet: string;
+  readonly urn: string | undefined;
+}
+
+/** Why a lambda's ingest data sets can't be read on the cube */
+export const V1_CUBE_INGEST_MESSAGE = {
+  NO_URN: (definition: string): string =>
+    `The ingest data set of ${definition} doesn't say which deployed definition it reads`,
+  NOT_SDLC: (definition: string): string =>
+    `Ingest definition ${definition} wasn't deployed from a project, which Cube doesn't read`,
+  OTHER_CLASS: (definition: string): string =>
+    `Ingest definition ${definition} is deployed to another environment than the cube's`,
+  OTHER_PATH: (definition: string): string =>
+    `The URN of ingest definition ${definition} names another definition`,
+} as const;
+
+/** The ingest data sets a lambda reads, in tree order */
+export const V1_collectCubeIngestAccessors = (
+  ir: IR,
+): V1_CubeIngestAccessor[] => {
+  const found: V1_CubeIngestAccessor[] = [];
+  const visit = (value: unknown): void => {
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (!value || typeof value !== 'object') {
+      return;
+    }
+    const node = value as PlainObject;
+    if (node.k === 'ingestAccessor') {
+      const [definition, dataSet] = node.path as [string, string];
+      found.push({
+        definition,
+        dataSet,
+        urn: typeof node.urn === 'string' ? node.urn : undefined,
+      });
+      return;
+    }
+    if (node.k !== 'raw') {
+      Object.values(node).forEach(visit);
+    }
+  };
+  visit(ir);
+  return found;
+};
+
+/** Whether a lambda reads an ingest data set, anywhere in it */
+export const V1_hasCubeIngestAccessor = (ir: IR): boolean =>
+  V1_collectCubeIngestAccessors(ir).length > 0;
+
+/**
+ * Why a lambda's ingest data sets can't be read on the cube, if they can't:
+ * each must say which SDLC-deployed definition it reads, deployed to the
+ * cube's class, under the path the query reads
+ */
+export const V1_checkCubeIngestAccessors = (
+  accessors: readonly V1_CubeIngestAccessor[],
+  settings: CubeIngestSettings,
+): string | undefined => {
+  for (const accessor of accessors) {
+    if (accessor.urn === undefined) {
+      return V1_CUBE_INGEST_MESSAGE.NO_URN(accessor.definition);
+    }
+    const urn = parseCubeIngestUrn(accessor.urn);
+    if (!urn) {
+      return V1_CUBE_INGEST_MESSAGE.NOT_SDLC(accessor.definition);
+    }
+    if (urn.environmentType !== settings.environmentType) {
+      return V1_CUBE_INGEST_MESSAGE.OTHER_CLASS(accessor.definition);
+    }
+    if (urn.definition !== accessor.definition) {
+      return V1_CUBE_INGEST_MESSAGE.OTHER_PATH(accessor.definition);
+    }
+  }
+  return undefined;
+};
+
+/** The URNs of the definitions lambdas read, each once */
+export const V1_getCubeIngestUrns = (
+  accessors: readonly V1_CubeIngestAccessor[],
+): string[] => [
+  ...new Set(
+    accessors
+      .map((accessor) => accessor.urn)
+      .filter((urn): urn is string => urn !== undefined),
+  ),
+];
+
+/** A definition as the model holds it: its test suites, which reference test data the model lacks, left out */
+const toModelElement = (element: PlainObject): PlainObject => {
+  const { testSuites, ...rest } = element;
+  return rest;
+};
+
+const splitPath = (path: string): { package: string; name: string } => {
+  const index = path.lastIndexOf('::');
+  return { package: path.slice(0, index), name: path.slice(index + 2) };
+};
+
+/** The model typing runs on: the definitions alone, with no runtime */
+export const V1_buildCubeIngestTypingContext = (
+  definitions: readonly PlainObject[],
+): PlainObject => ({
+  _type: 'data',
+  elements: definitions.map(toModelElement),
+});
+
+/**
+ * The model a run runs on: the definitions, then the lakehouse runtime at
+ * Cube's fixed path, with the environment and the warehouse, always given
+ */
+export const V1_buildCubeIngestExecutionContext = (
+  definitions: readonly PlainObject[],
+  environment: string,
+  warehouse: string,
+): PlainObject => ({
+  _type: 'data',
+  elements: [
+    ...definitions.map(toModelElement),
+    {
+      _type: 'runtime',
+      ...splitPath(CUBE_INGEST_RUNTIME_PATH),
+      runtimeValue: {
+        _type: 'LakehouseRuntime',
+        connectionStores: [],
+        connections: [],
+        mappings: [],
+        environment,
+        warehouse,
+      },
+    },
+  ],
+});

package/src/graph-manager/protocol/pure/v1/V1_CubeLakehouseEnvironmentResolver.ts

@@ -0,0 +1,141 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  V1_EntitlementsLakehouseEnvironmentType,
+  type V1_EntitlementsUserEnvResponse,
+} from '@finos/legend-graph';
+import {
+  decorateEnvWithRealm,
+  getEntitlementsLakehouseEnvironmentTypeCompatibleWithEnvName,
+  type LakehouseContractServerClient,
+  LakehouseEnvironmentType,
+} from '@finos/legend-server-lakehouse';
+import {
+  type CubeDataProductProject,
+  CubeDataProductEnvironmentType,
+  isCubeSnapshotVersion,
+} from '../../../CubeDataProduct.js';
+import { CubeEngineError, CubeEngineErrorKind } from '../../../CubeEngine.js';
+import type { CubeLakehouseEnvironment } from '../../../CubeLakehouseEnvironment.js';
+
+// The viewer's lakehouse environment, as Legend Query resolves it
+// (QueryEditorStore.resolveLakehouseEnvAndWarehouse): the environment the
+// host remembers for the viewer, else the first one the viewer's
+// entitlements name, read once per page visit. Query adds the
+// production-parallel realm for a snapshot version; Cube adds it for a
+// production-parallel deployment too, as Data Cube does. Query may
+// remember an environment with a realm already on it (its runtime dialog
+// saves the runtime's environment as it is), so Cube drops that realm first:
+// a production deployment never runs in the production-parallel realm
+
+export const V1_CUBE_NO_LAKEHOUSE_ENVIRONMENT =
+  'Unable to resolve lakehouse user environment. Please ensure your lakehouse entitlements are configured.';
+
+/** The environment without a realm, which `decorateEnvWithRealm` would add */
+const withoutRealm = (environment: string): string => {
+  switch (
+    getEntitlementsLakehouseEnvironmentTypeCompatibleWithEnvName(environment)
+  ) {
+    case V1_EntitlementsLakehouseEnvironmentType.DEVELOPMENT:
+      return environment.slice(
+        `${LakehouseEnvironmentType.DEVELOPMENT}-`.length,
+      );
+    case V1_EntitlementsLakehouseEnvironmentType.PRODUCTION_PARALLEL:
+      return environment.slice(
+        0,
+        -`-${LakehouseEnvironmentType.PRODUCTION_PARALLEL}`.length,
+      );
+    default:
+      return environment;
+  }
+};
+
+export class V1_CubeLakehouseEnvironmentResolver
+  implements CubeLakehouseEnvironment
+{
+  private readonly contractServerClient: LakehouseContractServerClient;
+  private readonly getAccessToken: () => string | undefined;
+  private readonly getCurrentUser: () => string;
+  private readonly getPreferredEnvironment: () => string | undefined;
+  /** The viewer's environment, read on the first run of the page visit */
+  private userEnvironment: Promise<string> | undefined;
+
+  constructor(
+    contractServerClient: LakehouseContractServerClient,
+    getAccessToken: () => string | undefined,
+    getCurrentUser: () => string,
+    getPreferredEnvironment: () => string | undefined = () => undefined,
+  ) {
+    this.contractServerClient = contractServerClient;
+    this.getAccessToken = getAccessToken;
+    this.getCurrentUser = getCurrentUser;
+    this.getPreferredEnvironment = getPreferredEnvironment;
+  }
+
+  private readUserEnvironment(): Promise<string> {
+    const preferred = this.getPreferredEnvironment();
+    if (preferred) {
+      return Promise.resolve(preferred);
+    }
+    this.userEnvironment ??= (async () => {
+      let response: V1_EntitlementsUserEnvResponse;
+      try {
+        response = await this.contractServerClient.getUserEntitlementEnvs(
+          this.getCurrentUser(),
+          this.getAccessToken(),
+        );
+      } catch (error) {
+        throw new CubeEngineError(
+          CubeEngineErrorKind.NETWORK,
+          `${V1_CUBE_NO_LAKEHOUSE_ENVIRONMENT}\n${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+      const environment = response.users
+        .map((user) => user.lakehouseEnvironment)
+        .find((name) => typeof name === 'string' && name.length > 0);
+      if (!environment) {
+        throw new CubeEngineError(
+          CubeEngineErrorKind.EXECUTION,
+          V1_CUBE_NO_LAKEHOUSE_ENVIRONMENT,
+        );
+      }
+      return environment;
+    })().catch((error: unknown) => {
+      // read again on the next run
+      this.userEnvironment = undefined;
+      throw error;
+    });
+    return this.userEnvironment;
+  }
+
+  /** The viewer's environment with no realm: what ingest environments are named by */
+  async resolveBaseEnvironment(): Promise<string> {
+    return withoutRealm(await this.readUserEnvironment());
+  }
+
+  async resolveEnvironment(project: CubeDataProductProject): Promise<string> {
+    const environment = await this.resolveBaseEnvironment();
+    return project.environmentType ===
+      CubeDataProductEnvironmentType.PRODUCTION_PARALLEL ||
+      isCubeSnapshotVersion(project.versionId)
+      ? decorateEnvWithRealm(
+          environment,
+          LakehouseEnvironmentType.PRODUCTION_PARALLEL,
+        )
+      : environment;
+  }
+}

package/src/graph-manager/protocol/pure/v1/V1_CubeLambdaSerializer.ts

@@ -66,6 +66,44 @@ const numberToken = (text: string): unknown => {
   return parseLosslessJSON(text);
 };
 
+/** The literal types whose value is a number, which Cube keeps as its digit string in an expression's JSON */
+const NUMBER_LITERAL_TYPES: readonly unknown[] = [
+  'integer',
+  'float',
+  'decimal',
+];
+
+/**
+ * An expression's lambda as the engine reads it (PLAN §11.7): each number
+ * literal written digit for digit from the string Cube keeps, and with an
+ * origin, any source information replaced by it on every value
+ * specification; with none, the JSON's own source information kept, so a
+ * lambda parsed with its locations is typed with them (the editor's Validate)
+ */
+const serializeLambdaJson = (
+  json: unknown,
+  origin: Origin | undefined,
+): unknown => {
+  if (Array.isArray(json)) {
+    return json.map((item) => serializeLambdaJson(item, origin));
+  }
+  if (typeof json !== 'object' || json === null) {
+    return json;
+  }
+  const object: PlainObject = Object.fromEntries(
+    Object.entries(json)
+      .filter(([key]) => !origin || key !== 'sourceInformation')
+      .map(([key, value]) => [key, serializeLambdaJson(value, origin)]),
+  );
+  if (
+    NUMBER_LITERAL_TYPES.includes(object._type) &&
+    typeof object.value === 'string'
+  ) {
+    object.value = numberToken(object.value);
+  }
+  return typeof object._type === 'string' ? stamped(object, origin) : object;
+};
+
 const serializeLiteral = (value: LiteralValue): PlainObject => {
   switch (value.kind) {
     case 'string':
@@ -193,6 +231,30 @@ const serialize = (ir: IR, inherited: Origin | undefined): PlainObject => {
         origin,
       );
     }
+    case 'dataProductAccessor':
+      // stamped on the outer instance only: the value is sent as legend-graph
+      // sends it, which Query and Data Cube already send to the engine
+      return stamped(
+        {
+          _type: 'classInstance',
+          type: 'P',
+          multiplicity: { lowerBound: 1, upperBound: 1 },
+          value: { path: [...ir.path], parameters: [] },
+        },
+        ir.origin ?? inherited,
+      );
+    case 'ingestAccessor':
+      // as legend-graph sends it, and Legend Query's ingest queries with it:
+      // the data set's rows, never its metadata
+      return stamped(
+        {
+          _type: 'classInstance',
+          type: 'I',
+          multiplicity: { lowerBound: 1, upperBound: 1 },
+          value: { path: [...ir.path], metadata: false },
+        },
+        ir.origin ?? inherited,
+      );
     case 'elementPtr':
       return stamped(
         { _type: 'packageableElementPtr', fullPath: ir.path },
@@ -232,12 +294,30 @@ const serialize = (ir: IR, inherited: Origin | undefined): PlainObject => {
         origin,
       );
     }
-    case 'let':
-    case 'block':
-      // window isolation (PLAN §8.6) comes with Partition, in M5
-      throw new Error(`Can't send "${ir.k}" yet`);
+    case 'let': {
+      // `let n = v` is the engine's `letFunction('n', v)`; the name is
+      // stamped too, so an error the engine reports on it lands on the node
+      // the let binds (PLAN §8.6)
+      const origin = ir.origin ?? inherited;
+      return stamped(
+        {
+          _type: 'func',
+          function: 'letFunction',
+          parameters: [
+            stamped({ _type: 'string', value: ir.name }, origin),
+            serialize(ir.value, origin),
+          ],
+        },
+        origin,
+      );
+    }
     case 'raw':
       return ir.json as PlainObject;
+    case 'lambdaJson':
+      return serializeLambdaJson(
+        ir.json,
+        ir.origin ?? inherited,
+      ) as PlainObject;
     default: {
       const unknown: never = ir;
       throw new Error(`Unknown IR ${JSON.stringify(unknown)}`);