@finos/legend-cube-builder 0.0.3 → 0.0.5

package/src/graph-manager/protocol/pure/v1/V1_CubeIngestModel.ts

@@ -0,0 +1,174 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { IR } from '@finos/legend-cube';
+import type { PlainObject } from '@finos/legend-shared';
+import {
+  CUBE_INGEST_RUNTIME_PATH,
+  type CubeIngestSettings,
+  parseCubeIngestUrn,
+} from '../../../CubeIngest.js';
+import type { CubeIngestDefinitionSource } from '../../../CubeIngestCatalog.js';
+import { CUBE_LAKEHOUSE_RUNTIME_KIND } from '../../../CubeLakehouseRuntime.js';
+
+// The models an ingest cube's engine calls run on (PLAN §6.7), built for
+// each call and never saved, as Data Cube builds its producer source's: the
+// definitions the call reads, as the ingest server serves them, and for a
+// run a lakehouse runtime at Cube's fixed path
+
+/**
+ * The definitions an ingest cube reads, as protocol JSON, and the
+ * environment its runs name: what the engine needs to type and run them
+ */
+export type V1_CubeIngestDefinitionSource = CubeIngestDefinitionSource;
+
+/** An ingest data set a lambda reads */
+export interface V1_CubeIngestAccessor {
+  readonly definition: string;
+  readonly dataSet: string;
+  readonly urn: string | undefined;
+}
+
+/** Why a lambda's ingest data sets can't be read on the cube */
+export const V1_CUBE_INGEST_MESSAGE = {
+  NO_URN: (definition: string): string =>
+    `The ingest data set of ${definition} doesn't say which deployed definition it reads`,
+  NOT_SDLC: (definition: string): string =>
+    `Ingest definition ${definition} wasn't deployed from a project, which Cube doesn't read`,
+  OTHER_CLASS: (definition: string): string =>
+    `Ingest definition ${definition} is deployed to another environment than the cube's`,
+  OTHER_PATH: (definition: string): string =>
+    `The URN of ingest definition ${definition} names another definition`,
+} as const;
+
+/** The ingest data sets a lambda reads, in tree order */
+export const V1_collectCubeIngestAccessors = (
+  ir: IR,
+): V1_CubeIngestAccessor[] => {
+  const found: V1_CubeIngestAccessor[] = [];
+  const visit = (value: unknown): void => {
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (!value || typeof value !== 'object') {
+      return;
+    }
+    const node = value as PlainObject;
+    if (node.k === 'ingestAccessor') {
+      const [definition, dataSet] = node.path as [string, string];
+      found.push({
+        definition,
+        dataSet,
+        urn: typeof node.urn === 'string' ? node.urn : undefined,
+      });
+      return;
+    }
+    if (node.k !== 'raw') {
+      Object.values(node).forEach(visit);
+    }
+  };
+  visit(ir);
+  return found;
+};
+
+/** Whether a lambda reads an ingest data set, anywhere in it */
+export const V1_hasCubeIngestAccessor = (ir: IR): boolean =>
+  V1_collectCubeIngestAccessors(ir).length > 0;
+
+/**
+ * Why a lambda's ingest data sets can't be read on the cube, if they can't:
+ * each must say which SDLC-deployed definition it reads, deployed to the
+ * cube's class, under the path the query reads
+ */
+export const V1_checkCubeIngestAccessors = (
+  accessors: readonly V1_CubeIngestAccessor[],
+  settings: CubeIngestSettings,
+): string | undefined => {
+  for (const accessor of accessors) {
+    if (accessor.urn === undefined) {
+      return V1_CUBE_INGEST_MESSAGE.NO_URN(accessor.definition);
+    }
+    const urn = parseCubeIngestUrn(accessor.urn);
+    if (!urn) {
+      return V1_CUBE_INGEST_MESSAGE.NOT_SDLC(accessor.definition);
+    }
+    if (urn.environmentType !== settings.environmentType) {
+      return V1_CUBE_INGEST_MESSAGE.OTHER_CLASS(accessor.definition);
+    }
+    if (urn.definition !== accessor.definition) {
+      return V1_CUBE_INGEST_MESSAGE.OTHER_PATH(accessor.definition);
+    }
+  }
+  return undefined;
+};
+
+/** The URNs of the definitions lambdas read, each once */
+export const V1_getCubeIngestUrns = (
+  accessors: readonly V1_CubeIngestAccessor[],
+): string[] => [
+  ...new Set(
+    accessors
+      .map((accessor) => accessor.urn)
+      .filter((urn): urn is string => urn !== undefined),
+  ),
+];
+
+/** A definition as the model holds it: its test suites, which reference test data the model lacks, left out */
+const toModelElement = (element: PlainObject): PlainObject => {
+  const { testSuites, ...rest } = element;
+  return rest;
+};
+
+const splitPath = (path: string): { package: string; name: string } => {
+  const index = path.lastIndexOf('::');
+  return { package: path.slice(0, index), name: path.slice(index + 2) };
+};
+
+/** The model typing runs on: the definitions alone, with no runtime */
+export const V1_buildCubeIngestTypingContext = (
+  definitions: readonly PlainObject[],
+): PlainObject => ({
+  _type: 'data',
+  elements: definitions.map(toModelElement),
+});
+
+/**
+ * The model a run runs on: the definitions, then the lakehouse runtime at
+ * Cube's fixed path, with the environment and the warehouse, always given
+ */
+export const V1_buildCubeIngestExecutionContext = (
+  definitions: readonly PlainObject[],
+  environment: string,
+  warehouse: string,
+): PlainObject => ({
+  _type: 'data',
+  elements: [
+    ...definitions.map(toModelElement),
+    {
+      _type: 'runtime',
+      ...splitPath(CUBE_INGEST_RUNTIME_PATH),
+      runtimeValue: {
+        _type: CUBE_LAKEHOUSE_RUNTIME_KIND,
+        connectionStores: [],
+        connections: [],
+        mappings: [],
+        environment,
+        warehouse,
+      },
+    },
+  ],
+});

package/src/graph-manager/protocol/pure/v1/V1_CubeLakehouseEnvironmentResolver.ts

@@ -0,0 +1,141 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  V1_EntitlementsLakehouseEnvironmentType,
+  type V1_EntitlementsUserEnvResponse,
+} from '@finos/legend-graph';
+import {
+  decorateEnvWithRealm,
+  getEntitlementsLakehouseEnvironmentTypeCompatibleWithEnvName,
+  type LakehouseContractServerClient,
+  LakehouseEnvironmentType,
+} from '@finos/legend-server-lakehouse';
+import {
+  type CubeDataProductProject,
+  CubeDataProductEnvironmentType,
+  isCubeSnapshotVersion,
+} from '../../../CubeDataProduct.js';
+import { CubeEngineError, CubeEngineErrorKind } from '../../../CubeEngine.js';
+import type { CubeLakehouseEnvironment } from '../../../CubeLakehouseEnvironment.js';
+
+// The viewer's lakehouse environment, as Legend Query resolves it
+// (QueryEditorStore.resolveLakehouseEnvAndWarehouse): the environment the
+// host remembers for the viewer, else the first one the viewer's
+// entitlements name, read once per page visit. Query adds the
+// production-parallel realm for a snapshot version; Cube adds it for a
+// production-parallel deployment too, as Data Cube does. Query may
+// remember an environment with a realm already on it (its runtime dialog
+// saves the runtime's environment as it is), so Cube drops that realm first:
+// a production deployment never runs in the production-parallel realm
+
+export const V1_CUBE_NO_LAKEHOUSE_ENVIRONMENT =
+  'Unable to resolve lakehouse user environment. Please ensure your lakehouse entitlements are configured.';
+
+/** The environment without a realm, which `decorateEnvWithRealm` would add */
+const withoutRealm = (environment: string): string => {
+  switch (
+    getEntitlementsLakehouseEnvironmentTypeCompatibleWithEnvName(environment)
+  ) {
+    case V1_EntitlementsLakehouseEnvironmentType.DEVELOPMENT:
+      return environment.slice(
+        `${LakehouseEnvironmentType.DEVELOPMENT}-`.length,
+      );
+    case V1_EntitlementsLakehouseEnvironmentType.PRODUCTION_PARALLEL:
+      return environment.slice(
+        0,
+        -`-${LakehouseEnvironmentType.PRODUCTION_PARALLEL}`.length,
+      );
+    default:
+      return environment;
+  }
+};
+
+export class V1_CubeLakehouseEnvironmentResolver
+  implements CubeLakehouseEnvironment
+{
+  private readonly contractServerClient: LakehouseContractServerClient;
+  private readonly getAccessToken: () => string | undefined;
+  private readonly getCurrentUser: () => string;
+  private readonly getPreferredEnvironment: () => string | undefined;
+  /** The viewer's environment, read on the first run of the page visit */
+  private userEnvironment: Promise<string> | undefined;
+
+  constructor(
+    contractServerClient: LakehouseContractServerClient,
+    getAccessToken: () => string | undefined,
+    getCurrentUser: () => string,
+    getPreferredEnvironment: () => string | undefined = () => undefined,
+  ) {
+    this.contractServerClient = contractServerClient;
+    this.getAccessToken = getAccessToken;
+    this.getCurrentUser = getCurrentUser;
+    this.getPreferredEnvironment = getPreferredEnvironment;
+  }
+
+  private readUserEnvironment(): Promise<string> {
+    const preferred = this.getPreferredEnvironment();
+    if (preferred) {
+      return Promise.resolve(preferred);
+    }
+    this.userEnvironment ??= (async () => {
+      let response: V1_EntitlementsUserEnvResponse;
+      try {
+        response = await this.contractServerClient.getUserEntitlementEnvs(
+          this.getCurrentUser(),
+          this.getAccessToken(),
+        );
+      } catch (error) {
+        throw new CubeEngineError(
+          CubeEngineErrorKind.NETWORK,
+          `${V1_CUBE_NO_LAKEHOUSE_ENVIRONMENT}\n${error instanceof Error ? error.message : String(error)}`,
+        );
+      }
+      const environment = response.users
+        .map((user) => user.lakehouseEnvironment)
+        .find((name) => typeof name === 'string' && name.length > 0);
+      if (!environment) {
+        throw new CubeEngineError(
+          CubeEngineErrorKind.EXECUTION,
+          V1_CUBE_NO_LAKEHOUSE_ENVIRONMENT,
+        );
+      }
+      return environment;
+    })().catch((error: unknown) => {
+      // read again on the next run
+      this.userEnvironment = undefined;
+      throw error;
+    });
+    return this.userEnvironment;
+  }
+
+  /** The viewer's environment with no realm: what ingest environments are named by */
+  async resolveBaseEnvironment(): Promise<string> {
+    return withoutRealm(await this.readUserEnvironment());
+  }
+
+  async resolveEnvironment(project: CubeDataProductProject): Promise<string> {
+    const environment = await this.resolveBaseEnvironment();
+    return project.environmentType ===
+      CubeDataProductEnvironmentType.PRODUCTION_PARALLEL ||
+      isCubeSnapshotVersion(project.versionId)
+      ? decorateEnvWithRealm(
+          environment,
+          LakehouseEnvironmentType.PRODUCTION_PARALLEL,
+        )
+      : environment;
+  }
+}