@finos/legend-cube-builder 0.0.3 → 0.0.4

package/src/graph-manager/protocol/pure/v1/V1_CubeDirectModel.ts

@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { IR } from '@finos/legend-cube';
+import type { PlainObject } from '@finos/legend-shared';
+import type { CubeDirectConnection } from '../../../CubeConnectionExplorer.js';
+import {
+  CUBE_DIRECT_CONNECTION_PATH,
+  CUBE_DIRECT_DATABASE_PATH,
+  CUBE_DIRECT_RUNTIME_PATH,
+} from '../../../CubeDirectConnection.js';
+import type { AccessorPath } from '../../../CubeEngine.js';
+import {
+  type V1_CubeExploredTablePath,
+  V1_quoteCubeName,
+} from './V1_CubeSchemaExploration.js';
+
+// The model a direct-connection cube's engine call runs on (PLAN §6.8): one
+// Database holding exactly the tables the call needs, the saved connection
+// bound to it, and a runtime with that one connection. It is built as protocol
+// JSON for each call, never saved: the DuckDB grammar has no setup SQL, so a
+// model in Pure text would lose it
+
+/** A table of the built Database, as read from the connection's database */
+export interface V1_CubeDirectTable {
+  /** As the database stores them */
+  path: V1_CubeExploredTablePath;
+  /** The table's definition, every name quoted, without the columns Cube hides */
+  definition: PlainObject;
+}
+
+const splitPath = (path: string): { package: string; name: string } => {
+  const index = path.lastIndexOf('::');
+  return { package: path.slice(0, index), name: path.slice(index + 2) };
+};
+
+/** By code point, the same in every locale */
+const compare = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
+
+/** The built Database's package and name, which schema exploration takes */
+export const V1_CUBE_DIRECT_DATABASE = splitPath(CUBE_DIRECT_DATABASE_PATH);
+
+/**
+ * The store accessors of a lambda, each once: found anywhere in it, so a new
+ * kind of IR node needs nothing here
+ */
+export const V1_collectCubeStoreAccessors = (
+  ir: IR,
+): readonly AccessorPath[] => {
+  const found = new Map<string, AccessorPath>();
+  const visit = (value: unknown): void => {
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (!value || typeof value !== 'object') {
+      return;
+    }
+    const node = value as PlainObject;
+    if (node.k === 'storeAccessor' && Array.isArray(node.path)) {
+      const path = node.path as unknown as AccessorPath;
+      found.set(JSON.stringify(path), path);
+      return;
+    }
+    if (node.k === 'raw') {
+      return;
+    }
+    Object.values(node).forEach(visit);
+  };
+  visit(ir);
+  return [...found.values()];
+};
+
+/**
+ * The model of one call: the tables in a fixed order (by schema, then table),
+ * so the same tables always give the same model
+ */
+export const V1_buildCubeDirectModelContext = (
+  connection: CubeDirectConnection,
+  tables: readonly V1_CubeDirectTable[],
+): PlainObject => {
+  const schemas = new Map<string, PlainObject[]>();
+  [...tables]
+    .sort(
+      (a, b) => compare(a.path[0], b.path[0]) || compare(a.path[1], b.path[1]),
+    )
+    .forEach(({ path: [schema], definition }) =>
+      schemas.set(schema, [...(schemas.get(schema) ?? []), definition]),
+    );
+  return {
+    _type: 'data',
+    elements: [
+      {
+        _type: 'relational',
+        ...splitPath(CUBE_DIRECT_DATABASE_PATH),
+        includedStores: [],
+        filters: [],
+        joins: [],
+        stereotypes: [],
+        schemas: [...schemas.entries()].map(([schema, definitions]) => ({
+          name: V1_quoteCubeName(schema),
+          tables: definitions,
+          views: [],
+          tabularFunctions: [],
+        })),
+      },
+      {
+        _type: 'connection',
+        ...splitPath(CUBE_DIRECT_CONNECTION_PATH),
+        // bound to the built Database: the engine needs the store it serves
+        connectionValue: { ...connection, element: CUBE_DIRECT_DATABASE_PATH },
+      },
+      {
+        _type: 'runtime',
+        ...splitPath(CUBE_DIRECT_RUNTIME_PATH),
+        runtimeValue: {
+          _type: 'engineRuntime',
+          mappings: [],
+          connectionStores: [],
+          connections: [
+            {
+              store: { type: 'STORE', path: CUBE_DIRECT_DATABASE_PATH },
+              storeConnections: [
+                {
+                  id: 'connection',
+                  connection: {
+                    _type: 'connectionPointer',
+                    connection: CUBE_DIRECT_CONNECTION_PATH,
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      },
+    ],
+  };
+};

package/src/graph-manager/protocol/pure/v1/V1_CubeEngineErrors.ts

@@ -18,6 +18,7 @@ import { NetworkClientError, type PlainObject } from '@finos/legend-shared';
 import {
   CubeEngineError,
   CubeEngineErrorKind,
+  type CubeSourceLocation,
   type NodeId,
 } from '../../../CubeEngine.js';
 import { V1_parseCubeSourceId } from './V1_CubeLambdaSerializer.js';
@@ -29,10 +30,43 @@ import { V1_parseCubeSourceId } from './V1_CubeLambdaSerializer.js';
 
 const COMPILATION_ERROR_TYPES = new Set(['COMPILATION', 'PARSER']);
 
+/**
+ * The engine's message when it can't fetch a pointer's project, e.g. a depot
+ * that is down or a version it doesn't have: HTML, with the URL it tried
+ */
+const DEPOT_FAILURE =
+  /unable to load information from the Pure SDLC(?: using: <a href='(?<url>[^']*)')?/u;
+
+/** Where the source information says the error is, when it gives every line and column */
+const readLocation = (
+  sourceInformation: PlainObject,
+): CubeSourceLocation | undefined => {
+  const { sourceId, startLine, startColumn, endLine, endColumn } =
+    sourceInformation;
+  const lines = [startLine, startColumn, endLine, endColumn];
+  return typeof sourceId === 'string' &&
+    lines.every((value) => Number.isInteger(Number(value)) && Number(value) > 0)
+    ? {
+        sourceId,
+        startLine: Number(startLine),
+        startColumn: Number(startColumn),
+        endLine: Number(endLine),
+        endColumn: Number(endColumn),
+      }
+    : undefined;
+};
+
 /** The engine's error payload: `{message, errorType?, sourceInformation?, trace?}` */
 const readPayload = (
   payload: unknown,
-): { message: string; errorType?: string; sourceId?: string } | undefined => {
+):
+  | {
+      message: string;
+      errorType?: string;
+      sourceId?: string;
+      location?: CubeSourceLocation;
+    }
+  | undefined => {
   if (!payload || typeof payload !== 'object') {
     return undefined;
   }
@@ -40,14 +74,20 @@ const readPayload = (
   if (typeof message !== 'string' || !message.trim()) {
     return undefined;
   }
-  const sourceId =
+  const depotFailure = DEPOT_FAILURE.exec(message);
+  const located =
     sourceInformation && typeof sourceInformation === 'object'
-      ? (sourceInformation as PlainObject).sourceId
+      ? (sourceInformation as PlainObject)
       : undefined;
+  const sourceId = located?.sourceId;
+  const location = located ? readLocation(located) : undefined;
   return {
-    message,
+    message: depotFailure
+      ? `The engine couldn't load the cube's project from its depot${depotFailure.groups?.url ? ` (${depotFailure.groups.url})` : ''}: check that the version is published and the depot can be reached`
+      : message,
     ...(typeof errorType === 'string' ? { errorType } : {}),
     ...(typeof sourceId === 'string' ? { sourceId } : {}),
+    ...(location ? { location } : {}),
   };
 };
 
@@ -72,6 +112,7 @@ export const V1_buildCubeEngineError = (
     read.message,
     origin?.nodeId ?? fallbackNodeId,
     origin?.role,
+    read.location,
   );
 };
 

package/src/graph-manager/protocol/pure/v1/V1_CubeExpression.ts

@@ -0,0 +1,89 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  isJsonObject,
+  type JsonObject,
+  type JsonValue,
+} from '@finos/legend-cube';
+import type { CubeParsedExpression } from '../../../CubeEngine.js';
+
+// An Extend column's expression (PLAN §11.7) as the engine parses it, read
+// losslessly: number literals keep their digits as strings, as Cube keeps
+// literal values (PLAN §4.9), and the serializer writes them back as numbers.
+
+const NUMBER_LITERAL_TYPES: readonly unknown[] = [
+  'integer',
+  'float',
+  'decimal',
+];
+
+const isLosslessNumber = (
+  value: unknown,
+): value is { isLosslessNumber: true; value: string } =>
+  typeof value === 'object' &&
+  value !== null &&
+  (value as { isLosslessNumber?: unknown }).isLosslessNumber === true;
+
+/** The JSON with every lossless number a JS number, but a number literal's value its digit string */
+const read = (json: unknown, inLiteral: boolean): JsonValue => {
+  if (isLosslessNumber(json)) {
+    return inLiteral ? json.value : Number(json.value);
+  }
+  if (Array.isArray(json)) {
+    return json.map((item) => read(item, false));
+  }
+  if (typeof json === 'object' && json !== null) {
+    const literal = NUMBER_LITERAL_TYPES.includes(
+      (json as { _type?: unknown })._type,
+    );
+    return Object.fromEntries(
+      Object.entries(json).map(([key, value]) => [
+        key,
+        read(value, literal && key === 'value'),
+      ]),
+    );
+  }
+  return json as JsonValue;
+};
+
+/** The JSON without any source information, at every depth */
+const withoutSourceInformation = (json: JsonValue): JsonValue =>
+  Array.isArray(json)
+    ? json.map(withoutSourceInformation)
+    : isJsonObject(json)
+      ? Object.fromEntries(
+          Object.entries(json)
+            .filter(([key]) => key !== 'sourceInformation')
+            .map(([key, value]) => [key, withoutSourceInformation(value)]),
+        )
+      : json;
+
+/**
+ * An expression as the engine's lossless JSON gives it: located, as parsed,
+ * and as a column stores it, without source information. Anything but an
+ * object is the engine's mistake.
+ */
+export const V1_readCubeExpression = (json: unknown): CubeParsedExpression => {
+  const located = read(json, false);
+  if (!isJsonObject(located)) {
+    throw new Error(`The engine parsed the expression as no object`);
+  }
+  return {
+    located,
+    lambda: withoutSourceInformation(located) as JsonObject,
+  };
+};

package/src/graph-manager/protocol/pure/v1/V1_CubeIngestDefinition.ts

@@ -0,0 +1,201 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  type CubeType,
+  OpaqueType,
+  PRIMITIVE_TYPE_PATH,
+  PrimitiveType,
+  resolveCubeType,
+  Schema,
+  SchemaColumn,
+} from '@finos/legend-cube';
+import {
+  V1_CInteger,
+  type V1_IngestDataset,
+  type V1_IngestDatasetSchema,
+  type V1_IngestDatasetSource,
+  V1_PackageableType,
+  type V1_RelationTypeColumn,
+  V1_WriteModeType,
+  type V1_WriteMode,
+  V1_deserializeIngestDefinitionContent,
+} from '@finos/legend-graph';
+import type { PlainObject } from '@finos/legend-shared';
+import { CubeIngestDataSet } from '../../../CubeIngestCatalog.js';
+
+// A deployed ingest definition's data sets, typed from what the definition
+// declares, with no engine call: each data set's columns, with their type
+// parameters, then the columns its write mode adds, as legend-graph types
+// them (#5598). A data set Cube can't read is listed with the reason, and
+// can't be picked
+
+/** A data set whose rows come from a query over other data sets */
+const MATERIALIZED_VIEW_SOURCE = 'FunctionSource';
+
+/** Why a data set can't be picked */
+export const V1_CUBE_INGEST_DATA_SET_REASON = {
+  MATERIALIZED_VIEW:
+    'It is a materialized view, which Cube does not support yet',
+  NO_COLUMNS: 'It declares no columns',
+  BAD_MULTIPLICITY: (column: string): string =>
+    `Column "${column}" holds more than one value, which Cube can't show`,
+  UNKNOWN_TYPE: (column: string, type: string): string =>
+    `Column "${column}" has a type Cube doesn't know: ${type}`,
+} as const;
+
+/** The columns a batch-milestoned data set adds after its own, by write mode */
+const milestoningColumns = (
+  writeMode: V1_WriteMode | undefined,
+): SchemaColumn[] => {
+  const type = writeMode?._type;
+  if (
+    type !== V1_WriteModeType.BATCH_MILESTONED &&
+    type !== V1_WriteModeType.BATCH_MILESTONED_BUSINESS_TEMPORAL
+  ) {
+    return [];
+  }
+  const integer = PrimitiveType.get(PRIMITIVE_TYPE_PATH.INTEGER);
+  const columns = [
+    new SchemaColumn('LAKE_IN_ID', integer, false),
+    new SchemaColumn('LAKE_OUT_ID', integer, false),
+    new SchemaColumn(
+      'LAKE_DIGEST',
+      PrimitiveType.get(PRIMITIVE_TYPE_PATH.STRING),
+      false,
+    ),
+  ];
+  if (type === V1_WriteModeType.BATCH_MILESTONED_BUSINESS_TEMPORAL) {
+    const timestamp = PrimitiveType.get(PRIMITIVE_TYPE_PATH.TIMESTAMP);
+    columns.push(
+      new SchemaColumn('LAKE_FROM', timestamp, false),
+      new SchemaColumn('LAKE_THRU', timestamp, false),
+    );
+  }
+  return columns;
+};
+
+/**
+ * A declared column's type, by its full path or, for a primitive whose path
+ * has a package (e.g. `meta::pure::metamodel::type::Integer`), its name;
+ * parameters kept. Any other type, such as an enumeration, gives the reason
+ * the data set can't be picked.
+ */
+const readColumnType = (
+  column: V1_RelationTypeColumn,
+): { type: CubeType } | { reason: string } => {
+  const { rawType, typeVariableValues } = column.genericType;
+  if (!(rawType instanceof V1_PackageableType)) {
+    return {
+      reason: V1_CUBE_INGEST_DATA_SET_REASON.UNKNOWN_TYPE(
+        column.name,
+        'not a named type',
+      ),
+    };
+  }
+  const params: number[] = [];
+  for (const value of typeVariableValues) {
+    if (!(value instanceof V1_CInteger)) {
+      return {
+        reason: V1_CUBE_INGEST_DATA_SET_REASON.UNKNOWN_TYPE(
+          column.name,
+          rawType.fullPath,
+        ),
+      };
+    }
+    params.push(value.value);
+  }
+  const name = rawType.fullPath.slice(rawType.fullPath.lastIndexOf(':') + 1);
+  const type = [rawType.fullPath, name]
+    .map((path) => resolveCubeType(path, params))
+    .find((candidate) => !(candidate instanceof OpaqueType));
+  return type
+    ? { type }
+    : {
+        reason: V1_CUBE_INGEST_DATA_SET_REASON.UNKNOWN_TYPE(
+          column.name,
+          params.length
+            ? `${rawType.fullPath}(${params.join(', ')})`
+            : rawType.fullPath,
+        ),
+      };
+};
+
+/** A data set's schema, or why it can't be picked */
+const readSchema = (
+  dataset: V1_IngestDataset,
+  definitionWriteMode: V1_WriteMode | undefined,
+): { schema: Schema } | { reason: string } => {
+  // the protocol types both as always there, but saved content may lack them
+  const source = dataset.source as V1_IngestDatasetSource | undefined;
+  if (source?._type === MATERIALIZED_VIEW_SOURCE) {
+    return { reason: V1_CUBE_INGEST_DATA_SET_REASON.MATERIALIZED_VIEW };
+  }
+  const schema: V1_IngestDatasetSchema | undefined = source?.schema;
+  const declared = schema?.columns ?? [];
+  if (!declared.length) {
+    return { reason: V1_CUBE_INGEST_DATA_SET_REASON.NO_COLUMNS };
+  }
+  const columns: SchemaColumn[] = [];
+  for (const column of declared) {
+    if (column.multiplicity.upperBound !== 1) {
+      return {
+        reason: V1_CUBE_INGEST_DATA_SET_REASON.BAD_MULTIPLICITY(column.name),
+      };
+    }
+    const read = readColumnType(column);
+    if ('reason' in read) {
+      return read;
+    }
+    columns.push(
+      new SchemaColumn(
+        column.name,
+        read.type,
+        column.multiplicity.lowerBound === 0,
+      ),
+    );
+  }
+  try {
+    return {
+      schema: new Schema([
+        ...columns,
+        ...milestoningColumns(dataset.writeMode ?? definitionWriteMode),
+      ]),
+    };
+  } catch (error) {
+    return { reason: error instanceof Error ? error.message : String(error) };
+  }
+};
+
+/**
+ * The data sets of an ingest definition's content (its protocol JSON), in
+ * order. Throws when the content can't be read at all.
+ */
+export const V1_readCubeIngestDataSets = (
+  content: PlainObject,
+): CubeIngestDataSet[] => {
+  const definition = V1_deserializeIngestDefinitionContent(content);
+  return (definition.datasets ?? []).map((dataset) => {
+    const read = readSchema(dataset, definition.writeMode);
+    return new CubeIngestDataSet({
+      name: dataset.name,
+      primaryKey: dataset.primaryKey,
+      ...('schema' in read
+        ? { schema: read.schema }
+        : { disabledReason: read.reason }),
+    });
+  });
+};