@finos/legend-cube-builder 0.0.3 → 0.0.5

package/src/graph-manager/protocol/pure/v1/V1_CubeDirectConnection.ts

@@ -0,0 +1,339 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { PlainObject } from '@finos/legend-shared';
+import {
+  CubeConnectionDatasourceKind,
+  CubeDirectDatabaseType,
+  type CubeConnectionDraft,
+  type CubeConnectionSummary,
+} from '../../../CubeConnectionExplorer.js';
+
+// A direct connection's protocol JSON (PLAN §6.8), read and written as plain
+// JSON. legend-graph's connection serializers are not used: they drop a DuckDB
+// connection's setup SQL and an H2 one's identifier setting, and a saved
+// connection must reach the engine exactly as it was saved. A connection is
+// checked against what Cube supports, and anything else is refused, never
+// dropped, since an imported cube is saved again unchanged
+
+const CONNECTION_TYPE = 'RelationalDatabaseConnection';
+const H2_DATASOURCE_TYPE = 'h2Local';
+const DUCKDB_DATASOURCE_TYPE = 'duckDB';
+const H2_DEFAULT_AUTHENTICATION_TYPE = 'h2Default';
+const TEST_AUTHENTICATION_TYPE = 'test';
+
+/** Where the engine records source positions: kept, and ignored when comparing */
+const SOURCE_INFORMATION_KEYS = [
+  'sourceInformation',
+  'elementSourceInformation',
+];
+/** The store the connection is bound to: set only in the model built for each call */
+const ELEMENT_KEY = 'element';
+
+/** The authentication strategies, as the engine names them, each database allows */
+const AUTHENTICATION_TYPES: Record<CubeDirectDatabaseType, readonly string[]> =
+  {
+    [CubeDirectDatabaseType.H2]: [
+      H2_DEFAULT_AUTHENTICATION_TYPE,
+      TEST_AUTHENTICATION_TYPE,
+    ],
+    [CubeDirectDatabaseType.DUCKDB]: [TEST_AUTHENTICATION_TYPE],
+  };
+
+const isEmptyList = (value: unknown): boolean =>
+  value === undefined ||
+  value === null ||
+  (Array.isArray(value) && value.length === 0);
+
+/**
+ * Connection settings Cube doesn't support yet, accepted only at the engine's
+ * default value: each changes how a query runs, in ways Cube doesn't show
+ */
+const UNSUPPORTED_SETTINGS: Record<string, (value: unknown) => boolean> = {
+  localMode: (value) =>
+    value === undefined || value === null || value === false,
+  quoteIdentifiers: (value) =>
+    value === undefined || value === null || value === false,
+  timeZone: (value) => value === undefined || value === null,
+  queryTimeOutInSeconds: (value) => value === undefined || value === null,
+  queryGenerationConfigs: (value) => isEmptyList(value),
+  // a schema mapper ran a query on another schema with no error
+  postProcessors: (value) => isEmptyList(value),
+  postProcessorWithParameter: (value) => isEmptyList(value),
+};
+
+const isObject = (value: unknown): value is PlainObject =>
+  Boolean(value) && typeof value === 'object' && !Array.isArray(value);
+
+const isDatabaseType = (value: unknown): value is CubeDirectDatabaseType =>
+  Object.values(CubeDirectDatabaseType).includes(
+    value as CubeDirectDatabaseType,
+  );
+
+const isStatementList = (value: unknown): value is string[] =>
+  Array.isArray(value) &&
+  value.every((statement) => typeof statement === 'string');
+
+/** The keys of an object that are neither expected nor source information */
+const unexpectedKeys = (
+  object: PlainObject,
+  expected: readonly string[],
+): string[] =>
+  Object.keys(object).filter(
+    (key) => !expected.includes(key) && !SOURCE_INFORMATION_KEYS.includes(key),
+  );
+
+const checkDatasource = (
+  databaseType: CubeDirectDatabaseType,
+  datasource: PlainObject,
+): string[] => {
+  const problems: string[] = [];
+  // a CSV setup is the engine's own SQL generation, which lost rows in tests
+  if (
+    datasource.testDataSetupCsv !== undefined &&
+    datasource.testDataSetupCsv !== null &&
+    datasource.testDataSetupCsv !== ''
+  ) {
+    problems.push(
+      "The connection sets up its data from CSV, which Cube doesn't support: use setup SQL statements",
+    );
+  }
+  if (databaseType === CubeDirectDatabaseType.H2) {
+    if (datasource._type !== H2_DATASOURCE_TYPE) {
+      problems.push(
+        'An H2 connection must use a local H2 database, created by its setup SQL',
+      );
+      return problems;
+    }
+    if (
+      datasource.disableDatabaseToUpper !== undefined &&
+      datasource.disableDatabaseToUpper !== null &&
+      datasource.disableDatabaseToUpper !== false
+    ) {
+      problems.push(
+        "The connection keeps H2 from upper-casing names, which Cube doesn't support",
+      );
+    }
+    if (
+      !isStatementList(datasource.testDataSetupSqls) ||
+      datasource.testDataSetupSqls.length === 0
+    ) {
+      // without setup SQL the engine connects to an H2 server that isn't there
+      problems.push('An H2 connection needs at least one setup SQL statement');
+    }
+    unexpectedKeys(datasource, [
+      '_type',
+      'testDataSetupSqls',
+      'testDataSetupCsv',
+      'disableDatabaseToUpper',
+    ]).forEach((key) =>
+      problems.push(
+        `The connection's database has a property Cube doesn't know: "${key}"`,
+      ),
+    );
+  } else {
+    if (datasource._type !== DUCKDB_DATASOURCE_TYPE) {
+      problems.push(
+        'A DuckDB connection must name a DuckDB database file, or none for an in-memory database',
+      );
+      return problems;
+    }
+    if (typeof datasource.path !== 'string') {
+      problems.push(
+        'A DuckDB connection must name a database file, or none for an in-memory database',
+      );
+    }
+    if (
+      datasource.testDataSetupSqls !== undefined &&
+      !isStatementList(datasource.testDataSetupSqls)
+    ) {
+      problems.push("The connection's setup SQL must be a list of statements");
+    }
+    unexpectedKeys(datasource, [
+      '_type',
+      'path',
+      'testDataSetupSqls',
+      'testDataSetupCsv',
+    ]).forEach((key) =>
+      problems.push(
+        `The connection's database has a property Cube doesn't know: "${key}"`,
+      ),
+    );
+  }
+  return problems;
+};
+
+const checkAuthentication = (
+  databaseType: CubeDirectDatabaseType,
+  authentication: PlainObject,
+): string[] => {
+  const allowed = AUTHENTICATION_TYPES[databaseType];
+  if (!allowed.includes(authentication._type as string)) {
+    return [
+      `${databaseType === CubeDirectDatabaseType.H2 ? 'An' : 'A'} ${databaseType} connection's authentication must be ${allowed
+        .map((type) => `"${type}"`)
+        .join(' or ')}`,
+    ];
+  }
+  // these strategies take no settings, so no secret can be saved with them
+  return unexpectedKeys(authentication, ['_type']).map(
+    (key) =>
+      `The connection's authentication has a property Cube doesn't know: "${key}"`,
+  );
+};
+
+/**
+ * What keeps Cube from using a connection, in words a user can act on: none
+ * when the connection is one Cube supports. Values are never repeated, only
+ * the names of properties, since a property Cube doesn't know could hold a
+ * secret
+ */
+export const V1_checkCubeDirectConnection = (
+  connection: unknown,
+): readonly string[] => {
+  if (!isObject(connection) || connection._type !== CONNECTION_TYPE) {
+    return ['The connection is not a relational database connection'];
+  }
+  const { type, databaseType } = connection;
+  if (!isDatabaseType(databaseType)) {
+    return [
+      `Cube connects to H2 and DuckDB databases only, not to "${String(databaseType)}"`,
+    ];
+  }
+  const problems: string[] = [];
+  if (type !== databaseType) {
+    problems.push(
+      "The connection's type and database type must be the same database",
+    );
+  }
+  Object.entries(UNSUPPORTED_SETTINGS).forEach(([key, isDefault]) => {
+    if (!isDefault(connection[key])) {
+      problems.push(`The connection sets "${key}", which Cube doesn't support`);
+    }
+  });
+  unexpectedKeys(connection, [
+    '_type',
+    'type',
+    'databaseType',
+    'datasourceSpecification',
+    'authenticationStrategy',
+    ELEMENT_KEY,
+    ...Object.keys(UNSUPPORTED_SETTINGS),
+  ]).forEach((key) =>
+    problems.push(`The connection has a property Cube doesn't know: "${key}"`),
+  );
+  if (isObject(connection.datasourceSpecification)) {
+    problems.push(
+      ...checkDatasource(databaseType, connection.datasourceSpecification),
+    );
+  } else {
+    problems.push("The connection doesn't say how to reach its database");
+  }
+  if (isObject(connection.authenticationStrategy)) {
+    problems.push(
+      ...checkAuthentication(databaseType, connection.authenticationStrategy),
+    );
+  } else {
+    problems.push("The connection doesn't say how to authenticate");
+  }
+  return problems;
+};
+
+/**
+ * The connection the form describes, as the engine reads it. It names no
+ * store: each call binds it to the Database built for that call
+ */
+export const V1_buildCubeDirectConnection = (
+  draft: CubeConnectionDraft,
+): PlainObject => {
+  const setupSqls = [...draft.setupSqls];
+  return draft.databaseType === CubeDirectDatabaseType.H2
+    ? {
+        _type: CONNECTION_TYPE,
+        type: CubeDirectDatabaseType.H2,
+        databaseType: CubeDirectDatabaseType.H2,
+        datasourceSpecification: {
+          _type: H2_DATASOURCE_TYPE,
+          testDataSetupSqls: setupSqls,
+        },
+        authenticationStrategy: { _type: H2_DEFAULT_AUTHENTICATION_TYPE },
+      }
+    : {
+        _type: CONNECTION_TYPE,
+        type: CubeDirectDatabaseType.DUCKDB,
+        databaseType: CubeDirectDatabaseType.DUCKDB,
+        datasourceSpecification: {
+          _type: DUCKDB_DATASOURCE_TYPE,
+          path: draft.path ?? '',
+          ...(setupSqls.length ? { testDataSetupSqls: setupSqls } : {}),
+        },
+        authenticationStrategy: { _type: TEST_AUTHENTICATION_TYPE },
+      };
+};
+
+/**
+ * What the page shows of a connection Cube supports (see
+ * `V1_checkCubeDirectConnection`): never its setup SQL
+ */
+export const V1_summarizeCubeDirectConnection = (
+  connection: PlainObject,
+): CubeConnectionSummary => {
+  const databaseType = connection.databaseType as CubeDirectDatabaseType;
+  const datasource = connection.datasourceSpecification as PlainObject;
+  const authentication = connection.authenticationStrategy as PlainObject;
+  const path = typeof datasource.path === 'string' ? datasource.path : '';
+  const setupSqls = datasource.testDataSetupSqls;
+  return {
+    databaseType,
+    datasourceKind:
+      databaseType === CubeDirectDatabaseType.H2
+        ? CubeConnectionDatasourceKind.LOCAL_H2
+        : path
+          ? CubeConnectionDatasourceKind.DUCKDB_FILE
+          : CubeConnectionDatasourceKind.DUCKDB_IN_MEMORY,
+    path:
+      databaseType === CubeDirectDatabaseType.DUCKDB && path ? path : undefined,
+    setupSqlCount: Array.isArray(setupSqls) ? setupSqls.length : 0,
+    authenticationKind: String(authentication._type),
+  };
+};
+
+const canonicalValue = (value: unknown): unknown => {
+  if (Array.isArray(value)) {
+    return value.map(canonicalValue);
+  }
+  if (isObject(value)) {
+    return Object.fromEntries(
+      Object.keys(value)
+        .filter(
+          (key) =>
+            key !== ELEMENT_KEY && !SOURCE_INFORMATION_KEYS.includes(key),
+        )
+        .sort()
+        .map((key) => [key, canonicalValue(value[key])]),
+    );
+  }
+  return value;
+};
+
+/**
+ * A connection as text that two connections share exactly when they reach the
+ * same database the same way: key order, the bound store and source
+ * positions don't count. Used to compare connections and to key caches
+ */
+export const V1_canonicalCubeDirectConnection = (
+  connection: PlainObject,
+): string => JSON.stringify(canonicalValue(connection));

package/src/graph-manager/protocol/pure/v1/V1_CubeDirectModel.ts

@@ -0,0 +1,151 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { IR } from '@finos/legend-cube';
+import type { PlainObject } from '@finos/legend-shared';
+import type { CubeDirectConnection } from '../../../CubeConnectionExplorer.js';
+import {
+  CUBE_DIRECT_CONNECTION_PATH,
+  CUBE_DIRECT_DATABASE_PATH,
+  CUBE_DIRECT_RUNTIME_PATH,
+} from '../../../CubeDirectConnection.js';
+import type { AccessorPath } from '../../../CubeEngine.js';
+import {
+  type V1_CubeExploredTablePath,
+  V1_quoteCubeName,
+} from './V1_CubeSchemaExploration.js';
+
+// The model a direct-connection cube's engine call runs on (PLAN §6.8): one
+// Database holding exactly the tables the call needs, the saved connection
+// bound to it, and a runtime with that one connection. It is built as protocol
+// JSON for each call, never saved: the DuckDB grammar has no setup SQL, so a
+// model in Pure text would lose it
+
+/** A table of the built Database, as read from the connection's database */
+export interface V1_CubeDirectTable {
+  /** As the database stores them */
+  path: V1_CubeExploredTablePath;
+  /** The table's definition, every name quoted, without the columns Cube hides */
+  definition: PlainObject;
+}
+
+const splitPath = (path: string): { package: string; name: string } => {
+  const index = path.lastIndexOf('::');
+  return { package: path.slice(0, index), name: path.slice(index + 2) };
+};
+
+/** By code point, the same in every locale */
+const compare = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
+
+/** The built Database's package and name, which schema exploration takes */
+export const V1_CUBE_DIRECT_DATABASE = splitPath(CUBE_DIRECT_DATABASE_PATH);
+
+/**
+ * The store accessors of a lambda, each once: found anywhere in it, so a new
+ * kind of IR node needs nothing here
+ */
+export const V1_collectCubeStoreAccessors = (
+  ir: IR,
+): readonly AccessorPath[] => {
+  const found = new Map<string, AccessorPath>();
+  const visit = (value: unknown): void => {
+    if (Array.isArray(value)) {
+      value.forEach(visit);
+      return;
+    }
+    if (!value || typeof value !== 'object') {
+      return;
+    }
+    const node = value as PlainObject;
+    if (node.k === 'storeAccessor' && Array.isArray(node.path)) {
+      const path = node.path as unknown as AccessorPath;
+      found.set(JSON.stringify(path), path);
+      return;
+    }
+    if (node.k === 'raw') {
+      return;
+    }
+    Object.values(node).forEach(visit);
+  };
+  visit(ir);
+  return [...found.values()];
+};
+
+/**
+ * The model of one call: the tables in a fixed order (by schema, then table),
+ * so the same tables always give the same model
+ */
+export const V1_buildCubeDirectModelContext = (
+  connection: CubeDirectConnection,
+  tables: readonly V1_CubeDirectTable[],
+): PlainObject => {
+  const schemas = new Map<string, PlainObject[]>();
+  [...tables]
+    .sort(
+      (a, b) => compare(a.path[0], b.path[0]) || compare(a.path[1], b.path[1]),
+    )
+    .forEach(({ path: [schema], definition }) =>
+      schemas.set(schema, [...(schemas.get(schema) ?? []), definition]),
+    );
+  return {
+    _type: 'data',
+    elements: [
+      {
+        _type: 'relational',
+        ...splitPath(CUBE_DIRECT_DATABASE_PATH),
+        includedStores: [],
+        filters: [],
+        joins: [],
+        stereotypes: [],
+        schemas: [...schemas.entries()].map(([schema, definitions]) => ({
+          name: V1_quoteCubeName(schema),
+          tables: definitions,
+          views: [],
+          tabularFunctions: [],
+        })),
+      },
+      {
+        _type: 'connection',
+        ...splitPath(CUBE_DIRECT_CONNECTION_PATH),
+        // bound to the built Database: the engine needs the store it serves
+        connectionValue: { ...connection, element: CUBE_DIRECT_DATABASE_PATH },
+      },
+      {
+        _type: 'runtime',
+        ...splitPath(CUBE_DIRECT_RUNTIME_PATH),
+        runtimeValue: {
+          _type: 'engineRuntime',
+          mappings: [],
+          connectionStores: [],
+          connections: [
+            {
+              store: { type: 'STORE', path: CUBE_DIRECT_DATABASE_PATH },
+              storeConnections: [
+                {
+                  id: 'connection',
+                  connection: {
+                    _type: 'connectionPointer',
+                    connection: CUBE_DIRECT_CONNECTION_PATH,
+                  },
+                },
+              ],
+            },
+          ],
+        },
+      },
+    ],
+  };
+};

package/src/graph-manager/protocol/pure/v1/V1_CubeEngineErrors.ts

@@ -18,6 +18,7 @@ import { NetworkClientError, type PlainObject } from '@finos/legend-shared';
 import {
   CubeEngineError,
   CubeEngineErrorKind,
+  type CubeSourceLocation,
   type NodeId,
 } from '../../../CubeEngine.js';
 import { V1_parseCubeSourceId } from './V1_CubeLambdaSerializer.js';
@@ -29,10 +30,43 @@ import { V1_parseCubeSourceId } from './V1_CubeLambdaSerializer.js';
 
 const COMPILATION_ERROR_TYPES = new Set(['COMPILATION', 'PARSER']);
 
+/**
+ * The engine's message when it can't fetch a pointer's project, e.g. a depot
+ * that is down or a version it doesn't have: HTML, with the URL it tried
+ */
+const DEPOT_FAILURE =
+  /unable to load information from the Pure SDLC(?: using: <a href='(?<url>[^']*)')?/u;
+
+/** Where the source information says the error is, when it gives every line and column */
+const readLocation = (
+  sourceInformation: PlainObject,
+): CubeSourceLocation | undefined => {
+  const { sourceId, startLine, startColumn, endLine, endColumn } =
+    sourceInformation;
+  const lines = [startLine, startColumn, endLine, endColumn];
+  return typeof sourceId === 'string' &&
+    lines.every((value) => Number.isInteger(Number(value)) && Number(value) > 0)
+    ? {
+        sourceId,
+        startLine: Number(startLine),
+        startColumn: Number(startColumn),
+        endLine: Number(endLine),
+        endColumn: Number(endColumn),
+      }
+    : undefined;
+};
+
 /** The engine's error payload: `{message, errorType?, sourceInformation?, trace?}` */
 const readPayload = (
   payload: unknown,
-): { message: string; errorType?: string; sourceId?: string } | undefined => {
+):
+  | {
+      message: string;
+      errorType?: string;
+      sourceId?: string;
+      location?: CubeSourceLocation;
+    }
+  | undefined => {
   if (!payload || typeof payload !== 'object') {
     return undefined;
   }
@@ -40,14 +74,20 @@ const readPayload = (
   if (typeof message !== 'string' || !message.trim()) {
     return undefined;
   }
-  const sourceId =
+  const depotFailure = DEPOT_FAILURE.exec(message);
+  const located =
     sourceInformation && typeof sourceInformation === 'object'
-      ? (sourceInformation as PlainObject).sourceId
+      ? (sourceInformation as PlainObject)
       : undefined;
+  const sourceId = located?.sourceId;
+  const location = located ? readLocation(located) : undefined;
   return {
-    message,
+    message: depotFailure
+      ? `The engine couldn't load the cube's project from its depot${depotFailure.groups?.url ? ` (${depotFailure.groups.url})` : ''}: check that the version is published and the depot can be reached`
+      : message,
     ...(typeof errorType === 'string' ? { errorType } : {}),
     ...(typeof sourceId === 'string' ? { sourceId } : {}),
+    ...(location ? { location } : {}),
   };
 };
 
@@ -72,6 +112,7 @@ export const V1_buildCubeEngineError = (
     read.message,
     origin?.nodeId ?? fallbackNodeId,
     origin?.role,
+    read.location,
   );
 };
 

package/src/graph-manager/protocol/pure/v1/V1_CubeExpression.ts

@@ -0,0 +1,89 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  isJsonObject,
+  type JsonObject,
+  type JsonValue,
+} from '@finos/legend-cube';
+import type { CubeParsedExpression } from '../../../CubeEngine.js';
+
+// An Extend column's expression (PLAN §11.7) as the engine parses it, read
+// losslessly: number literals keep their digits as strings, as Cube keeps
+// literal values (PLAN §4.9), and the serializer writes them back as numbers.
+
+const NUMBER_LITERAL_TYPES: readonly unknown[] = [
+  'integer',
+  'float',
+  'decimal',
+];
+
+const isLosslessNumber = (
+  value: unknown,
+): value is { isLosslessNumber: true; value: string } =>
+  typeof value === 'object' &&
+  value !== null &&
+  (value as { isLosslessNumber?: unknown }).isLosslessNumber === true;
+
+/** The JSON with every lossless number a JS number, but a number literal's value its digit string */
+const read = (json: unknown, inLiteral: boolean): JsonValue => {
+  if (isLosslessNumber(json)) {
+    return inLiteral ? json.value : Number(json.value);
+  }
+  if (Array.isArray(json)) {
+    return json.map((item) => read(item, false));
+  }
+  if (typeof json === 'object' && json !== null) {
+    const literal = NUMBER_LITERAL_TYPES.includes(
+      (json as { _type?: unknown })._type,
+    );
+    return Object.fromEntries(
+      Object.entries(json).map(([key, value]) => [
+        key,
+        read(value, literal && key === 'value'),
+      ]),
+    );
+  }
+  return json as JsonValue;
+};
+
+/** The JSON without any source information, at every depth */
+const withoutSourceInformation = (json: JsonValue): JsonValue =>
+  Array.isArray(json)
+    ? json.map(withoutSourceInformation)
+    : isJsonObject(json)
+      ? Object.fromEntries(
+          Object.entries(json)
+            .filter(([key]) => key !== 'sourceInformation')
+            .map(([key, value]) => [key, withoutSourceInformation(value)]),
+        )
+      : json;
+
+/**
+ * An expression as the engine's lossless JSON gives it: located, as parsed,
+ * and as a column stores it, without source information. Anything but an
+ * object is the engine's mistake.
+ */
+export const V1_readCubeExpression = (json: unknown): CubeParsedExpression => {
+  const located = read(json, false);
+  if (!isJsonObject(located)) {
+    throw new Error(`The engine parsed the expression as no object`);
+  }
+  return {
+    located,
+    lambda: withoutSourceInformation(located) as JsonObject,
+  };
+};