@finos/legend-cube-builder 0.0.3 → 0.0.5

package/src/graph-manager/protocol/pure/v1/V1_CubeFunctionOutlineBuilder.ts

@@ -0,0 +1,312 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { PlainObject } from '@finos/legend-shared';
+import type {
+  CubeOutlineFunction,
+  CubeOutlineFunctionColumn,
+  CubeOutlineFunctionParameter,
+} from '../../../CubeEngine.js';
+
+// The functions of a parsed model that return a relation (PLAN §6.5), read
+// as plain JSON like the rest of the outline: their parameters, the columns
+// they declare, and the runtime their body reads through, if any. A cube on
+// a function with its own `from(runtime)` must run on that runtime: run on
+// another, the query reads different databases depending on its shape.
+
+const FUNCTION_TYPE = 'function';
+const RELATION_TYPES = [
+  'Relation',
+  'meta::pure::metamodel::relation::Relation',
+];
+const FROM_FUNCTIONS = ['from', 'meta::pure::mapping::from'];
+/** How deep calls are followed, for a runtime; deeper calls aren't read */
+const MAX_CALL_DEPTH = 20;
+
+/** The primitive types a parameter can take a value of (user, 2026-10-12) */
+const VALUE_TYPES = [
+  'String',
+  'Integer',
+  'Float',
+  'Decimal',
+  'Boolean',
+  'StrictDate',
+  'DateTime',
+  'Date',
+];
+
+export const V1_CUBE_FUNCTION_PROBLEM = {
+  PARAMETER_TYPE: (name: string, type: string): string =>
+    `Its parameter "${name}" is of type ${type}, which Cube can't take a value for yet`,
+  PARAMETER_MULTIPLICITY: (name: string, multiplicity: string): string =>
+    `Its parameter "${name}" takes ${multiplicity} values; Cube takes one value, or none`,
+  RUNTIME_NOT_AN_ELEMENT:
+    "It reads through a runtime given by an expression, which Cube can't tell",
+  TWO_RUNTIMES: (runtimes: readonly string[]): string =>
+    `It reads through more than one runtime: ${runtimes.join(', ')}`,
+} as const;
+
+const asObject = (value: unknown): PlainObject =>
+  value && typeof value === 'object' && !Array.isArray(value)
+    ? (value as PlainObject)
+    : {};
+
+const asList = (value: unknown): unknown[] =>
+  Array.isArray(value) ? value : [];
+
+const asString = (value: unknown): string | undefined =>
+  typeof value === 'string' ? value : undefined;
+
+const lastSegment = (path: string): string => {
+  const index = path.lastIndexOf('::');
+  return index < 0 ? path : path.slice(index + 2);
+};
+
+/** `[1]`, `[0..1]`, `[*]`, `[1..*]`, as the grammar writes it */
+const multiplicityText = (json: unknown): string => {
+  const { lowerBound, upperBound } = asObject(json);
+  const lower = typeof lowerBound === 'number' ? lowerBound : 0;
+  const upper = typeof upperBound === 'number' ? upperBound : undefined;
+  if (upper === undefined) {
+    return lower === 0 ? '[*]' : `[${lower}..*]`;
+  }
+  return lower === upper ? `[${lower}]` : `[${lower}..${upper}]`;
+};
+
+/** The engine's suffix for a multiplicity in a function's full name */
+const multiplicitySuffix = (json: unknown): string => {
+  const { lowerBound, upperBound } = asObject(json);
+  const lower = typeof lowerBound === 'number' ? lowerBound : 0;
+  const upper = typeof upperBound === 'number' ? upperBound : undefined;
+  if (upper === undefined) {
+    return lower === 0 ? 'MANY' : `$${lower}_MANY$`;
+  }
+  return lower === upper ? String(lower) : `$${lower}_${upper}$`;
+};
+
+const typePathOf = (genericType: unknown): string =>
+  asString(asObject(asObject(genericType).rawType).fullPath) ?? '';
+
+const returnsRelation = (element: PlainObject): boolean =>
+  RELATION_TYPES.includes(typePathOf(element.returnGenericType)) &&
+  multiplicityText(element.returnMultiplicity) === '[1]';
+
+/**
+ * The path a function is called by: its full name without the signature the
+ * engine appends (`ordersAbove_Float_1__Relation_1_` → `ordersAbove`)
+ */
+const callPathOf = (element: PlainObject): string => {
+  const name = asString(element.name) ?? '';
+  const pkg = asString(element.package);
+  const suffix = `_${asList(element.parameters)
+    .map((parameter) => {
+      const json = asObject(parameter);
+      return `${lastSegment(typePathOf(json.genericType))}_${multiplicitySuffix(json.multiplicity)}_`;
+    })
+    .join(
+      '',
+    )}_${lastSegment(typePathOf(element.returnGenericType))}_${multiplicitySuffix(element.returnMultiplicity)}_`;
+  const base = name.endsWith(suffix)
+    ? name.slice(0, -suffix.length)
+    : // a name the engine wrote otherwise: its first segment
+      (name.split('_')[0] ?? name);
+  return pkg ? `${pkg}::${base}` : base;
+};
+
+const signatureOf = (element: PlainObject): string => {
+  const name = asString(element.name) ?? '';
+  const pkg = asString(element.package);
+  return pkg ? `${pkg}::${name}` : name;
+};
+
+const buildParameter = (json: unknown): CubeOutlineFunctionParameter => {
+  const parameter = asObject(json);
+  return {
+    name: asString(parameter.name) ?? '',
+    type: typePathOf(parameter.genericType),
+    multiplicity: multiplicityText(parameter.multiplicity),
+  };
+};
+
+const buildColumns = (element: PlainObject): CubeOutlineFunctionColumn[] => {
+  const [relationType] = asList(
+    asObject(element.returnGenericType).typeArguments,
+  );
+  return asList(asObject(asObject(relationType).rawType).columns).map(
+    (json) => {
+      const columnJson = asObject(json);
+      return {
+        name: asString(columnJson.name) ?? '',
+        type: typePathOf(columnJson.genericType),
+      };
+    },
+  );
+};
+
+/** Why a parameter can't be given a value, if it can't */
+const parameterProblemOf = (
+  parameter: CubeOutlineFunctionParameter,
+): string | undefined =>
+  !VALUE_TYPES.includes(parameter.type)
+    ? V1_CUBE_FUNCTION_PROBLEM.PARAMETER_TYPE(
+        parameter.name,
+        lastSegment(parameter.type) || '(unknown)',
+      )
+    : !['[1]', '[0..1]'].includes(parameter.multiplicity)
+      ? V1_CUBE_FUNCTION_PROBLEM.PARAMETER_MULTIPLICITY(
+          parameter.name,
+          parameter.multiplicity,
+        )
+      : undefined;
+
+/** What a function's body reads through: runtime paths, and whether one is an expression */
+interface RuntimeUse {
+  readonly runtimes: Set<string>;
+  notAnElement: boolean;
+}
+
+/**
+ * Walks the model's functions for the runtimes their `from()` calls name,
+ * following calls to the model's functions (by path, or by name when only one
+ * function has it), each function read once
+ */
+class V1_CubeRuntimeFinder {
+  private readonly byPath = new Map<string, PlainObject[]>();
+  private readonly byName = new Map<string, PlainObject[]>();
+  private readonly uses = new Map<PlainObject, RuntimeUse>();
+
+  constructor(functions: readonly PlainObject[]) {
+    functions.forEach((element) => {
+      const path = callPathOf(element);
+      this.byPath.set(path, [...(this.byPath.get(path) ?? []), element]);
+      const name = lastSegment(path);
+      this.byName.set(name, [...(this.byName.get(name) ?? []), element]);
+    });
+  }
+
+  useOf(element: PlainObject): RuntimeUse {
+    const use: RuntimeUse = { runtimes: new Set(), notAnElement: false };
+    this.walkFunction(element, use, new Set(), 0);
+    return use;
+  }
+
+  private walkFunction(
+    element: PlainObject,
+    use: RuntimeUse,
+    visiting: Set<PlainObject>,
+    depth: number,
+  ): void {
+    const known = this.uses.get(element);
+    if (known) {
+      known.runtimes.forEach((runtime) => use.runtimes.add(runtime));
+      use.notAnElement ||= known.notAnElement;
+      return;
+    }
+    if (visiting.has(element) || depth > MAX_CALL_DEPTH) {
+      return;
+    }
+    visiting.add(element);
+    const own: RuntimeUse = { runtimes: new Set(), notAnElement: false };
+    asList(element.body).forEach((value) =>
+      this.walkValue(value, own, visiting, depth),
+    );
+    visiting.delete(element);
+    this.uses.set(element, own);
+    own.runtimes.forEach((runtime) => use.runtimes.add(runtime));
+    use.notAnElement ||= own.notAnElement;
+  }
+
+  private calledFunctions(name: string): readonly PlainObject[] {
+    return (
+      this.byPath.get(name) ??
+      (!name.includes('::') && this.byName.get(name)?.length === 1
+        ? (this.byName.get(name) ?? [])
+        : [])
+    );
+  }
+
+  private walkValue(
+    value: unknown,
+    use: RuntimeUse,
+    visiting: Set<PlainObject>,
+    depth: number,
+  ): void {
+    if (Array.isArray(value)) {
+      value.forEach((item) => this.walkValue(item, use, visiting, depth));
+      return;
+    }
+    const json = asObject(value);
+    if (json._type === 'func') {
+      const name = asString(json.function) ?? '';
+      const parameters = asList(json.parameters);
+      if (FROM_FUNCTIONS.includes(name) && parameters.length >= 2) {
+        const runtime = asObject(parameters[parameters.length - 1]);
+        const path = asString(runtime.fullPath);
+        if (runtime._type === 'packageableElementPtr' && path) {
+          use.runtimes.add(path);
+        } else {
+          use.notAnElement = true;
+        }
+      }
+      this.calledFunctions(name).forEach((called) =>
+        this.walkFunction(called, use, visiting, depth + 1),
+      );
+    }
+    Object.values(json).forEach((child) => {
+      if (child && typeof child === 'object') {
+        this.walkValue(child, use, visiting, depth);
+      }
+    });
+  }
+}
+
+/**
+ * The model's functions that return a relation, `[1]`, in model order: those
+ * of `listed`, with the runtimes read through every function of `all` (a
+ * project's own functions, and its dependencies' too)
+ */
+export const V1_buildCubeFunctionOutlines = (
+  listed: readonly PlainObject[],
+  all: readonly PlainObject[] = listed,
+): CubeOutlineFunction[] => {
+  const finder = new V1_CubeRuntimeFinder(
+    all.filter((element) => element._type === FUNCTION_TYPE),
+  );
+  return listed
+    .filter(
+      (element) => element._type === FUNCTION_TYPE && returnsRelation(element),
+    )
+    .map((element) => {
+      const parameters = asList(element.parameters).map(buildParameter);
+      const use = finder.useOf(element);
+      const runtimes = [...use.runtimes];
+      const problem =
+        parameters.map(parameterProblemOf).find((each) => each !== undefined) ??
+        (use.notAnElement
+          ? V1_CUBE_FUNCTION_PROBLEM.RUNTIME_NOT_AN_ELEMENT
+          : runtimes.length > 1
+            ? V1_CUBE_FUNCTION_PROBLEM.TWO_RUNTIMES(runtimes)
+            : undefined);
+      return {
+        path: callPathOf(element),
+        signature: signatureOf(element),
+        parameters,
+        columns: buildColumns(element),
+        runtime: runtimes.length === 1 ? runtimes[0] : undefined,
+        problem,
+      };
+    });
+};

package/src/graph-manager/protocol/pure/v1/V1_CubeIngestDefinition.ts

@@ -0,0 +1,227 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  type CubeType,
+  OpaqueType,
+  PRIMITIVE_TYPE_PATH,
+  PrimitiveType,
+  resolveCubeType,
+  Schema,
+  SchemaColumn,
+} from '@finos/legend-cube';
+import {
+  MILESTONE_INGEST_COLUMNS,
+  V1_CInteger,
+  type V1_IngestDataset,
+  type V1_IngestDatasetSchema,
+  type V1_IngestDatasetSource,
+  V1_PackageableType,
+  type V1_RelationTypeColumn,
+  V1_WriteModeType,
+  type V1_WriteMode,
+  V1_deserializeIngestDefinitionContent,
+} from '@finos/legend-graph';
+import type { PlainObject } from '@finos/legend-shared';
+import { CubeIngestDataSet } from '../../../CubeIngestCatalog.js';
+
+// A deployed ingest definition's data sets, typed from what the definition
+// declares, with no engine call: each data set's columns, with their type
+// parameters, then the columns its write mode adds, typed as legend-graph
+// types them (#5598) and in the order the engine returns them. A data set
+// Cube can't read is listed with the reason, and can't be picked
+
+/** A data set whose rows come from a query over other data sets */
+const MATERIALIZED_VIEW_SOURCE = 'FunctionSource';
+
+/** Why a data set can't be picked */
+export const V1_CUBE_INGEST_DATA_SET_REASON = {
+  MATERIALIZED_VIEW:
+    'It is a materialized view, which Cube does not support yet',
+  NO_COLUMNS: 'It declares no columns',
+  BAD_MULTIPLICITY: (column: string): string =>
+    `Column "${column}" holds more than one value, which Cube can't show`,
+  UNKNOWN_TYPE: (column: string, type: string): string =>
+    `Column "${column}" has a type Cube doesn't know: ${type}`,
+} as const;
+
+/**
+ * The columns a batch-milestoned data set adds after its own, by write mode,
+ * in the order the engine returns them, since a run's rows are read by
+ * position: the digest, then the batches the row came in and went out with,
+ * then, business temporal, the dates it holds from and through. That is the
+ * order Legend's persistence components add them to a table in (digest,
+ * batch in and out, validity from and thru), and the engine gives
+ * LAKE_DIGEST right after a data set's own columns. legend-graph lists them
+ * LAKE_IN_ID first, which Legend Query doesn't mind: it reads them by name.
+ */
+const milestoningColumns = (
+  writeMode: V1_WriteMode | undefined,
+): SchemaColumn[] => {
+  const type = writeMode?._type;
+  if (
+    type !== V1_WriteModeType.BATCH_MILESTONED &&
+    type !== V1_WriteModeType.BATCH_MILESTONED_BUSINESS_TEMPORAL
+  ) {
+    return [];
+  }
+  const integer = PrimitiveType.get(PRIMITIVE_TYPE_PATH.INTEGER);
+  const columns = [
+    new SchemaColumn(
+      MILESTONE_INGEST_COLUMNS.INGEST_LAKE_DIGEST,
+      PrimitiveType.get(PRIMITIVE_TYPE_PATH.STRING),
+      false,
+    ),
+    new SchemaColumn(
+      MILESTONE_INGEST_COLUMNS.INGEST_LAKE_IN_ID,
+      integer,
+      false,
+    ),
+    new SchemaColumn(
+      MILESTONE_INGEST_COLUMNS.INGEST_LAKE_OUT_ID,
+      integer,
+      false,
+    ),
+  ];
+  if (type === V1_WriteModeType.BATCH_MILESTONED_BUSINESS_TEMPORAL) {
+    const timestamp = PrimitiveType.get(PRIMITIVE_TYPE_PATH.TIMESTAMP);
+    columns.push(
+      new SchemaColumn(
+        MILESTONE_INGEST_COLUMNS.INGEST_LAKE_FROM,
+        timestamp,
+        false,
+      ),
+      new SchemaColumn(
+        MILESTONE_INGEST_COLUMNS.INGEST_LAKE_THRU,
+        timestamp,
+        false,
+      ),
+    );
+  }
+  return columns;
+};
+
+/**
+ * A declared column's type, by its full path or, for a primitive whose path
+ * has a package (e.g. `meta::pure::metamodel::type::Integer`), its name;
+ * parameters kept. Any other type, such as an enumeration, gives the reason
+ * the data set can't be picked.
+ */
+const readColumnType = (
+  column: V1_RelationTypeColumn,
+): { type: CubeType } | { reason: string } => {
+  const { rawType, typeVariableValues } = column.genericType;
+  if (!(rawType instanceof V1_PackageableType)) {
+    return {
+      reason: V1_CUBE_INGEST_DATA_SET_REASON.UNKNOWN_TYPE(
+        column.name,
+        'not a named type',
+      ),
+    };
+  }
+  const params: number[] = [];
+  for (const value of typeVariableValues) {
+    if (!(value instanceof V1_CInteger)) {
+      return {
+        reason: V1_CUBE_INGEST_DATA_SET_REASON.UNKNOWN_TYPE(
+          column.name,
+          rawType.fullPath,
+        ),
+      };
+    }
+    params.push(value.value);
+  }
+  const name = rawType.fullPath.slice(rawType.fullPath.lastIndexOf(':') + 1);
+  const type = [rawType.fullPath, name]
+    .map((path) => resolveCubeType(path, params))
+    .find((candidate) => !(candidate instanceof OpaqueType));
+  return type
+    ? { type }
+    : {
+        reason: V1_CUBE_INGEST_DATA_SET_REASON.UNKNOWN_TYPE(
+          column.name,
+          params.length
+            ? `${rawType.fullPath}(${params.join(', ')})`
+            : rawType.fullPath,
+        ),
+      };
+};
+
+/** A data set's schema, or why it can't be picked */
+const readSchema = (
+  dataset: V1_IngestDataset,
+  definitionWriteMode: V1_WriteMode | undefined,
+): { schema: Schema } | { reason: string } => {
+  // the protocol types both as always there, but saved content may lack them
+  const source = dataset.source as V1_IngestDatasetSource | undefined;
+  if (source?._type === MATERIALIZED_VIEW_SOURCE) {
+    return { reason: V1_CUBE_INGEST_DATA_SET_REASON.MATERIALIZED_VIEW };
+  }
+  const schema: V1_IngestDatasetSchema | undefined = source?.schema;
+  const declared = schema?.columns ?? [];
+  if (!declared.length) {
+    return { reason: V1_CUBE_INGEST_DATA_SET_REASON.NO_COLUMNS };
+  }
+  const columns: SchemaColumn[] = [];
+  for (const column of declared) {
+    if (column.multiplicity.upperBound !== 1) {
+      return {
+        reason: V1_CUBE_INGEST_DATA_SET_REASON.BAD_MULTIPLICITY(column.name),
+      };
+    }
+    const read = readColumnType(column);
+    if ('reason' in read) {
+      return read;
+    }
+    columns.push(
+      new SchemaColumn(
+        column.name,
+        read.type,
+        column.multiplicity.lowerBound === 0,
+      ),
+    );
+  }
+  try {
+    return {
+      schema: new Schema([
+        ...columns,
+        ...milestoningColumns(dataset.writeMode ?? definitionWriteMode),
+      ]),
+    };
+  } catch (error) {
+    return { reason: error instanceof Error ? error.message : String(error) };
+  }
+};
+
+/**
+ * The data sets of an ingest definition's content (its protocol JSON), in
+ * order. Throws when the content can't be read at all.
+ */
+export const V1_readCubeIngestDataSets = (
+  content: PlainObject,
+): CubeIngestDataSet[] => {
+  const definition = V1_deserializeIngestDefinitionContent(content);
+  return (definition.datasets ?? []).map((dataset) => {
+    const read = readSchema(dataset, definition.writeMode);
+    return new CubeIngestDataSet({
+      name: dataset.name,
+      primaryKey: dataset.primaryKey,
+      ...('schema' in read
+        ? { schema: read.schema }
+        : { disabledReason: read.reason }),
+    });
+  });
+};