@finos/legend-cube-builder 0.0.5 → 0.0.6

package/src/stores/variant/CubeJsonFieldsState.ts

@@ -0,0 +1,252 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  getVariantDatabaseProblem,
+  type Schema,
+  type SchemaColumn,
+} from '@finos/legend-cube';
+import { assertErrorThrown, type GeneratorFn } from '@finos/legend-shared';
+import {
+  action,
+  computed,
+  flow,
+  flowResult,
+  makeObservable,
+  observable,
+} from 'mobx';
+import { CubeEngineError } from '../../graph-manager/CubeEngine.js';
+import type { CubeEditorState } from '../CubeEditorState.js';
+import { CubeQueryRunAborted } from '../CubeQueryRun.js';
+import {
+  type CubeJsonField,
+  filterCubeJsonFields,
+  getCubeJsonFields,
+} from './CubeJsonOffers.js';
+import { readCubeJsonSample } from './CubeJsonSampling.js';
+import type { CubeJsonSample } from './CubeJsonShape.js';
+
+/** A field's path as one key */
+const pathKey = (path: readonly string[]): string => JSON.stringify(path);
+
+/**
+ * A Variant input column's fields in an expression editor, Extend's or
+ * Filter by Expression's (PLAN §11.20, V.9): opened from the column's chip,
+ * read from a sample of the column just before the node being edited, so no
+ * run of the cube is needed and stale rows don't matter. The sample is read
+ * as Explore JSON reads one (listed in Runs, and it can be stopped); a field
+ * clicked writes its expression into the editor's.
+ */
+export class CubeJsonFieldsState {
+  readonly editorState: CubeEditorState;
+  /** The node being edited, whose input is read */
+  readonly nodeId: string;
+
+  /** The column whose fields are open; `undefined` while closed */
+  column: SchemaColumn | undefined;
+  /** What was read, and of which column */
+  sample:
+    | { readonly column: string; readonly sample: CubeJsonSample }
+    | undefined;
+  isReading = false;
+  /** Why the column couldn't be read */
+  error: string | undefined;
+  search = '';
+  /** The fields folded, by path; every field starts open */
+  readonly folded = observable.set<string>();
+  /** The field whose other forms show, by path */
+  formsOf: string | undefined;
+
+  private controller: AbortController | undefined;
+
+  constructor(editorState: CubeEditorState, nodeId: string) {
+    makeObservable(this, {
+      column: observable.ref,
+      sample: observable.ref,
+      isReading: observable,
+      error: observable,
+      search: observable,
+      formsOf: observable,
+      input: computed,
+      databaseProblem: computed,
+      fields: computed,
+      shownFields: computed,
+      toggle: flow,
+      read: flow,
+      stop: action,
+      close: action,
+      setSearch: action,
+      toggleFolded: action,
+      toggleForms: action,
+    });
+    this.editorState = editorState;
+    this.nodeId = nodeId;
+  }
+
+  /** The node's input: its id and schema, once known */
+  get input(): { id: string; schema: Schema } | undefined {
+    const [id] = this.editorState.document.query.getInputIds(this.nodeId);
+    const schema =
+      id === undefined ? undefined : this.editorState.analysis.schemas.get(id);
+    return id === undefined || !schema ? undefined : { id, schema };
+  }
+
+  /**
+   * Why the fields can't be read: the database the node's input runs on has
+   * no Variant functions; `undefined` when they can, or it isn't known yet
+   */
+  get databaseProblem(): string | undefined {
+    const { editorState, input } = this;
+    const { context, query } = editorState.document;
+    return input === undefined || context?.runtime === undefined
+      ? undefined
+      : getVariantDatabaseProblem(
+          editorState.getRunDatabaseType(
+            query,
+            input.id,
+            context.model,
+            context.runtime,
+          ),
+        );
+  }
+
+  /** The open column's fields, from what was read of it */
+  get fields(): CubeJsonField | undefined {
+    const { column, sample } = this;
+    return column && sample?.column === column.name
+      ? getCubeJsonFields(column.name, sample.sample)
+      : undefined;
+  }
+
+  /** The fields matching the search, with the fields they are in */
+  get shownFields(): CubeJsonField | undefined {
+    return this.fields && filterCubeJsonFields(this.fields, this.search);
+  }
+
+  isOpen(column: string): boolean {
+    return this.column?.name === column;
+  }
+
+  isFolded(path: readonly string[]): boolean {
+    return this.folded.has(pathKey(path));
+  }
+
+  areFormsShown(path: readonly string[]): boolean {
+    return this.formsOf === pathKey(path);
+  }
+
+  /**
+   * Opens a column's fields, reading its first values unless they were read
+   * already, or closes them if open. The model's databases are loaded first:
+   * a database without JSON functions is refused, with the reason.
+   */
+  *toggle(column: SchemaColumn): GeneratorFn<void> {
+    if (this.isOpen(column.name)) {
+      this.close();
+      return;
+    }
+    this.stop();
+    this.column = column;
+    this.error = undefined;
+    this.search = '';
+    this.formsOf = undefined;
+    if (this.sample?.column === column.name) {
+      return;
+    }
+    const { context } = this.editorState.document;
+    if (context) {
+      yield flowResult(this.editorState.loadModelOutline(context.model));
+    }
+    const problem = this.databaseProblem;
+    if (problem) {
+      this.error = problem;
+      return;
+    }
+    yield flowResult(this.read(false));
+  }
+
+  /** Reads the open column's first values, or every row, before the node */
+  *read(everyRow: boolean): GeneratorFn<void> {
+    const { editorState, column, input } = this;
+    if (!column || !input || this.isReading) {
+      return;
+    }
+    const controller = new AbortController();
+    this.controller = controller;
+    this.isReading = true;
+    this.error = undefined;
+    try {
+      const sample = (yield readCubeJsonSample(editorState, {
+        document: editorState.document,
+        afterId: input.id,
+        column,
+        everyRow,
+        abortController: controller,
+      })) as CubeJsonSample | undefined;
+      if (this.controller === controller && sample) {
+        this.sample = { column: column.name, sample };
+      }
+    } catch (error) {
+      if (
+        this.controller !== controller ||
+        error instanceof CubeQueryRunAborted
+      ) {
+        return;
+      }
+      assertErrorThrown(error);
+      this.error =
+        error instanceof CubeEngineError ? error.firstLine : error.message;
+    } finally {
+      if (this.controller === controller) {
+        this.controller = undefined;
+        this.isReading = false;
+      }
+    }
+  }
+
+  /** Stops a read in flight, keeping what was read before it */
+  stop(): void {
+    this.controller?.abort();
+    this.controller = undefined;
+    this.isReading = false;
+  }
+
+  /** Closes the fields, keeping what was read for when they open again */
+  close(): void {
+    this.stop();
+    this.column = undefined;
+    this.error = undefined;
+    this.formsOf = undefined;
+  }
+
+  setSearch(search: string): void {
+    this.search = search;
+  }
+
+  toggleFolded(path: readonly string[]): void {
+    const key = pathKey(path);
+    if (this.folded.has(key)) {
+      this.folded.delete(key);
+    } else {
+      this.folded.add(key);
+    }
+  }
+
+  toggleForms(path: readonly string[]): void {
+    const key = pathKey(path);
+    this.formsOf = this.formsOf === key ? undefined : key;
+  }
+}

package/src/stores/variant/CubeJsonOffers.ts

@@ -0,0 +1,359 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { foldColumnName } from '@finos/legend-cube';
+import { columnAccessOf, pureStringLiteral } from '../CubePureText.js';
+import {
+  CUBE_JSON_PURE_TYPE,
+  CubeJsonScalarKind,
+  type CubeJsonSample,
+  type CubeJsonShape,
+  describeCubeJsonShape,
+  getCubeJsonScalarType,
+  getCubeJsonTopValues,
+  isCubeJsonScalarShape,
+} from './CubeJsonShape.js';
+
+// The fields of a Variant column's documents, and the columns each can become
+// (PLAN §11.20), after legend-lite's Data Cube: picking one writes the Pure
+// text of an Extend column, which the engine parses and types like any typed
+// there. Only forms run on legend-engine are offered: `contains` goes through
+// `toMany(@Variant)->map(…to(@T))`, since `toMany(@String)->contains` writes
+// invalid SQL on DuckDB; no `exists` (an engine error); no tuple explode
+// (`toVariant(newMap(…))`, nested maps excluded on Snowflake and DuckDB).
+
+/** The Variant type, as an expression names it */
+const VARIANT = 'Variant';
+
+/** How many of an array's top values get a "contains" offer */
+const CONTAINS_OFFERS = 8;
+
+/** One column a field can become */
+export interface CubeJsonOffer {
+  /** Unique among the field's offers, e.g. `value`, `count`, `contains:FR` */
+  readonly key: string;
+  /** What the dialog shows, e.g. `the value, as Integer` */
+  readonly label: string;
+  /** The column's default name, e.g. `payload_user_tier` */
+  readonly name: string;
+  /** The Pure type it should take, `Variant` for JSON: a suggestion, the engine types the column */
+  readonly type: string;
+  /** An Extend column's text, `x | …`; `undefined` for an explode */
+  readonly code: string | undefined;
+  /** One row per element of the array at this path: an Explode Array node */
+  readonly explode?:
+    | { readonly column: string; readonly path: readonly string[] }
+    | undefined;
+}
+
+/** A position in the documents, and what it can become */
+export interface CubeJsonField {
+  /** The keys from the column to it; empty for the column itself */
+  readonly path: readonly string[];
+  /** What it held, e.g. `text`, `list of object` */
+  readonly description: string;
+  /** Its share, 0 to 1, of the places it could be: the rows read, or its object's occurrences */
+  readonly presence: number;
+  /** Its most frequent values, for a scalar */
+  readonly topValues: readonly string[];
+  readonly offers: readonly CubeJsonOffer[];
+  readonly children: readonly CubeJsonField[];
+}
+
+/** A column name from parts: joined by `_`, letters, digits and `_` only */
+const nameOf = (parts: readonly string[]): string =>
+  parts
+    .join('_')
+    .replace(/[^A-Za-z0-9_]+/gu, '_')
+    .replace(/_+/gu, '_')
+    .replace(/^_|_$/gu, '') || 'value';
+
+/** The default name of an array's element, as Explode Array makes it: its last key's, or its column's, then `_element` */
+export const getCubeJsonElementName = (
+  column: string,
+  path: readonly string[],
+): string => nameOf([path[path.length - 1] ?? column, 'element']);
+
+/** `$x.c->get('a')->get('b')` */
+const pathOf = (column: string, path: readonly string[]): string =>
+  path.reduce(
+    (expression, key) => `${expression}->get(${pureStringLiteral(key)})`,
+    columnAccessOf(column),
+  );
+
+/** A function of the row */
+const ofRow = (body: string): string => `x | ${body}`;
+
+/**
+ * A seen value as a Pure literal of the kind it is read as: a number in a
+ * Float list with a point (`2.0`); `undefined` for a number Pure can't write
+ * as a literal (an exponent)
+ */
+const literalOf = (
+  kind: CubeJsonScalarKind,
+  text: string,
+): string | undefined =>
+  kind === CubeJsonScalarKind.INTEGER || kind === CubeJsonScalarKind.FLOAT
+    ? !/^-?\d+(?:\.\d+)?$/u.test(text)
+      ? undefined
+      : kind === CubeJsonScalarKind.FLOAT && !text.includes('.')
+        ? `${text}.0`
+        : text
+    : kind === CubeJsonScalarKind.BOOLEAN
+      ? String(text === 'true')
+      : pureStringLiteral(text);
+
+/** The kinds "contains" compares as typed literals: text, numbers and booleans */
+const COMPARABLE_KINDS = new Set([
+  CubeJsonScalarKind.TEXT,
+  CubeJsonScalarKind.INTEGER,
+  CubeJsonScalarKind.FLOAT,
+  CubeJsonScalarKind.BOOLEAN,
+]);
+
+const arrayOffers = (
+  column: string,
+  path: readonly string[],
+  shape: CubeJsonShape,
+): CubeJsonOffer[] => {
+  const array = pathOf(column, path);
+  const names = [column, ...path];
+  const many = `${array}->toMany(@${VARIANT})`;
+  const offers: CubeJsonOffer[] = [
+    {
+      key: 'explode',
+      label: 'one row per element',
+      name: getCubeJsonElementName(column, path),
+      type: VARIANT,
+      code: undefined,
+      explode: { column, path },
+    },
+    {
+      key: 'count',
+      label: 'number of elements',
+      name: nameOf([...names, 'count']),
+      type: 'Integer',
+      code: ofRow(`${many}->size()`),
+    },
+  ];
+  const { element } = shape;
+  if (!element) {
+    return offers;
+  }
+  const elementKind = getCubeJsonScalarType(element);
+  if (elementKind && isCubeJsonScalarShape(element)) {
+    offers.push({
+      key: 'list',
+      label: 'all values, as text',
+      name: nameOf([...names, 'list']),
+      type: 'String',
+      code: ofRow(`${many}->map(e | $e->to(@String))->joinStrings(', ')`),
+    });
+    if (COMPARABLE_KINDS.has(elementKind)) {
+      const type = CUBE_JSON_PURE_TYPE[elementKind];
+      getCubeJsonTopValues(element, CONTAINS_OFFERS).forEach((value) => {
+        const literal = literalOf(elementKind, value);
+        if (literal !== undefined) {
+          offers.push({
+            key: `contains:${value}`,
+            label: `contains ${value}`,
+            name: nameOf([...names, 'has', value]),
+            type: 'Boolean',
+            code: ofRow(
+              `${many}->map(e | $e->to(@${type}))->contains(${literal})`,
+            ),
+          });
+        }
+      });
+    }
+  }
+  if (element.objects > 0) {
+    offers.push({
+      key: 'first',
+      label: 'first element, as JSON',
+      name: nameOf([...names, 'first']),
+      type: VARIANT,
+      code: ofRow(`${array}->get(0)`),
+    });
+    [...element.fields.entries()]
+      .sort(([left], [right]) => left.localeCompare(right))
+      .forEach(([key, field]) => {
+        const kind = getCubeJsonScalarType(field);
+        if (!kind || !isCubeJsonScalarShape(field)) {
+          return;
+        }
+        const type = CUBE_JSON_PURE_TYPE[kind];
+        offers.push({
+          key: `first:${key}`,
+          label: `first element's ${key}`,
+          name: nameOf([...names, 'first', key]),
+          type,
+          code: ofRow(
+            `${array}->get(0)->get(${pureStringLiteral(key)})->to(@${type})`,
+          ),
+        });
+        if (
+          kind === CubeJsonScalarKind.INTEGER ||
+          kind === CubeJsonScalarKind.FLOAT
+        ) {
+          offers.push({
+            key: `total:${key}`,
+            label: `total of ${key}`,
+            name: nameOf([...names, key, 'total']),
+            type,
+            code: ofRow(
+              `${many}->map(e | $e->get(${pureStringLiteral(key)})->to(@${type}))->sum()`,
+            ),
+          });
+        }
+      });
+  }
+  return offers;
+};
+
+const offersOf = (
+  column: string,
+  path: readonly string[],
+  shape: CubeJsonShape,
+): CubeJsonOffer[] => {
+  const names = [column, ...path];
+  const offers: CubeJsonOffer[] = [];
+  const kind = getCubeJsonScalarType(shape);
+  if (kind && isCubeJsonScalarShape(shape)) {
+    const type = CUBE_JSON_PURE_TYPE[kind];
+    offers.push({
+      key: 'value',
+      label: `the value, as ${type}`,
+      name: nameOf(path.length ? names : [...names, 'value']),
+      type,
+      code: ofRow(`${pathOf(column, path)}->to(@${type})`),
+    });
+  }
+  // a nested object or array, as a Variant column of its own, to explore again
+  if (path.length > 0 && (shape.objects > 0 || shape.arrays > 0)) {
+    offers.push({
+      key: 'json',
+      label: 'as JSON',
+      name: nameOf(names),
+      type: VARIANT,
+      code: ofRow(pathOf(column, path)),
+    });
+  }
+  if (shape.objects > 0) {
+    offers.push({
+      key: 'keys',
+      label: 'its keys, as text',
+      name: nameOf([...names, 'keys']),
+      type: 'String',
+      code: ofRow(
+        `${pathOf(column, path)}->to(@Map<String, ${VARIANT}>)->toOne()->keys()->joinStrings(', ')`,
+      ),
+    });
+  }
+  if (shape.arrays > 0) {
+    offers.push(...arrayOffers(column, path, shape));
+  }
+  return offers;
+};
+
+/**
+ * A column's fields, from a sample of its documents: the column itself, its
+ * object's fields, theirs, and so on, each with what it held, how often it was
+ * there and what it can become. An array's elements' fields are reached
+ * through the array's offers (first element's, total of), not as fields.
+ */
+export const getCubeJsonFields = (
+  column: string,
+  sample: CubeJsonSample,
+): CubeJsonField => {
+  const build = (
+    shape: CubeJsonShape,
+    path: readonly string[],
+    places: number,
+  ): CubeJsonField => ({
+    path,
+    description: describeCubeJsonShape(shape),
+    presence: Math.min(1, (shape.present - shape.nulls) / Math.max(1, places)),
+    topValues: isCubeJsonScalarShape(shape) ? getCubeJsonTopValues(shape) : [],
+    offers: offersOf(column, path, shape),
+    children:
+      shape.objects > 0
+        ? [...shape.fields.entries()]
+            .sort(([left], [right]) => left.localeCompare(right))
+            .map(([key, field]) => build(field, [...path, key], shape.objects))
+        : [],
+  });
+  return build(sample.shape, [], sample.rows);
+};
+
+/** A name no taken column has, compared folded: `name`, else `name_2`, `name_3`, … */
+export const getFreeCubeColumnName = (
+  name: string,
+  taken: readonly string[],
+): string => {
+  const folded = new Set(taken.map(foldColumnName));
+  if (!folded.has(foldColumnName(name))) {
+    return name;
+  }
+  let index = 2;
+  while (folded.has(foldColumnName(`${name}_${index}`))) {
+    index += 1;
+  }
+  return `${name}_${index}`;
+};
+
+/** An offer's expression of the row, as an editor writes it: its text without `x | ` */
+export const getCubeJsonOfferExpression = (
+  offer: CubeJsonOffer,
+): string | undefined => offer.code?.replace(/^x \| /u, '');
+
+/**
+ * What clicking a field in an expression editor writes (PLAN §11.20, V.9):
+ * a value as its suggested type, anything else as JSON (the column itself
+ * for its root); and the column "+ column" would add, but for the root,
+ * which is the column already
+ */
+export const getCubeJsonFieldDefault = (
+  column: string,
+  field: CubeJsonField,
+): { expression: string; column?: { name: string; code: string } } => {
+  const offer = field.offers.find(
+    ({ key }) => key === 'value' || key === 'json',
+  );
+  const expression =
+    (offer && getCubeJsonOfferExpression(offer)) ?? pathOf(column, field.path);
+  return offer?.code && field.path.length
+    ? { expression, column: { name: offer.name, code: offer.code } }
+    : { expression };
+};
+
+/** The fields whose keys match a search, with the fields they are in; all without a search */
+export const filterCubeJsonFields = (
+  field: CubeJsonField,
+  search: string,
+): CubeJsonField | undefined => {
+  const needle = search.trim().toLowerCase();
+  if (!needle) {
+    return field;
+  }
+  const children = field.children
+    .map((child) => filterCubeJsonFields(child, search))
+    .filter((child): child is CubeJsonField => child !== undefined);
+  return children.length ||
+    field.path.some((key) => key.toLowerCase().includes(needle))
+    ? { ...field, children }
+    : undefined;
+};

package/src/stores/variant/CubeJsonParse.ts

@@ -0,0 +1,75 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Extend } from '@finos/legend-cube';
+import type { CubeEditorState } from '../CubeEditorState.js';
+import { columnAccessOf } from '../CubePureText.js';
+import { readCubeJson } from './CubeJson.js';
+import { getFreeCubeColumnName } from './CubeJsonOffers.js';
+
+// Parse as JSON (PLAN §11.20): a text column holding JSON documents read as a
+// Variant column, `fromJson()`, so Explore JSON and Explode Array work on it.
+// JSON stored as text is common, and it is the only way to a Variant column on
+// a local source: H2 has none, and a direct connection hides a JSON column.
+
+/** The column a text column's documents become: `<name>_json` */
+export const getCubeJsonParseName = (column: string): string =>
+  `${column}_json`;
+
+/** The Extend column's text: the text read as a document (`fromJson` takes one value) */
+export const getCubeJsonParseCode = (column: string): string =>
+  `x | ${columnAccessOf(column)}->toOne()->fromJson()`;
+
+/** Whether a text cell holds a JSON object or array, which Parse as JSON is offered on */
+export const isCubeJsonDocumentText = (text: string): boolean => {
+  const node = readCubeJson(text);
+  return node?.kind === 'object' || node?.kind === 'array';
+};
+
+/**
+ * Adds Parse as JSON's column after the node that ran, as one undo step, once
+ * the engine parses its text; nothing if the cube changed meanwhile
+ */
+export const addCubeJsonParseColumn = async (
+  editorState: CubeEditorState,
+  column: string,
+): Promise<void> => {
+  const { document } = editorState;
+  const { query } = document;
+  const afterId = query.selected;
+  if (afterId === undefined) {
+    return;
+  }
+  const extendId = query.generateId(Extend.TYPE);
+  const code = getCubeJsonParseCode(column);
+  const name = getFreeCubeColumnName(
+    getCubeJsonParseName(column),
+    (editorState.analysis.schemas.get(afterId)?.columns ?? []).map(
+      ({ name: taken }) => taken,
+    ),
+  );
+  const { lambda } = await editorState.host.engine.parseExpression(
+    code,
+    `${extendId}:0`,
+  );
+  // as the menu found it: nothing changed while the engine parsed
+  if (editorState.document === document) {
+    editorState.addConfiguredNode(
+      new Extend(extendId, [{ name, code, lambda }]),
+      afterId,
+    );
+  }
+};