@finos/legend-cube-builder 0.0.5 → 0.0.6

package/src/stores/variant/CubeJsonSampling.ts

@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import {
+  ColumnComparisonFilter,
+  type CubeDocument,
+  Filter,
+  FilterOperator,
+  Restrict,
+  Schema,
+  type SchemaColumn,
+} from '@finos/legend-cube';
+import type { CubeEditorState } from '../CubeEditorState.js';
+import { runCubeQuery } from '../CubeQueryRun.js';
+import { type CubeJsonSample, inferCubeJsonShape } from './CubeJsonShape.js';
+
+/** How many values Explore JSON reads first (user, 2026-10-14) */
+export const CUBE_JSON_SAMPLE_SIZE = 1000;
+
+/** No limit: Read every row (the run's `limit(rowLimit + 1)` then takes them all) */
+const EVERY_ROW = Number.MAX_SAFE_INTEGER;
+
+/** The runs that read a Variant column's documents, as Runs lists them */
+export const CUBE_JSON_RUN_LABEL = {
+  sample: (column: string): string => `Sample of ${column}`,
+  everyRow: (column: string): string => `Every value of ${column}`,
+};
+
+/**
+ * Reads a Variant column's documents (PLAN §11.20) through the cube's own
+ * query path, so the read is listed in Runs and can be stopped: the query up
+ * to a node, then the column alone and its non-empty values, its first
+ * `CUBE_JSON_SAMPLE_SIZE` or every row. Explore JSON reads after the node
+ * that ran, the Explode Array editor after its input. `undefined` for a cube
+ * with no runtime; a failed or stopped run throws, as `runCubeQuery` does.
+ */
+export const readCubeJsonSample = async (
+  editorState: CubeEditorState,
+  options: {
+    document: CubeDocument;
+    afterId: string;
+    column: SchemaColumn;
+    everyRow: boolean;
+    abortController: AbortController;
+  },
+): Promise<CubeJsonSample | undefined> => {
+  const { document, afterId, column, everyRow, abortController } = options;
+  const { context, query } = document;
+  if (context?.runtime === undefined) {
+    return undefined;
+  }
+  // each after the one before; the query isn't kept
+  const restrict = new Restrict(query.generateId(Restrict.TYPE), [column.name]);
+  const nonEmpty = new Filter(
+    query.generateId(Filter.TYPE),
+    new ColumnComparisonFilter(column.name, FilterOperator.IS_NOT_EMPTY),
+  );
+  const values = [restrict, nonEmpty].reduce(
+    (built, node) => built.add(node, built.selected),
+    query.selected === afterId ? query : query.select(afterId),
+  );
+  const result = await runCubeQuery(editorState, {
+    query: values,
+    model: context.model,
+    runtime: context.runtime,
+    schema: new Schema([column]),
+    rowLimit: everyRow ? EVERY_ROW : CUBE_JSON_SAMPLE_SIZE,
+    abortController,
+    label: everyRow
+      ? CUBE_JSON_RUN_LABEL.everyRow(column.name)
+      : CUBE_JSON_RUN_LABEL.sample(column.name),
+  });
+  return inferCubeJsonShape(
+    result.rows.map((row) => row[0] ?? null),
+    !result.limited,
+  );
+};

package/src/stores/variant/CubeJsonShape.ts

@@ -0,0 +1,279 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { CubeResultValue } from '../../graph-manager/CubeEngine.js';
+import { type CubeJsonNode, readCubeJson } from './CubeJson.js';
+
+// What a Variant column's documents hold (PLAN §11.20), inferred from a
+// sample of them, as legend-lite's Data Cube does: nothing in the model says,
+// since the column is declared only SEMISTRUCTURED. The inferred type only
+// picks the `to(@T)` an offer writes; the engine then types the column. A
+// wrong pick isn't always an error, though: on Snowflake, DuckDB, Postgres
+// and Databricks `to(@Integer)` of `1.5` returns a value, so the type is
+// picked conservatively (`getCubeJsonScalarType`).
+
+/** What a scalar looked like, which picks its `to(@T)` */
+export enum CubeJsonScalarKind {
+  INTEGER = 'integer',
+  FLOAT = 'float',
+  BOOLEAN = 'boolean',
+  DATE = 'date',
+  DATETIME = 'datetime',
+  TEXT = 'text',
+}
+
+/** The Pure type each kind is read as */
+export const CUBE_JSON_PURE_TYPE: Record<CubeJsonScalarKind, string> = {
+  [CubeJsonScalarKind.INTEGER]: 'Integer',
+  [CubeJsonScalarKind.FLOAT]: 'Float',
+  [CubeJsonScalarKind.BOOLEAN]: 'Boolean',
+  [CubeJsonScalarKind.DATE]: 'StrictDate',
+  [CubeJsonScalarKind.DATETIME]: 'DateTime',
+  [CubeJsonScalarKind.TEXT]: 'String',
+};
+
+/** A kind as the dialog names it */
+export const CUBE_JSON_KIND_LABEL: Record<CubeJsonScalarKind, string> = {
+  [CubeJsonScalarKind.INTEGER]: 'whole number',
+  [CubeJsonScalarKind.FLOAT]: 'number',
+  [CubeJsonScalarKind.BOOLEAN]: 'true or false',
+  [CubeJsonScalarKind.DATE]: 'date',
+  [CubeJsonScalarKind.DATETIME]: 'date and time',
+  [CubeJsonScalarKind.TEXT]: 'text',
+};
+
+const DATE = /^\d{4}-\d{2}-\d{2}$/u;
+const DATETIME =
+  /^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:?\d{2})?$/u;
+const WHOLE_NUMBER = /^-?\d+$/u;
+/** The largest whole number a 64-bit `to(@Integer)` holds */
+const MAX_INTEGER = 9223372036854775807n;
+
+/** At most this many distinct values of a position are counted */
+const MAX_VALUES = 50;
+
+const kindOf = (node: CubeJsonNode): CubeJsonScalarKind | undefined => {
+  switch (node.kind) {
+    case 'number':
+      // a whole number past 64 bits would overflow `to(@Integer)`: text
+      return WHOLE_NUMBER.test(node.text)
+        ? BigInt(node.text) <= MAX_INTEGER && BigInt(node.text) >= -MAX_INTEGER
+          ? CubeJsonScalarKind.INTEGER
+          : CubeJsonScalarKind.TEXT
+        : CubeJsonScalarKind.FLOAT;
+    case 'boolean':
+      return CubeJsonScalarKind.BOOLEAN;
+    case 'string':
+      return DATE.test(node.value)
+        ? CubeJsonScalarKind.DATE
+        : DATETIME.test(node.value)
+          ? CubeJsonScalarKind.DATETIME
+          : CubeJsonScalarKind.TEXT;
+    default:
+      return undefined;
+  }
+};
+
+/** A scalar's text, as it is shown and counted */
+const textOf = (node: CubeJsonNode): string =>
+  node.kind === 'number'
+    ? node.text
+    : node.kind === 'string'
+      ? node.value
+      : node.kind === 'boolean'
+        ? String(node.value)
+        : '';
+
+/** Everything seen at one position of the documents */
+export interface CubeJsonShape {
+  /** Times the position was there, empty included */
+  present: number;
+  /** Times it held null */
+  nulls: number;
+  /** The kinds of scalar seen, and how often */
+  readonly scalars: Map<CubeJsonScalarKind, number>;
+  /** Distinct scalar values and how often, up to `MAX_VALUES` of them */
+  readonly values: Map<string, number>;
+  /** Times it held an object */
+  objects: number;
+  /** The objects' fields, in the order first seen */
+  readonly fields: Map<string, CubeJsonShape>;
+  /** Times it held an array */
+  arrays: number;
+  /** Every element of every array seen here */
+  element: CubeJsonShape | undefined;
+  minLength: number;
+  maxLength: number;
+}
+
+const emptyShape = (): CubeJsonShape => ({
+  present: 0,
+  nulls: 0,
+  scalars: new Map(),
+  values: new Map(),
+  objects: 0,
+  fields: new Map(),
+  arrays: 0,
+  element: undefined,
+  minLength: Infinity,
+  maxLength: 0,
+});
+
+const observe = (shape: CubeJsonShape, node: CubeJsonNode): void => {
+  shape.present += 1;
+  if (node.kind === 'null') {
+    shape.nulls += 1;
+    return;
+  }
+  if (node.kind === 'object') {
+    shape.objects += 1;
+    node.entries.forEach(([key, value]) => {
+      let field = shape.fields.get(key);
+      if (!field) {
+        field = emptyShape();
+        shape.fields.set(key, field);
+      }
+      observe(field, value);
+    });
+    return;
+  }
+  if (node.kind === 'array') {
+    shape.arrays += 1;
+    shape.minLength = Math.min(shape.minLength, node.items.length);
+    shape.maxLength = Math.max(shape.maxLength, node.items.length);
+    const element = shape.element ?? emptyShape();
+    shape.element = element;
+    node.items.forEach((item) => observe(element, item));
+    return;
+  }
+  const kind = kindOf(node);
+  if (kind) {
+    shape.scalars.set(kind, (shape.scalars.get(kind) ?? 0) + 1);
+  }
+  const text = textOf(node);
+  if (shape.values.has(text) || shape.values.size < MAX_VALUES) {
+    shape.values.set(text, (shape.values.get(text) ?? 0) + 1);
+  }
+};
+
+/** What a sample of a column showed */
+export interface CubeJsonSample {
+  /** The shape of the column's documents */
+  readonly shape: CubeJsonShape;
+  /** Cells read, empty ones included */
+  readonly rows: number;
+  /** Cells that couldn't be read as JSON */
+  readonly unreadable: number;
+  /** Every row of the column was read: the shape is the whole column's */
+  readonly complete: boolean;
+}
+
+/** A column's shape, built as its cells arrive */
+export class CubeJsonShapeReader {
+  private readonly shape = emptyShape();
+  private rows = 0;
+  private unreadable = 0;
+
+  /** A cell as the grid receives it: the document's JSON text, or null */
+  add(cell: CubeResultValue): void {
+    this.rows += 1;
+    if (cell === null) {
+      return;
+    }
+    const node = readCubeJson(String(cell));
+    if (node) {
+      observe(this.shape, node);
+    } else {
+      this.unreadable += 1;
+    }
+  }
+
+  result(complete: boolean): CubeJsonSample {
+    return {
+      shape: this.shape,
+      rows: this.rows,
+      unreadable: this.unreadable,
+      complete,
+    };
+  }
+}
+
+/** The shape of a sample of a column's cells */
+export const inferCubeJsonShape = (
+  cells: readonly CubeResultValue[],
+  complete = false,
+): CubeJsonSample => {
+  const reader = new CubeJsonShapeReader();
+  cells.forEach((cell) => reader.add(cell));
+  return reader.result(complete);
+};
+
+/**
+ * The one scalar type a position is read as, picked so a wrong guess can't
+ * give wrong rows: one kind is that kind; whole numbers and fractions
+ * together are numbers (`Float`); any other mix, dates with date-times
+ * included, is text, which every value reads as. `undefined` when no scalar
+ * was seen.
+ */
+export const getCubeJsonScalarType = (
+  shape: CubeJsonShape,
+): CubeJsonScalarKind | undefined => {
+  const kinds = [...shape.scalars.keys()];
+  if (kinds.length <= 1) {
+    return kinds[0];
+  }
+  return kinds.every(
+    (kind) =>
+      kind === CubeJsonScalarKind.INTEGER || kind === CubeJsonScalarKind.FLOAT,
+  )
+    ? CubeJsonScalarKind.FLOAT
+    : CubeJsonScalarKind.TEXT;
+};
+
+/** A position holds only scalars (and nulls): no object or array */
+export const isCubeJsonScalarShape = (shape: CubeJsonShape): boolean =>
+  shape.objects === 0 && shape.arrays === 0 && shape.scalars.size > 0;
+
+/** A position's most frequent values, most frequent first, then by text */
+export const getCubeJsonTopValues = (
+  shape: CubeJsonShape,
+  count = 12,
+): string[] =>
+  [...shape.values.entries()]
+    .sort(
+      (left, right) => right[1] - left[1] || left[0].localeCompare(right[0]),
+    )
+    .slice(0, count)
+    .map(([value]) => value);
+
+/** What a position held, e.g. `text`, `object`, `list of object`, `number or object`; `always empty` for nothing but nulls */
+export const describeCubeJsonShape = (shape: CubeJsonShape): string => {
+  const parts: string[] = [];
+  const scalar = getCubeJsonScalarType(shape);
+  if (scalar) {
+    parts.push(CUBE_JSON_KIND_LABEL[scalar]);
+  }
+  if (shape.objects > 0) {
+    parts.push('object');
+  }
+  if (shape.arrays > 0) {
+    const { element } = shape;
+    parts.push(
+      `list of ${!element || element.present === element.nulls ? 'nothing' : describeCubeJsonShape(element)}`,
+    );
+  }
+  return parts.length ? parts.join(' or ') : 'always empty';
+};

package/src/stores/variant/CubeJsonViewerState.ts

@@ -0,0 +1,59 @@
+/**
+ * Copyright (c) 2026-present, Goldman Sachs
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { action, computed, makeObservable, observable } from 'mobx';
+import { type CubeJsonNode, readCubeJson } from './CubeJson.js';
+
+/**
+ * Open JSON (PLAN §11.20): one Variant cell's document, shown as a tree in a
+ * dialog, from the cell's menu
+ */
+export class CubeJsonViewerState {
+  /** The cell's column, for the title; `undefined` while closed */
+  column: string | undefined;
+  /** The cell's text, as the engine sent it */
+  text: string | undefined;
+
+  constructor() {
+    makeObservable(this, {
+      column: observable,
+      text: observable,
+      isOpen: computed,
+      document: computed,
+      open: action,
+      close: action,
+    });
+  }
+
+  get isOpen(): boolean {
+    return this.text !== undefined;
+  }
+
+  /** The document; `undefined` for text that isn't JSON, shown as it is */
+  get document(): CubeJsonNode | undefined {
+    return this.text === undefined ? undefined : readCubeJson(this.text);
+  }
+
+  open(column: string, text: string): void {
+    this.column = column;
+    this.text = text;
+  }
+
+  close(): void {
+    this.column = undefined;
+    this.text = undefined;
+  }
+}

package/tsconfig.json

@@ -54,20 +54,24 @@
     "./src/__lib__/LegendCubeHelpText.ts",
     "./src/__lib__/LegendCubeIngestLabels.ts",
     "./src/__lib__/LegendCubeLabels.ts",
+    "./src/__lib__/LegendCubeNavigation.ts",
     "./src/__lib__/LegendCubeProjectLabels.ts",
     "./src/__lib__/LegendCubeProjectSourceLabels.ts",
     "./src/__lib__/LegendCubeServiceLabels.ts",
     "./src/__lib__/LegendCubeTesting.ts",
+    "./src/__lib__/LegendCubeTour.ts",
     "./src/__test-utils__/CubeCanvasTestUtils.ts",
     "./src/__test-utils__/CubeConnectionTestSupport.ts",
     "./src/__test-utils__/CubeDirectConnectionFixtures.ts",
     "./src/__test-utils__/CubeEngineTestSupport.ts",
+    "./src/__test-utils__/CubeExampleTestUtils.ts",
     "./src/__test-utils__/CubeExpressionTestUtils.ts",
     "./src/__test-utils__/CubeFunctionTestOutline.ts",
     "./src/__test-utils__/CubeLambdaCorpus.ts",
     "./src/__test-utils__/CubeNorthwindTestQueries.ts",
     "./src/__test-utils__/CubeOperationsTestUtils.ts",
     "./src/__test-utils__/CubeTestApplication.ts",
+    "./src/__test-utils__/CubeVariantTestUtils.ts",
     "./src/__test-utils__/FakeCubeConnectionExplorer.ts",
     "./src/__test-utils__/FakeCubeDataProductCatalog.ts",
     "./src/__test-utils__/FakeCubeEngine.ts",
@@ -150,6 +154,7 @@
     "./src/stores/CubeLambdaImportState.ts",
     "./src/stores/CubeLambdaSources.ts",
     "./src/stores/CubeNodeEditorState.ts",
+    "./src/stores/CubePureText.ts",
     "./src/stores/CubeQueryRun.ts",
     "./src/stores/CubeRunStatsState.ts",
     "./src/stores/CubeRunningStepLambda.ts",
@@ -157,10 +162,13 @@
     "./src/stores/CubeShowPureState.ts",
     "./src/stores/CubeSourcePickerState.ts",
     "./src/stores/CubeSpecTransferState.ts",
+    "./src/stores/CubeTourState.ts",
+    "./src/stores/LegendCubeHost.ts",
     "./src/stores/LocalModelCatalog.ts",
     "./src/stores/editors/CubeAggregationRows.ts",
     "./src/stores/editors/CubeConcatDraft.ts",
     "./src/stores/editors/CubeDifferenceDraft.ts",
+    "./src/stores/editors/CubeExplodeDraft.ts",
     "./src/stores/editors/CubeExpressionFilterDraft.ts",
     "./src/stores/editors/CubeExtendDraft.ts",
     "./src/stores/editors/CubeFilterDraft.ts",
@@ -192,6 +200,14 @@
     "./src/stores/source-picker/CubeProjectTabState.ts",
     "./src/stores/source-picker/CubeServiceTabState.ts",
     "./src/stores/source-picker/CubeSourcePickerTab.ts",
+    "./src/stores/variant/CubeJson.ts",
+    "./src/stores/variant/CubeJsonExplorerState.ts",
+    "./src/stores/variant/CubeJsonFieldsState.ts",
+    "./src/stores/variant/CubeJsonOffers.ts",
+    "./src/stores/variant/CubeJsonParse.ts",
+    "./src/stores/variant/CubeJsonSampling.ts",
+    "./src/stores/variant/CubeJsonShape.ts",
+    "./src/stores/variant/CubeJsonViewerState.ts",
     "./src/__test-utils__/CubePageTestUtils.tsx",
     "./src/components/CubeAddItems.tsx",
     "./src/components/CubeButton.tsx",
@@ -213,6 +229,7 @@
     "./src/components/editors/CubeDataProductSourceEditor.tsx",
     "./src/components/editors/CubeDifferenceEditor.tsx",
     "./src/components/editors/CubeDistinctEditor.tsx",
+    "./src/components/editors/CubeExplodeEditor.tsx",
     "./src/components/editors/CubeExpressionFilterEditor.tsx",
     "./src/components/editors/CubeExtendEditor.tsx",
     "./src/components/editors/CubeFilterEditor.tsx",
@@ -222,6 +239,7 @@
     "./src/components/editors/CubeIntegerField.tsx",
     "./src/components/editors/CubeJoinEditor.tsx",
     "./src/components/editors/CubeJoinKeyRows.tsx",
+    "./src/components/editors/CubeJsonFieldsTree.tsx",
     "./src/components/editors/CubeNodeEditorPanel.tsx",
     "./src/components/editors/CubeNodeEditorPopper.tsx",
     "./src/components/editors/CubePartitionEditor.tsx",
@@ -240,6 +258,8 @@
     "./src/components/grid/CubeColumnFormatDialog.tsx",
     "./src/components/grid/CubeDrilldownBar.tsx",
     "./src/components/grid/CubeGridRegion.tsx",
+    "./src/components/grid/CubeJsonExplorerDialog.tsx",
+    "./src/components/grid/CubeJsonViewerDialog.tsx",
     "./src/components/grid/CubeResultGrid.tsx",
     "./src/components/grid/CubeRunStatsPanel.tsx",
     "./src/components/grid/CubeServerSideGrid.tsx",
@@ -258,7 +278,8 @@
     "./src/components/source-picker/CubeSourcePicker.tsx",
     "./src/components/source-picker/CubeTablePreview.tsx",
     "./src/components/spec-transfer/CubeImportPureDialog.tsx",
-    "./src/components/spec-transfer/CubeSpecTransferDialog.tsx"
+    "./src/components/spec-transfer/CubeSpecTransferDialog.tsx",
+    "./src/components/tour/CubeTour.tsx"
   ],
   "include": [
     "src/**/*.ts",