@finos/legend-cube-builder 0.0.4 → 0.0.6

package/src/stores/source-picker/CubeIngestTabState.ts

@@ -53,6 +53,7 @@ import type { CubeEditorState } from '../CubeEditorState.js';
 import {
   type CubeSourcePickerTab,
   CubeSourcePickerTabKey,
+  generateCubeNodeIds,
 } from './CubeSourcePickerTab.js';
 
 /** What failed in the tab: its first line, and the rest on demand */
@@ -66,6 +67,12 @@ export const CUBE_INGEST_TAB_MESSAGE = {
     'The cube changed while the data set was being added; pick it again.',
 } as const;
 
+/** A data set picked to add: the URN of its definition, and its name */
+export interface CubeIngestPick {
+  readonly definitionUrn: string;
+  readonly dataSet: string;
+}
+
 const toError = (error: unknown): CubeIngestTabError => {
   if (error instanceof CubeEngineError) {
     return {
@@ -79,10 +86,11 @@ const toError = (error: unknown): CubeIngestTabError => {
 /**
  * The source dialog's Ingest tab (PLAN §6.7), as Data Cube's producer
  * source picks: the Mode (class), the viewer's environment for it, a
- * producer deployment, one of its SDLC-deployed definitions, then a data
- * set, typed from what the definition declares. The first Add saves the
- * class, the producer deployment and the warehouse as the cube's model;
- * then the tab offers only that deployment's definitions.
+ * producer deployment, one of its SDLC-deployed definitions, or several
+ * ticked, then one or several of their data sets, typed from what each
+ * definition declares. The first Add saves the class, the producer
+ * deployment and the warehouse as the cube's model; then the tab offers only
+ * that deployment's definitions.
  */
 export class CubeIngestTabState implements CubeSourcePickerTab {
   readonly key = CubeSourcePickerTabKey.INGEST;
@@ -95,19 +103,24 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
   producerDeploymentId: string | undefined;
   definitionList: CubeIngestDefinitionList | undefined;
   definitionSearch = '';
-  definition: CubeIngestDefinitionCandidate | undefined;
-  dataSets: readonly CubeIngestDataSet[] | undefined;
-  dataSetName: string | undefined;
+  /** The definitions opened, in the order opened: one picked alone, or several ticked */
+  definitions: readonly CubeIngestDefinitionCandidate[] = [];
+  /** The data sets of each opened definition, by its URN, once read */
+  dataSetsByDefinition: ReadonlyMap<string, readonly CubeIngestDataSet[]> =
+    new Map();
+  /** The picked data sets, in the order picked, of any opened definition; several are added at once */
+  picks: readonly CubeIngestPick[] = [];
   /** The warehouse of a cube without one yet: the one the viewer last picked, else the default */
   warehouse = CUBE_DEFAULT_CONSUMER_WAREHOUSE;
   isLoadingEnvironment = false;
   isListingDefinitions = false;
-  isDescribing = false;
   isAdding = false;
   error: CubeIngestTabError | undefined;
 
   /** The class the environment and producers were read for */
   private loadedEnvironmentType: CubeDataProductEnvironmentType | undefined;
+  /** The definitions whose data sets are being read, by URN, each with its request */
+  private describing: ReadonlyMap<string, number> = new Map();
   /** Each counts its calls: a later call, or closing the dialog, drops an earlier answer */
   private environmentRequest = 0;
   private definitionsRequest = 0;
@@ -115,43 +128,55 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
   private confirmRequest = 0;
 
   constructor(editorState: CubeEditorState) {
-    makeObservable<CubeIngestTabState, 'loadedEnvironmentType'>(this, {
-      environmentType: observable,
-      environment: observable.ref,
-      producers: observable.ref,
-      producerDeploymentId: observable,
-      definitionList: observable.ref,
-      definitionSearch: observable,
-      definition: observable.ref,
-      dataSets: observable.ref,
-      dataSetName: observable,
-      warehouse: observable,
-      isLoadingEnvironment: observable,
-      isListingDefinitions: observable,
-      isDescribing: observable,
-      isAdding: observable,
-      error: observable.ref,
-      loadedEnvironmentType: observable,
-      fixedSettings: computed,
-      isAvailable: computed,
-      isBusy: computed,
-      shownDefinitions: computed,
-      dataSet: computed,
-      canConfirm: computed,
-      setEnvironmentType: action,
-      selectProducer: action,
-      setDefinitionSearch: action,
-      selectDefinition: action,
-      selectDataSet: action,
-      setWarehouse: action,
-      retry: action,
-      open: action,
-      close: action,
-      loadEnvironment: flow,
-      listDefinitions: flow,
-      describeDefinition: flow,
-      confirm: flow,
-    });
+    makeObservable<CubeIngestTabState, 'loadedEnvironmentType' | 'describing'>(
+      this,
+      {
+        environmentType: observable,
+        environment: observable.ref,
+        producers: observable.ref,
+        producerDeploymentId: observable,
+        definitionList: observable.ref,
+        definitionSearch: observable,
+        definitions: observable.ref,
+        dataSetsByDefinition: observable.ref,
+        picks: observable.ref,
+        definition: computed,
+        dataSets: computed,
+        dataSetNames: computed,
+        dataSetName: computed,
+        pickedDataSets: computed,
+        previewedDefinition: computed,
+        warehouse: observable,
+        isLoadingEnvironment: observable,
+        isListingDefinitions: observable,
+        isDescribing: computed,
+        isAdding: observable,
+        error: observable.ref,
+        loadedEnvironmentType: observable,
+        describing: observable.ref,
+        fixedSettings: computed,
+        isAvailable: computed,
+        isBusy: computed,
+        shownDefinitions: computed,
+        dataSet: computed,
+        canConfirm: computed,
+        setEnvironmentType: action,
+        selectProducer: action,
+        setDefinitionSearch: action,
+        selectDefinition: action,
+        toggleDefinition: action,
+        selectDataSet: action,
+        toggleDataSet: action,
+        setWarehouse: action,
+        retry: action,
+        open: action,
+        close: action,
+        loadEnvironment: flow,
+        listDefinitions: flow,
+        describeDefinition: flow,
+        confirm: flow,
+      },
+    );
     this.editorState = editorState;
   }
 
@@ -170,6 +195,11 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
     return model ? getCubeIngestSettings(model) : undefined;
   }
 
+  /** Whether data sets of an opened definition are being read */
+  get isDescribing(): boolean {
+    return this.describing.size > 0;
+  }
+
   get isBusy(): boolean {
     return (
       this.isLoadingEnvironment ||
@@ -194,8 +224,77 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
     );
   }
 
+  /** The definition opened last */
+  get definition(): CubeIngestDefinitionCandidate | undefined {
+    return this.definitions.at(-1);
+  }
+
+  /** The data sets of the definition opened last, once read */
+  get dataSets(): readonly CubeIngestDataSet[] | undefined {
+    const { definition } = this;
+    return definition
+      ? this.dataSetsByDefinition.get(definition.urn)
+      : undefined;
+  }
+
+  isDefinitionOpened(candidate: CubeIngestDefinitionCandidate): boolean {
+    return this.definitions.some((each) => each.urn === candidate.urn);
+  }
+
+  /** The data sets of an opened definition, once read */
+  getDataSets(definitionUrn: string): readonly CubeIngestDataSet[] | undefined {
+    return this.dataSetsByDefinition.get(definitionUrn);
+  }
+
+  /** The names of the picked data sets, in the order picked */
+  get dataSetNames(): readonly string[] {
+    return this.picks.map((pick) => pick.dataSet);
+  }
+
+  /** The data set picked last, the one previewed */
+  get dataSetName(): string | undefined {
+    return this.picks.at(-1)?.dataSet;
+  }
+
+  /** The data set picked last, the one previewed */
   get dataSet(): CubeIngestDataSet | undefined {
-    return this.dataSets?.find((each) => each.name === this.dataSetName);
+    return this.findDataSet(this.picks.at(-1));
+  }
+
+  /** The definition of the data set picked last */
+  get previewedDefinition(): CubeIngestDefinitionCandidate | undefined {
+    const urn = this.picks.at(-1)?.definitionUrn;
+    return this.definitions.find((each) => each.urn === urn);
+  }
+
+  /** The picked data sets, in the order picked */
+  get pickedDataSets(): readonly (CubeIngestDataSet | undefined)[] {
+    return this.picks.map((pick) => this.findDataSet(pick));
+  }
+
+  /** How many data sets Add adds */
+  get pickedCount(): number {
+    return this.picks.length;
+  }
+
+  /** Whether a data set of an opened definition, by default the one opened last, is picked */
+  isDataSetPicked(
+    name: string,
+    definitionUrn: string | undefined = this.definition?.urn,
+  ): boolean {
+    return this.picks.some(
+      (pick) => pick.definitionUrn === definitionUrn && pick.dataSet === name,
+    );
+  }
+
+  private findDataSet(
+    pick: CubeIngestPick | undefined,
+  ): CubeIngestDataSet | undefined {
+    return pick
+      ? this.dataSetsByDefinition
+          .get(pick.definitionUrn)
+          ?.find((each) => each.name === pick.dataSet)
+      : undefined;
   }
 
   get canConfirm(): boolean {
@@ -205,8 +304,8 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
       !this.isDescribing &&
       !this.isAdding &&
       this.producerDeploymentId !== undefined &&
-      this.definition !== undefined &&
-      this.dataSet?.isPickable === true &&
+      this.pickedDataSets.length > 0 &&
+      this.pickedDataSets.every((each) => each?.isPickable === true) &&
       this.warehouse.trim() !== ''
     );
   }
@@ -241,24 +340,66 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
     this.definitionSearch = search;
   }
 
-  /** Picks a definition and reads its data sets */
+  /**
+   * Opens a definition alone, in place of any opened, and reads its data
+   * sets; one already opened among others keeps its data sets and picks
+   */
   selectDefinition(candidate: CubeIngestDefinitionCandidate | undefined): void {
-    if (candidate?.urn === this.definition?.urn) {
+    if (
+      this.definitions.length <= 1 &&
+      candidate?.urn === this.definition?.urn
+    ) {
+      return;
+    }
+    if (candidate && this.isDefinitionOpened(candidate)) {
+      this.closeDefinitions((each) => each.urn !== candidate.urn);
       return;
     }
     this.resetDefinition();
-    this.definition = candidate;
     if (candidate) {
-      flowResult(this.describeDefinition(candidate)).catch(
-        this.editorState.host.applicationStore.alertUnhandledError,
-      );
+      this.openDefinition(candidate);
     }
   }
 
-  /** Picks a data set that can be added */
-  selectDataSet(name: string): void {
-    if (this.dataSets?.find((each) => each.name === name)?.isPickable) {
-      this.dataSetName = name;
+  /**
+   * Opens a definition beside those opened, to add data sets of several at
+   * once, and reads its data sets; or closes it, its picks dropped
+   */
+  toggleDefinition(candidate: CubeIngestDefinitionCandidate): void {
+    if (this.isDefinitionOpened(candidate)) {
+      this.closeDefinitions((each) => each.urn === candidate.urn);
+    } else {
+      this.openDefinition(candidate);
+    }
+  }
+
+  /** Picks a data set that can be added, alone, in place of any picked */
+  selectDataSet(
+    name: string,
+    definitionUrn: string | undefined = this.definition?.urn,
+  ): void {
+    if (
+      definitionUrn !== undefined &&
+      this.findDataSet({ definitionUrn, dataSet: name })?.isPickable
+    ) {
+      this.picks = [{ definitionUrn, dataSet: name }];
+    }
+  }
+
+  /** Adds a data set that can be added to the picked ones, or takes it out */
+  toggleDataSet(
+    name: string,
+    definitionUrn: string | undefined = this.definition?.urn,
+  ): void {
+    if (definitionUrn === undefined) {
+      return;
+    }
+    if (this.isDataSetPicked(name, definitionUrn)) {
+      this.picks = this.picks.filter(
+        (pick) => pick.definitionUrn !== definitionUrn || pick.dataSet !== name,
+      );
+    } else if (this.findDataSet({ definitionUrn, dataSet: name })?.isPickable) {
+      this.picks = [...this.picks, { definitionUrn, dataSet: name }];
     }
   }
 
@@ -280,10 +421,16 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
       !this.definitionList
     ) {
       flowResult(this.listDefinitions()).catch(alertUnhandledError);
-    } else if (this.definition && !this.dataSets) {
-      flowResult(this.describeDefinition(this.definition)).catch(
-        alertUnhandledError,
-      );
+    } else {
+      this.definitions
+        .filter(
+          (each) =>
+            !this.dataSetsByDefinition.has(each.urn) &&
+            !this.describing.has(each.urn),
+        )
+        .forEach((each) => {
+          flowResult(this.describeDefinition(each)).catch(alertUnhandledError);
+        });
     }
   }
 
@@ -333,10 +480,8 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
     this.isListingDefinitions = false;
     this.isAdding = false;
     // a definition whose data sets haven't arrived would show none
-    if (this.definition && !this.dataSets) {
-      this.resetDefinition();
-    }
-    this.isDescribing = false;
+    this.describing = new Map();
+    this.closeDefinitions((each) => !this.dataSetsByDefinition.has(each.urn));
     // read again on reopening
     if (!this.environment || !this.producers) {
       this.loadedEnvironmentType = undefined;
@@ -364,6 +509,20 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
       }
       this.environment = environment;
       this.producers = producers;
+      // a new cube starts on the environment's own warehouse, when it has one
+      if (!this.fixedSettings && environment.warehouse) {
+        this.warehouse = environment.warehouse;
+      }
+      // the only producer deployment is the one to read
+      const [onlyProducer] = producers;
+      if (
+        this.producerDeploymentId === undefined &&
+        producers.length === 1 &&
+        onlyProducer
+      ) {
+        this.producerDeploymentId = onlyProducer.deploymentId;
+        this.resetDefinitions();
+      }
     } catch (error) {
       if (request === this.environmentRequest) {
         this.loadedEnvironmentType = undefined;
@@ -408,7 +567,7 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
     }
   }
 
-  /** Reads the definition's data sets */
+  /** Reads an opened definition's data sets */
   *describeDefinition(
     candidate: CubeIngestDefinitionCandidate,
   ): GeneratorFn<void> {
@@ -417,43 +576,67 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
       return;
     }
     const request = ++this.describeRequest;
-    this.isDescribing = true;
+    this.describing = new Map([...this.describing, [candidate.urn, request]]);
     this.error = undefined;
+    // an answer counts while the definition is open and this is its last read
+    const isCurrent = (): boolean =>
+      this.describing.get(candidate.urn) === request;
     try {
       const dataSets = (yield catalog.describe(
         candidate.urn,
         this.environmentType,
       )) as readonly CubeIngestDataSet[];
-      if (request === this.describeRequest) {
-        this.dataSets = dataSets;
-        // the only data set that can be added is picked
-        const pickable = dataSets.filter((each) => each.isPickable);
-        this.dataSetName =
-          pickable.length === 1 ? pickable[0]?.name : undefined;
+      if (isCurrent()) {
+        this.dataSetsByDefinition = new Map([
+          ...this.dataSetsByDefinition,
+          [candidate.urn, dataSets],
+        ]);
+        // the only data set of it that can be added is picked
+        const [onlyPickable, ...otherPickable] = dataSets.filter(
+          (each) => each.isPickable,
+        );
+        if (
+          onlyPickable &&
+          !otherPickable.length &&
+          !this.isDataSetPicked(onlyPickable.name, candidate.urn)
+        ) {
+          this.picks = [
+            ...this.picks,
+            { definitionUrn: candidate.urn, dataSet: onlyPickable.name },
+          ];
+        }
       }
     } catch (error) {
-      if (request === this.describeRequest) {
+      if (isCurrent()) {
         this.error = toError(error);
       }
     } finally {
-      if (request === this.describeRequest) {
-        this.isDescribing = false;
+      if (isCurrent()) {
+        this.describing = new Map(
+          [...this.describing].filter(([urn]) => urn !== candidate.urn),
+        );
       }
     }
   }
 
   /**
-   * Adds the data set, already typed; the first saves the class, the
-   * producer deployment and the warehouse as the cube's model, in the same
-   * undo step, and remembers the warehouse for the viewer's next cubes
+   * Adds the picked data sets, already typed, in one undo step; the first
+   * saves the class, the producer deployment and the warehouse as the cube's
+   * model, in the same undo step, and remembers the warehouse for the
+   * viewer's next cubes
    */
   *confirm(): GeneratorFn<boolean> {
     if (!this.canConfirm) {
       return false;
     }
     const { editorState } = this;
-    const definition = this.definition as CubeIngestDefinitionCandidate;
-    const dataSet = this.dataSet as CubeIngestDataSet;
+    // each picked data set with its definition, as picked
+    const picked = this.picks.map((pick) => ({
+      definition: this.definitions.find(
+        (each) => each.urn === pick.definitionUrn,
+      ) as CubeIngestDefinitionCandidate,
+      dataSet: this.findDataSet(pick) as CubeIngestDataSet,
+    }));
     const producerDeploymentId = this.producerDeploymentId as string;
     const warehouse = this.warehouse.trim();
     const request = ++this.confirmRequest;
@@ -466,29 +649,45 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
         return false;
       }
       const { document } = editorState;
-      const id = document.query.generateId(IngestDatasetSource.TYPE);
-      const node = INGEST_DATASET_SOURCE_DEFINITION.resolve(
-        INGEST_DATASET_SOURCE_DEFINITION.fromCoordinates(id, {
-          ingestDefinitionUrn: definition.urn,
-          ingestDefinition: definition.definition,
-          dataSet: dataSet.name,
-        }),
-        {
-          kind: 'resolved',
-          schema: dataSet.schema as NonNullable<CubeIngestDataSet['schema']>,
-        },
+      const ids = generateCubeNodeIds(
+        document.query,
+        IngestDatasetSource.TYPE,
+        picked.length,
+      );
+      const nodes = picked.map(({ definition, dataSet }, index) =>
+        INGEST_DATASET_SOURCE_DEFINITION.resolve(
+          INGEST_DATASET_SOURCE_DEFINITION.fromCoordinates(
+            ids[index] as string,
+            {
+              ingestDefinitionUrn: definition.urn,
+              ingestDefinition: definition.definition,
+              dataSet: dataSet.name,
+            },
+          ),
+          {
+            kind: 'resolved',
+            schema: dataSet.schema as NonNullable<CubeIngestDataSet['schema']>,
+          },
+        ),
       );
       const settings = this.fixedSettings;
       if (
-        (document.context !== undefined &&
-          (!settings ||
-            settings.environmentType !== this.environmentType ||
-            settings.producerDeploymentId !== producerDeploymentId)) ||
-        !document.query.canAdd(node)
+        document.context !== undefined &&
+        (!settings ||
+          settings.environmentType !== this.environmentType ||
+          settings.producerDeploymentId !== producerDeploymentId)
       ) {
         throw new Error(CUBE_INGEST_TAB_MESSAGE.CUBE_CHANGED);
       }
-      const query = document.query.add(node);
+      // each data set is added in the order picked; the first one picked is
+      // the one that runs on an empty cube, as a single one would be
+      let query = document.query;
+      for (const node of nodes) {
+        if (!query.canAdd(node)) {
+          throw new Error(CUBE_INGEST_TAB_MESSAGE.CUBE_CHANGED);
+        }
+        query = query.add(node);
+      }
       editorState.applyDocument(
         document.context
           ? document.withQuery(query)
@@ -506,8 +705,8 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
       if (!settings) {
         editorState.dataProductRuntime.remember(warehouse);
       }
-      // the definition stays open, to add another of its data sets
-      this.dataSetName = undefined;
+      // the definitions stay open, to add others of their data sets
+      this.picks = [];
       return true;
     } catch (error) {
       if (request === this.confirmRequest) {
@@ -542,10 +741,36 @@ export class CubeIngestTabState implements CubeSourcePickerTab {
   }
 
   private resetDefinition(): void {
-    this.describeRequest++;
-    this.isDescribing = false;
-    this.definition = undefined;
-    this.dataSets = undefined;
-    this.dataSetName = undefined;
+    this.describing = new Map();
+    this.definitions = [];
+    this.dataSetsByDefinition = new Map();
+    this.picks = [];
+  }
+
+  private openDefinition(candidate: CubeIngestDefinitionCandidate): void {
+    this.definitions = [...this.definitions, candidate];
+    flowResult(this.describeDefinition(candidate)).catch(
+      this.editorState.host.applicationStore.alertUnhandledError,
+    );
+  }
+
+  /** Closes the opened definitions given, their data sets, reads and picks dropped */
+  private closeDefinitions(
+    closes: (candidate: CubeIngestDefinitionCandidate) => boolean,
+  ): void {
+    const closed = new Set(
+      this.definitions.filter(closes).map((each) => each.urn),
+    );
+    if (!closed.size) {
+      return;
+    }
+    this.definitions = this.definitions.filter((each) => !closed.has(each.urn));
+    this.dataSetsByDefinition = new Map(
+      [...this.dataSetsByDefinition].filter(([urn]) => !closed.has(urn)),
+    );
+    this.describing = new Map(
+      [...this.describing].filter(([urn]) => !closed.has(urn)),
+    );
+    this.picks = this.picks.filter((pick) => !closed.has(pick.definitionUrn));
   }
 }