@platforma-sdk/model 1.53.15 → 1.54.8

package/src/components/PlDataTable/state-migration.ts

@@ -0,0 +1,285 @@
+import type {
+  AxisId,
+  CanonicalizedJson,
+  PObjectId,
+  PTableColumnId,
+  PTableColumnSpec,
+  PTableRecordFilter,
+  PTableSorting,
+} from "@milaboratories/pl-model-common";
+import { canonicalizeJson } from "@milaboratories/pl-model-common";
+import { distillFilterSpec } from "../../filters";
+import type { PlDataTableFilterState, PlTableFilter } from "./v4";
+import type {
+  PlDataTableFiltersWithMeta,
+  PlDataTableGridStateCore,
+  PlDataTableSheetState,
+  PlDataTableStateV2CacheEntry,
+  PlDataTableStateV2Normalized,
+  PTableParamsV2,
+} from "./v5";
+
+/**
+ * PlDataTableV2 persisted state
+ */
+export type PlDataTableStateV2 =
+  // Old versions of the state
+  | {
+      // no version
+      gridState: {
+        columnOrder?: {
+          orderedColIds: CanonicalizedJson<PTableColumnSpec>[];
+        };
+        sort?: {
+          sortModel: {
+            colId: CanonicalizedJson<PTableColumnSpec>;
+            sort: "asc" | "desc";
+          }[];
+        };
+        columnVisibility?: {
+          hiddenColIds: CanonicalizedJson<PTableColumnSpec>[];
+        };
+        sourceId?: string;
+        sheets?: Record<CanonicalizedJson<AxisId>, string | number>;
+      };
+      pTableParams?: {
+        sorting?: PTableSorting[];
+        filters?: PTableRecordFilter[];
+      };
+    }
+  | {
+      version: 2;
+      stateCache: {
+        sourceId: string;
+        gridState: {
+          columnOrder?: {
+            orderedColIds: CanonicalizedJson<PTableColumnSpec>[];
+          };
+          sort?: {
+            sortModel: {
+              colId: CanonicalizedJson<PTableColumnSpec>;
+              sort: "asc" | "desc";
+            }[];
+          };
+          columnVisibility?: {
+            hiddenColIds: CanonicalizedJson<PTableColumnSpec>[];
+          };
+        };
+        sheetsState: PlDataTableSheetState[];
+      }[];
+      pTableParams: {
+        hiddenColIds: PObjectId[] | null;
+        filters: PTableRecordFilter[];
+        sorting: PTableSorting[];
+      };
+    }
+  | {
+      version: 3;
+      stateCache: {
+        sourceId: string;
+        gridState: {
+          columnOrder?: {
+            orderedColIds: CanonicalizedJson<PTableColumnSpec>[];
+          };
+          sort?: {
+            sortModel: {
+              colId: CanonicalizedJson<PTableColumnSpec>;
+              sort: "asc" | "desc";
+            }[];
+          };
+          columnVisibility?: {
+            hiddenColIds: CanonicalizedJson<PTableColumnSpec>[];
+          };
+        };
+        sheetsState: PlDataTableSheetState[];
+        filtersState: PlDataTableFilterState[];
+      }[];
+      pTableParams: PTableParamsV2;
+    }
+  | {
+      version: 4;
+      stateCache: {
+        sourceId: string;
+        gridState: PlDataTableGridStateCore;
+        sheetsState: PlDataTableSheetState[];
+        filtersState: PlDataTableFilterState[];
+      }[];
+      /** Old format; only fields used in migration are typed */
+      pTableParams: {
+        sourceId: string | null;
+        hiddenColIds: PObjectId[] | null;
+        sorting: PTableSorting[];
+      };
+    }
+  // Normalized state
+  | PlDataTableStateV2Normalized;
+
+/** Upgrade PlDataTableStateV2 to the latest version */
+export function upgradePlDataTableStateV2(
+  state: PlDataTableStateV2 | undefined,
+): PlDataTableStateV2Normalized {
+  // Block just added, had no state, model started earlier than the UI
+  if (!state) {
+    return createPlDataTableStateV2();
+  }
+  // v1 -> v2
+  if (!("version" in state)) {
+    // Non upgradeable as sourceId calculation algorithm has changed, resetting state to default
+    state = createPlDataTableStateV2();
+  }
+  // v2 -> v3
+  if (state.version === 2) {
+    state = {
+      version: 3,
+      stateCache: state.stateCache.map((entry) => ({
+        ...entry,
+        filtersState: [],
+      })),
+      pTableParams: createDefaultPTableParams(),
+    };
+  }
+  // v3 -> v4
+  if (state.version === 3) {
+    // Non upgradeable as column ids calculation algorithm has changed, resetting state to default
+    state = createPlDataTableStateV2();
+  }
+  // v4 -> v5: migrate per-column filters to tree-based format
+  if (state.version === 4) {
+    state = migrateV4toV5(state);
+  }
+  return state;
+}
+
+/** Migrate v4 state to v5: convert per-column filters to tree-based format */
+function migrateV4toV5(
+  state: Extract<PlDataTableStateV2, { version: 4 }>,
+): PlDataTableStateV2Normalized {
+  let idCounter = 0;
+  const nextId = () => ++idCounter;
+
+  const migratedCache: PlDataTableStateV2CacheEntry[] = state.stateCache.map((entry) => {
+    const leaves: PlDataTableFiltersWithMeta[] = [];
+    for (const f of entry.filtersState) {
+      if (f.filter !== null && !f.filter.disabled) {
+        const column = canonicalizeJson<PTableColumnId>(f.id);
+        leaves.push(migrateTableFilter(column, f.filter.value, nextId));
+      }
+    }
+    const filtersState: PlDataTableFiltersWithMeta | null =
+      leaves.length > 0 ? { id: nextId(), type: "and", filters: leaves } : null;
+
+    return {
+      sourceId: entry.sourceId,
+      gridState: entry.gridState,
+      sheetsState: entry.sheetsState,
+      filtersState,
+    };
+  });
+
+  const oldSourceId = state.pTableParams.sourceId;
+  const currentCache = oldSourceId
+    ? migratedCache.find((e) => e.sourceId === oldSourceId)
+    : undefined;
+
+  return {
+    version: 5,
+    stateCache: migratedCache,
+    pTableParams:
+      currentCache && oldSourceId
+        ? {
+            sourceId: oldSourceId,
+            hiddenColIds: state.pTableParams.hiddenColIds,
+            filters: distillFilterSpec(currentCache.filtersState),
+            sorting: state.pTableParams.sorting,
+          }
+        : createDefaultPTableParams(),
+  };
+}
+
+/** Migrate a single per-column PlTableFilter to a tree-based FilterSpec node */
+function migrateTableFilter(
+  column: string,
+  filter: PlTableFilter,
+  nextId: () => number,
+): PlDataTableFiltersWithMeta {
+  const id = nextId();
+  switch (filter.type) {
+    case "isNA":
+      return { id, type: "isNA", column };
+    case "isNotNA":
+      return { id, type: "isNotNA", column };
+    case "number_equals":
+      return { id, type: "equal", column, x: filter.reference };
+    case "number_notEquals":
+      return { id, type: "notEqual", column, x: filter.reference };
+    case "number_greaterThan":
+      return { id, type: "greaterThan", column, x: filter.reference };
+    case "number_greaterThanOrEqualTo":
+      return { id, type: "greaterThanOrEqual", column, x: filter.reference };
+    case "number_lessThan":
+      return { id, type: "lessThan", column, x: filter.reference };
+    case "number_lessThanOrEqualTo":
+      return { id, type: "lessThanOrEqual", column, x: filter.reference };
+    case "number_between":
+      return {
+        id,
+        type: "and",
+        filters: [
+          filter.includeLowerBound
+            ? { id: nextId(), type: "greaterThanOrEqual" as const, column, x: filter.lowerBound }
+            : { id: nextId(), type: "greaterThan" as const, column, x: filter.lowerBound },
+          filter.includeUpperBound
+            ? { id: nextId(), type: "lessThanOrEqual" as const, column, x: filter.upperBound }
+            : { id: nextId(), type: "lessThan" as const, column, x: filter.upperBound },
+        ],
+      };
+    case "string_equals":
+      return { id, type: "patternEquals", column, value: filter.reference };
+    case "string_notEquals":
+      return { id, type: "patternNotEquals", column, value: filter.reference };
+    case "string_contains":
+      return { id, type: "patternContainSubsequence", column, value: filter.reference };
+    case "string_doesNotContain":
+      return { id, type: "patternNotContainSubsequence", column, value: filter.reference };
+    case "string_matches":
+      return { id, type: "patternMatchesRegularExpression", column, value: filter.reference };
+    case "string_doesNotMatch":
+      return {
+        id,
+        type: "not",
+        filter: {
+          id: nextId(),
+          type: "patternMatchesRegularExpression",
+          column,
+          value: filter.reference,
+        },
+      };
+    case "string_containsFuzzyMatch":
+      return {
+        id,
+        type: "patternFuzzyContainSubsequence",
+        column,
+        value: filter.reference,
+        maxEdits: filter.maxEdits,
+        substitutionsOnly: filter.substitutionsOnly,
+        ...(filter.wildcard !== undefined ? { wildcard: filter.wildcard } : {}),
+      };
+  }
+}
+
+export function createDefaultPTableParams(): PTableParamsV2 {
+  return {
+    sourceId: null,
+    hiddenColIds: null,
+    filters: null,
+    sorting: [],
+  };
+}
+
+export function createPlDataTableStateV2(): PlDataTableStateV2Normalized {
+  return {
+    version: 5,
+    stateCache: [],
+    pTableParams: createDefaultPTableParams(),
+  };
+}

package/src/components/PlDataTable/table.ts

@@ -0,0 +1,278 @@
+import type {
+  AxisId,
+  AxisSpec,
+  DataInfo,
+  PColumn,
+  PColumnIdAndSpec,
+  PColumnValues,
+  PObjectId,
+  PTableColumnId,
+  PTableColumnIdAxis,
+  PTableColumnIdColumn,
+  PTableDefV2,
+  PTableSorting,
+  SpecQuery,
+  SingleAxisSelector,
+  SpecQueryExpression,
+  SpecQueryJoinEntry,
+} from "@milaboratories/pl-model-common";
+import {
+  Annotation,
+  canonicalizeJson,
+  getAxisId,
+  getColumnIdAndSpec,
+  isLinkerColumn,
+  readAnnotation,
+  uniqueBy,
+  isBooleanExpression,
+} from "@milaboratories/pl-model-common";
+import { filterSpecToSpecQueryExpr } from "../../filters";
+import type { RenderCtxBase, TreeNodeAccessor, PColumnDataUniversal } from "../../render";
+import { allPColumnsReady, deriveLabels } from "../../render";
+import { identity, isFunction, isNil } from "es-toolkit";
+import { distillFilterSpec } from "../../filters/distill";
+import type { CreatePlDataTableOps, PlDataTableFilters, PlDataTableModel } from "./v5";
+import { upgradePlDataTableStateV2 } from "./state-migration";
+import type { PlDataTableStateV2 } from "./state-migration";
+import type { PlDataTableSheet } from "./v5";
+import { getAllLabelColumns, getMatchingLabelColumns } from "./labels";
+
+/** Convert a PTableColumnId to a SpecQueryExpression reference. */
+function columnIdToExpr(col: PTableColumnId): SpecQueryExpression {
+  if (col.type === "axis") {
+    return { type: "axisRef", value: col.id as SingleAxisSelector };
+  }
+  return { type: "columnRef", value: col.id };
+}
+
+/** Wrap a SpecQuery as a SpecQueryJoinEntry with empty qualifications. */
+function joinEntry<C>(input: SpecQuery<C>): SpecQueryJoinEntry<C> {
+  return { entry: input, qualifications: [] };
+}
+
+function createPTableDef(params: {
+  columns: PColumn<PColumnDataUniversal>[];
+  labelColumns: PColumn<PColumnDataUniversal>[];
+  coreJoinType: "inner" | "full";
+  filters: null | PlDataTableFilters;
+  sorting: PTableSorting[];
+  coreColumnPredicate?: (spec: PColumnIdAndSpec) => boolean;
+}): PTableDefV2<PColumn<TreeNodeAccessor | PColumnValues | DataInfo<TreeNodeAccessor>>> {
+  let coreColumns = params.columns;
+  const secondaryColumns: typeof params.columns = [];
+
+  if (isFunction(params.coreColumnPredicate)) {
+    coreColumns = [];
+    for (const c of params.columns)
+      if (params.coreColumnPredicate(getColumnIdAndSpec(c))) coreColumns.push(c);
+      else secondaryColumns.push(c);
+  }
+
+  secondaryColumns.push(...params.labelColumns);
+
+  // Build SpecQuery directly from columns
+  const coreJoinQuery: SpecQuery<
+    PColumn<TreeNodeAccessor | PColumnValues | DataInfo<TreeNodeAccessor>>
+  > = {
+    type: params.coreJoinType === "inner" ? "innerJoin" : "fullJoin",
+    entries: coreColumns.map((c) => joinEntry({ type: "column", column: c })),
+  };
+
+  let query: SpecQuery<PColumn<TreeNodeAccessor | PColumnValues | DataInfo<TreeNodeAccessor>>> = {
+    type: "outerJoin",
+    primary: joinEntry(coreJoinQuery),
+    secondary: secondaryColumns.map((c) => joinEntry({ type: "column", column: c })),
+  };
+
+  // Apply filters
+  if (params.filters !== null) {
+    const nonEmpty = distillFilterSpec(params.filters);
+
+    if (!isNil(nonEmpty)) {
+      const pridicate = filterSpecToSpecQueryExpr(nonEmpty);
+      if (!isBooleanExpression(pridicate)) {
+        throw new Error(
+          `Filter conversion produced a non-boolean expression (got type "${pridicate.type}"), expected a boolean predicate for query filtering`,
+        );
+      }
+      query = {
+        type: "filter",
+        input: query,
+        predicate: pridicate,
+      };
+    }
+  }
+
+  // Apply sorting
+  if (params.sorting.length > 0) {
+    query = {
+      type: "sort",
+      input: query,
+      sortBy: params.sorting.map((s) => ({
+        expression: columnIdToExpr(s.column),
+        ascending: s.ascending,
+        nullsFirst: s.ascending === s.naAndAbsentAreLeastValues,
+      })),
+    };
+  }
+
+  return { query };
+}
+
+/** Check if column should be omitted from the table */
+export function isColumnHidden(spec: { annotations?: Annotation }): boolean {
+  return readAnnotation(spec, Annotation.Table.Visibility) === "hidden";
+}
+
+/** Check if column is hidden by default */
+export function isColumnOptional(spec: { annotations?: Annotation }): boolean {
+  return readAnnotation(spec, Annotation.Table.Visibility) === "optional";
+}
+
+/**
+ * Create p-table spec and handle given ui table state
+ *
+ * @param ctx context
+ * @param columns column list
+ * @param tableState table ui state
+ * @returns PlAgDataTableV2 table source
+ */
+export function createPlDataTableV2<A, U>(
+  ctx: RenderCtxBase<A, U>,
+  columns: PColumn<PColumnDataUniversal>[],
+  tableState: PlDataTableStateV2 | undefined,
+  ops?: CreatePlDataTableOps,
+): PlDataTableModel | undefined {
+  if (columns.length === 0) return undefined;
+
+  const tableStateNormalized = upgradePlDataTableStateV2(tableState);
+
+  const allLabelColumns = getAllLabelColumns(ctx.resultPool);
+  if (!allLabelColumns) return undefined;
+
+  let fullLabelColumns = getMatchingLabelColumns(columns.map(getColumnIdAndSpec), allLabelColumns);
+  fullLabelColumns = deriveLabels(fullLabelColumns, identity, { includeNativeLabel: true }).map(
+    (v) => {
+      return {
+        ...v.value,
+        spec: {
+          ...v.value.spec,
+          annotations: {
+            ...v.value.spec.annotations,
+            [Annotation.Label]: v.label,
+          },
+        },
+      };
+    },
+  );
+
+  const fullColumns = [...columns, ...fullLabelColumns];
+
+  const fullColumnsAxes = uniqueBy(
+    fullColumns.flatMap((c) => c.spec.axesSpec.map((a) => getAxisId(a))),
+    (a) => canonicalizeJson<AxisId>(a),
+  );
+  const fullColumnsIds: PTableColumnId[] = [
+    ...fullColumnsAxes.map((a) => ({ type: "axis", id: a }) satisfies PTableColumnIdAxis),
+    ...fullColumns.map((c) => ({ type: "column", id: c.id }) satisfies PTableColumnIdColumn),
+  ];
+  const fullColumnsIdsSet = new Set(fullColumnsIds.map((c) => canonicalizeJson<PTableColumnId>(c)));
+  const isValidColumnId = (id: PTableColumnId): boolean =>
+    fullColumnsIdsSet.has(canonicalizeJson<PTableColumnId>(id));
+
+  const coreJoinType = ops?.coreJoinType ?? "full";
+  const filters = tableStateNormalized.pTableParams.filters;
+  const sorting: PTableSorting[] = uniqueBy(
+    [...tableStateNormalized.pTableParams.sorting, ...(ops?.sorting ?? [])],
+    (s) => canonicalizeJson<PTableColumnId>(s.column),
+  ).filter((s) => {
+    const valid = isValidColumnId(s.column);
+    if (!valid)
+      ctx.logWarn(`Sorting ${JSON.stringify(s)} does not match provided columns, skipping`);
+    return valid;
+  });
+
+  const fullDef = createPTableDef({
+    columns,
+    labelColumns: fullLabelColumns,
+    coreJoinType,
+    filters,
+    sorting,
+    coreColumnPredicate: ops?.coreColumnPredicate,
+  });
+  const fullHandle = ctx.createPTableV2(fullDef);
+  if (!fullHandle) return undefined;
+
+  const hiddenColumns = new Set<PObjectId>(
+    ((): PObjectId[] => {
+      // Inner join works as a filter - all columns must be present
+      if (coreJoinType === "inner") return [];
+
+      const hiddenColIds = tableStateNormalized.pTableParams.hiddenColIds;
+      if (hiddenColIds) return hiddenColIds;
+
+      return columns.filter((c) => isColumnOptional(c.spec)).map((c) => c.id);
+    })(),
+  );
+
+  // Preserve linker columns
+  columns.filter((c) => isLinkerColumn(c.spec)).forEach((c) => hiddenColumns.delete(c.id));
+
+  // Preserve core columns as they change the shape of join.
+  const coreColumnPredicate = ops?.coreColumnPredicate;
+  if (coreColumnPredicate) {
+    const coreColumns = columns.flatMap((c) =>
+      coreColumnPredicate(getColumnIdAndSpec(c)) ? [c.id] : [],
+    );
+    coreColumns.forEach((c) => hiddenColumns.delete(c));
+  }
+
+  // Sorting changes the order of result rows — preserve sorted columns from being hidden
+  sorting
+    .map((s) => s.column)
+    .filter((c): c is PTableColumnIdColumn => c.type === "column")
+    .forEach((c) => hiddenColumns.delete(c.id));
+
+  const visibleColumns = columns.filter((c) => !hiddenColumns.has(c.id));
+  const visibleLabelColumns = getMatchingLabelColumns(
+    visibleColumns.map(getColumnIdAndSpec),
+    allLabelColumns,
+  );
+
+  // if at least one column is not yet computed, we can't show the table
+  if (!allPColumnsReady([...visibleColumns, ...visibleLabelColumns])) return undefined;
+
+  const visibleDef = createPTableDef({
+    columns: visibleColumns,
+    labelColumns: visibleLabelColumns,
+    coreJoinType,
+    filters,
+    sorting,
+    coreColumnPredicate,
+  });
+  const visibleHandle = ctx.createPTableV2(visibleDef);
+  if (!visibleHandle) return undefined;
+
+  return {
+    sourceId: tableStateNormalized.pTableParams.sourceId,
+    fullTableHandle: fullHandle,
+    visibleTableHandle: visibleHandle,
+  } satisfies PlDataTableModel;
+}
+
+/** Create sheet entries for PlDataTable */
+export function createPlDataTableSheet<A, U>(
+  ctx: RenderCtxBase<A, U>,
+  axis: AxisSpec,
+  values: (string | number)[],
+): PlDataTableSheet {
+  const labels = ctx.resultPool.findLabels(axis);
+  return {
+    axis,
+    options: values.map((v) => ({
+      value: v,
+      label: labels?.[v] ?? v.toString(),
+    })),
+    defaultValue: values[0],
+  };
+}