@platforma-sdk/model 1.63.1 → 1.64.0

package/src/components/PlDataTable/createPlDataTable/createPlDataTableV3.ts

@@ -1,380 +1,569 @@
 import type {
   AxisId,
   CanonicalizedJson,
-  DataInfo,
+  FilterSpecNode,
   PColumn,
-  PColumnIdAndSpec,
-  PColumnValues,
   PObjectId,
   PTableColumnId,
   PTableColumnIdAxis,
   PTableColumnIdColumn,
-  PTableDefV2,
   PTableSorting,
-  SpecQuery,
-  SingleAxisSelector,
-  SpecQueryExpression,
-  SpecQueryJoinEntry,
   PColumnSpec,
-  PlRef,
   MultiColumnSelector,
+  SUniversalPColumnId,
 } from "@milaboratories/pl-model-common";
 import {
-  Annotation,
   canonicalizeJson,
   getAxisId,
   getColumnIdAndSpec,
-  isLinkerColumn,
-  readAnnotation,
-  isBooleanExpression,
   parseJson,
   uniqueBy,
 } from "@milaboratories/pl-model-common";
-import { filterSpecToSpecQueryExpr } from "../../../filters";
-import { collectFilterSpecColumns } from "../../../filters/traverse";
-import type { RenderCtxBase, TreeNodeAccessor, PColumnDataUniversal } from "../../../render";
-import { allPColumnsReady } from "../../../render";
-import { isFunction, isNil } from "es-toolkit";
+import { collectFilterSpecColumns, traverseFilterSpec } from "../../../filters/traverse";
+import type { RenderCtxBase, PColumnDataUniversal } from "../../../render";
 import { isEmpty } from "es-toolkit/compat";
-import { distillFilterSpec } from "../../../filters/distill";
-import type { PlDataTableFilters, PlDataTableModel } from "../typesV5";
+import type { PlDataTableFilters, PlDataTableFilterSpecLeaf, PlDataTableModel } from "../typesV5";
 import { upgradePlDataTableStateV2 } from "../state-migration";
 import type { PlDataTableStateV2 } from "../state-migration";
-import type { ColumnSource, MatchingMode } from "../../../columns";
+import type { ColumnMatch, ColumnSnapshot, MatchingMode } from "../../../columns";
 import { Services, type RequireServices } from "@milaboratories/pl-model-common";
-import { ColumnCollectionBuilder } from "../../../columns";
-import { isColumnSnapshotProvider } from "../../../columns/column_snapshot_provider";
-import { collectCtxColumnSnapshotProviders } from "../../../columns/ctx_column_sources";
 import { getAllLabelColumns, getMatchingLabelColumns } from "../labels";
-
-/** Convert a PTableColumnId to a SpecQueryExpression reference. */
-export function columnIdToExpr(col: PTableColumnId): SpecQueryExpression {
-  if (col.type === "axis") {
-    return { type: "axisRef", value: col.id as SingleAxisSelector };
-  }
-  return { type: "columnRef", value: col.id };
-}
-
-/** Wrap a SpecQuery as a SpecQueryJoinEntry with empty qualifications. */
-export function joinEntry<C>(input: SpecQuery<C>): SpecQueryJoinEntry<C> {
-  return { entry: input, qualifications: [] };
-}
-
-export function createPTableDef(params: {
-  columns: PColumn<PColumnDataUniversal>[];
-  labelColumns: PColumn<PColumnDataUniversal>[];
-  coreJoinType: "inner" | "full";
-  filters: null | PlDataTableFilters;
-  sorting: PTableSorting[];
-  coreColumnPredicate?: (spec: PColumnIdAndSpec) => boolean;
-}): PTableDefV2<PColumn<TreeNodeAccessor | PColumnValues | DataInfo<TreeNodeAccessor>>> {
-  let coreColumns = params.columns;
-  const secondaryColumns: typeof params.columns = [];
-
-  if (isFunction(params.coreColumnPredicate)) {
-    coreColumns = [];
-    for (const c of params.columns)
-      if (params.coreColumnPredicate(getColumnIdAndSpec(c))) coreColumns.push(c);
-      else secondaryColumns.push(c);
-  }
-
-  secondaryColumns.push(...params.labelColumns);
-
-  // Build SpecQuery directly from columns
-  const coreJoinQuery: SpecQuery<
-    PColumn<TreeNodeAccessor | PColumnValues | DataInfo<TreeNodeAccessor>>
-  > = {
-    type: params.coreJoinType === "inner" ? "innerJoin" : "fullJoin",
-    entries: coreColumns.map((c) => joinEntry({ type: "column", column: c })),
-  };
-
-  let query: SpecQuery<PColumn<TreeNodeAccessor | PColumnValues | DataInfo<TreeNodeAccessor>>> = {
-    type: "outerJoin",
-    primary: joinEntry(coreJoinQuery),
-    secondary: secondaryColumns.map((c) => joinEntry({ type: "column", column: c })),
-  };
-
-  // Apply filters
-  if (params.filters !== null) {
-    const nonEmpty = distillFilterSpec(params.filters);
-
-    if (!isNil(nonEmpty)) {
-      const pridicate = filterSpecToSpecQueryExpr(nonEmpty);
-      if (!isBooleanExpression(pridicate)) {
-        throw new Error(
-          `Filter conversion produced a non-boolean expression (got type "${pridicate.type}"), expected a boolean predicate for query filtering`,
-        );
-      }
-      query = {
-        type: "filter",
-        input: query,
-        predicate: pridicate,
-      };
+import type { DeriveLabelsOptions } from "../../../labels/derive_distinct_labels";
+import {
+  deriveAllLabels,
+  isColumnHidden,
+  isColumnOptional,
+  withLabelAnnotations,
+  withTableVisualAnnotations,
+} from "./utils";
+import { createPTableDefV3 } from "./createPTableDefV3";
+import { discoverTableColumnSnaphots, type DiscoveredTableColumnOptions } from "./discoverColumns";
+import { isNil, RequiredBy, throwError, type Nil } from "@milaboratories/helpers";
+
+export type createPlDataTableOptionsV3 = (
+  | {
+      discoverColumnOptions: DiscoveredTableColumnOptions;
     }
-  }
-
-  // Apply sorting
-  if (params.sorting.length > 0) {
-    query = {
-      type: "sort",
-      input: query,
-      sortBy: params.sorting.map((s) => ({
-        expression: columnIdToExpr(s.column),
-        ascending: s.ascending,
-        nullsFirst: !s.naAndAbsentAreLeastValues,
-      })),
-    };
-  }
-
-  return { query };
-}
-
-/** Check if column should be omitted from the table */
-export function isColumnHidden(spec: { annotations?: Annotation }): boolean {
-  return readAnnotation(spec, Annotation.Table.Visibility) === "hidden";
-}
+  | {
+      columns: Nil | TableColumnSnapshot<SUniversalPColumnId>[];
+    }
+) & {
+  filters?: PlDataTableFilters;
+  sorting?: PTableSorting[];
+  primaryJoinType?: "inner" | "full";
 
-/** Check if column is hidden by default */
-export function isColumnOptional(spec: { annotations?: Annotation }): boolean {
-  return readAnnotation(spec, Annotation.Table.Visibility) === "optional";
-}
+  tableState?: PlDataTableStateV2;
+  labelsOptions?: DeriveLabelsOptions;
+  columnsDisplayOptions?: ColumnsDisplayOptions;
+};
 
 /** Structured source config — selectors/anchors instead of raw ColumnSource. */
-type ColumnsSelectorConfig = {
+export type ColumnsSelectorConfig = {
   include?: MultiColumnSelector | MultiColumnSelector[];
   exclude?: MultiColumnSelector | MultiColumnSelector[];
-  anchors?: Record<string, PlRef | PObjectId | PColumnSpec>;
   mode?: MatchingMode;
   maxHops?: number;
 };
 
-export type createPlDataTableOptionsV3 = {
-  source?: ColumnSource | ColumnSource[];
-  columns: ColumnsSelectorConfig;
+export type ColumnsDisplayOptions = {
+  /** Column ordering rules. Higher priority = further left. First matching rule wins. */
+  ordering?: ColumnOrderRule[];
+  /** Column visibility rules. First matching rule wins. Unmatched columns use default visibility. */
+  visibility?: ColumnVisibilityRule[];
+};
 
-  // Existing from V2
-  filters?: PlDataTableFilters;
-  sorting?: PTableSorting[];
-  coreJoinType?: "inner" | "full";
-  coreColumnPredicate?: (spec: PColumnIdAndSpec) => boolean;
+export type ColumnOrderRule = {
+  match: ColumnMatcher;
+  /** Higher number = further left in table */
+  priority: number;
+};
 
-  state?: PlDataTableStateV2;
+export type ColumnVisibilityRule = {
+  match: ColumnMatcher;
+  visibility: "default" | "optional" | "hidden";
 };
 
-// interface ColumnDisplayConfig {
-//   /** Column ordering rules. Higher priority = further left. First matching rule wins. */
-//   ordering?: ColumnOrderRule[];
-//   /** Column visibility rules. First matching rule wins. Unmatched columns use default visibility. */
-//   visibility?: ColumnVisibilityRule[];
-// }
-
-// interface ColumnOrderRule {
-//   match: ColumnMatcher;
-//   /** Higher number = further left in table */
-//   priority: number;
-// }
-
-// interface ColumnVisibilityRule {
-//   match: ColumnMatcher;
-//   visibility: "default" | "optional" | "hidden";
-// }
-
-// type ColumnMatcher =
-//   | ((spec: PColumnSpec) => boolean)
-//   | { name: string | string[] }
-//   | { annotation: Record<string, string> }
-//   | { ids: Set<string> };
+export type ColumnMatcher = (spec: PColumnSpec) => boolean;
+
+// Main Function
 
 export function createPlDataTableV3<A, U, S extends RequireServices<typeof Services.PFrameSpec>>(
   ctx: RenderCtxBase<A, U, S>,
   options: createPlDataTableOptionsV3,
 ): PlDataTableModel | undefined {
-  const providers = options.source
-    ? normalizeSourceList(options.source).filter(isColumnSnapshotProvider)
-    : collectCtxColumnSnapshotProviders(ctx);
-
-  if (providers.length === 0) return undefined;
-
-  // Step 1: Build collection from sources
-  const builder = new ColumnCollectionBuilder(ctx.services.pframeSpec).addSources(providers);
-  const anchors = options.columns.anchors;
-  const collection = isNil(anchors) ? builder.build() : builder.build({ anchors });
-
-  if (!collection) return undefined;
-  try {
-    // Step 2: Get data columns, excluding annotation-hidden ones
-    const findOptions = options.columns
-      ? {
-          include: options.columns.include,
-          exclude: options.columns.exclude,
-          mode: options.columns.mode,
-          maxHops: options.columns.maxHops,
-        }
-      : undefined;
-    const findResult = collection.findColumns(findOptions);
-    const snapshots = findResult.map((v) => ("column" in v ? v.column : v));
-    const dataSnapshots = snapshots.filter((s) => !isColumnHidden(s.spec));
-    if (dataSnapshots.length === 0) return undefined;
-
-    // Convert snapshots to PColumn<PColumnDataUniversal>[]
-    const columns: PColumn<PColumnDataUniversal>[] = [];
-    for (const snap of dataSnapshots) {
-      if (!snap.data) return undefined;
-      const data = snap.data.get();
-      if (data === undefined) return undefined;
-      columns.push({ id: snap.id, spec: snap.spec, data });
-    }
+  const state = upgradePlDataTableStateV2(options.tableState);
+  const primaryJoinType = options.primaryJoinType ?? "full";
+
+  const discovered =
+    "discoverColumnOptions" in options
+      ? discoverTableColumnSnaphots(ctx, options.discoverColumnOptions)
+      : options.columns;
+  if (isNil(discovered) || discovered.length === 0) return undefined;
+
+  const splited = splitDiscoveredColumns(discovered);
+  const resolved = resolveDiscoveredColumns(splited, discovered);
+
+  const labelColumns = getMatchingLabelColumns(resolved.all, getAllLabelColumns(ctx));
+
+  const derivedLabels = deriveAllLabels({
+    columns: discovered.map((dc) => ({
+      id: dc.id,
+      spec: dc.spec,
+      linkerPath: dc.linkerPath?.map((lp) => ({ spec: lp.linker.spec })),
+    })),
+    labelColumns,
+    deriveLabelsOptions: {
+      includeNativeLabel: true,
+      ...options.labelsOptions,
+    },
+  });
+
+  const annotated = annotateColumnGroups(
+    resolved,
+    labelColumns,
+    derivedLabels,
+    options.columnsDisplayOptions,
+  );
+
+  const columnIsAvailable = createColumnValidationById([
+    ...annotated.direct,
+    ...annotated.linked.flatMap((lc) => [...(annotated.linkers.get(lc.id) ?? []), lc]),
+    ...annotated.labels,
+  ]);
+
+  const filters = mergeFilters(
+    state.pTableParams.filters,
+    remapFilterColumnIds(options.filters, discovered),
+  );
+  validateFilters(filters, columnIsAvailable);
+
+  const sorting = resolveSorting(
+    state.pTableParams.sorting,
+    remapSortingColumnIds(options.sorting, discovered),
+  );
+  validateSorting(sorting, columnIsAvailable);
+
+  const primaryColumnIds = new Set<PObjectId>(
+    discovered.filter((dc) => dc.isPrimary).map((dc) => dc.id),
+  );
+  const primaryColumns = annotated.direct.filter((c) => primaryColumnIds.has(c.id));
+  const secondaryColumns = annotated.direct.filter((c) => !primaryColumnIds.has(c.id));
+
+  if (primaryColumns.length === 0) return undefined;
+
+  const fullDef = createPTableDefV3({
+    primaryJoinType,
+    primaryColumns,
+    secondaryGroups: [
+      ...secondaryColumns.map((c) => [c]),
+      ...annotated.linked.map((lc) => [...(annotated.linkers.get(lc.id) ?? []), lc]),
+      ...annotated.labels.map((c) => [c]),
+    ],
+    filters,
+    sorting,
+  });
+
+  const fullHandle = ctx.createPTableV2(fullDef);
+  const pframeHandle = ctx.createPFrame([
+    ...annotated.direct,
+    ...annotated.linked,
+    ...annotated.labels,
+    ...uniqueBy([...annotated.linkers.values()].flat(), (c) => c.id),
+  ]);
+  if (!fullHandle || !pframeHandle) return undefined;
+
+  const hiddenSpecs = state.pTableParams.hiddenColIds;
+
+  const hiddenColumnIds = computeHiddenColumns(
+    [...annotated.direct, ...annotated.linked],
+    sorting,
+    filters,
+    hiddenSpecs,
+  );
+
+  const visible = buildVisibleColumns(annotated, hiddenColumnIds, labelColumns);
+  const visibleNonCoreDirect = secondaryColumns.filter((c) => !hiddenColumnIds.has(c.id));
+  const visibleLinkedGroups = buildVisibleLinkedGroups(
+    visible.direct,
+    visible.linked,
+    annotated.linkers,
+    hiddenSpecs,
+  );
+
+  const visibleDef = createPTableDefV3({
+    primaryJoinType,
+    primaryColumns,
+    secondaryGroups: [
+      ...visibleNonCoreDirect.map((c) => [c]),
+      ...visibleLinkedGroups,
+      ...visible.labels.map((c) => [c]),
+    ],
+    filters,
+    sorting,
+  });
+  const visibleHandle = ctx.createPTableV2(visibleDef);
+  if (!visibleHandle) return undefined;
+
+  return {
+    sourceId: state.pTableParams.sourceId,
+    fullTableHandle: fullHandle,
+    fullPframeHandle: pframeHandle,
+    visibleTableHandle: visibleHandle,
+  } satisfies PlDataTableModel;
+}
 
-    // Step 3: Normalize table state
-    const tableStateNormalized = upgradePlDataTableStateV2(options.state);
+/** A single column discovered from sources — normalized from raw ColumnSnapshot/ColumnMatch. */
+export type TableColumnSnapshot<Id extends PObjectId | SUniversalPColumnId> = ColumnSnapshot<Id> & {
+  readonly isPrimary?: boolean;
+  readonly originalId?: PObjectId;
+  readonly linkerPath?: ColumnMatch["path"];
+};
 
-    // Step 4: Get label columns from result pool and match to data columns
-    const allLabelColumns = getAllLabelColumns(ctx.resultPool);
-    if (!allLabelColumns) return undefined;
+type TableColumn = PColumn<undefined | PColumnDataUniversal>;
 
-    const fullLabelColumns = getMatchingLabelColumns(
-      columns.map(getColumnIdAndSpec),
-      allLabelColumns,
-    );
+type SplitDiscoveredColumns = {
+  readonly direct: TableColumnSnapshot<SUniversalPColumnId>[];
+  readonly linked: TableColumnSnapshot<SUniversalPColumnId>[];
+};
 
-    const fullColumns = [...columns, ...fullLabelColumns];
+type ResolvedColumns = {
+  readonly direct: TableColumn[];
+  readonly linked: TableColumn[];
+  readonly linkers: Map<PObjectId, TableColumn[]>;
+  readonly all: TableColumn[];
+};
 
-    // Step 5: Build column ID set for filter/sorting validation
-    const fullColumnsAxes = uniqueBy(
-      fullColumns.flatMap((c) => c.spec.axesSpec.map((a) => getAxisId(a))),
-      (a) => canonicalizeJson<AxisId>(a),
-    );
-    const fullColumnsIds: PTableColumnId[] = [
-      ...fullColumnsAxes.map((a) => ({ type: "axis", id: a }) satisfies PTableColumnIdAxis),
-      ...fullColumns.map((c) => ({ type: "column", id: c.id }) satisfies PTableColumnIdColumn),
-    ];
-    const fullColumnsIdsSet = new Set(
-      fullColumnsIds.map((c) => canonicalizeJson<PTableColumnId>(c)),
-    );
-    const isValidColumnId = (id: string): boolean =>
-      fullColumnsIdsSet.has(id as CanonicalizedJson<PTableColumnId>);
-
-    // Step 6: Filtering validation
-    const stateFilters = tableStateNormalized.pTableParams.filters;
-    const opsFilters = options?.filters ?? null;
-    const filters: null | PlDataTableFilters =
-      stateFilters != null && opsFilters != null
-        ? { type: "and", filters: [stateFilters, opsFilters] }
-        : (stateFilters ?? opsFilters);
-    const filterColumns = filters ? collectFilterSpecColumns(filters) : [];
-    const firstInvalidFilterColumn = filterColumns.find((col) => !isValidColumnId(col));
-    if (firstInvalidFilterColumn)
-      throw new Error(
-        `Invalid filter column ${firstInvalidFilterColumn}: column reference does not match the table columns`,
-      );
-
-    // Step 7: Sorting validation
-    const userSorting = tableStateNormalized.pTableParams.sorting;
-    const sorting = (isEmpty(userSorting) ? options?.sorting : userSorting) ?? [];
-    const firstInvalidSortingColumn = sorting.find(
-      (s) => !isValidColumnId(canonicalizeJson<PTableColumnId>(s.column)),
-    );
-    if (firstInvalidSortingColumn)
-      throw new Error(
-        `Invalid sorting column ${JSON.stringify(firstInvalidSortingColumn.column)}: column reference does not match the table columns`,
-      );
-
-    // Step 8: Build full table definition and handles
-    const coreJoinType = options?.coreJoinType ?? "full";
-    const fullDef = createPTableDef({
-      columns,
-      labelColumns: fullLabelColumns,
-      coreJoinType,
-      filters,
-      sorting,
-      coreColumnPredicate: options?.coreColumnPredicate,
-    });
+type AnnotatedColumnGroups = {
+  readonly direct: TableColumn[];
+  readonly linked: TableColumn[];
+  readonly linkers: Map<PObjectId, TableColumn[]>;
+  readonly labels: TableColumn[];
+};
 
-    const fullHandle = ctx.createPTableV2(fullDef);
-    const pframeHandle = ctx.createPFrame(fullColumns);
-    if (!fullHandle || !pframeHandle) return undefined;
+type VisibleColumns = {
+  readonly direct: TableColumn[];
+  readonly linked: TableColumn[];
+  readonly labels: PColumn<PColumnDataUniversal>[];
+};
 
-    // Step 9: Determine hidden columns
-    const hiddenColumns = new Set<PObjectId>(
-      ((): PObjectId[] => {
-        // Inner join works as a filter — all columns must be present
-        if (coreJoinType === "inner") return [];
+/** Split discovered columns into direct (no linker path) and linked (with linker path). */
+function splitDiscoveredColumns(
+  columns: TableColumnSnapshot<SUniversalPColumnId>[],
+): SplitDiscoveredColumns {
+  return {
+    direct: columns.filter((dc) => isNil(dc.linkerPath) || dc.linkerPath.length === 0),
+    linked: columns.filter((dc) => !isNil(dc.linkerPath) && dc.linkerPath.length > 0),
+  };
+}
+
+/** Resolve DiscoveredColumn snapshots into PColumn objects with lazily-evaluated data. */
+function resolveDiscoveredColumns(
+  split: SplitDiscoveredColumns,
+  allDiscovered: TableColumnSnapshot<SUniversalPColumnId>[],
+): ResolvedColumns {
+  const linked = split.linked.map(resolveSnapshot);
+  const linkers = new Map<PObjectId, TableColumn[]>(
+    split.linked
+      .filter(
+        (dc): dc is RequiredBy<TableColumnSnapshot<SUniversalPColumnId>, "linkerPath"> =>
+          !isNil(dc.linkerPath),
+      )
+      .map((dc, i) => [linked[i].id, dc.linkerPath.map((s) => resolveSnapshot(s.linker))]),
+  );
+
+  return {
+    all: allDiscovered.map(resolveSnapshot),
+    direct: split.direct.map(resolveSnapshot),
+    linked,
+    linkers,
+  };
+}
+
+/** Annotate all column groups with derived labels and display options. */
+function annotateColumnGroups(
+  resolved: ResolvedColumns,
+  labelColumns: PColumn<PColumnDataUniversal>[],
+  derivedLabels: Record<string, string>,
+  displayOptions: ColumnsDisplayOptions | undefined,
+): AnnotatedColumnGroups {
+  const direct = withTableVisualAnnotations(
+    displayOptions,
+    withLabelAnnotations(derivedLabels, resolved.direct),
+  );
+  const linked = withTableVisualAnnotations(
+    displayOptions,
+    withLabelAnnotations(derivedLabels, resolved.linked),
+  );
+  const linkers = new Map<PObjectId, TableColumn[]>(
+    [...resolved.linkers].map(([id, cols]) => [id, withLabelAnnotations(derivedLabels, cols)]),
+  );
+  const labels = withLabelAnnotations(derivedLabels, labelColumns);
+
+  return { direct, linked, linkers, labels };
+}
 
-        const hiddenColIds = tableStateNormalized.pTableParams.hiddenColIds;
-        if (hiddenColIds) return hiddenColIds;
+/** Build an index of all valid column IDs (axes + columns) for filter/sorting validation. */
+function createColumnValidationById(fullColumns: TableColumn[]) {
+  const axisIds = uniqueBy(
+    fullColumns.flatMap((c) => c.spec.axesSpec.map(getAxisId)),
+    (a) => canonicalizeJson<AxisId>(a),
+  );
 
-        return columns.filter((c) => isColumnOptional(c.spec)).map((c) => c.id);
-      })(),
+  const allIds: PTableColumnId[] = [
+    ...axisIds.map((a) => ({ type: "axis", id: a }) satisfies PTableColumnIdAxis),
+    ...fullColumns.map((c) => ({ type: "column", id: c.id }) satisfies PTableColumnIdColumn),
+  ];
+
+  const validIdSet = new Set(allIds.map((c) => canonicalizeJson<PTableColumnId>(c)));
+
+  return (id: string): boolean => {
+    return validIdSet.has(id as CanonicalizedJson<PTableColumnId>);
+  };
+}
+
+/** Merge filters from table state and options into a single filter spec. */
+function mergeFilters(
+  stateFilters: PlDataTableFilters | null,
+  optionsFilters: PlDataTableFilters | null | undefined,
+): Nil | PlDataTableFilters {
+  const normalized = optionsFilters ?? null;
+  return stateFilters !== null && normalized !== null
+    ? { type: "and", filters: [stateFilters, normalized] }
+    : (stateFilters ?? normalized);
+}
+
+/** Validate that all column references in filters exist in the table. */
+function validateFilters(
+  filters: Nil | PlDataTableFilters,
+  isValidColumnId: (id: string) => boolean,
+): void {
+  if (filters == null) return;
+  const filterColumns = collectFilterSpecColumns(filters);
+  const firstInvalid = filterColumns.find((col) => !isValidColumnId(col));
+  if (firstInvalid !== undefined) {
+    throw new Error(
+      `Invalid filter column ${firstInvalid}: column reference does not match the table columns`,
     );
+  }
+}
+
+/** Pick user sorting from state if non-empty, otherwise fall back to options default. */
+function resolveSorting(
+  userSorting: PTableSorting[],
+  defaultSorting: Nil | PTableSorting[],
+): PTableSorting[] {
+  return (isEmpty(userSorting) ? defaultSorting : userSorting) ?? [];
+}
+
+/** Validate that all column references in sorting exist in the table. */
+function validateSorting(sorting: PTableSorting[], isValidColumnId: (id: string) => boolean): void {
+  const firstInvalid = sorting.find(
+    (s) => !isValidColumnId(canonicalizeJson<PTableColumnId>(s.column)),
+  );
+  if (firstInvalid !== undefined) {
+    throw new Error(
+      `Invalid sorting column ${JSON.stringify(firstInvalid.column)}: column reference does not match the table columns`,
+    );
+  }
+}
+
+/** Determine which columns should be hidden based on state or optional-column defaults. */
+function computeHiddenColumns(
+  columns: TableColumn[],
+  sorting: Nil | PTableSorting[],
+  filters: Nil | PlDataTableFilters,
+  hiddenSpecs: Nil | PTableColumnId[],
+): Set<PObjectId> {
+  const alwaysHidden = columns.filter((c) => isColumnHidden(c.spec)).map((c) => c.id);
+  const optionalHidden = !isNil(hiddenSpecs)
+    ? hiddenSpecs.filter((s): s is PTableColumnIdColumn => s.type === "column").map((s) => s.id)
+    : columns.filter((c) => isColumnOptional(c.spec)).map((c) => c.id);
+  const initial = [...alwaysHidden, ...optionalHidden];
+  const preserved = collectPreservedColumnIds(sorting, filters);
+
+  return new Set(initial.filter((id) => !preserved.has(id)));
+}
 
-    // Preserve linker columns
-    columns.filter((c) => isLinkerColumn(c.spec)).forEach((c) => hiddenColumns.delete(c.id));
+/**
+ * Build visible linked column groups. Non-hidden groups are included fully;
+ * hidden groups are trimmed to only the prefix that brings axes not yet
+ * covered by earlier groups (visible or previously trimmed).
+ */
+function buildVisibleLinkedGroups(
+  direct: TableColumn[],
+  linked: TableColumn[],
+  linkers: Map<PObjectId, TableColumn[]>,
+  hiddenSpecs: PTableColumnId[] | null,
+): TableColumn[][] {
+  const result: TableColumn[][] = [];
+  const coveredAxisIds = new Set<CanonicalizedJson<AxisId>>();
+
+  const collectAxes = (group: TableColumn[]) => {
+    for (const col of group) {
+      for (const as of col.spec.axesSpec) {
+        coveredAxisIds.add(canonicalizeJson(getAxisId(as)));
+      }
+    }
+  };
+
+  collectAxes(direct);
+
+  for (const lc of linked) {
+    const group = [...(linkers.get(lc.id) ?? []), lc];
+    result.push(group);
+    collectAxes(group);
+  }
 
-    // Preserve core columns as they change the shape of join
-    const coreColumnPredicate = options?.coreColumnPredicate;
-    if (coreColumnPredicate) {
-      const coreColumns = columns.flatMap((c) =>
-        coreColumnPredicate(getColumnIdAndSpec(c)) ? [c.id] : [],
-      );
-      coreColumns.forEach((c) => hiddenColumns.delete(c));
+  for (const group of linkers.values()) {
+    const trimmed = trimGroupByVisibleAxes(group, hiddenSpecs, coveredAxisIds);
+    if (trimmed.length > 0) {
+      result.push(trimmed);
+      collectAxes(trimmed);
     }
+  }
+
+  return result;
+}
 
-    // Preserve sorted columns from being hidden
-    sorting
-      .map((s) => s.column)
-      .filter((c): c is PTableColumnIdColumn => c.type === "column")
-      .forEach((c) => hiddenColumns.delete(c.id));
-
-    // Preserve filter columns from being hidden
-    if (filters) {
-      collectFilterSpecColumns(filters)
-        .flatMap((c) => {
-          const obj = parseJson(c);
-          return obj.type === "column" ? [obj.id] : [];
-        })
-        .forEach((c) => hiddenColumns.delete(c));
+/**
+ * For a linked column group [linker1, ..., linkerN, column], find the rightmost
+ * element that has at least one non-hidden and not-yet-covered axis.
+ * Return the prefix up to and including that element.
+ * If no element has such an axis, return empty array.
+ */
+function trimGroupByVisibleAxes(
+  group: TableColumn[],
+  hiddenSpecs: PTableColumnId[] | null,
+  coveredAxisIds: Set<CanonicalizedJson<AxisId>>,
+): TableColumn[] {
+  if (hiddenSpecs === null) return group;
+
+  const hiddenAxisIds = new Set(
+    hiddenSpecs
+      .filter((s): s is PTableColumnIdAxis => s.type === "axis")
+      .map((s) => canonicalizeJson(s.id)),
+  );
+
+  const uncoveredAxisIds = new Set(
+    group
+      .flatMap((c) => c.spec.axesSpec.map((as) => canonicalizeJson(getAxisId(as))))
+      .filter((id) => !hiddenAxisIds.has(id) && !coveredAxisIds.has(id)),
+  );
+
+  let lastNeeded = -1;
+  for (let i = 0; i < group.length; i++) {
+    const newAxes = group[i].spec.axesSpec
+      .map((as) => canonicalizeJson(getAxisId(as)))
+      .filter((id) => uncoveredAxisIds.has(id));
+    if (newAxes.length > 0) {
+      for (const id of newAxes) uncoveredAxisIds.delete(id);
+      lastNeeded = i;
     }
+  }
+  return lastNeeded === -1 ? [] : group.slice(0, lastNeeded + 1);
+}
 
-    // Step 10: Build visible table definition
-    const visibleColumns = columns.filter((c) => !hiddenColumns.has(c.id));
-    const visibleLabelColumns = getMatchingLabelColumns(
-      visibleColumns.map(getColumnIdAndSpec),
-      allLabelColumns,
-    );
+/** Collect IDs of columns that must remain visible (sorted, filtered). */
+function collectPreservedColumnIds(
+  sorting: Nil | PTableSorting[],
+  filters: Nil | PlDataTableFilters,
+): Set<PObjectId> {
+  const sortedIds = (sorting ?? [])
+    .map((s) => s.column)
+    .filter((c): c is PTableColumnIdColumn => c.type === "column")
+    .map((c) => c.id);
+
+  const filterIds = !isNil(filters)
+    ? collectFilterSpecColumns(filters).flatMap((c) => {
+        const obj = parseJson(c);
+        return obj.type === "column" ? [obj.id] : [];
+      })
+    : [];
+
+  return new Set<PObjectId>([...sortedIds, ...filterIds]);
+}
 
-    if (!allPColumnsReady([...visibleColumns, ...visibleLabelColumns])) return undefined;
+/** Filter annotated columns to only visible ones, re-matching label columns for the visible subset. */
+function buildVisibleColumns(
+  annotated: AnnotatedColumnGroups,
+  hiddenColumns: Set<PObjectId>,
+  originalLabelColumns: PColumn<PColumnDataUniversal>[],
+): VisibleColumns {
+  const direct = annotated.direct.filter((c) => !hiddenColumns.has(c.id));
+  const linked = annotated.linked.filter((c) => !hiddenColumns.has(c.id));
+  const labels = getMatchingLabelColumns(
+    [...direct, ...linked].map(getColumnIdAndSpec),
+    originalLabelColumns,
+  );
+  return { direct, linked, labels };
+}
 
-    const visibleDef = createPTableDef({
-      columns: visibleColumns,
-      labelColumns: visibleLabelColumns,
-      coreJoinType,
-      filters,
-      sorting,
-      coreColumnPredicate,
-    });
-    const visibleHandle = ctx.createPTableV2(visibleDef);
+/** Resolve a ColumnSnapshot to a PColumn with lazily-evaluated data. */
+function resolveSnapshot(
+  snap: ColumnSnapshot<PObjectId>,
+): PColumn<undefined | PColumnDataUniversal> {
+  return { id: snap.id, spec: snap.spec, data: snap.data?.get() };
+}
+
+/** Remap column references in sorting entries. */
+function remapSortingColumnIds(
+  sorting: Nil | PTableSorting[],
+  columns: TableColumnSnapshot<PObjectId | SUniversalPColumnId>[],
+): Nil | PTableSorting[] {
+  return sorting?.map((s) => {
+    if (s.column.type === "axis") return s; // Axis references are unaffected by column ID remapping
 
-    if (!visibleHandle) return undefined;
+    const id = s.column.id;
+    const column =
+      columns.find((c) => (c.originalId ?? c.id) === id) ??
+      throwError(`Column ID "${id}" in sorting does not match any discovered column`);
 
     return {
-      sourceId: tableStateNormalized.pTableParams.sourceId,
-      fullTableHandle: fullHandle,
-      fullPframeHandle: pframeHandle,
-      visibleTableHandle: visibleHandle,
-    } satisfies PlDataTableModel;
-  } finally {
-    collection.dispose();
-  }
+      ...s,
+      column: {
+        type: "column",
+        id: column.id,
+      },
+    };
+  });
 }
 
-/** Normalize raw ColumnSource | ColumnSource[] into a flat list of sources. */
-function normalizeSourceList(source: ColumnSource | ColumnSource[]): ColumnSource[] {
-  if (
-    Array.isArray(source) &&
-    source.length > 0 &&
-    (Array.isArray(source[0]) || isColumnSnapshotProvider(source[0]))
-  ) {
-    return source as ColumnSource[];
-  }
-  return [source as ColumnSource];
+type PlDataTableFilterNode = FilterSpecNode<PlDataTableFilterSpecLeaf>;
+
+/** Remap column references in a filter tree. */
+function remapFilterColumnIds(
+  filters: Nil | PlDataTableFilters,
+  columns: TableColumnSnapshot<PObjectId | SUniversalPColumnId>[],
+): Nil | PlDataTableFilters {
+  if (isNil(filters)) return filters;
+
+  const map = (
+    tableColumnId: CanonicalizedJson<PTableColumnId>,
+  ): CanonicalizedJson<PTableColumnId> => {
+    const parsed = parseJson<PTableColumnId>(tableColumnId);
+    if (parsed.type === "axis") return tableColumnId; // Axis references are unaffected by column ID remapping
+
+    const originalId = parsed.id;
+    const column =
+      columns.find((c) => (c.originalId ?? c.id) === originalId) ??
+      throwError(`Column ID "${parsed.id}" in filters does not match any discovered column`);
+
+    return canonicalizeJson<PTableColumnId>({
+      type: "column",
+      id: column.id,
+    });
+  };
+
+  return traverseFilterSpec(filters, {
+    leaf: (leaf): PlDataTableFilterNode => {
+      if (leaf.type === undefined) return leaf;
+      const result = { ...leaf };
+      if ("column" in result) result.column = map(result.column);
+      if ("rhs" in result) result.rhs = map(result.rhs);
+      return result;
+    },
+    and: (results): PlDataTableFilterNode => ({ type: "and", filters: results }),
+    or: (results): PlDataTableFilterNode => ({ type: "or", filters: results }),
+    not: (result): PlDataTableFilterNode => ({ type: "not", filter: result }),
+  }) as PlDataTableFilters;
 }