@nshiab/simple-data-analysis-core 2.1.11 → 2.1.13

package/src/methods/summarize.ts

@@ -0,0 +1,196 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import getStatExpression, {
+  allStats,
+  isTemporalStatType,
+  type Stat,
+} from "../helpers/getStatExpression.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queryDB from "../helpers/queryDB.ts";
+import queueOp from "../helpers/queueOp.ts";
+import resolveOutputTable from "../helpers/resolveOutputTable.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+
+type SummarizeOptions = {
+  outputTable?: string | boolean;
+  columns?: string | string[];
+  by?: string | string[];
+  stats?: Stat | Stat[] | { [key: string]: Stat };
+  decimals?: number;
+  datesToMs?: boolean;
+};
+
+export default function summarize(
+  simpleTable: SimpleTable,
+  options: SummarizeOptions = {},
+): SimpleTable {
+  options = structuredClone(options);
+  options.outputTable = resolveOutputTable(simpleTable, options.outputTable);
+  const columns = getSummarizeColumns(options);
+  const boundColumns = columns.length > 1 ? columns : [];
+
+  if (typeof options.outputTable === "string") {
+    // The output table instance is created at call time so it can be
+    // returned synchronously and chained on right away.
+    const outputTable = simpleTable.sdb.newTable(options.outputTable);
+    queueOp(outputTable, {
+      kind: "barrier",
+      method: "summarize()",
+      parameters: { options },
+      execute: async () => {
+        await queryDB(
+          simpleTable,
+          `CREATE OR REPLACE TABLE ${quoteIdentifier(outputTable.name)} AS ${summarizeSelect(
+            `${quoteIdentifier(simpleTable.name)}`,
+            await simpleTable.getTypes(),
+            options,
+          )}`,
+          mergeOptions(simpleTable, {
+            table: outputTable.name,
+            method: "summarize()",
+            parameters: { options },
+            values: boundColumns,
+          }),
+        );
+      },
+    });
+    return outputTable;
+  }
+
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "summarize()",
+    parameters: { options },
+    needsSchema: true,
+    values: boundColumns,
+    buildSelect: (input, types) => summarizeSelect(input, types, options),
+  });
+  return simpleTable;
+}
+
+function getSummarizeColumns(options: SummarizeOptions): string[] {
+  const by = options.by ? stringToArray(options.by) : [];
+  return [
+    ...new Set(
+      (options.columns ? stringToArray(options.columns) : []).filter(
+        (column) => !by.includes(column),
+      ),
+    ),
+  ];
+}
+
+function summarizeSelect(input: string, types: TableSchema, options: SummarizeOptions): string {
+  const by = options.by ? stringToArray(options.by) : [];
+  // Duplicated columns would duplicate rows now that the branches are combined
+  // with UNION ALL instead of a deduplicating UNION.
+  const columns = getSummarizeColumns(options);
+
+  let stats: Stat[];
+  let outputColumns: string[] | undefined;
+  if (options.stats === undefined) {
+    stats = columns.length === 0 ? ["count"] : [...allStats];
+  } else if (typeof options.stats === "string") {
+    stats = [options.stats];
+  } else if (Array.isArray(options.stats)) {
+    stats = options.stats.length === 0 ? [...allStats] : options.stats;
+  } else {
+    const entries = Object.entries(options.stats);
+    outputColumns = entries.map((d) => d[0]);
+    stats = entries.map((d) => d[1]);
+  }
+
+  // The `column` output column is a row key that distinguishes rows for
+  // different input columns, so it appears only when there is more than one.
+  const columnColumn = columns.length > 1;
+
+  // With datesToMs, dates are converted to milliseconds inside the aggregate
+  // expressions (same conversions as convert()), so the input table is left
+  // untouched instead of being rewritten.
+  const references: { [column: string]: string } = {};
+  const effectiveTypes: TableSchema = {};
+  for (const column of columns) {
+    const type = types[column];
+    if (
+      options.datesToMs === true &&
+      typeof type === "string" &&
+      (type.includes("TIME") || type.includes("DATE"))
+    ) {
+      references[column] = type.includes("TIME")
+        ? `(date_part('epoch', ${quoteIdentifier(column)}) * 1000)`
+        : `(epoch(${quoteIdentifier(column)}) * 1000)`;
+      effectiveTypes[column] = "BIGINT";
+    } else {
+      references[column] = `${quoteIdentifier(column)}`;
+      effectiveTypes[column] = type;
+    }
+  }
+
+  const columnTypes = columns.map((d) => effectiveTypes[d]);
+  const doubleAndDate = columnTypes.includes("DOUBLE") && columnTypes.some(isTemporalStatType);
+  if (doubleAndDate) {
+    throw new Error(
+      "You are trying to summarize numbers and timestamps/dates/times. You can specify columns in the options (just numbers or just timestamps/dates/times) or convert your timestamps/dates/times to the number of ms since 1970-01-01 00:00:00 by passing the option { datesToMs: true }.",
+    );
+  }
+
+  const bySelect = by.map((d) => `${quoteIdentifier(d)}`).join(", ");
+  const groupBy = by.length > 0 ? `\nGROUP BY ${bySelect}` : "";
+
+  // With no columns, the only meaningful statistic is the row count, which
+  // doesn't need an input-column label (or any temporary column) at all.
+  if (columns.length === 0) {
+    const statColumns = stats.map((stat, i) => {
+      const name = outputColumns ? outputColumns[i] : stat;
+      return stat === "count"
+        ? `CAST(COUNT(*) AS INTEGER) AS ${quoteIdentifier(name)}`
+        : `NULL AS ${quoteIdentifier(name)}`;
+    });
+    const orderBy =
+      by.length > 0 ? `\nORDER BY ${by.map((d) => `${quoteIdentifier(d)} ASC`).join(", ")}` : "";
+    return `SELECT ${[...by.map((d) => `${quoteIdentifier(d)}`), ...statColumns].join(
+      ", ",
+    )}\nFROM ${input}${groupBy}${orderBy}`;
+  }
+
+  // One UNION ALL branch per input column. DuckDB runs the branches as
+  // concurrent pipelines, which parallelizes the expensive aggregates
+  // (medians, distinct counts) better than computing them all in one scan.
+  const branches: string[] = [];
+  for (const column of columns) {
+    let hasAggregate = false;
+    const projections = stats.map((stat, j) => {
+      const name = outputColumns ? outputColumns[j] : stat;
+      const expression = getStatExpression(
+        stat,
+        effectiveTypes[column],
+        references[column],
+        options.decimals,
+      );
+      if (expression === null) {
+        return `NULL AS ${quoteIdentifier(name)}`;
+      }
+      hasAggregate = true;
+      return `${expression} AS ${quoteIdentifier(name)}`;
+    });
+    const select = `SELECT ${
+      columnColumn ? `? AS ${quoteIdentifier("column")}, ` : ""
+    }${bySelect === "" ? "" : `${bySelect}, `}${projections.join(", ")}`;
+    if (by.length === 0 && !hasAggregate) {
+      // Every statistic is NULL and there is nothing to group by: the branch
+      // is a single constant row, so the input is not even scanned.
+      branches.push(select);
+    } else {
+      branches.push(`${select}\nFROM ${input}${groupBy}`);
+    }
+  }
+
+  const orderByColumns = columnColumn ? ["column", ...by] : by;
+  const orderBy =
+    orderByColumns.length > 0
+      ? `\nORDER BY ${orderByColumns.map((d) => `${quoteIdentifier(d)} ASC`).join(", ")}`
+      : "";
+
+  return `${branches.join("\nUNION ALL\n")}${orderBy}`;
+}

package/src/methods/topologicalSort.ts

@@ -0,0 +1,189 @@
+import buildWeakGraphComponentsSql from "../helpers/buildWeakGraphComponentsSql.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import getGraphEndpointColumns from "../helpers/getGraphEndpointColumns.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+import { prepareGraphSql } from "../helpers/prepareGraphTraversal.ts";
+import queueGraphResult from "../helpers/queueGraphResult.ts";
+import queueOp from "../helpers/queueOp.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+
+type TopologicalSortOptions = {
+  outputTable?: string | boolean;
+};
+
+export default function topologicalSort(
+  simpleTable: SimpleTable,
+  sourceColumn: string,
+  targetColumn: string,
+  options: TopologicalSortOptions = {},
+): SimpleTable {
+  if (typeof sourceColumn !== "string") {
+    throw new TypeError("topologicalSort() sourceColumn must be a string.");
+  }
+  if (typeof targetColumn !== "string") {
+    throw new TypeError("topologicalSort() targetColumn must be a string.");
+  }
+  if (options === null || typeof options !== "object" || Array.isArray(options)) {
+    throw new TypeError("topologicalSort() options must be an object.");
+  }
+  if (
+    options.outputTable !== undefined &&
+    typeof options.outputTable !== "string" &&
+    typeof options.outputTable !== "boolean"
+  ) {
+    throw new TypeError("topologicalSort() options.outputTable must be a string or boolean.");
+  }
+
+  options = structuredClone(options);
+  const parameters = { sourceColumn, targetColumn, options };
+
+  const result = queueGraphResult(simpleTable, {
+    method: "topologicalSort()",
+    parameters,
+    outputTable: options.outputTable,
+    values: (schema) => {
+      validateEndpoints(schema, sourceColumn, targetColumn);
+      return [];
+    },
+    buildSelect: (input, schema) =>
+      topologicalSortSelect(input, schema, sourceColumn, targetColumn),
+    outputSchema: (schema) => ({
+      node: validateEndpoints(schema, sourceColumn, targetColumn).idType,
+      componentId: "BIGINT",
+      order: "BIGINT",
+    }),
+  });
+  // Materialize the sorting computation before downstream operations: DuckDB
+  // can otherwise discard even its terminal cycle check for LIMIT 0 or FALSE.
+  // This barrier adds no query or separate validation traversal, and upstream
+  // preparation still fuses into the sorting statement.
+  queueOp(result, {
+    kind: "barrier",
+    method: "topologicalSort()",
+    parameters,
+    execute: () => Promise.resolve(),
+  });
+  return result;
+}
+
+function validateEndpoints(schema: TableSchema, source: string, target: string) {
+  return getGraphEndpointColumns(schema, source, target, "topologicalSort()");
+}
+
+function topologicalSortSelect(
+  input: string,
+  schema: TableSchema,
+  source: string,
+  target: string,
+): string {
+  const prepared = prepareGraphSql(input, schema, source, target, "topologicalSort()");
+  const relations = prepared.relationNames([
+    "graph_edges",
+    "graph_nodes",
+    "graph_topological_order",
+    "graph_topological_result",
+    "graph_component_edges",
+    "graph_component_labels",
+    "graph_component_roots",
+    "graph_ordered_nodes",
+  ]);
+  const edges = relations.graph_edges;
+  const nodes = relations.graph_nodes;
+  const ordering = relations.graph_topological_order;
+  const result = relations.graph_topological_result;
+  const componentEdges = relations.graph_component_edges;
+  const labels = relations.graph_component_labels;
+  const roots = relations.graph_component_roots;
+  const orderedNodes = relations.graph_ordered_nodes;
+  const q = quoteIdentifier;
+
+  return `WITH RECURSIVE ${edges} AS (
+      ${prepared.edges("outgoing")}
+    ), ${nodes} AS (
+      SELECT ${q("__from")} AS ${q("node")},
+        ${q("__from_key")} AS ${q("__key")}
+      FROM ${edges}
+      UNION
+      SELECT ${q("__to")} AS ${q("node")},
+        ${q("__to_key")} AS ${q("__key")}
+      FROM ${edges}
+    ), ${componentEdges} AS (
+      SELECT * FROM ${edges}
+      UNION ALL
+      SELECT ${q("__to")}, ${q("__from")},
+        ${q("__to_key")}, ${q("__from_key")}
+      FROM ${edges}
+    ), ${buildWeakGraphComponentsSql(componentEdges, nodes, labels, roots)},
+    ${ordering}(
+      ${q("node")}, ${q("__key")}, ${q("order")}, ${q("__ordered")}
+    ) AS (
+      SELECT ${q("eligible")}.${q("node")},
+        ${q("eligible")}.${q("__key")}, CAST(1 AS BIGINT),
+        list_value(${q("eligible")}.${q("__key")})
+      FROM (
+        SELECT ${q("candidate")}.${q("node")},
+          ${q("candidate")}.${q("__key")}
+        FROM ${nodes} AS ${q("candidate")}
+        WHERE NOT EXISTS (
+          SELECT 1 FROM ${edges} AS ${q("edge")}
+          WHERE ${q("edge")}.${q("__to_key")} =
+            ${q("candidate")}.${q("__key")}
+        )
+        ORDER BY ${q("candidate")}.${q("__key")}
+        LIMIT 1
+      ) AS ${q("eligible")}
+      UNION ALL
+      SELECT ${q("eligible")}.${q("node")},
+        ${q("eligible")}.${q("__key")},
+        ${q("ordered")}.${q("order")} + 1,
+        list_append(
+          ${q("ordered")}.${q("__ordered")},
+          ${q("eligible")}.${q("__key")}
+        )
+      FROM ${ordering} AS ${q("ordered")}
+      CROSS JOIN LATERAL (
+        SELECT ${q("candidate")}.${q("node")},
+          ${q("candidate")}.${q("__key")}
+        FROM ${nodes} AS ${q("candidate")}
+        WHERE NOT list_contains(
+          ${q("ordered")}.${q("__ordered")},
+          ${q("candidate")}.${q("__key")}
+        )
+          AND NOT EXISTS (
+            SELECT 1 FROM ${edges} AS ${q("edge")}
+            WHERE ${q("edge")}.${q("__to_key")} =
+                ${q("candidate")}.${q("__key")}
+              AND NOT list_contains(
+                ${q("ordered")}.${q("__ordered")},
+                ${q("edge")}.${q("__from_key")}
+              )
+          )
+        ORDER BY ${q("candidate")}.${q("__key")}
+        LIMIT 1
+      ) AS ${q("eligible")}
+    ), ${result} AS (
+      SELECT CASE WHEN count(*) < (SELECT count(*) FROM ${nodes})
+        THEN error(
+          'topologicalSort() cannot order the graph because it contains a directed cycle.'
+        )
+        ELSE list(${q("node")} ORDER BY ${q("order")})
+      END AS ${q("__nodes")}
+      FROM ${ordering}
+    ), ${orderedNodes} AS (
+      SELECT unnest(${q("__nodes")}) AS ${q("node")},
+        generate_subscripts(${q("__nodes")}, 1) AS ${q("__order")}
+      FROM ${result}
+    )
+    SELECT ${q("ordered")}.${q("node")}, ${q("roots")}.${q("componentId")},
+      row_number() OVER (
+        PARTITION BY ${q("roots")}.${q("componentId")}
+        ORDER BY ${q("ordered")}.${q("__order")}
+      ) AS ${q("order")}
+    FROM ${orderedNodes} AS ${q("ordered")}
+    INNER JOIN ${labels} AS ${q("labels")}
+      ON ${prepared.key(`${q("ordered")}.${q("node")}`)} =
+        ${q("labels")}.${q("__node_key")}
+    INNER JOIN ${roots} AS ${q("roots")}
+      ON ${q("labels")}.${q("__root_key")} = ${q("roots")}.${q("__root_key")}
+    ORDER BY ${q("componentId")}, ${q("order")}`;
+}

package/src/methods/trim.ts

@@ -0,0 +1,50 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+
+export default function trim(
+  simpleTable: SimpleTable,
+  columns: string | string[],
+  options: {
+    character?: string;
+    side?: "left" | "right" | "both";
+  } = {},
+) {
+  const cols = columns === "all" ? null : stringToArray(columns);
+  const resolveColumns = (schema: TableSchema) =>
+    cols ?? Object.keys(schema).filter((column) => schema[column] === "VARCHAR");
+  const side = options.side ?? "both";
+  const fn =
+    side === "left" ? "LTRIM" : side === "right" ? "RTRIM" : side === "both" ? "TRIM" : null;
+  if (fn === null) {
+    throw new Error(
+      `trim() options.side must be "left", "right", or "both". Received ${JSON.stringify(
+        options.side,
+      )}.`,
+    );
+  }
+  const character = options.character;
+  const specialCharacter = typeof character === "string" ? ", ?" : "";
+
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "trim()",
+    parameters: { columns, options },
+    values:
+      typeof character === "string"
+        ? (schema) => resolveColumns(schema).map(() => character)
+        : undefined,
+    needsSchema: columns === "all",
+    buildSelect: (input, schema) => {
+      const selectedColumns = resolveColumns(schema);
+      if (columns === "all" && selectedColumns.length === 0) {
+        return `SELECT * FROM ${input}`;
+      }
+      return `SELECT * REPLACE (${selectedColumns
+        .map((c) => `${fn}(${quoteIdentifier(c)}${specialCharacter}) AS ${quoteIdentifier(c)}`)
+        .join(", ")}) FROM ${input}`;
+    },
+  });
+}

package/src/methods/truncate.ts

@@ -0,0 +1,16 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function truncate(simpleTable: SimpleTable, column: string, length: number) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "truncate()",
+    parameters: { column, length },
+    needsSchema: false,
+    buildSelect: (input) =>
+      `SELECT * REPLACE (LEFT(${quoteIdentifier(column)}, ${length}) AS ${quoteIdentifier(
+        column,
+      )}) FROM ${input}`,
+  });
+}

package/src/methods/umap.ts

@@ -0,0 +1,186 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import queueOp from "../helpers/queueOp.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import queryDB from "../helpers/queryDB.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import buildUmapGraph from "../helpers/buildUmapGraph.ts";
+import prepareNumericFeatures from "../helpers/prepareNumericFeatures.ts";
+import prepareUmapLayoutOptions from "../helpers/prepareUmapLayoutOptions.ts";
+import readScalarNumber from "../helpers/readScalarNumber.ts";
+import optimizeUmapLayout from "../helpers/optimizeUmapLayout.ts";
+import initialUmapCoordinates from "../helpers/initialUmapCoordinates.ts";
+import appendColumnBatches from "../helpers/appendColumnBatches.ts";
+import { DOUBLE, INTEGER } from "@duckdb/node-api";
+
+type Options = NonNullable<Parameters<SimpleTable["umap"]>[1]>;
+
+export default function umap(
+  table: SimpleTable,
+  columns: string | string[],
+  options: Options = {},
+) {
+  const selected = typeof columns === "string" ? columns : [...columns];
+  const settings = structuredClone(options);
+  queueOp(table, {
+    kind: "barrier",
+    method: "umap()",
+    parameters: { columns: selected, options: settings },
+    execute: () => execute(table, selected, settings),
+  });
+}
+
+async function execute(table: SimpleTable, columns: string | string[], options: Options) {
+  const { neighbors = 15, metric = "euclidean" } = options;
+  if (!Number.isSafeInteger(neighbors) || neighbors < 2) {
+    throw new Error("neighbors must be a safe integer of at least 2.");
+  }
+  const layoutOptions = prepareUmapLayoutOptions(options);
+  if (!["euclidean", "cosine"].includes(metric)) {
+    throw new Error("metric must be euclidean or cosine.");
+  }
+  const sourceColumns = await table.getColumns();
+  if (sourceColumns.some((c) => ["umapx", "umapy"].includes(c.toLowerCase()))) {
+    throw new Error("UMAP output columns umapX and umapY must not already exist.");
+  }
+
+  const connection = table.connection!;
+  const [sourceOid, sourceTemporary] =
+    (
+      await connection.runAndReadAll(
+        `SELECT table_oid,temporary FROM duckdb_tables()
+      WHERE lower(table_name)=lower($1) AND schema_name=current_schema()
+        AND database_name IN (current_database(),'temp')
+      ORDER BY temporary DESC LIMIT 1`,
+        [table.name],
+      )
+    ).getRowsJS()[0] ?? [];
+  if (typeof sourceOid !== "number" && typeof sourceOid !== "bigint") {
+    throw new Error("UMAP requires a materialized source table.");
+  }
+  const q = quoteIdentifier;
+  const count = await readScalarNumber(connection, `SELECT count(*) FROM ${q(table.name)}`);
+  if (count < 3 || count > 0x7fffffff) {
+    throw new Error("UMAP requires between 3 and 2147483647 rows.");
+  }
+  const prepared = await prepareNumericFeatures(
+    table,
+    typeof columns === "string"
+      ? { kind: "vector", column: columns }
+      : { kind: "scalars", columns },
+    { method: "umap()" },
+  );
+  const dimensions = prepared.dimensions;
+  const suffix = crypto.randomUUID().replaceAll("-", "");
+  const name = (part: string) => `__sda_umap_${suffix}_${part}`;
+  const ordinal = q(prepared.rowIdColumn);
+  const source = q(table.name),
+    snapshot = q(prepared.relation);
+  const layoutName = name("layout"),
+    layout = q(layoutName);
+  const names = {
+    rows: q(name("rows")),
+    knn: q(name("knn")),
+    search: q(name("search")),
+    hnsw: q(name("hnsw")),
+    candidates: q(name("candidates")),
+    scales: q(name("scales")),
+    directed: q(name("directed")),
+    graph: q(name("graph")),
+  };
+  const scratch = [
+    layout,
+    ...Object.entries(names)
+      .filter(([key]) => key !== "hnsw")
+      .map(([, value]) => value),
+  ];
+  try {
+    await connection.run(`CREATE TEMP TABLE ${names.rows} AS
+      SELECT (row_number() OVER (ORDER BY ${ordinal})-1)::INTEGER AS vertex,
+        ${ordinal} AS ordinal, ${q(prepared.vectorColumn)} AS vec FROM ${snapshot}`);
+    await connection.run(`ALTER TABLE ${snapshot} DROP COLUMN ${q(prepared.vectorColumn)}`);
+    if (
+      await readScalarNumber(
+        connection,
+        `SELECT count(*) FROM ${names.rows}
+      WHERE NOT isfinite(array_inner_product(vec,vec)) OR array_inner_product(vec,vec)>1e300
+      ${metric === "cosine" ? "OR array_inner_product(vec,vec)=0" : ""}`,
+      )
+    ) {
+      throw new Error("Vector norms overflow, or cosine distance received a zero vector.");
+    }
+    const graph = await buildUmapGraph(
+      connection,
+      {
+        count,
+        dimensions,
+        neighbors: Math.min(neighbors, count - 1),
+      },
+      {
+        metric,
+        search: count <= 1000 ? "exact" : "hnsw",
+      },
+      names,
+    );
+    // Release index and vector scratch before allocating optimizer state.
+    for (const relation of [
+      names.search,
+      names.candidates,
+      names.knn,
+      names.directed,
+      names.scales,
+      names.graph,
+    ]) {
+      await connection.run(`DROP TABLE IF EXISTS ${relation}`);
+    }
+    await connection.run(`ALTER TABLE ${names.rows} DROP COLUMN vec`);
+    const coordinates = optimizeUmapLayout(
+      graph,
+      initialUmapCoordinates(count, layoutOptions.seed),
+      layoutOptions,
+    );
+    await connection.run(`CREATE TEMP TABLE ${layout} (vertex INTEGER,x DOUBLE,y DOUBLE)`);
+    await appendColumnBatches(
+      connection,
+      layoutName,
+      [INTEGER, DOUBLE, DOUBLE],
+      count,
+      (column, start, end) =>
+        Array.from({ length: end - start }, (_, row) =>
+          column === 0 ? start + row : coordinates[2 * (start + row) + column - 1],
+        ),
+    );
+    // Replace the table and recreate existing native indexes, which
+    // CREATE OR REPLACE drops.
+    const indexes = (
+      await connection.runAndReadAll("SELECT sql FROM duckdb_indexes() WHERE table_oid=$1", [
+        sourceOid,
+      ])
+    )
+      .getRowsJS()
+      .map((row) => {
+        if (typeof row[0] !== "string") throw new Error("Expected index SQL to be a string.");
+        return row[0];
+      });
+    await queryDB(
+      table,
+      `CREATE OR REPLACE ${sourceTemporary ? "TEMP " : ""}TABLE ${source} AS
+      SELECT ${prepared.sourceColumns
+        .map((c) => `s.${q(c)}`)
+        .join(",")}, l.x AS "umapX", l.y AS "umapY"
+      FROM ${snapshot} s JOIN ${names.rows} r ON s.${ordinal}=r.ordinal
+      JOIN ${layout} l ON r.vertex=l.vertex ORDER BY s.${ordinal}`,
+      mergeOptions(table, {
+        table: table.name,
+        method: "umap()",
+        parameters: { columns, options },
+        noClean: true,
+      }),
+    );
+    for (const sql of indexes) await connection.run(sql);
+  } finally {
+    for (const relation of scratch) {
+      await connection.run(`DROP TABLE IF EXISTS ${relation}`);
+    }
+    await prepared.cleanup();
+  }
+}

package/src/methods/union.ts

@@ -0,0 +1,28 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function union(
+  simpleTable: SimpleTable,
+  column1: string,
+  column2: string,
+  newColumn: string,
+) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "union()",
+    parameters: { column1, column2, newColumn },
+    needsSchema: true,
+    needsSpatial: true,
+    buildSelect: (input, types) => {
+      // The schema type carries the projection (e.g. GEOMETRY('EPSG:4326')),
+      // so the cast keeps it on the new geometries.
+      const expression = `ST_Union(${quoteIdentifier(column1)}, ${quoteIdentifier(
+        column2,
+      )})::${types[column1]}`;
+      return Object.keys(types).includes(newColumn)
+        ? `SELECT * REPLACE (${expression} AS ${quoteIdentifier(newColumn)}) FROM ${input}`
+        : `SELECT *, ${expression} AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+    },
+  });
+}

package/src/methods/unnest.ts

@@ -0,0 +1,18 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function unnest(simpleTable: SimpleTable, column: string, separator: string) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "unnest()",
+    parameters: { column, separator },
+    values: [separator],
+    needsSchema: false,
+    buildSelect: (input) =>
+      `SELECT
+  * EXCLUDE ${quoteIdentifier(column)},
+  TRIM(UNNEST(SPLIT(${quoteIdentifier(column)}, ?))) AS ${quoteIdentifier(column)}
+FROM ${input}`,
+  });
+}