@nshiab/simple-data-analysis-core 2.1.11 → 2.1.13

package/src/methods/clone.ts

@@ -0,0 +1,55 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import queueOp from "../helpers/queueOp.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function clone(
+  simpleTable: SimpleTable,
+  nameOrOptions:
+    | string
+    | {
+        name?: string;
+        conditions?: string;
+        columns?: string | string[];
+        limit?: number;
+        offset?: number;
+      } = {},
+): SimpleTable {
+  const columns =
+    typeof nameOrOptions === "object" && nameOrOptions.columns
+      ? stringToArray(nameOrOptions.columns)
+      : [];
+
+  // The cloned table instance is created at call time so it can be returned
+  // synchronously and chained on right away. Delegating to sdb.newTable()
+  // makes subclasses using tableClass work correctly.
+  let clonedTable: SimpleTable;
+  const options =
+    typeof nameOrOptions === "string" ? { name: nameOrOptions } : { ...nameOrOptions };
+  if (typeof options.name === "string") {
+    clonedTable = simpleTable.sdb.newTable(options.name);
+  } else {
+    clonedTable = simpleTable.sdb.newTable(undefined);
+  }
+
+  const selectClause =
+    columns.length > 0 ? columns.map((col) => `${quoteIdentifier(col)}`).join(", ") : "*";
+
+  queueOp(clonedTable, {
+    kind: "source",
+    method: "clone()",
+    parameters: { options },
+    // The clone reads simpleTable by name rather than through clonedTable's
+    // own (nonexistent) chain, so simpleTable's pending work must close and
+    // execute before this SELECT runs, as it would at this call position.
+    rawSQL: [`${quoteIdentifier(simpleTable.name)}`],
+    buildSelect: () =>
+      `SELECT ${selectClause} FROM ${quoteIdentifier(simpleTable.name)}${
+        options.conditions ? ` WHERE ${options.conditions}` : ""
+      }${typeof options.limit === "number" ? ` LIMIT ${options.limit}` : ""}${
+        typeof options.offset === "number" ? ` OFFSET ${options.offset}` : ""
+      }`,
+  });
+
+  return clonedTable;
+}

package/src/methods/cloneColumn.ts

@@ -0,0 +1,20 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function cloneColumn(simpleTable: SimpleTable, column: string, newColumn: string) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "cloneColumn()",
+    parameters: { column, newColumn },
+    needsSchema: true,
+    buildSelect: (input, types) => {
+      if (!types[column]) {
+        throw new Error(`Can't find type of ${column}`);
+      }
+      assertNewColumns(types, [newColumn], "cloneColumn()");
+      return `SELECT *, ${quoteIdentifier(column)} AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+    },
+  });
+}

package/src/methods/cloneColumnWithOffset.ts

@@ -0,0 +1,58 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queryDB from "../helpers/queryDB.ts";
+import queueOp from "../helpers/queueOp.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function cloneColumnWithOffset(
+  simpleTable: SimpleTable,
+  column: string,
+  newColumn: string,
+  options: {
+    offset?: number;
+    by?: string | string[];
+  } = {},
+) {
+  // The offset is based on rowid, which only exists on the materialized
+  // table, not on the output of a fused step: it executes as a barrier.
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "barrier",
+    method: "cloneColumnWithOffset()",
+    parameters: { column, newColumn, options },
+    execute: async () => {
+      // A SELECT *, expr AS col colliding with an existing column would be
+      // silently renamed by DuckDB (col -> col_1) instead of erroring.
+      assertNewColumns(await simpleTable.getTypes(), [newColumn], "cloneColumnWithOffset()");
+
+      const offset = options.offset ?? 1;
+      const by = options.by ? stringToArray(options.by) : [];
+      const partition =
+        by.length > 0 ? `PARTITION BY ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")}` : "";
+
+      // When by are specified, also sort the final result by
+      // by.
+      await queryDB(
+        simpleTable,
+        `CREATE OR REPLACE TABLE ${quoteIdentifier(
+          simpleTable.name,
+        )} AS SELECT *, LEAD(${quoteIdentifier(
+          column,
+        )}, ${offset}) OVER(${partition} ORDER BY rowid) AS ${quoteIdentifier(
+          newColumn,
+        )} FROM ${quoteIdentifier(simpleTable.name)}${
+          by.length > 0
+            ? ` ORDER BY ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")}, rowid`
+            : ` ORDER BY rowid`
+        };`,
+        mergeOptions(simpleTable, {
+          table: simpleTable.name,
+          method: "cloneColumnWithOffset()",
+          parameters: { column, newColumn },
+        }),
+      );
+    },
+  });
+}

package/src/methods/columnProportions.ts

@@ -0,0 +1,44 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function columnProportions(
+  simpleTable: SimpleTable,
+  column: string,
+  newColumn: string,
+  options: {
+    by?: string | string[];
+    base?: number;
+    decimals?: number;
+  } = {},
+) {
+  if (options.base !== undefined && (!Number.isFinite(options.base) || options.base <= 0)) {
+    throw new Error("columnProportions() options.base must be a finite number greater than 0.");
+  }
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "columnProportions()",
+    parameters: { column, newColumn, options },
+    needsSchema: true,
+    buildSelect: (input, schema) => {
+      assertNewColumns(schema, [newColumn], "columnProportions()");
+
+      const by = options.by ? stringToArray(options.by) : [];
+
+      const partition =
+        by.length === 0 ? "" : `PARTITION BY ${by.map((d) => `${quoteIdentifier(d)}`).join(",")}`;
+      const proportion = `${quoteIdentifier(column)} / sum(${quoteIdentifier(
+        column,
+      )}) OVER(${partition}) * ${options.base ?? 1}`;
+
+      return typeof options.decimals === "number"
+        ? `SELECT *, ROUND(${proportion}, ${options.decimals}) AS ${quoteIdentifier(
+            newColumn,
+          )} FROM ${input}`
+        : `SELECT *, ${proportion} AS ${quoteIdentifier(newColumn)} FROM ${input}`;
+    },
+  });
+}

package/src/methods/commonNeighbors.ts

@@ -0,0 +1,157 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+import prepareGraphStarts, {
+  type GraphId,
+  type PreparedGraphStarts,
+} from "../helpers/prepareGraphStarts.ts";
+import prepareGraphTraversal, {
+  type GraphDirection,
+  validateGraphStarts,
+} from "../helpers/prepareGraphTraversal.ts";
+import queueGraphResult from "../helpers/queueGraphResult.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+
+type CommonNeighborsOptions = {
+  direction?: GraphDirection;
+  outputTable?: string | boolean;
+};
+
+export default function commonNeighbors(
+  simpleTable: SimpleTable,
+  sourceColumn: string,
+  targetColumn: string,
+  nodeA: GraphId,
+  nodeB: GraphId,
+  options: CommonNeighborsOptions = {},
+): SimpleTable {
+  if (typeof sourceColumn !== "string") {
+    throw new TypeError("commonNeighbors() sourceColumn must be a string.");
+  }
+  if (typeof targetColumn !== "string") {
+    throw new TypeError("commonNeighbors() targetColumn must be a string.");
+  }
+  if (options === null || typeof options !== "object" || Array.isArray(options)) {
+    throw new TypeError("commonNeighbors() options must be an object.");
+  }
+  if (
+    options.direction !== undefined &&
+    !["outgoing", "incoming", "both"].includes(options.direction)
+  ) {
+    throw new TypeError(
+      'commonNeighbors() options.direction must be "outgoing", "incoming", or "both".',
+    );
+  }
+  if (
+    options.outputTable !== undefined &&
+    typeof options.outputTable !== "string" &&
+    typeof options.outputTable !== "boolean"
+  ) {
+    throw new TypeError("commonNeighbors() options.outputTable must be a string or boolean.");
+  }
+
+  const nodes = prepareNodes(nodeA, nodeB);
+  options = structuredClone(options);
+  const direction = options.direction ?? "outgoing";
+  const parameters = { sourceColumn, targetColumn, nodeA, nodeB, options };
+
+  return queueGraphResult(simpleTable, {
+    method: "commonNeighbors()",
+    parameters,
+    outputTable: options.outputTable,
+    values: (schema) => {
+      validateNodes(schema, sourceColumn, targetColumn, nodes);
+      return nodes.values;
+    },
+    buildSelect: (input, schema) =>
+      commonNeighborsSelect(input, schema, sourceColumn, targetColumn, nodes, direction),
+    outputSchema: (schema) => ({
+      node: validateNodes(schema, sourceColumn, targetColumn, nodes).idType,
+    }),
+  });
+}
+
+function prepareNodes(nodeA: GraphId, nodeB: GraphId): PreparedGraphStarts {
+  if (Array.isArray(nodeA)) {
+    throw new TypeError("commonNeighbors() nodeA must be one node ID.");
+  }
+  if (Array.isArray(nodeB)) {
+    throw new TypeError("commonNeighbors() nodeB must be one node ID.");
+  }
+  const first = prepareGraphStarts(nodeA, "commonNeighbors()", "nodeA", true);
+  const second = prepareGraphStarts(nodeB, "commonNeighbors()", "nodeB", true);
+  if (first.family !== second.family) {
+    throw new TypeError(
+      "commonNeighbors() nodeA and nodeB must both be string IDs or both be numeric IDs.",
+    );
+  }
+  if (first.values[0] === second.values[0]) {
+    throw new TypeError("commonNeighbors() nodeA and nodeB must be different.");
+  }
+  return {
+    family: first.family,
+    numericValues: [...first.numericValues, ...second.numericValues],
+    values: [first.values[0], second.values[0]],
+  };
+}
+
+function validateNodes(
+  schema: TableSchema,
+  source: string,
+  target: string,
+  nodes: PreparedGraphStarts,
+) {
+  return validateGraphStarts(schema, source, target, nodes, "commonNeighbors()", "nodeA and nodeB");
+}
+
+function commonNeighborsSelect(
+  input: string,
+  schema: TableSchema,
+  source: string,
+  target: string,
+  nodes: PreparedGraphStarts,
+  direction: GraphDirection,
+): string {
+  const prepared = prepareGraphTraversal(
+    input,
+    schema,
+    source,
+    target,
+    nodes,
+    "commonNeighbors()",
+    "nodeA and nodeB",
+  );
+  const relations = prepared.relationNames([
+    "graph_requested_nodes",
+    "graph_edges",
+    "graph_neighbor_memberships",
+    "graph_common_neighbors",
+  ]);
+  const requested = relations.graph_requested_nodes;
+  const edges = relations.graph_edges;
+  const memberships = relations.graph_neighbor_memberships;
+  const common = relations.graph_common_neighbors;
+  const q = quoteIdentifier;
+
+  return `WITH ${requested}(${q("node")}) AS (
+      VALUES ${prepared.startValues}
+    ), ${edges} AS (
+      ${prepared.edges(direction)}
+    ), ${memberships} AS (
+      SELECT ${q("requested")}.${q("node")} AS ${q("requested_node")},
+        ${q("edges")}.${q("__from_key")} AS ${q("__requested_key")},
+        ${q("edges")}.${q("__to")} AS ${q("node")},
+        ${q("edges")}.${q("__to_key")} AS ${q("__node_key")}
+      FROM ${edges} AS ${q("edges")}
+      INNER JOIN ${requested} AS ${q("requested")}
+        ON ${q("edges")}.${q("__from_key")} =
+          ${prepared.key(`${q("requested")}.${q("node")}`)}
+    ), ${common} AS (
+      SELECT MIN(${q("node")}) AS ${q("node")}, ${q("__node_key")}
+      FROM ${memberships}
+      GROUP BY ${q("__node_key")}
+      HAVING COUNT(DISTINCT ${q("__requested_key")}) = 2
+    )
+    SELECT ${q("node")}
+    FROM ${common}
+    ORDER BY ${q("__node_key")}`;
+}

package/src/methods/concatenate.ts

@@ -0,0 +1,33 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function concatenate(
+  simpleTable: SimpleTable,
+  columns: string[],
+  newColumn: string,
+  options: {
+    separator?: string;
+  } = {},
+) {
+  columns = [...columns];
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "concatenate()",
+    parameters: { columns, newColumn, options },
+    needsSchema: true,
+    values: typeof options.separator === "string" ? [options.separator] : undefined,
+    buildSelect: (input, types) => {
+      assertNewColumns(types, [newColumn], "concatenate()");
+      const expression =
+        typeof options.separator === "string"
+          ? `CONCAT_WS(?, ${columns.map((d) => `${quoteIdentifier(d)}`).join(", ")})`
+          : `CONCAT(${columns.map((d) => `${quoteIdentifier(d)}`).join(", ")})`;
+      return `SELECT *, CAST(${expression} AS VARCHAR) AS ${quoteIdentifier(
+        newColumn,
+      )} FROM ${input}`;
+    },
+  });
+}

package/src/methods/connectedComponents.ts

@@ -0,0 +1,195 @@
+import buildChronologicalGraphComponentsSql from "../helpers/buildChronologicalGraphComponentsSql.ts";
+import buildWeakGraphComponentsSql from "../helpers/buildWeakGraphComponentsSql.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import getGraphEndpointColumns from "../helpers/getGraphEndpointColumns.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+import prepareGraphTemporalSql, {
+  type GraphTemporalOptions,
+  type PreparedGraphTemporalOptions,
+  prepareGraphTemporalOptions,
+} from "../helpers/prepareGraphTemporalSql.ts";
+import { prepareGraphSql } from "../helpers/prepareGraphTraversal.ts";
+import queueGraphResult from "../helpers/queueGraphResult.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import validateGraphTemporalEvents from "../helpers/validateGraphTemporalEvents.ts";
+
+type ConnectedComponentsOptions = GraphTemporalOptions & {
+  mode?: "weak" | "strong";
+  outputTable?: string | boolean;
+};
+
+export default function connectedComponents(
+  simpleTable: SimpleTable,
+  sourceColumn: string,
+  targetColumn: string,
+  options: ConnectedComponentsOptions = {},
+): SimpleTable {
+  if (typeof sourceColumn !== "string") {
+    throw new TypeError("connectedComponents() sourceColumn must be a string.");
+  }
+  if (typeof targetColumn !== "string") {
+    throw new TypeError("connectedComponents() targetColumn must be a string.");
+  }
+  if (options === null || typeof options !== "object" || Array.isArray(options)) {
+    throw new TypeError("connectedComponents() options must be an object.");
+  }
+  if (options.mode !== undefined && !["weak", "strong"].includes(options.mode)) {
+    throw new TypeError('connectedComponents() options.mode must be "weak" or "strong".');
+  }
+  if (
+    options.outputTable !== undefined &&
+    typeof options.outputTable !== "string" &&
+    typeof options.outputTable !== "boolean"
+  ) {
+    throw new TypeError("connectedComponents() options.outputTable must be a string or boolean.");
+  }
+
+  const temporalOptions = prepareGraphTemporalOptions(options, undefined, "connectedComponents()");
+  if (temporalOptions !== undefined && options.mode !== "strong") {
+    throw new TypeError(
+      'connectedComponents() chronological options require options.mode to be explicitly set to "strong".',
+    );
+  }
+  options = structuredClone(options);
+  const mode = options.mode ?? "weak";
+  const parameters = { sourceColumn, targetColumn, options };
+
+  return queueGraphResult(simpleTable, {
+    method: "connectedComponents()",
+    parameters,
+    outputTable: options.outputTable,
+    preflight:
+      temporalOptions === undefined
+        ? undefined
+        : (input) =>
+            validateGraphTemporalEvents(
+              input,
+              temporalOptions,
+              "connectedComponents()",
+              parameters,
+            ),
+    values: (schema) => {
+      const temporal = validateInputs(schema, sourceColumn, targetColumn, temporalOptions).temporal;
+      return temporal === undefined ? [] : [temporal.gapParameter];
+    },
+    buildSelect: (input, schema) =>
+      connectedComponentsSelect(input, schema, sourceColumn, targetColumn, mode, temporalOptions),
+    outputSchema: (schema) => ({
+      componentId: "BIGINT",
+      node: validateInputs(schema, sourceColumn, targetColumn, temporalOptions).endpoints.idType,
+    }),
+  });
+}
+
+function validateEndpoints(schema: TableSchema, source: string, target: string) {
+  return getGraphEndpointColumns(schema, source, target, "connectedComponents()");
+}
+
+function validateInputs(
+  schema: TableSchema,
+  source: string,
+  target: string,
+  temporalOptions: PreparedGraphTemporalOptions | undefined,
+) {
+  const endpoints = validateEndpoints(schema, source, target);
+  const temporal =
+    temporalOptions === undefined
+      ? undefined
+      : prepareGraphTemporalSql(schema, temporalOptions, "connectedComponents()");
+  return { endpoints, temporal };
+}
+
+function connectedComponentsSelect(
+  input: string,
+  schema: TableSchema,
+  source: string,
+  target: string,
+  mode: "weak" | "strong",
+  temporalOptions: PreparedGraphTemporalOptions | undefined,
+): string {
+  const prepared = prepareGraphSql(input, schema, source, target, "connectedComponents()");
+  if (temporalOptions !== undefined) {
+    return buildChronologicalGraphComponentsSql(
+      prepared,
+      prepareGraphTemporalSql(schema, temporalOptions, "connectedComponents()"),
+    );
+  }
+  const relationNames =
+    mode === "weak"
+      ? ["graph_edges", "graph_nodes", "graph_component_labels", "graph_component_roots"]
+      : [
+          "graph_edges",
+          "graph_nodes",
+          "graph_reachable",
+          "graph_component_members",
+          "graph_component_roots",
+        ];
+  const relations = prepared.relationNames(relationNames);
+  const edges = relations.graph_edges;
+  const nodes = relations.graph_nodes;
+  const roots = relations.graph_component_roots;
+  const q = quoteIdentifier;
+
+  const common = `${edges} AS (
+      ${prepared.edges(mode === "weak" ? "both" : "outgoing")}
+    ), ${nodes} AS (
+      SELECT ${q("__from")} AS ${q("node")},
+        ${q("__from_key")} AS ${q("__key")}
+      FROM ${edges}
+      UNION
+      SELECT ${q("__to")} AS ${q("node")},
+        ${q("__to_key")} AS ${q("__key")}
+      FROM ${edges}
+    )`;
+
+  if (mode === "weak") {
+    const labels = relations.graph_component_labels;
+    return `WITH RECURSIVE ${common}, ${buildWeakGraphComponentsSql(edges, nodes, labels, roots)}
+    SELECT ${q("roots")}.${q("componentId")}, ${q("labels")}.${q("node")}
+    FROM ${labels} AS ${q("labels")}
+    INNER JOIN ${roots} AS ${q("roots")}
+      ON ${q("labels")}.${q("__root_key")} =
+        ${q("roots")}.${q("__root_key")}
+    ORDER BY ${q("labels")}.${q("__node_key")}`;
+  }
+
+  const reachable = relations.graph_reachable;
+  const members = relations.graph_component_members;
+  return `WITH RECURSIVE ${common}, ${reachable}(
+      ${q("__start_key")}, ${q("__node_key")}
+    ) AS (
+      SELECT ${q("__key")}, ${q("__key")} FROM ${nodes}
+      UNION
+      SELECT ${q("reached")}.${q("__start_key")},
+        ${q("edges")}.${q("__to_key")}
+      FROM ${reachable} AS ${q("reached")}
+      INNER JOIN ${edges} AS ${q("edges")}
+        ON ${q("reached")}.${q("__node_key")} =
+          ${q("edges")}.${q("__from_key")}
+    ), ${members} AS (
+      SELECT ${q("nodes")}.${q("node")}, ${q("nodes")}.${q("__key")},
+        MIN(${q("forward")}.${q("__start_key")}) AS ${q("__root_key")}
+      FROM ${nodes} AS ${q("nodes")}
+      INNER JOIN ${reachable} AS ${q("forward")}
+        ON ${q("nodes")}.${q("__key")} =
+          ${q("forward")}.${q("__node_key")}
+      INNER JOIN ${reachable} AS ${q("reverse")}
+        ON ${q("forward")}.${q("__start_key")} =
+          ${q("reverse")}.${q("__node_key")}
+        AND ${q("nodes")}.${q("__key")} =
+          ${q("reverse")}.${q("__start_key")}
+      GROUP BY ${q("nodes")}.${q("node")}, ${q("nodes")}.${q("__key")}
+    ), ${roots} AS (
+      SELECT ${q("__root_key")},
+        CAST(row_number() OVER (ORDER BY ${q("__root_key")}) - 1 AS BIGINT)
+          AS ${q("componentId")}
+      FROM (SELECT DISTINCT ${q("__root_key")} FROM ${members})
+        AS ${q("distinct_roots")}
+    )
+    SELECT ${q("roots")}.${q("componentId")}, ${q("members")}.${q("node")}
+    FROM ${members} AS ${q("members")}
+    INNER JOIN ${roots} AS ${q("roots")}
+      ON ${q("members")}.${q("__root_key")} =
+        ${q("roots")}.${q("__root_key")}
+    ORDER BY ${q("members")}.${q("__key")}`;
+}