@nshiab/simple-data-analysis-core 2.1.12 → 2.1.13

package/src/methods/loadStatCanData.ts

@@ -0,0 +1,183 @@
+import crypto from "node:crypto";
+import { createWriteStream, mkdirSync, rmSync } from "node:fs";
+import { pipeline } from "node:stream/promises";
+import { Readable } from "node:stream";
+import type SimpleTable from "../class/SimpleTable.ts";
+import SDAError from "../class/SDAError.ts";
+import extractZipEntryToFile from "../helpers/extractZipEntryToFile.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queueOp from "../helpers/queueOp.ts";
+import queryDB from "../helpers/queryDB.ts";
+import { loadDataQuery } from "./loadData.ts";
+import { createStatCanCacheId, useStatCanCache } from "../helpers/statCanCache.ts";
+
+type LoadStatCanOptions = {
+  lang?: "en" | "fr";
+  cache?: boolean;
+  ttl?: number;
+};
+
+type StatCanDownloadResponse = {
+  status?: unknown;
+  object?: unknown;
+};
+
+const DOWNLOAD_ENDPOINT = "https://www150.statcan.gc.ca/t1/wds/rest/getFullTableDownloadCSV";
+
+// Adapted from getStatCanTable() in @nshiab/journalism-web-scraping.
+
+export default function loadStatCanData(
+  table: SimpleTable,
+  pid: string,
+  options: LoadStatCanOptions = {},
+): void {
+  pid = normalizeStatCanPid(pid);
+  options = structuredClone(options);
+  assertOptions(options);
+  const lang = options.lang ?? "en";
+  const parameters = { pid, options };
+
+  queueOp(table, {
+    kind: "barrier",
+    method: "loadStatCanData()",
+    parameters,
+    execute: () => executeLoadStatCanData(table, pid, lang, options, parameters),
+  });
+}
+
+async function executeLoadStatCanData(
+  table: SimpleTable,
+  pid: string,
+  lang: "en" | "fr",
+  options: LoadStatCanOptions,
+  parameters: { [key: string]: unknown },
+): Promise<void> {
+  const cacheEnabled = options.cache ?? true;
+  const cacheId = createStatCanCacheId(pid, lang);
+  try {
+    await useStatCanCache(table, cacheId, cacheEnabled, options.ttl, parameters, async () => {
+      const downloadUrl = await getDownloadUrl(pid, lang);
+      const temporaryDirectory = ".sda-cache/tmp";
+      mkdirSync(temporaryDirectory, { recursive: true });
+      const prefix = `${cacheId}.${crypto.randomUUID()}`;
+      const archive = `${temporaryDirectory}/${prefix}.zip`;
+      const csv = `${temporaryDirectory}/${prefix}.csv`;
+
+      try {
+        await downloadToFile(downloadUrl, archive);
+        await extractZipEntryToFile(archive, `${pid}.csv`, csv);
+        await queryDB(
+          table,
+          loadDataQuery(table.name, [csv], { fileType: "csv" }),
+          mergeOptions(table, {
+            table: table.name,
+            method: "loadStatCanData()",
+            parameters,
+          }),
+        );
+      } finally {
+        rmSync(archive, { force: true });
+        rmSync(csv, { force: true });
+      }
+    });
+  } catch (error) {
+    if (error instanceof SDAError) {
+      throw error;
+    }
+    throw new SDAError({
+      method: "loadStatCanData()",
+      parameters,
+      query: "",
+      cause: error,
+    });
+  }
+}
+
+async function getDownloadUrl(pid: string, lang: "en" | "fr"): Promise<string> {
+  const url = `${DOWNLOAD_ENDPOINT}/${pid}/${lang}`;
+  const response = await fetch(url, {
+    headers: {
+      "user-agent":
+        "simple-data-analysis-core (https://github.com/nshiab/simple-data-analysis-core)",
+    },
+  });
+  if (!response.ok) {
+    throw await httpError("retrieve the Statistics Canada download URL", url, response);
+  }
+
+  const result = (await response.json()) as StatCanDownloadResponse;
+  if (result.status !== "SUCCESS" || typeof result.object !== "string") {
+    throw new Error(`Statistics Canada did not return a CSV download URL for PID ${pid}.`);
+  }
+  const downloadUrl = new URL(result.object);
+  if (downloadUrl.protocol !== "http:" && downloadUrl.protocol !== "https:") {
+    throw new Error(`Statistics Canada returned an invalid CSV download URL for PID ${pid}.`);
+  }
+  return downloadUrl.href;
+}
+
+async function downloadToFile(url: string, file: string): Promise<void> {
+  const response = await fetch(url, {
+    headers: {
+      "user-agent":
+        "simple-data-analysis-core (https://github.com/nshiab/simple-data-analysis-core)",
+    },
+  });
+  if (!response.ok) {
+    throw await httpError("download Statistics Canada data", url, response);
+  }
+  if (response.body === null) {
+    throw new Error(
+      `Could not download Statistics Canada data from ${JSON.stringify(
+        url,
+      )}: the response body is empty.`,
+    );
+  }
+  try {
+    await pipeline(
+      Readable.fromWeb(response.body as Parameters<typeof Readable.fromWeb>[0]),
+      createWriteStream(file, { flags: "wx" }),
+    );
+  } catch (error) {
+    rmSync(file, { force: true });
+    throw error;
+  }
+}
+
+async function httpError(action: string, url: string, response: Response): Promise<Error> {
+  const responseText = (await response.text()).slice(0, 500).trim();
+  return new Error(
+    `Could not ${action} from ${JSON.stringify(
+      url,
+    )}: HTTP ${response.status} ${response.statusText}${
+      responseText.length > 0 ? `\n${responseText}` : ""
+    }`,
+  );
+}
+
+export function normalizeStatCanPid(pid: string): string {
+  if (typeof pid !== "string") {
+    throw new TypeError("loadStatCanData() pid must be a string.");
+  }
+  const normalized = pid.replaceAll("-", "");
+  if (!/^\d{8}(?:\d{2})?$/.test(normalized)) {
+    throw new Error(
+      'loadStatCanData() pid must contain 8 or 10 digits, optionally formatted like "36-10-0402-01".',
+    );
+  }
+  return normalized.slice(0, 8);
+}
+
+function assertOptions(options: LoadStatCanOptions): void {
+  if (options.lang !== undefined && options.lang !== "en" && options.lang !== "fr") {
+    throw new Error('loadStatCanData() options.lang must be "en" or "fr".');
+  }
+  if (options.cache !== undefined && typeof options.cache !== "boolean") {
+    throw new TypeError("loadStatCanData() options.cache must be a boolean.");
+  }
+  if (options.ttl !== undefined && (!Number.isFinite(options.ttl) || options.ttl < 0)) {
+    throw new Error(
+      "loadStatCanData() options.ttl must be a finite, non-negative number of seconds.",
+    );
+  }
+}

package/src/methods/loadYahooFinanceData.ts

@@ -0,0 +1,205 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import SDAError from "../class/SDAError.ts";
+import queueOp from "../helpers/queueOp.ts";
+import { executePreparedArray, prepareArray } from "./loadArray.ts";
+
+const INTERVAL_SECONDS = {
+  "1d": 24 * 60 * 60,
+  "1h": 60 * 60,
+  "1m": 60,
+} as const;
+
+const COLUMN_TYPES = {
+  datetime: "TIMESTAMP",
+  open: "DOUBLE",
+  high: "DOUBLE",
+  low: "DOUBLE",
+  close: "DOUBLE",
+  adjustedClose: "DOUBLE",
+  volume: "DOUBLE",
+} as const;
+
+type YahooInterval = keyof typeof INTERVAL_SECONDS;
+type YahooQuoteVariable = "open" | "high" | "low" | "close" | "volume";
+type YahooValues = Array<number | null>;
+
+type YahooFinanceResult = {
+  timestamp?: number[];
+  indicators?: {
+    adjclose?: Array<{ adjclose?: YahooValues }>;
+    quote?: Array<Partial<Record<YahooQuoteVariable, YahooValues>>>;
+  };
+};
+
+type YahooFinanceResponse = {
+  chart?: {
+    error?: { description?: string } | null;
+    result?: YahooFinanceResult[] | null;
+  };
+};
+
+type YahooFinanceRow = {
+  datetime: Date;
+  open: number | null;
+  high: number | null;
+  low: number | null;
+  close: number | null;
+  adjustedClose: number | null;
+  volume: number | null;
+};
+
+export default function loadYahooFinanceData(
+  table: SimpleTable,
+  symbol: string,
+  startDate: Date,
+  endDate: Date,
+  interval: YahooInterval,
+): void {
+  assertArguments(symbol, startDate, endDate, interval);
+  const request = {
+    symbol,
+    startDate: new Date(startDate),
+    endDate: new Date(endDate),
+    interval,
+  };
+
+  queueOp(table, {
+    kind: "barrier",
+    method: "loadYahooFinanceData()",
+    parameters: request,
+    execute: () => executeLoadYahooFinanceData(table, request),
+  });
+}
+
+async function executeLoadYahooFinanceData(
+  table: SimpleTable,
+  request: {
+    symbol: string;
+    startDate: Date;
+    endDate: Date;
+    interval: YahooInterval;
+  },
+): Promise<void> {
+  try {
+    const rows = await getYahooFinanceData(request);
+    await executePreparedArray(table, prepareArray(rows, COLUMN_TYPES));
+  } catch (error) {
+    if (error instanceof SDAError) {
+      throw error;
+    }
+    throw new SDAError({
+      method: "loadYahooFinanceData()",
+      parameters: request,
+      query: "",
+      cause: error,
+    });
+  }
+}
+
+async function getYahooFinanceData(request: {
+  symbol: string;
+  startDate: Date;
+  endDate: Date;
+  interval: YahooInterval;
+}): Promise<YahooFinanceRow[]> {
+  const { symbol, startDate, endDate, interval } = request;
+  const startTime = startDate.getTime();
+  const exclusiveEndTime = getExclusiveEndTime(endDate, interval);
+  const url = new URL(
+    `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}`,
+  );
+  url.search = new URLSearchParams({
+    includeAdjustedClose: "true",
+    interval,
+    period1: String(Math.floor(startTime / 1000)),
+    period2: String(Math.ceil(exclusiveEndTime / 1000)),
+  }).toString();
+
+  const response = await fetch(url);
+  if (!response.ok) {
+    const text = (await response.text()).slice(0, 500).trim();
+    throw new Error(
+      `Failed to fetch Yahoo Finance data: ${response.status} ${response.statusText}${
+        text.length > 0 ? `. ${text}` : ""
+      }. Yahoo may have changed, rate-limited, or disabled this undocumented endpoint.`,
+    );
+  }
+
+  const data = (await response.json()) as YahooFinanceResponse;
+  if (data.chart?.error) {
+    throw new Error(data.chart.error.description ?? "Yahoo Finance returned an unknown error.");
+  }
+
+  const result = data.chart?.result?.[0];
+  const timestamps = result?.timestamp;
+  if (!result || !timestamps?.length) {
+    throw new Error("No Yahoo Finance data found.");
+  }
+
+  const quote = result.indicators?.quote?.[0];
+  const adjustedClose = result.indicators?.adjclose?.[0]?.adjclose;
+  const rows: YahooFinanceRow[] = [];
+  for (let index = 0; index < timestamps.length; index++) {
+    const timestamp = timestamps[index] * 1000;
+    if (Number.isFinite(timestamp) && timestamp >= startTime && timestamp < exclusiveEndTime) {
+      rows.push({
+        datetime: new Date(timestamp),
+        open: getFiniteValue(quote?.open, index),
+        high: getFiniteValue(quote?.high, index),
+        low: getFiniteValue(quote?.low, index),
+        close: getFiniteValue(quote?.close, index),
+        adjustedClose: getFiniteValue(adjustedClose, index),
+        volume: getFiniteValue(quote?.volume, index),
+      });
+    }
+  }
+
+  if (rows.length === 0) {
+    throw new Error(`No Yahoo Finance data found for ${symbol}.`);
+  }
+  return rows;
+}
+
+function getFiniteValue(values: YahooValues | undefined, index: number): number | null {
+  const value = values?.[index];
+  return typeof value === "number" && Number.isFinite(value) ? value : null;
+}
+
+function getExclusiveEndTime(endDate: Date, interval: YahooInterval): number {
+  const end = new Date(endDate);
+  if (interval === "1d") {
+    end.setUTCHours(0, 0, 0, 0);
+  } else if (interval === "1h") {
+    end.setUTCMinutes(0, 0, 0);
+  } else {
+    end.setUTCSeconds(0, 0);
+  }
+  return end.getTime() + INTERVAL_SECONDS[interval] * 1000;
+}
+
+function assertArguments(
+  symbol: string,
+  startDate: Date,
+  endDate: Date,
+  interval: YahooInterval,
+): void {
+  if (typeof symbol !== "string" || symbol.trim().length === 0) {
+    throw new TypeError("loadYahooFinanceData() symbol must be a non-empty string.");
+  }
+  if (
+    !(startDate instanceof Date) ||
+    !(endDate instanceof Date) ||
+    !Number.isFinite(startDate.getTime()) ||
+    !Number.isFinite(endDate.getTime())
+  ) {
+    throw new RangeError("loadYahooFinanceData() startDate and endDate must be valid dates.");
+  }
+  if (endDate.getTime() < startDate.getTime()) {
+    throw new RangeError(
+      "loadYahooFinanceData() endDate must be equal to or later than startDate.",
+    );
+  }
+  if (interval !== "1d" && interval !== "1h" && interval !== "1m") {
+    throw new Error('loadYahooFinanceData() interval must be "1d", "1h", or "1m".');
+  }
+}

package/src/methods/log.ts

@@ -0,0 +1,76 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import logData from "../helpers/logData.ts";
+import getData from "./getData.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import formatNumber from "../helpers/formatNumber.ts";
+
+export default async function log(
+  simpleTable: SimpleTable,
+  options:
+    | "all"
+    | number
+    | {
+        count?: number | "all";
+        types?: boolean;
+        conditions?: string;
+      } = {},
+) {
+  if (simpleTable.connection === undefined) {
+    await simpleTable.sdb.start();
+    simpleTable.db = simpleTable.sdb.db;
+    simpleTable.connection = simpleTable.sdb.connection;
+  }
+  if (simpleTable.connection === undefined) {
+    throw new Error("simpleTable.connection is undefined");
+  }
+
+  let count: number;
+  if (typeof options === "number") {
+    count = options;
+  } else if (options === "all") {
+    count = await simpleTable.getRowCount();
+  } else if (typeof options === "object") {
+    if (options.count === "all") {
+      count = await simpleTable.getRowCount();
+    } else if (typeof options.count === "number") {
+      count = options.count;
+    } else {
+      count = simpleTable.rowsToLog;
+    }
+  } else {
+    count = simpleTable.rowsToLog;
+  }
+  const types =
+    typeof options === "object"
+      ? (options.types ?? simpleTable.typesToLog)
+      : simpleTable.typesToLog;
+  const conditions = typeof options === "object" ? (options.conditions ?? undefined) : undefined;
+
+  if (simpleTable.connection === undefined || !(await simpleTable.sdb.hasTable(simpleTable.name))) {
+    console.log(`\nTable ${simpleTable.name}: no data`);
+  } else {
+    console.log(`\nTable ${simpleTable.name}:`);
+    if (conditions) {
+      console.log(`Conditions: ${conditions}`);
+    }
+    const data = await getData(simpleTable, { limit: count, conditions }, "log()");
+    logData(types ? await simpleTable.getTypes() : null, data, simpleTable.charsToLog);
+    const rowCount = conditions
+      ? parseInt(
+          (
+            (await simpleTable.sdb.customQuery(
+              `select count(*) as count from ${quoteIdentifier(
+                simpleTable.name,
+              )} where ${conditions}`,
+              { returnData: true },
+            )) as { count: string }[]
+          )[0].count,
+        )
+      : await simpleTable.getRowCount();
+    const displayedRows = data.length;
+    const showing =
+      displayedRows < rowCount ? ` / showing ${formatNumber(displayedRows)} rows` : "";
+    console.log(`${formatNumber(rowCount)} rows in total${showing}`);
+  }
+  return;
+}

package/src/methods/logBottom.ts

@@ -0,0 +1,15 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import logData from "../helpers/logData.ts";
+import getBottom from "./getBottom.ts";
+
+export default async function logBottom(
+  simpleTable: SimpleTable,
+  count?: number,
+  options: { originalOrder?: boolean } = {},
+) {
+  const rows = count ?? simpleTable.rowsToLog;
+  console.log(`\nTable ${simpleTable.name} (${rows} bottom rows):`);
+  const data = await getBottom(simpleTable, rows, options, "logBottom()");
+  logData(null, data, simpleTable.charsToLog);
+  return;
+}

package/src/methods/longer.ts

@@ -0,0 +1,33 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function longer(
+  simpleTable: SimpleTable,
+  columns: string[],
+  namesTo: string,
+  valuesTo: string,
+) {
+  columns = [...columns];
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "longer()",
+    parameters: { columns, namesTo, valuesTo },
+    needsSchema: true,
+    buildSelect: (input, schema) => {
+      // The pivoted columns are consumed by UNPIVOT and don't appear in the
+      // output, so namesTo/valuesTo may reuse one of those names.
+      assertNewColumns(schema, [namesTo, valuesTo], "longer()", columns);
+
+      return `SELECT * FROM (
+            FROM ${input} UNPIVOT INCLUDE NULLS (
+            ${quoteIdentifier(valuesTo)}
+            for ${quoteIdentifier(namesTo)} in (${columns
+              .map((d) => `${quoteIdentifier(d)}`)
+              .join(", ")})
+            )
+        )`;
+    },
+  });
+}

package/src/methods/lower.ts

@@ -0,0 +1,18 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function lower(simpleTable: SimpleTable, columns: string | string[]) {
+  const cols = stringToArray(columns);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "lower()",
+    parameters: { columns },
+    needsSchema: false,
+    buildSelect: (input) =>
+      `SELECT * REPLACE (${cols
+        .map((c) => `LOWER(${quoteIdentifier(c)}) AS ${quoteIdentifier(c)}`)
+        .join(", ")}) FROM ${input}`,
+  });
+}

package/src/methods/neighbors.ts

@@ -0,0 +1,133 @@
+import prepareGraphStarts, {
+  type GraphId,
+  type PreparedGraphStarts,
+} from "../helpers/prepareGraphStarts.ts";
+import prepareGraphTraversal, {
+  type GraphDirection,
+  validateGraphStarts,
+} from "../helpers/prepareGraphTraversal.ts";
+import queueGraphResult from "../helpers/queueGraphResult.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { TableSchema } from "../helpers/pendingOps.ts";
+
+type NeighborsOptions = {
+  direction?: "outgoing" | "incoming" | "both";
+  outputTable?: string | boolean;
+};
+
+export default function neighbors(
+  simpleTable: SimpleTable,
+  sourceColumn: string,
+  targetColumn: string,
+  startNodes: GraphId | GraphId[],
+  options: NeighborsOptions = {},
+): SimpleTable {
+  if (typeof sourceColumn !== "string") {
+    throw new TypeError("neighbors() sourceColumn must be a string.");
+  }
+  if (typeof targetColumn !== "string") {
+    throw new TypeError("neighbors() targetColumn must be a string.");
+  }
+  if (options === null || typeof options !== "object" || Array.isArray(options)) {
+    throw new TypeError("neighbors() options must be an object.");
+  }
+  if (
+    options.direction !== undefined &&
+    !["outgoing", "incoming", "both"].includes(options.direction)
+  ) {
+    throw new TypeError('neighbors() options.direction must be "outgoing", "incoming", or "both".');
+  }
+  if (
+    options.outputTable !== undefined &&
+    typeof options.outputTable !== "string" &&
+    typeof options.outputTable !== "boolean"
+  ) {
+    throw new TypeError("neighbors() options.outputTable must be a string or boolean.");
+  }
+
+  const preparedStarts = prepareGraphStarts(startNodes, "neighbors()", "startNodes");
+  options = structuredClone(options);
+  const direction = options.direction ?? "outgoing";
+  const parameters = {
+    sourceColumn,
+    targetColumn,
+    startNodes: structuredClone(startNodes),
+    options,
+  };
+
+  return queueGraphResult(simpleTable, {
+    method: "neighbors()",
+    parameters,
+    outputTable: options.outputTable,
+    values: (schema) => {
+      validateStarts(schema, sourceColumn, targetColumn, preparedStarts);
+      return preparedStarts.values;
+    },
+    buildSelect: (input, schema) =>
+      neighborsSelect(input, schema, sourceColumn, targetColumn, preparedStarts, direction),
+    outputSchema: (schema) => {
+      const endpoints = validateStarts(schema, sourceColumn, targetColumn, preparedStarts);
+      return { start: endpoints.idType, node: endpoints.idType };
+    },
+  });
+}
+
+function validateStarts(
+  schema: TableSchema,
+  source: string,
+  target: string,
+  starts: PreparedGraphStarts,
+) {
+  return validateGraphStarts(schema, source, target, starts, "neighbors()", "startNodes");
+}
+
+function neighborsSelect(
+  input: string,
+  schema: TableSchema,
+  source: string,
+  target: string,
+  starts: PreparedGraphStarts,
+  direction: GraphDirection,
+): string {
+  const prepared = prepareGraphTraversal(
+    input,
+    schema,
+    source,
+    target,
+    starts,
+    "neighbors()",
+    "startNodes",
+  );
+  const relations = prepared.relationNames(["graph_starts", "graph_edges", "neighbor_matches"]);
+  const startsRelation = relations.graph_starts;
+  const edgesRelation = relations.graph_edges;
+  const matchesRelation = relations.neighbor_matches;
+
+  return `WITH ${startsRelation}(${quoteIdentifier("start")}) AS (
+      VALUES ${prepared.startValues}
+    ), ${edgesRelation} AS (
+      ${prepared.edges(direction)}
+    ), ${matchesRelation} AS (
+      SELECT ${quoteIdentifier("starts")}.${quoteIdentifier("start")},
+        ${quoteIdentifier("edges")}.${quoteIdentifier("__to")} AS ${quoteIdentifier("node")},
+        ${quoteIdentifier("edges")}.${quoteIdentifier("__from_key")} AS ${quoteIdentifier(
+          "__start_key",
+        )},
+        ${quoteIdentifier("edges")}.${quoteIdentifier("__to_key")} AS ${quoteIdentifier(
+          "__node_key",
+        )}
+      FROM ${edgesRelation} AS ${quoteIdentifier("edges")}
+      INNER JOIN ${startsRelation} AS ${quoteIdentifier("starts")}
+        ON ${quoteIdentifier("edges")}.${quoteIdentifier("__from_key")} =
+          ${prepared.key(`${quoteIdentifier("starts")}.${quoteIdentifier("start")}`)}
+    )
+    SELECT ${quoteIdentifier("start")}, ${quoteIdentifier("node")}
+    FROM (
+      SELECT DISTINCT ${quoteIdentifier("start")}, ${quoteIdentifier(
+        "node",
+      )}, ${quoteIdentifier("__start_key")}, ${quoteIdentifier("__node_key")}
+      FROM ${matchesRelation}
+    ) AS ${quoteIdentifier("distinct_neighbors")}
+    ORDER BY ${quoteIdentifier("__start_key")}, ${quoteIdentifier("__node_key")}`;
+}

package/src/methods/nest.ts

@@ -0,0 +1,31 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import queueOp from "../helpers/queueOp.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function nest(
+  simpleTable: SimpleTable,
+  column: string,
+  separator: string,
+  by: string | string[],
+) {
+  by = Array.isArray(by) ? [...by] : by;
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "nest()",
+    parameters: { column, separator, by },
+    values: [separator],
+    needsSchema: false,
+    buildSelect: (input) => {
+      const cats = stringToArray(by);
+      const groupBy = cats.map((d) => `${quoteIdentifier(d)}`).join(", ");
+      const selectColumns = `${groupBy}, `;
+      const orderBy = `\nORDER BY ${cats.map((d) => `${quoteIdentifier(d)} ASC`).join(", ")}`;
+
+      return `SELECT
+  ${selectColumns}STRING_AGG(${quoteIdentifier(column)}, ?) AS ${quoteIdentifier(column)}
+FROM ${input}
+GROUP BY ${groupBy}${orderBy}`;
+    },
+  });
+}