@nshiab/simple-data-analysis-core 2.1.12 → 2.1.14

package/src/methods/unnestGeo.ts

@@ -0,0 +1,21 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import findGeoColumnFromSchema from "../helpers/findGeoColumnFromSchema.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function unnestGeo(simpleTable: SimpleTable, column?: string) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "unnestGeo()",
+    parameters: { column },
+    needsSchema: true,
+    needsSpatial: true,
+    buildSelect: (input, types) => {
+      const col = column ?? findGeoColumnFromSchema(types, "unnestGeo()", simpleTable.name);
+      // The recursive UNNEST adds a path column, removed by the outer SELECT.
+      return `SELECT * EXCLUDE(path) FROM (SELECT * EXCLUDE(${quoteIdentifier(
+        col,
+      )}), UNNEST(ST_Dump(${quoteIdentifier(col)}), recursive := TRUE) FROM ${input})`;
+    },
+  });
+}

package/src/methods/updateColumn.ts

@@ -0,0 +1,20 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function updateColumn(simpleTable: SimpleTable, column: string, definition: string) {
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "updateColumn()",
+    parameters: { column, definition },
+    // The schema is needed to keep the column's type: UPDATE casts the
+    // definition to the column's type on assignment.
+    needsSchema: true,
+    rawSQL: [definition],
+    preservesSchema: true,
+    buildSelect: (input, types) =>
+      `SELECT * REPLACE (CAST((${definition}) AS ${types[column]}) AS ${quoteIdentifier(
+        column,
+      )}) FROM ${input}`,
+  });
+}

package/src/methods/updateWithJS.ts

@@ -0,0 +1,291 @@
+import prepareGeometry from "../helpers/prepareGeometry.ts";
+import geometryFromJSON from "../helpers/geometryFromJSON.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import { retainRegisteredTables } from "../helpers/tableRegistry.ts";
+import queueOp from "../helpers/queueOp.ts";
+import { prepareArray } from "./loadArray.ts";
+import appendColumnBatches from "../helpers/appendColumnBatches.ts";
+import readMutationRows from "../helpers/readMutationRows.ts";
+import parseDuckDBType from "../helpers/parseDuckDBType.ts";
+
+type Row = { [key: string]: unknown };
+
+type DataModifier = (rows: Row[]) => Row[] | Promise<Row[]>;
+
+type UpdateWithJSOptions = NonNullable<Parameters<SimpleTable["updateWithJS"]>[1]>;
+
+export default function updateWithJS(
+  simpleTable: SimpleTable,
+  dataModifier: DataModifier,
+  options: UpdateWithJSOptions = {},
+): void {
+  if (
+    options.batchSize !== undefined &&
+    (!Number.isInteger(options.batchSize) || options.batchSize < 1)
+  ) {
+    throw new Error("updateWithJS() batchSize must be a positive integer.");
+  }
+
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "barrier",
+    method: "updateWithJS()",
+    parameters: { options },
+    execute: () => executeUpdateWithJS(simpleTable, dataModifier, options),
+  });
+}
+
+async function executeUpdateWithJS(
+  simpleTable: SimpleTable,
+  dataModifier: DataModifier,
+  options: UpdateWithJSOptions,
+): Promise<void> {
+  const types = await simpleTable.getTypes();
+  const geometryColumns = Object.keys(types).filter((key) =>
+    types[key].toUpperCase().startsWith("GEOMETRY"),
+  );
+  for (const column of geometryColumns) {
+    if (types[column].toUpperCase() !== "GEOMETRY('EPSG:4326')") {
+      throw new Error(
+        `Column ${JSON.stringify(column)} has CRS ${
+          types[column]
+        }. Call .reproject("EPSG:4326", { column: ${JSON.stringify(
+          column,
+        )} }) before updateWithJS().`,
+      );
+    }
+    const name = quoteIdentifier(column);
+    // Compare the actual binary representation, including dimensions and double
+    // precision, rather than assuming GeoJSON serialization is lossless.
+    const result = await simpleTable.connection!.runAndReadAll(
+      `SELECT count(*) AS invalid FROM ${quoteIdentifier(simpleTable.name)}
+       WHERE ${name} IS NOT NULL AND
+       ST_AsWKB(${name}) != ST_AsWKB(${geometryFromJSON(`ST_AsGeoJSON(${name})`)})`,
+    );
+    if (result.getRows()[0][0] !== 0n) {
+      throw new Error(
+        `Column ${JSON.stringify(
+          column,
+        )} contains geometry that cannot round-trip through GeoJSON without losing precision or dimensions; unsupported representations cannot be updated with updateWithJS().`,
+      );
+    }
+  }
+  for (const key of Object.keys(options.columnTypes ?? {})) {
+    if (Object.keys(types).some((name) => name.toLowerCase() === key.toLowerCase())) {
+      throw new Error(
+        `updateWithJS() columnTypes may only declare new columns; ${JSON.stringify(
+          key,
+        )} already exists.`,
+      );
+    }
+  }
+
+  const batchSize = options.batchSize;
+  const geometryBatch = batchSize !== undefined && geometryColumns.length > 0;
+  if (batchSize !== undefined) {
+    // DuckDB resolves identifiers case-insensitively, and a real rowid column
+    // shadows its hidden row identifier used for pagination.
+    const conflictingColumn = Object.keys(types).find((column) =>
+      ["rowid", "__sda_rowid"].includes(column.toLowerCase()),
+    );
+    if (conflictingColumn !== undefined) {
+      throw new Error(
+        `The table has a column named "${conflictingColumn}", which conflicts with the internal column used by the batchSize option. Rename it or run updateWithJS without batchSize.`,
+      );
+    }
+  }
+
+  // Rows are pulled in batches by rowid, passed through the modifier, and
+  // accumulated in a temporary table, so only one batch of rows is
+  // materialized in JS memory at a time.
+  const suffix = Math.random().toString(36).slice(2, 10);
+  const scratch = simpleTable.sdb.newTable(`updateWithJS_scratch_${suffix}`);
+  const accumulator = `updateWithJS_accumulator_${suffix}`;
+
+  try {
+    let first = true;
+    let accumulatorSchema: string | undefined;
+    let lastRowid: bigint | null = null;
+    let sawRows = false;
+    let outputOffset = 0;
+    let inputOffset = 0;
+
+    while (true) {
+      const page = `${quoteIdentifier(simpleTable.name)}${
+        lastRowid === null ? "" : ` WHERE rowid > ${lastRowid}`
+      }${batchSize === undefined ? "" : ` ORDER BY rowid LIMIT ${batchSize}`}`;
+      // Select the batch before serializing geometry. Otherwise DuckDB can
+      // evaluate ST_AsGeoJSON on remaining rows before applying the LIMIT,
+      // repeatedly converting polygons that belong to later batches.
+      const from = geometryBatch
+        ? `(SELECT *, rowid AS __sda_rowid FROM ${page}) AS __sda_batch ORDER BY __sda_rowid`
+        : page;
+      const source = await readMutationRows(
+        simpleTable.connection!,
+        `SELECT ${Object.keys(types)
+          .map((key) =>
+            geometryColumns.includes(key)
+              ? `ST_AsGeoJSON(${quoteIdentifier(key)})::VARCHAR AS ${quoteIdentifier(key)}`
+              : quoteIdentifier(key),
+          )
+          .join(", ")}${
+          batchSize === undefined ? "" : geometryBatch ? ", __sda_rowid" : ", rowid AS __sda_rowid"
+        } FROM ${from}`,
+        batchSize === undefined ? undefined : "__sda_rowid",
+      );
+      const batch = source.rows;
+      const inputCount = batch.length;
+      if (inputCount === 0 && sawRows) break;
+      sawRows ||= inputCount > 0;
+      if (batchSize !== undefined && batch.length > 0) {
+        // Keep the cursor separate: adding and deleting it on every row also
+        // makes ordinary callback operations such as object spread slower.
+        lastRowid = source.lastCursorValue as bigint;
+      }
+
+      for (const [index, row] of batch.entries()) {
+        for (const key of geometryColumns) {
+          row[key] = row[key] === null ? null : JSON.parse(row[key] as string);
+          prepareGeometry(row[key], key, inputOffset + index + 1);
+        }
+      }
+      inputOffset += inputCount;
+      const modified = await dataModifier(batch);
+      if (modified.length === 0) {
+        if (!sawRows) return;
+        if (batchSize === undefined) break;
+        continue;
+      }
+      const keys = Object.keys(modified[0]);
+      for (const key of Object.keys(options.columnTypes ?? {})) {
+        if (!keys.includes(key)) {
+          throw new Error(`Unknown column ${JSON.stringify(key)} in updateWithJS columnTypes.`);
+        }
+      }
+      const newKeys = keys.filter((key) => !source.types.has(key));
+      const added =
+        newKeys.length === 0
+          ? undefined
+          : prepareArray(
+              modified.map((row) => Object.fromEntries(newKeys.map((key) => [key, row[key]]))),
+              options.columnTypes,
+            );
+      const outputGeometry = keys.filter(
+        (key) =>
+          geometryColumns.includes(key) ||
+          (added && added.types[newKeys.indexOf(key)] === "GEOMETRY('EPSG:4326')"),
+      );
+      const outputJSON = keys.filter(
+        (key) => types[key] === "JSON" || (added && added.types[newKeys.indexOf(key)] === "JSON"),
+      );
+      if (outputGeometry.length && !simpleTable.sdb.spatialLoaded) {
+        await simpleTable.sdb.customQuery(
+          "INSTALL spatial; LOAD spatial; SET geometry_always_xy = true;",
+        );
+        simpleTable.sdb.spatialLoaded = true;
+      }
+      const geometryData = new Map(
+        outputGeometry.map((key) => [
+          key,
+          geometryColumns.includes(key)
+            ? modified.map((row, i) => prepareGeometry(row[key], key, outputOffset + i + 1))
+            : added!.columnsData[newKeys.indexOf(key)],
+        ]),
+      );
+      outputOffset += modified.length;
+      const columnTypes = keys.map((key) =>
+        outputGeometry.includes(key) || outputJSON.includes(key)
+          ? parseDuckDBType("VARCHAR")
+          : (source.types.get(key) ?? parseDuckDBType(added!.types[newKeys.indexOf(key)])),
+      );
+      const schema = JSON.stringify(keys.map((key, i) => [key, columnTypes[i].toString()]));
+      // Append matching native batches directly. SQL still handles conversion
+      // and BY NAME alignment when a callback changes the batch schema.
+      const direct =
+        outputGeometry.length === 0 &&
+        outputJSON.length === 0 &&
+        (first || schema === accumulatorSchema);
+      const destination = direct ? accumulator : scratch.name;
+      if (!direct || first) {
+        await simpleTable.sdb.customQuery(
+          `CREATE OR REPLACE TABLE ${quoteIdentifier(destination)} (${keys
+            .map((key, i) => `${quoteIdentifier(key)} ${columnTypes[i].toString()}`)
+            .join(", ")})`,
+        );
+      }
+      await appendColumnBatches(
+        simpleTable.connection!,
+        destination,
+        columnTypes,
+        modified.length,
+        (column, start, end) => {
+          const key = keys[column];
+          return outputGeometry.includes(key)
+            ? geometryData.get(key)!.slice(start, end)
+            : source.types.has(key)
+              ? modified
+                  .slice(start, end)
+                  .map((row) => source.toNative(row[key], columnTypes[column]))
+              : added!.columnsData[newKeys.indexOf(key)].slice(start, end);
+        },
+      );
+      const projection = keys
+        .map((key) =>
+          outputGeometry.includes(key)
+            ? `${geometryFromJSON(quoteIdentifier(key))} AS ${quoteIdentifier(key)}`
+            : outputJSON.includes(key)
+              ? `${quoteIdentifier(key)}::JSON AS ${quoteIdentifier(key)}`
+              : quoteIdentifier(key),
+        )
+        .join(", ");
+      if (direct) {
+        accumulatorSchema = schema;
+        first = false;
+      } else if (first) {
+        await simpleTable.sdb.customQuery(
+          `CREATE OR REPLACE TABLE ${quoteIdentifier(
+            accumulator,
+          )} AS SELECT ${projection} FROM ${quoteIdentifier(scratch.name)}`,
+        );
+        first = false;
+      } else {
+        await simpleTable.sdb.customQuery(
+          `INSERT INTO ${quoteIdentifier(
+            accumulator,
+          )} BY NAME SELECT ${projection} FROM ${quoteIdentifier(scratch.name)}`,
+        );
+      }
+      if (batchSize === undefined || inputCount === 0) break;
+    }
+
+    if (first) {
+      throw new Error(
+        "The dataModifier returned no rows. updateWithJS can't infer the table schema from zero rows.",
+      );
+    }
+
+    // Publish the completed ordinary table without copying its rows again.
+    // Qualify the creation namespace: a temporary table or a later search-path
+    // schema can shadow the name without being the destination of replacement.
+    const namespace = await simpleTable.connection!.runAndReadAll(
+      "SELECT current_database(), current_schema()",
+    );
+    const prefix = namespace
+      .getRows()[0]
+      .map((name) => quoteIdentifier(String(name)))
+      .join(".");
+    await simpleTable.sdb.customQuery(
+      `DROP TABLE IF EXISTS ${prefix}.${quoteIdentifier(simpleTable.name)};
+ALTER TABLE ${prefix}.${quoteIdentifier(accumulator)} RENAME TO ${quoteIdentifier(
+        simpleTable.name,
+      )};`,
+    );
+  } finally {
+    await simpleTable.sdb.customQuery(
+      `DROP TABLE IF EXISTS ${quoteIdentifier(accumulator)};
+DROP TABLE IF EXISTS ${quoteIdentifier(scratch.name)};`,
+    );
+    retainRegisteredTables(simpleTable.sdb, (table) => table !== scratch);
+  }
+}

package/src/methods/upper.ts

@@ -0,0 +1,18 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function upper(simpleTable: SimpleTable, columns: string | string[]) {
+  const cols = stringToArray(columns);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "upper()",
+    parameters: { columns },
+    needsSchema: false,
+    buildSelect: (input) =>
+      `SELECT * REPLACE (${cols
+        .map((c) => `UPPER(${quoteIdentifier(c)}) AS ${quoteIdentifier(c)}`)
+        .join(", ")}) FROM ${input}`,
+  });
+}

package/src/methods/wider.ts

@@ -0,0 +1,41 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queryDB from "../helpers/queryDB.ts";
+import queueOp from "../helpers/queueOp.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function wider(
+  simpleTable: SimpleTable,
+  namesFrom: string,
+  valuesFrom: string,
+  options: {
+    stat?: "sum" | "count" | "min" | "max" | "mean" | "median" | "first";
+  } = {},
+) {
+  const stat = options.stat ?? "sum";
+  const aggregateFunction = stat === "mean" ? "avg" : stat;
+
+  // DuckDB rewrites PIVOT into multiple statements internally, so it can't
+  // be part of a fused chain: it executes as a barrier.
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "barrier",
+    method: "wider()",
+    parameters: { namesFrom, valuesFrom, options },
+    execute: async () => {
+      await queryDB(
+        simpleTable,
+        `CREATE OR REPLACE TABLE ${quoteIdentifier(
+          simpleTable.name,
+        )} AS SELECT * FROM (PIVOT ${quoteIdentifier(simpleTable.name)} ON ${quoteIdentifier(
+          namesFrom,
+        )} USING ${aggregateFunction}(${quoteIdentifier(valuesFrom)}));`,
+        mergeOptions(simpleTable, {
+          table: simpleTable.name,
+          method: "wider()",
+          parameters: { namesFrom, valuesFrom, options },
+        }),
+      );
+    },
+  });
+}

package/src/methods/writeDB.ts

@@ -0,0 +1,127 @@
+import queryDbFile from "../helpers/queryDbFile.ts";
+import {
+  existsSync,
+  linkSync,
+  lstatSync,
+  mkdtempSync,
+  renameSync,
+  rmSync,
+  statSync,
+} from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import type SimpleDB from "../class/SimpleDB.ts";
+import cleanPath from "../helpers/cleanPath.ts";
+import createDirectory from "../helpers/createDirectory.ts";
+import getCurrentDatabase from "../helpers/getCurrentDatabase.ts";
+import getDbFileType from "../helpers/getDbFileType.ts";
+import getDbIndexes from "../helpers/getDbIndexes.ts";
+import parseValue from "../helpers/parseValue.ts";
+import prepareDbExtensions from "../helpers/prepareDbExtensions.ts";
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import readDbMetadata from "../helpers/readDbMetadata.ts";
+import writeDbMetadata from "../helpers/writeDbMetadata.ts";
+
+export default async function writeDB(
+  sdb: SimpleDB,
+  file: string,
+  options: { overwrite?: boolean; metadata?: boolean } = {},
+) {
+  const type = getDbFileType(file);
+  const output = resolve(file);
+  assertDestination(output, options.overwrite ?? false);
+  const database = await getCurrentDatabase(sdb);
+  const source = quoteIdentifier(database);
+  const databases = await queryDbFile(
+    sdb,
+    "SELECT path FROM duckdb_databases() WHERE path IS NOT NULL;",
+    { returnData: true },
+  );
+  for (const { path } of databases!) {
+    if (typeof path !== "string" || !existsSync(path)) continue;
+    const original = statSync(path);
+    const target = existsSync(output) ? statSync(output) : undefined;
+    if (
+      resolve(path) === output ||
+      (target?.dev === original.dev && target?.ino === original.ino)
+    ) {
+      throw new Error(
+        "writeDB() cannot replace an open database file. Export to a different path.",
+      );
+    }
+  }
+
+  const indexes = getDbIndexes(sdb);
+  await readDbMetadata(sdb, database);
+  await prepareDbExtensions(sdb, database, indexes);
+  if (type === "sqlite") await queryDbFile(sdb, "INSTALL sqlite; LOAD sqlite;");
+  createDirectory(output);
+  const directory = mkdtempSync(join(dirname(output), ".sda-export-"));
+  const temporary = join(directory, "snapshot.db");
+  const name = `__sda_export_${crypto.randomUUID().replaceAll("-", "")}`;
+  const target = quoteIdentifier(name);
+  let attached = false;
+  try {
+    await queryDbFile(
+      sdb,
+      `ATTACH '${cleanPath(temporary)}' AS ${target} (TYPE ${type}, READ_WRITE);`,
+    );
+    attached = true;
+    if (type === "duckdb") {
+      await queryDbFile(sdb, `COPY FROM DATABASE ${source} TO ${target};`);
+      await writeDbMetadata(sdb, name, options.metadata === false ? {} : indexes);
+      await queryDbFile(sdb, `CHECKPOINT ${target};`);
+    } else {
+      // SQLite is an interchange format: materialize main-schema relations,
+      // without DuckDB extension schemas, indexes, or SDA metadata.
+      const tables = await queryDbFile(
+        sdb,
+        `SELECT table_name FROM information_schema.tables
+         WHERE table_catalog = ${parseValue(database)} AND table_schema = 'main';`,
+        { returnData: true },
+      );
+      for (const { table_name } of tables!) {
+        const table = quoteIdentifier(String(table_name));
+        await queryDbFile(
+          sdb,
+          `CREATE TABLE ${target}.${table} AS SELECT * FROM ${source}.main.${table};`,
+        );
+      }
+    }
+    await queryDbFile(sdb, `DETACH ${target};`);
+    attached = false;
+    assertDestination(output, options.overwrite ?? false);
+    if (options.overwrite) {
+      // Do not fall back to unlinking the original if atomic replacement fails.
+      renameSync(temporary, output);
+    } else {
+      // An exclusive link also protects a destination created during the export.
+      linkSync(temporary, output);
+    }
+  } finally {
+    try {
+      if (attached) await queryDbFile(sdb, `DETACH ${target};`);
+    } finally {
+      rmSync(directory, { recursive: true, force: true });
+    }
+  }
+}
+
+function assertDestination(file: string, overwrite: boolean): void {
+  let existing: ReturnType<typeof lstatSync>;
+  try {
+    existing = lstatSync(file);
+  } catch (error) {
+    if (error instanceof Error && "code" in error && error.code === "ENOENT") {
+      return;
+    }
+    throw error;
+  }
+  if (!existing.isFile()) {
+    throw new Error(
+      `The output path ${file} must be a regular file, not a directory or symbolic link.`,
+    );
+  }
+  if (!overwrite) {
+    throw new Error(`The file ${file} already exists. Set overwrite to true to replace it.`);
+  }
+}

package/src/methods/writeData.ts

@@ -0,0 +1,109 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import cleanPath from "../helpers/cleanPath.ts";
+import createDirectory from "../helpers/createDirectory.ts";
+import getExtension from "../helpers/getExtension.ts";
+import hasGeometryColumn from "../helpers/hasGeometryColumn.ts";
+import mergeOptions from "../helpers/mergeOptions.ts";
+import queryDB from "../helpers/queryDB.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import writeDataAsArrays from "../helpers/writeDataAsArrays.ts";
+import { existsSync, rmSync } from "node:fs";
+
+export default async function writeData(
+  simpleTable: SimpleTable,
+  file: string,
+  options: {
+    compression?: boolean;
+    dataAsArrays?: boolean;
+    formatDates?: boolean;
+  } = {},
+) {
+  const extension = getExtension(file);
+  assertWriteDataOptions(extension, options);
+
+  if (await hasGeometryColumn(simpleTable)) {
+    throw new Error("Table contains geometry columns. Use writeGeoData() instead.");
+  }
+
+  createDirectory(file);
+
+  if (options.dataAsArrays) {
+    await writeDataAsArrays(simpleTable, file);
+  } else {
+    await queryDB(
+      simpleTable,
+      writeDataQuery(simpleTable.name, file, extension, options),
+      mergeOptions(simpleTable, {
+        table: simpleTable.name,
+        method: "writeData()",
+        parameters: { file, options },
+      }),
+    );
+  }
+}
+
+function assertWriteDataOptions(extension: string, options: { dataAsArrays?: boolean }): void {
+  const supported = ["csv", "json", "parquet", "db", "sqlite"];
+  if (!supported.includes(extension)) {
+    throw new Error(
+      `writeData() does not support the extension ${JSON.stringify(
+        extension,
+      )}. Use .csv, .json, .parquet, .db, or .sqlite.`,
+    );
+  }
+  if (options.dataAsArrays === true && extension !== "json") {
+    throw new Error("writeData() dataAsArrays works only with JSON files.");
+  }
+}
+
+function writeDataQuery(
+  table: string,
+  file: string,
+  fileExtension: string,
+  options: { compression?: boolean; formatDates?: boolean },
+) {
+  const cleanedFile = cleanPath(file);
+  if (fileExtension === "csv") {
+    return `COPY ${quoteIdentifier(table)} TO '${
+      options.compression ? cleanedFile + ".gz" : cleanedFile
+    }' (DELIMITER ',', HEADER TRUE${options.compression ? ", COMPRESSION GZIP" : ""}${
+      options.formatDates ? ", DATEFORMAT '%xT%X.%gZ', TIMESTAMPFORMAT '%xT%X.%gZ'" : ""
+    });`;
+  } else if (fileExtension === "json") {
+    return `COPY ${quoteIdentifier(table)} TO '${
+      options.compression ? cleanedFile + ".gz" : cleanedFile
+    }' (FORMAT JSON, ARRAY TRUE${options.compression ? ", COMPRESSION GZIP" : ""}${
+      options.formatDates ? ", DATEFORMAT '%xT%X.%gZ', TIMESTAMPFORMAT '%xT%X.%gZ'" : ""
+    });`;
+  } else if (fileExtension === "parquet") {
+    if (options.compression) {
+      return `COPY ${quoteIdentifier(
+        table,
+      )} TO '${cleanedFile}' (FORMAT PARQUET, COMPRESSION ZSTD);`;
+    } else {
+      return `COPY ${quoteIdentifier(table)} TO '${cleanedFile}' (FORMAT PARQUET);`;
+    }
+  } else if (fileExtension === "db") {
+    if (existsSync(file)) {
+      rmSync(file);
+    }
+    const database = quoteIdentifier("my_database");
+    return `ATTACH '${cleanedFile}' AS ${database};
+COPY FROM DATABASE ${quoteIdentifier("memory")} TO ${database};
+CREATE OR REPLACE TABLE ${database}.${quoteIdentifier(
+      table,
+    )} AS SELECT * FROM ${quoteIdentifier(table)};
+DETACH ${database};`;
+  } else if (fileExtension === "sqlite") {
+    if (existsSync(file)) {
+      rmSync(file);
+    }
+    const database = quoteIdentifier("my_sqlite_db");
+    return `INSTALL sqlite; LOAD sqlite;
+    ATTACH '${cleanedFile}' AS ${database} (TYPE SQLITE);
+    CREATE TABLE ${database}.${quoteIdentifier(table)} AS SELECT * FROM ${quoteIdentifier(table)};
+    DETACH ${database};`;
+  } else {
+    throw new Error(`writeData() does not support the extension ${JSON.stringify(fileExtension)}.`);
+  }
+}

package/src/methods/zScore.ts

@@ -0,0 +1,44 @@
+import quoteIdentifier from "../helpers/quoteIdentifier.ts";
+import assertNewColumns from "../helpers/assertNewColumns.ts";
+import queueOp from "../helpers/queueOp.ts";
+import stringToArray from "../helpers/stringToArray.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default function zScore(
+  simpleTable: SimpleTable,
+  column: string,
+  newColumn: string,
+  options: {
+    by?: string | string[];
+    decimals?: number;
+  } = {},
+) {
+  options = structuredClone(options);
+  queueOp(simpleTable, {
+    kind: "fusable",
+    method: "zScore()",
+    parameters: { column, newColumn, options },
+    needsSchema: true,
+    buildSelect: (input, schema) => {
+      assertNewColumns(schema, [newColumn], "zScore()");
+
+      const by = options.by ? stringToArray(options.by) : [];
+      const partition =
+        by.length > 0 ? `PARTITION BY ${by.map((d) => `${quoteIdentifier(d)}`).join(", ")}` : "";
+
+      const tempQuery = `(${quoteIdentifier(column)}-AVG(${quoteIdentifier(
+        column,
+      )}) OVER(${partition}))
+            /
+            STDDEV_POP(${quoteIdentifier(column)}) OVER(${partition})`;
+      return `SELECT *, (
+        ${
+          typeof options.decimals === "number"
+            ? `ROUND(${tempQuery}, ${options.decimals})`
+            : tempQuery
+        }
+        ) AS ${quoteIdentifier(newColumn)},
+    FROM ${input}`;
+    },
+  });
+}