@nshiab/simple-data-analysis-core 2.1.11 → 2.1.13

package/src/helpers/flushAllTables.ts

@@ -0,0 +1,500 @@
+import { AsyncLocalStorage } from "node:async_hooks";
+import type SimpleDB from "../class/SimpleDB.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { FusableOp, PendingOp, RelationalOp, TableSchema } from "./pendingOps.ts";
+import cleanSQL from "./cleanSQL.ts";
+import referencedTables from "./referencedTables.ts";
+import ensureSpatial from "./ensureSpatial.ts";
+import extractTypes from "./extractTypes.ts";
+import mergeOptions from "./mergeOptions.ts";
+import queryDB from "./queryDB.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+import { getRegisteredTables } from "./tableRegistry.ts";
+import {
+  type AsyncOperationFrame,
+  getAsyncOperationFrame,
+  isDrainingAsyncOperationFrame,
+  type PendingEntry,
+  runWhileDrainingAsyncOperationFrame,
+  runWithAsyncOperationFrame,
+} from "./asyncOperationContext.ts";
+
+/**
+ * The databases whose flush is executing on the current async call chain.
+ * Queries issued by the flush itself (fused statements, barriers, DESCRIBE)
+ * re-enter flushAllTables through queryDB and must not trigger a new flush,
+ * while queries from concurrent observers must wait for the in-flight flush
+ * instead of reading mid-flush state.
+ */
+const flushContext = new AsyncLocalStorage<Set<SimpleDB>>();
+
+/**
+ * Runs `fn` with `sdb` exempt from the top-level flush. Queries issued by an
+ * asynchronous barrier can still drain operations queued inside that barrier.
+ * Used by the flush itself and by connection setup, whose queries must run
+ * before any queued operation.
+ */
+export function runExemptFromFlush<T>(sdb: SimpleDB, fn: () => Promise<T>): Promise<T> {
+  const exempt = new Set(flushContext.getStore());
+  exempt.add(sdb);
+  return flushContext.run(exempt, fn);
+}
+
+/**
+ * Executes the pending operations of all tables in program order: every
+ * queued operation carries a database-wide sequence number, so operations
+ * on different tables replay exactly in the order the user queued them,
+ * like immediate execution would. Consecutive fusable operations on the
+ * same table compose as CTEs into a single statement; barriers execute
+ * in order between fused segments.
+ *
+ * Every query execution point goes through this (via queryDB). Only one
+ * flush runs at a time: concurrent observers wait for the in-flight flush,
+ * then flush whatever queued in the meantime, so an awaited observer always
+ * answers from fully executed state.
+ */
+export default async function flushAllTables(sdb: SimpleDB): Promise<void> {
+  const frame = getAsyncOperationFrame(sdb);
+  if (frame !== undefined) {
+    if (!isDrainingAsyncOperationFrame(frame)) {
+      await drainAsyncOperationFrame(frame);
+    }
+    return;
+  }
+  if (flushContext.getStore()?.has(sdb)) {
+    // A query issued by the top-level flush itself.
+    return;
+  }
+  while (sdb.flushPromise !== null) {
+    // Another observer's flush is running: wait for it instead of reading
+    // mid-flush state, then flush anything queued in the meantime.
+    try {
+      await sdb.flushPromise;
+    } catch {
+      // The error belongs to the observer that started that flush; this
+      // observer only cares that the flush is over.
+    }
+  }
+  if (sdb.pendingCount === 0) {
+    return;
+  }
+  const promise = runExemptFromFlush(sdb, () => flush(sdb));
+  sdb.flushPromise = promise;
+  try {
+    await promise;
+  } finally {
+    sdb.flushPromise = null;
+  }
+}
+
+/**
+ * An open run of consecutive fusable operations on one table, accumulated
+ * while the flush walks the program-order replay. A segment closes when the
+ * replay moves to another table, reaches a barrier or dependency through
+ * user-supplied SQL, or reaches the end. Keeping only contiguous operations
+ * together preserves database-wide failure order.
+ */
+type Segment = {
+  table: SimpleTable;
+  ops: RelationalOp[];
+};
+
+type CompiledCte = {
+  alias: string;
+  select: string;
+  values: import("@duckdb/node-api").DuckDBValue[];
+};
+
+type CompiledOp = {
+  cte: CompiledCte;
+  schema: TableSchema | null;
+};
+
+async function flush(sdb: SimpleDB): Promise<void> {
+  // Take ownership of every queue upfront, in program order.
+  const entries: PendingEntry[] = [];
+  for (const table of getRegisteredTables(sdb)) {
+    for (const op of table.pendingOps.splice(0, table.pendingOps.length)) {
+      entries.push({ table, op });
+    }
+  }
+  sdb.pendingCount = 0;
+  await replayEntries(sdb, entries);
+}
+
+async function replayEntries(sdb: SimpleDB, pendingEntries: PendingEntry[]): Promise<void> {
+  const entries: (PendingEntry | null)[] = [...pendingEntries].sort(
+    (a, b) => a.op.sequence - b.op.sequence,
+  );
+
+  let open: Segment | null = null;
+  // The table whose operation is executing, for error attribution: on
+  // failure, that table's chain aborts and the other tables' pending work
+  // goes back to their queues.
+  let executing: SimpleTable | null = null;
+  // The entry being dispatched, if it is neither executed nor part of an
+  // open segment yet.
+  let current: { table: SimpleTable; op: PendingOp } | null = null;
+
+  const runOpenSegment = async () => {
+    if (open === null) {
+      return;
+    }
+    const segment = open;
+    open = null;
+    executing = segment.table;
+    await runSegment(segment.table, segment.ops);
+    executing = null;
+  };
+
+  let index = 0;
+  try {
+    for (; index < entries.length; index++) {
+      const entry = entries[index]!;
+      // Release the reference: a consumed operation (and whatever its
+      // closures capture, like loadArray data) must not stay alive for the
+      // rest of the flush.
+      entries[index] = null;
+      current = entry;
+      const { table, op } = entry;
+
+      if (open !== null && open.table !== table) {
+        await runOpenSegment();
+      }
+
+      if (op.kind === "barrier") {
+        // A barrier is multi-statement by nature and can read or write any
+        // table, so everything pending executes first.
+        await runOpenSegment();
+        executing = table;
+        await op.execute();
+        executing = null;
+        current = null;
+        continue;
+      }
+
+      if (op.kind === "asyncBarrier") {
+        // An asynchronous extension barrier also captures and drains builder
+        // operations queued by its callback before replay continues.
+        await runOpenSegment();
+        executing = table;
+        await runAsyncBarrier(table, op);
+        executing = null;
+        current = null;
+        continue;
+      }
+
+      const reads = op.rawSQL === undefined ? [] : referencedTables(op.rawSQL, sdb);
+
+      if (op.kind === "fusable" && reads.includes(table)) {
+        // User-supplied SQL referencing the operation's own table must read
+        // the state produced by the previous steps, which a fused chain
+        // can't provide (the name would resolve to the pre-chain table). So
+        // the chain materializes and the operation runs on its own.
+        await runOpenSegment();
+        executing = table;
+        await runStepwise(table, op);
+        executing = null;
+        current = null;
+        continue;
+      }
+
+      const sourceSegment: Segment | null = open;
+      const segmentSource = sourceSegment?.ops[0];
+      if (
+        op.kind === "fusable" &&
+        segmentSource?.kind === "source" &&
+        (op.requiresMaterializedInput || (op.needsSpatial && segmentSource.schema === undefined))
+      ) {
+        // Materialize only the external source when this operation needs a
+        // stable physical input, or when a spatial operation would otherwise
+        // make DuckDB scan an unknown source once for schema inference and
+        // again for execution. Earlier transforms remain open and fuse with
+        // this operation.
+        const source: RelationalOp = segmentSource;
+        const pending: RelationalOp[] = sourceSegment?.ops.slice(1) ?? [];
+        open = null;
+        executing = table;
+        await runSegment(table, [source]);
+        executing = null;
+        open = { table, ops: [...pending, op] };
+        current = null;
+        continue;
+      }
+
+      if (op.kind === "source") {
+        // A source replaces the target table from an external relation. It
+        // starts a new segment rather than consuming any segment already open
+        // on the same table. A source may read that materialized table itself
+        // (for example, an in-place join) and still fuse with downstream steps.
+        await runOpenSegment();
+        open = { table, ops: [op] };
+        current = null;
+        continue;
+      }
+
+      if (open === null) {
+        open = { table, ops: [] };
+      }
+      open.ops.push(op);
+      current = null;
+    }
+    await runOpenSegment();
+  } catch (error) {
+    // The failing table's chain aborts, like a v1 script stopping at the
+    // failing line. The other tables' pending work was queued successfully
+    // and goes back to their queues, to execute at their next observation.
+    const failing = executing;
+    const requeue = (table: SimpleTable, op: PendingOp) => {
+      if (table !== failing) {
+        table.pendingOps.push(op);
+        sdb.pendingCount++;
+      }
+    };
+    if (current !== null) {
+      requeue(current.table, current.op);
+    }
+    if (open !== null) {
+      for (const op of open.ops) {
+        requeue(open.table, op);
+      }
+    }
+    for (let i = index + 1; i < entries.length; i++) {
+      const entry = entries[i];
+      if (entry !== null) {
+        requeue(entry.table, entry.op);
+      }
+    }
+    throw error;
+  }
+}
+
+async function runAsyncBarrier(
+  table: SimpleTable,
+  op: Extract<PendingOp, { kind: "asyncBarrier" }>,
+): Promise<void> {
+  await runWithAsyncOperationFrame(table.sdb, async (frame) => {
+    await op.execute();
+    await drainAsyncOperationFrame(frame);
+  });
+}
+
+async function drainAsyncOperationFrame(frame: AsyncOperationFrame): Promise<void> {
+  while (frame.drainPromise !== null) {
+    try {
+      await frame.drainPromise;
+    } catch {
+      // The observer that started the drain receives its failure. A concurrent
+      // observer waits for that drain to finish, then handles remaining work.
+    }
+  }
+  if (frame.entries.length === 0) {
+    return;
+  }
+
+  // Start on the next microtask so drainPromise is visible before replay can
+  // re-enter user code or another observer can attempt the same drain.
+  const promise = Promise.resolve().then(() =>
+    runWhileDrainingAsyncOperationFrame(frame, async () => {
+      while (frame.entries.length > 0) {
+        const entries = frame.entries.splice(0, frame.entries.length);
+        await replayEntries(frame.sdb, entries);
+      }
+    }),
+  );
+  frame.drainPromise = promise;
+  try {
+    await promise;
+  } finally {
+    if (frame.drainPromise === promise) {
+      frame.drainPromise = null;
+    }
+  }
+}
+
+async function runSegment(table: SimpleTable, ops: RelationalOp[]): Promise<void> {
+  if (ops.length === 1) {
+    await runStepwise(table, ops[0]);
+    return;
+  }
+
+  if (ops.some(needsSpatial)) {
+    await ensureSpatial(table);
+  }
+
+  // Normalize each fragment once using the database syntax before composition.
+  const ctes: CompiledCte[] = [];
+  // The schema of the next fragment's input, reused while operations declare
+  // they preserve it, so a run of same-shape steps costs at most one
+  // DESCRIBE round-trip.
+  let schema: TableSchema | null = null;
+  for (let i = 0; i < ops.length; i++) {
+    const op = ops[i];
+    try {
+      const compiled = await compileOp(table, op, ctes, schema);
+      ctes.push(compiled.cte);
+      schema = compiled.schema;
+    } catch (error) {
+      // Flush-time validation (or a broken earlier fragment surfacing at
+      // DESCRIBE) fails the chain at this operation. The steps before it
+      // already succeeded in program order, so they execute before the
+      // error propagates — like a v1 script failing at this line.
+      if (ctes.length > 0) {
+        await executeFused(table, ops.slice(0, i), ctes);
+      }
+      throw error;
+    }
+  }
+  await executeFused(table, ops, ctes);
+}
+
+async function compileOp(
+  table: SimpleTable,
+  op: RelationalOp,
+  ctes: CompiledCte[],
+  schema: TableSchema | null,
+): Promise<CompiledOp> {
+  // Source relations can read tables named s1, s2, etc. A generated CTE must
+  // not shadow those physical inputs, including in later chain fragments.
+  const usedNames = new Set([
+    ...getRegisteredTables(table.sdb).map((input) => input.name.toLowerCase()),
+    ...ctes.map((cte) => cte.alias),
+  ]);
+  let aliasIndex = ctes.length + 1;
+  while (usedNames.has(`s${aliasIndex}`)) aliasIndex++;
+  const alias = `s${aliasIndex}`;
+  if (op.kind === "source") {
+    return {
+      cte: {
+        alias,
+        select: cleanSQL(await op.buildSelect(), table.sdb.expressionSyntax),
+        values: [],
+      },
+      schema: op.schema ?? null,
+    };
+  }
+
+  const input =
+    ctes.length === 0 ? quoteIdentifier(table.name) : quoteIdentifier(ctes[ctes.length - 1].alias);
+  const inputSchema = op.needsSchema && schema === null ? await describeChain(table, ctes) : schema;
+  const buildSchema = inputSchema ?? {};
+  const outputSchema =
+    inputSchema === null
+      ? null
+      : op.outputSchema !== undefined
+        ? op.outputSchema(inputSchema)
+        : op.preservesSchema === true
+          ? inputSchema
+          : null;
+  return {
+    cte: {
+      alias,
+      select: cleanSQL(op.buildSelect(input, buildSchema), table.sdb.expressionSyntax),
+      values: resolveValues(op, buildSchema),
+    },
+    schema: outputSchema,
+  };
+}
+
+async function executeFused(
+  table: SimpleTable,
+  ops: RelationalOp[],
+  ctes: CompiledCte[],
+): Promise<void> {
+  // A trailing line comment must end before the generated CTE delimiter.
+  const query = `CREATE OR REPLACE TABLE ${quoteIdentifier(table.name)} AS WITH ${ctes
+    .map((c) => `${quoteIdentifier(c.alias)} AS (${c.select}\n)`)
+    .join(", ")} SELECT * FROM ${quoteIdentifier(ctes[ctes.length - 1].alias)}`;
+
+  try {
+    await queryDB(
+      table,
+      query,
+      mergeOptions(table, {
+        table: table.name,
+        method: ops.map((op) => op.method).join(" + "),
+        parameters: Object.fromEntries(
+          ops.map((op, i) => [`${i + 1}. ${op.method}`, op.parameters]),
+        ),
+        values: ctes.flatMap((cte) => cte.values),
+        noClean: true,
+      }),
+    );
+  } catch (fusedError) {
+    // Re-run the segment step by step to pinpoint the culprit method and
+    // report it exactly as immediate execution would have.
+    for (const op of ops) {
+      await runStepwise(table, op);
+    }
+    // Every step succeeded on its own: the failure came from the fusion
+    // itself. The table now holds the correct stepwise result, so we warn
+    // instead of throwing.
+    console.warn(
+      `The fused query for ${ops
+        .map((op) => op.method)
+        .join(
+          " + ",
+        )} failed, but running the methods one by one succeeded. Please report this as a bug to https://github.com/nshiab/simple-data-analysis-core/issues.\n${
+        fusedError instanceof Error ? fusedError.message : String(fusedError)
+      }`,
+    );
+  }
+}
+
+async function runStepwise(table: SimpleTable, op: RelationalOp): Promise<void> {
+  if (needsSpatial(op)) {
+    await ensureSpatial(table);
+  }
+  // Compilation is shared with the fused path, so both paths execute the same
+  // cleaned fragment with the same bound values and schema handling.
+  const compiled = await compileOp(table, op, [], null);
+  await queryDB(
+    table,
+    `CREATE OR REPLACE TABLE ${quoteIdentifier(table.name)} AS ${compiled.cte.select}`,
+    mergeOptions(table, {
+      table: table.name,
+      method: op.method,
+      parameters: op.parameters,
+      values: compiled.cte.values,
+      noClean: true,
+    }),
+  );
+}
+
+function needsSpatial(op: RelationalOp): boolean {
+  return op.needsSpatial === true;
+}
+
+function resolveValues(op: FusableOp, schema: TableSchema) {
+  return typeof op.values === "function" ? op.values(schema) : (op.values ?? []);
+}
+
+/**
+ * Returns the schema at the current point in the fused chain. With no CTEs
+ * yet, this describes the table itself; otherwise it describes the output of
+ * the last fragment. DESCRIBE is planning-only, so no data is computed.
+ */
+async function describeChain(table: SimpleTable, ctes: CompiledCte[]): Promise<TableSchema> {
+  const select =
+    ctes.length === 0
+      ? `SELECT * FROM ${quoteIdentifier(table.name)}`
+      : `WITH ${ctes
+          .map((c) => `${quoteIdentifier(c.alias)} AS (${c.select}\n)`)
+          .join(", ")} SELECT * FROM ${quoteIdentifier(ctes[ctes.length - 1].alias)}`;
+  const query =
+    ctes.length === 0 ? `DESCRIBE ${quoteIdentifier(table.name)}` : `DESCRIBE ${select}`;
+  const values = ctes.flatMap((cte) => cte.values);
+  const types = await queryDB(
+    table,
+    query,
+    mergeOptions(table, {
+      table: table.name,
+      method: null,
+      parameters: null,
+      returnData: true,
+      values,
+      noClean: true,
+    }),
+  );
+  return extractTypes(types);
+}

package/src/helpers/foldIdentifier.ts

@@ -0,0 +1,4 @@
+/** Matches DuckDB's ASCII case folding for quoted and unquoted identifiers. */
+export default function foldIdentifier(identifier: string): string {
+  return identifier.replace(/[A-Z]/g, (letter) => letter.toLowerCase());
+}

package/src/helpers/formatDate.ts

@@ -0,0 +1,76 @@
+const MONTHS = [
+  "January",
+  "February",
+  "March",
+  "April",
+  "May",
+  "June",
+  "July",
+  "August",
+  "September",
+  "October",
+  "November",
+  "December",
+];
+
+/**
+ * Formats a `Date` object into a human-readable string using only the format
+ * strings actually used in the project.
+ *
+ * @example
+ * ```ts
+ * // Format a date with time (local time).
+ * const date = new Date("2023-01-01T01:35:00");
+ * const formatted = formatDate(date, "Month DD, YYYY, at HH:MM period");
+ * console.log(formatted); // "January 1, 2023, at 1:35 a.m."
+ * ```
+ *
+ * @example
+ * ```ts
+ * // Format a date as month and day only (UTC).
+ * const formatted = formatDate(new Date("2023-01-01"), "Month DD", { utc: true });
+ * console.log(formatted); // "January 1"
+ * ```
+ *
+ * @example
+ * ```ts
+ * // Minutes of ":00" are omitted.
+ * const formatted = formatDate(new Date("2023-01-01T01:00:00"), "Month DD, YYYY, at HH:MM period", { utc: true });
+ * console.log(formatted); // "January 1, 2023, at 1 a.m."
+ * ```
+ *
+ * @param date - The `Date` object to format.
+ *
+ * @param format - The format string to use. Only `"Month DD, YYYY, at HH:MM period"` and `"Month DD"` are supported.
+ *
+ * @param options - Optional settings.
+ *
+ * @param options.utc - If `true`, format the date in UTC instead of local time. Defaults to `false`.
+ *
+ * @returns The formatted date string.
+ */
+export default function formatDate(
+  date: Date,
+  format: "Month DD, YYYY, at HH:MM period" | "Month DD",
+  options: { utc?: boolean } = {},
+): string {
+  const year = options.utc ? date.getUTCFullYear() : date.getFullYear();
+  const month = MONTHS[options.utc ? date.getUTCMonth() : date.getMonth()];
+  const day = options.utc ? date.getUTCDate() : date.getDate();
+  const hours = options.utc ? date.getUTCHours() : date.getHours();
+  const minutes = options.utc ? date.getUTCMinutes() : date.getMinutes();
+
+  const period = hours >= 12 ? "p.m." : "a.m.";
+  const displayHours = hours % 12 || 12;
+  const displayMinutes = minutes.toString().padStart(2, "0");
+
+  if (format === "Month DD, YYYY, at HH:MM period") {
+    const time =
+      minutes === 0 ? `${displayHours} ${period}` : `${displayHours}:${displayMinutes} ${period}`;
+    return `${month} ${day}, ${year}, at ${time}`;
+  } else if (format === "Month DD") {
+    return `${month} ${day}`;
+  }
+
+  return "";
+}

package/src/helpers/formatMissingTables.ts

@@ -0,0 +1,39 @@
+import quoteIdentifier from "./quoteIdentifier.ts";
+
+/**
+ * Formats a missing-table error with the requested and available table names.
+ *
+ * @example
+ * ```ts
+ * const message = formatMissingTables(
+ *   "getTable()",
+ *   ["missing"],
+ *   ["votes", "districts"],
+ * );
+ * ```
+ *
+ * @param method - The SDA method looking up the tables.
+ *
+ * @param missingTables - The requested table names that were not found.
+ *
+ * @param availableTables - The table names currently available.
+ *
+ * @returns An actionable missing-table error message.
+ */
+export default function formatMissingTables(
+  method: string,
+  missingTables: string[],
+  availableTables: string[],
+): string {
+  const missing = [...missingTables].sort();
+  const available = [...availableTables].sort();
+  const missingDescription =
+    missing.length === 1
+      ? `table ${quoteIdentifier(missing[0])}`
+      : `tables ${missing.map(quoteIdentifier).join(", ")}`;
+  const availableDescription =
+    available.length === 0
+      ? "No tables are currently available."
+      : `Available tables: ${available.map(quoteIdentifier).join(", ")}.`;
+  return `${method} could not find ${missingDescription}. ${availableDescription}`;
+}