@nshiab/simple-data-analysis-core 2.1.11 → 2.1.13

package/src/helpers/queueOp.ts

@@ -0,0 +1,84 @@
+import type SimpleDB from "../class/SimpleDB.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import type { PendingOpInput } from "./pendingOps.ts";
+import { ensureTableRegistered, getRegisteredTables } from "./tableRegistry.ts";
+import { markTableChanged } from "./tableGeneration.ts";
+import {
+  assertCacheTableMutation,
+  captureCacheTableOperation,
+  recordCacheTableOperation,
+  recordCacheTableReferences,
+} from "./cacheTableDependencies.ts";
+import { captureAsyncOperation } from "./asyncOperationContext.ts";
+
+/**
+ * Queues an internal relational operation, barrier, or asynchronous barrier.
+ *
+ * @internal
+ */
+export default function queueOp(simpleTable: SimpleTable, op: PendingOpInput): void {
+  assertCacheTableMutation(simpleTable);
+  const sdb = simpleTable.sdb;
+  if (sdb.lifecycleState !== "open") {
+    throw new Error(
+      `${op.method} cannot queue work because its SimpleDB is ${sdb.lifecycleState}.`,
+    );
+  }
+  let capturedOp: PendingOpInput = captureCacheTableOperation({
+    ...op,
+    parameters: op.parameters === null ? null : structuredClone(op.parameters),
+  });
+  if (capturedOp.kind === "fusable" || capturedOp.kind === "source") {
+    capturedOp = {
+      ...capturedOp,
+      rawSQL: capturedOp.rawSQL === undefined ? undefined : [...capturedOp.rawSQL],
+    };
+    if (capturedOp.rawSQL !== undefined) {
+      recordCacheTableReferences(simpleTable, capturedOp.rawSQL);
+    }
+  }
+  if (
+    capturedOp.kind === "fusable" &&
+    capturedOp.values !== undefined &&
+    typeof capturedOp.values !== "function"
+  ) {
+    capturedOp = {
+      ...capturedOp,
+      values: [...capturedOp.values],
+    };
+  }
+  // A table de-registered by removeTable()/removeTables()/selectTables() that
+  // queues new work comes back under the flush's responsibility, like v1
+  // methods recreating a removed table.
+  ensureTableRegistered(sdb, simpleTable);
+  markTableChanged(simpleTable);
+  const queuedOp = {
+    ...capturedOp,
+    sequence: sdb.opSequence++,
+  };
+  recordCacheTableOperation(simpleTable, queuedOp);
+  if (captureAsyncOperation(sdb, { table: simpleTable, op: queuedOp })) {
+    return;
+  }
+  simpleTable.pendingOps.push(queuedOp);
+  sdb.pendingCount++;
+}
+
+/**
+ * Empties every table's pending queue without executing it and returns what
+ * was dropped, so close() can report forgotten work after cleaning up.
+ */
+export function discardAllPending(sdb: SimpleDB): { table: SimpleTable; methods: string[] }[] {
+  const discarded: { table: SimpleTable; methods: string[] }[] = [];
+  for (const table of getRegisteredTables(sdb)) {
+    if (table.pendingOps.length > 0) {
+      discarded.push({
+        table,
+        methods: table.pendingOps.map((op) => op.method),
+      });
+      sdb.pendingCount -= table.pendingOps.length;
+      table.pendingOps.length = 0;
+    }
+  }
+  return discarded;
+}

package/src/helpers/quoteIdentifier.ts

@@ -0,0 +1,18 @@
+/**
+ * Quotes one SQL identifier and escapes embedded double quotes.
+ * Use this for library-generated table, column, schema, database, and alias
+ * names. User-authored SQL expressions must remain unmodified.
+ *
+ * @example
+ * ```ts
+ * quoteIdentifier('people "archive"');
+ * // '"people ""archive"""'
+ * ```
+ *
+ * @param identifier - The identifier to quote.
+ *
+ * @returns The identifier wrapped in double quotes with embedded quotes doubled.
+ */
+export default function quoteIdentifier(identifier: string): string {
+  return `"${identifier.replaceAll('"', '""')}"`;
+}

package/src/helpers/quoteQualifiedIdentifier.ts

@@ -0,0 +1,23 @@
+import quoteIdentifier from "./quoteIdentifier.ts";
+
+/**
+ * Quotes a SQL identifier together with the relation that owns it.
+ *
+ * Qualifying column references prevents DuckDB from resolving a relation name
+ * as its row struct when a missing column has the same name as the relation.
+ *
+ * @example
+ * ```ts
+ * quoteQualifiedIdentifier("people", "full name");
+ * // '"people"."full name"'
+ * ```
+ *
+ * @param relation - The table or relation containing the identifier.
+ *
+ * @param identifier - The identifier to qualify.
+ *
+ * @returns The separately quoted relation and identifier joined by a period.
+ */
+export default function quoteQualifiedIdentifier(relation: string, identifier: string): string {
+  return `${quoteIdentifier(relation)}.${quoteIdentifier(identifier)}`;
+}

package/src/helpers/readCacheSources.ts

@@ -0,0 +1,72 @@
+import { readFileSync } from "node:fs";
+import type { TableGenerationId } from "./tableGeneration.ts";
+import type { IndexDefinition } from "./indexDefinitions.ts";
+import type { CacheTableDependency } from "./cacheTableDependencies.ts";
+import { validateIndexes } from "./readDbMetadata.ts";
+
+export type CacheSource = {
+  file: string | null;
+  creation: number;
+  duration: number;
+  geo: boolean;
+  entryGeneration?: TableGenerationId | null;
+  generationId?: TableGenerationId;
+  indexes?: IndexDefinition[];
+  formatVersion?: string;
+  codeHash?: string;
+  inputHashes?: string[];
+  tableDependencies?: CacheTableDependency[];
+};
+
+export type CacheSources = {
+  [key: string]: CacheSource;
+};
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function isArray(value: unknown): value is unknown[] {
+  return Array.isArray(value);
+}
+
+function isCacheSource(value: unknown): value is CacheSource {
+  if (!isRecord(value)) return false;
+  if (value.indexes !== undefined) validateIndexes({ source: value.indexes });
+  return (
+    (value.file === null || typeof value.file === "string") &&
+    typeof value.creation === "number" &&
+    typeof value.duration === "number" &&
+    typeof value.geo === "boolean" &&
+    (value.entryGeneration === undefined ||
+      value.entryGeneration === null ||
+      typeof value.entryGeneration === "string") &&
+    (value.generationId === undefined || typeof value.generationId === "string") &&
+    (value.formatVersion === undefined || typeof value.formatVersion === "string") &&
+    (value.codeHash === undefined || typeof value.codeHash === "string") &&
+    (value.inputHashes === undefined ||
+      (isArray(value.inputHashes) &&
+        value.inputHashes.every((hash) => typeof hash === "string"))) &&
+    (value.tableDependencies === undefined ||
+      (isArray(value.tableDependencies) &&
+        value.tableDependencies.every(
+          (dependency) =>
+            isRecord(dependency) &&
+            typeof dependency.tableName === "string" &&
+            typeof dependency.generationId === "string",
+        )))
+  );
+}
+
+/** Reads persisted table-cache metadata, validating JSON before it is used. */
+export default function readCacheSources(path: string): CacheSources {
+  const value: unknown = JSON.parse(readFileSync(path, "utf8"));
+  if (!isRecord(value)) throw new Error("Invalid SDA table cache metadata.");
+  const sources: [string, CacheSource][] = [];
+  for (const [id, source] of Object.entries(value)) {
+    if (!isCacheSource(source))
+      throw new Error(`Invalid SDA table cache entry ${JSON.stringify(id)}.`);
+    sources.push([id, source]);
+  }
+  return Object.fromEntries(sources);
+}

package/src/helpers/readDbMetadata.ts

@@ -0,0 +1,81 @@
+import queryDbFile from "./queryDbFile.ts";
+import type SimpleDB from "../class/SimpleDB.ts";
+import type { IndexDefinition } from "./indexDefinitions.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+import parseValue from "./parseValue.ts";
+
+/** Reads embedded SDA metadata when present. */
+export default async function readDbMetadata(
+  sdb: SimpleDB,
+  database: string,
+): Promise<Record<string, IndexDefinition[]>> {
+  const schemas = await queryDbFile(
+    sdb,
+    `SELECT schema_name FROM duckdb_schemas()
+     WHERE database_name = ${parseValue(database)} AND schema_name = '__sda';`,
+    { returnData: true },
+  );
+  if (schemas!.length > 0) {
+    const rows = await queryDbFile(
+      sdb,
+      `SELECT version, indexes FROM ${quoteIdentifier(database)}.__sda.metadata;`,
+      { returnData: true },
+    );
+    if (rows!.length !== 1 || rows![0].version !== 1 || typeof rows![0].indexes !== "string") {
+      throw new Error("Unsupported SDA database metadata in the reserved __sda schema.");
+    }
+    return validateIndexes(JSON.parse(rows![0].indexes));
+  }
+  return {};
+}
+
+function isRecord(value: unknown): value is Record<string, unknown> {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+export function validateIndexes(value: unknown): Record<string, IndexDefinition[]> {
+  if (!isRecord(value)) throw new Error("Invalid SDA index metadata.");
+  for (const definitions of Object.values(value)) {
+    if (!Array.isArray(definitions)) {
+      throw new Error("Invalid SDA index metadata.");
+    }
+    for (const definition of definitions) {
+      if (
+        !isRecord(definition) ||
+        typeof definition.name !== "string" ||
+        !isRecord(definition.options)
+      ) {
+        throw new Error("Invalid SDA index definition.");
+      }
+      const options = definition.options;
+      if (definition.kind === "fts") {
+        if (
+          typeof definition.idColumn !== "string" ||
+          typeof definition.textColumn !== "string" ||
+          ["stemmer", "stopwords", "ignore"].some(
+            (key) => options[key] !== undefined && typeof options[key] !== "string",
+          ) ||
+          ["stripAccents", "lower"].some(
+            (key) => options[key] !== undefined && typeof options[key] !== "boolean",
+          )
+        ) {
+          throw new Error("Invalid SDA FTS index definition.");
+        }
+      } else if (definition.kind === "vss") {
+        if (
+          typeof definition.column !== "string" ||
+          ["efConstruction", "efSearch", "M"].some(
+            (key) =>
+              options[key] !== undefined &&
+              (typeof options[key] !== "number" || !Number.isFinite(options[key])),
+          )
+        ) {
+          throw new Error("Invalid SDA VSS index definition.");
+        }
+      } else {
+        throw new Error("Unsupported SDA index definition.");
+      }
+    }
+  }
+  return value as Record<string, IndexDefinition[]>;
+}

package/src/helpers/readMutationRows.ts

@@ -0,0 +1,107 @@
+import {
+  DuckDBArrayValue,
+  type DuckDBConnection,
+  DuckDBDateValue,
+  DuckDBListValue,
+  DuckDBStructValue,
+  DuckDBTimestampValue,
+  type DuckDBType,
+  type DuckDBValue,
+} from "@duckdb/node-api";
+
+/** Reads mutation values, optionally keeping a pagination cursor out of rows. */
+export default async function readMutationRows(
+  connection: DuckDBConnection,
+  query: string,
+  cursorColumn?: string,
+) {
+  const result = await connection.run(query);
+  const names = result.deduplicatedColumnNames();
+  const types = result.columnTypes();
+  const cursorIndex = cursorColumn === undefined ? -1 : names.indexOf(cursorColumn);
+  let lastCursorValue: DuckDBValue | undefined;
+  const dates = new WeakMap<Date, { millis: number; value: DuckDBValue }>();
+  const convert = (value: DuckDBValue): unknown => {
+    if (value instanceof DuckDBDateValue || value instanceof DuckDBTimestampValue) {
+      const millis =
+        value instanceof DuckDBDateValue
+          ? value.days * 86400000
+          : Number(value.micros >= 0n ? value.micros / 1000n : (value.micros - 999n) / 1000n);
+      const date = new Date(millis);
+      // Infinite/out-of-range dates retain their native representation.
+      if (!Number.isFinite(date.getTime())) return value;
+      dates.set(date, { millis, value });
+      return date;
+    }
+    if (value instanceof DuckDBArrayValue || value instanceof DuckDBListValue) {
+      return value.items.map(convert);
+    }
+    if (value instanceof DuckDBStructValue) {
+      return Object.fromEntries(
+        Object.entries(value.entries).map(([key, item]) => [key, convert(item)]),
+      );
+    }
+    // In particular, decimals and high-resolution temporal values stay native.
+    return value;
+  };
+  const rows: Record<string, unknown>[] = [];
+  while (true) {
+    const chunk = await result.fetchChunk();
+    if (!chunk || chunk.rowCount === 0) break;
+    const offset = rows.length;
+    for (let i = 0; i < chunk.rowCount; i++) rows.push({});
+    for (let i = 0; i < names.length; i++) {
+      if (i === cursorIndex) {
+        lastCursorValue = chunk.getColumnVector(i).getItem(chunk.rowCount - 1);
+        continue;
+      }
+      chunk.visitColumnValues(i, (value, row) => {
+        rows[offset + row][names[i]] = convert(value);
+      });
+    }
+  }
+  const toNative = (value: unknown, type: DuckDBType): DuckDBValue => {
+    if (value === null || value === undefined) return null;
+    if (value instanceof Date) {
+      const original = dates.get(value);
+      if (
+        original &&
+        value.getTime() === original.millis &&
+        type.toString() === (original.value instanceof DuckDBDateValue ? "DATE" : "TIMESTAMP")
+      ) {
+        return original.value;
+      }
+      if (type.toString() === "DATE") {
+        if (value.getTime() % 86400000 !== 0) {
+          throw new Error("DATE values must represent midnight UTC.");
+        }
+        return new DuckDBDateValue(value.getTime() / 86400000);
+      }
+      return new DuckDBTimestampValue(BigInt(value.getTime()) * 1000n);
+    }
+    if (Array.isArray(value) && "valueType" in type) {
+      if ("length" in type && value.length !== type.length) {
+        throw new Error(`Expected ${type.length} array elements for ${type.toString()}.`);
+      }
+      const items = value.map((item: unknown) => toNative(item, type.valueType));
+      return "length" in type ? new DuckDBArrayValue(items) : new DuckDBListValue(items);
+    }
+    if (typeof value === "object" && "entryNames" in type && "entryTypes" in type) {
+      return new DuckDBStructValue(
+        Object.fromEntries(
+          type.entryNames.map((name, i) => [
+            name,
+            toNative((value as Record<string, unknown>)[name], type.entryTypes[i]),
+          ]),
+        ),
+      );
+    }
+    return value as DuckDBValue;
+  };
+  return {
+    rows,
+    types: new Map(names.flatMap((name, i) => (i === cursorIndex ? [] : [[name, types[i]]]))),
+    lastCursorValue,
+    toNative,
+  };
+}

package/src/helpers/readScalarNumber.ts

@@ -0,0 +1,6 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+
+// For internal queries returning a single numeric value, such as row counts.
+export default async function readScalarNumber(connection: DuckDBConnection, sql: string) {
+  return Number((await connection.runAndReadAll(sql)).getRowsJS()[0][0]);
+}

package/src/helpers/referencedTables.ts

@@ -0,0 +1,24 @@
+import type SimpleDB from "../class/SimpleDB.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import scanSQL from "./scanSQL.ts";
+import foldIdentifier from "./foldIdentifier.ts";
+import { getRegisteredTables } from "./tableRegistry.ts";
+
+/**
+ * Finds registered table identifiers in SQL, excluding literals and comments.
+ * Matching is conservative: column names and aliases also count as references.
+ * DuckDB folds ASCII letters in both quoted and unquoted identifiers.
+ */
+export default function referencedTables(sql: readonly string[], sdb: SimpleDB): SimpleTable[] {
+  const identifiers = new Set<string>();
+  for (const fragment of sql) {
+    for (const token of scanSQL(fragment)) {
+      if (token.kind === "quoted" && token.text.startsWith('"')) {
+        identifiers.add(foldIdentifier(token.text.slice(1, -1).replaceAll('""', '"')));
+      } else if (token.kind === "sql" && /^[\p{L}_][\p{L}\p{N}_$]*$/u.test(token.text)) {
+        identifiers.add(foldIdentifier(token.text));
+      }
+    }
+  }
+  return getRegisteredTables(sdb).filter((table) => identifiers.has(foldIdentifier(table.name)));
+}

package/src/helpers/removeColumnsNow.ts

@@ -0,0 +1,27 @@
+import quoteIdentifier from "./quoteIdentifier.ts";
+import mergeOptions from "./mergeOptions.ts";
+import queryDB from "./queryDB.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+/**
+ * Drops columns immediately, without queueing. For use inside a barrier's
+ * execute, which runs during a flush: a sync builder method called there
+ * would queue an operation for the next flush instead of running now.
+ */
+export default async function removeColumnsNow(
+  simpleTable: SimpleTable,
+  columns: string[],
+  method: string,
+): Promise<void> {
+  await queryDB(
+    simpleTable,
+    columns
+      .map((d) => `ALTER TABLE ${quoteIdentifier(simpleTable.name)} DROP ${quoteIdentifier(d)};`)
+      .join("\n"),
+    mergeOptions(simpleTable, {
+      table: simpleTable.name,
+      method,
+      parameters: { columns },
+    }),
+  );
+}

package/src/helpers/resolveOutputTable.ts

@@ -0,0 +1,19 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+
+/**
+ * Resolves `outputTable: true` into an auto-generated table name
+ * ("table0", "table1", ...), incrementing the database's shared counter so
+ * concurrent auto-named tables never collide. A string, `false`, or
+ * `undefined` value passes through unchanged.
+ */
+export default function resolveOutputTable(
+  simpleTable: SimpleTable,
+  outputTable: string | boolean | undefined,
+): string | boolean | undefined {
+  if (outputTable === true) {
+    const name = `table${simpleTable.sdb.tableIncrement}`;
+    simpleTable.sdb.tableIncrement += 1;
+    return name;
+  }
+  return outputTable;
+}