@nshiab/simple-data-analysis-core 2.1.12 → 2.1.13

package/src/helpers/serializeCacheInputs.ts

@@ -0,0 +1,117 @@
+import SimpleTable from "../class/SimpleTable.ts";
+import { getTableGeneration } from "./tableGeneration.ts";
+import normalizeCacheCode from "./normalizeCacheCode.ts";
+
+/**
+ * Converts cache inputs to a deterministic, type-tagged representation.
+ * SimpleTable values are represented by their name and current generation.
+ */
+export default function serializeCacheInputs(inputs: readonly unknown[]): string {
+  if (!Array.isArray(inputs)) {
+    throw new TypeError("cache() options.inputs must be an array.");
+  }
+  return serializeValue(inputs, "inputs", new Set<object>());
+}
+
+/** Returns a user-facing cache diagnostic label for supported input values. */
+export function getCacheInputLabel(value: unknown): string | undefined {
+  return value instanceof SimpleTable ? `table ${JSON.stringify(value.name)}` : undefined;
+}
+
+function serializeValue(value: unknown, path: string, ancestors: Set<object>): string {
+  if (value === null) {
+    return JSON.stringify(["null"]);
+  }
+
+  switch (typeof value) {
+    case "undefined":
+      return JSON.stringify(["undefined"]);
+    case "boolean":
+      return JSON.stringify(["boolean", value]);
+    case "string":
+      return JSON.stringify(["string", value]);
+    case "number":
+      return serializeNumber(value);
+    case "bigint":
+      return JSON.stringify(["bigint", value.toString()]);
+    case "function":
+      return JSON.stringify([
+        "function",
+        normalizeCacheCode(Function.prototype.toString.call(value)),
+      ]);
+    case "symbol":
+      throw new TypeError(`cache() ${path} cannot be a symbol.`);
+  }
+
+  if (value instanceof SimpleTable) {
+    return JSON.stringify(["SimpleTable", value.name, getTableGeneration(value)]);
+  }
+  if (value instanceof Date) {
+    if (Number.isNaN(value.getTime())) {
+      throw new TypeError(`cache() ${path} is an invalid Date.`);
+    }
+    return JSON.stringify(["Date", value.toISOString()]);
+  }
+
+  if (ancestors.has(value)) {
+    throw new TypeError(`cache() ${path} is cyclic.`);
+  }
+  ancestors.add(value);
+  try {
+    if (Array.isArray(value)) {
+      const serializedItems: string[] = [];
+      for (let i = 0; i < value.length; i++) {
+        serializedItems.push(serializeValue(value[i], `${path}[${i}]`, ancestors));
+      }
+      return JSON.stringify(["array", serializedItems]);
+    }
+
+    if (!isPlainObject(value)) {
+      const name = value.constructor?.name || "unknown";
+      throw new TypeError(
+        `cache() ${path} is an unsupported ${name} instance. Pass its deterministic state instead.`,
+      );
+    }
+
+    if (Object.getOwnPropertySymbols(value).length > 0) {
+      throw new TypeError(`cache() ${path} cannot have symbol properties.`);
+    }
+    const serializedEntries: [string, string][] = [];
+    for (const key of Object.keys(value).sort()) {
+      serializedEntries.push([key, serializeValue(value[key], propertyPath(path, key), ancestors)]);
+    }
+    return JSON.stringify(["object", serializedEntries]);
+  } finally {
+    ancestors.delete(value);
+  }
+}
+
+function serializeNumber(value: number): string {
+  if (Number.isNaN(value)) {
+    return JSON.stringify(["number", "NaN"]);
+  }
+  if (value === Infinity) {
+    return JSON.stringify(["number", "Infinity"]);
+  }
+  if (value === -Infinity) {
+    return JSON.stringify(["number", "-Infinity"]);
+  }
+  if (Object.is(value, -0)) {
+    return JSON.stringify(["number", "-0"]);
+  }
+  return JSON.stringify(["number", value.toString()]);
+}
+
+function isPlainObject(value: unknown): value is Readonly<Record<string, unknown>> {
+  if (value === null || typeof value !== "object") {
+    return false;
+  }
+  const prototype: unknown = Object.getPrototypeOf(value);
+  return prototype === Object.prototype || prototype === null;
+}
+
+function propertyPath(path: string, key: string): string {
+  return /^[A-Za-z_$][A-Za-z0-9_$]*$/.test(key)
+    ? `${path}.${key}`
+    : `${path}[${JSON.stringify(key)}]`;
+}

package/src/helpers/setDbProps.ts

@@ -0,0 +1,31 @@
+import type SimpleDB from "../class/SimpleDB.ts";
+import type { IndexDefinition } from "./indexDefinitions.ts";
+import { markTableChanged } from "./tableGeneration.ts";
+import { registerTable } from "./tableRegistry.ts";
+
+export default async function setDbProps(
+  simpleDB: SimpleDB,
+  indexes: { [table: string]: IndexDefinition[] },
+) {
+  for (const table of await simpleDB.getTableNames()) {
+    if (!simpleDB.getTables().some((registered) => registered.name === table)) {
+      const instance = new simpleDB.tableClass(table, simpleDB, {
+        rowsToLog: simpleDB.rowsToLog,
+        charsToLog: simpleDB.charsToLog,
+        typesToLog: simpleDB.typesToLog,
+      });
+      registerTable(simpleDB, instance);
+      markTableChanged(instance);
+    }
+  }
+
+  for (const table of simpleDB.getTables()) {
+    if (indexes[table.name]) {
+      table.indexes = structuredClone(indexes[table.name]);
+    }
+  }
+
+  while (simpleDB.getTables().some((table) => table.name === `table${simpleDB.tableIncrement}`)) {
+    simpleDB.tableIncrement++;
+  }
+}

package/src/helpers/splitGraphemes.ts

@@ -0,0 +1,7 @@
+const segmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" });
+
+// Keep combining characters attached to their base without normalizing text.
+// Grapheme counts are not terminal display widths for wide characters or emoji.
+export default function splitGraphemes(text: string): string[] {
+  return Array.from(segmenter.segment(text), ({ segment }) => segment);
+}

package/src/helpers/stabilizeCosineVectors.ts

@@ -0,0 +1,44 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+import readScalarNumber from "./readScalarNumber.ts";
+
+/**
+ * Stabilizes cosine arithmetic in a caller-owned private vector relation.
+ * The already-quoted relation has unique vertex ids and a fixed-size DOUBLE
+ * vec column. Source data must be retained separately by the caller.
+ *
+ * Safe vectors keep their existing arithmetic and tie behavior. Outside the
+ * safe range, divide by the largest absolute component, then by the resulting
+ * length. This preserves direction without overflowing or underflowing norms.
+ * The bounds keep squared-norm products between 1e-300 and 1e300, with room
+ * for rounding. Only the private vec column changes; no scratch tables remain.
+ */
+export default async function stabilizeCosineVectors(
+  connection: DuckDBConnection,
+  rows: string,
+): Promise<void> {
+  if (
+    await readScalarNumber(
+      connection,
+      `SELECT count(*) FROM ${rows} WHERE vec IS NULL
+        OR len(vec)=0 OR list_count(vec)!=len(vec)
+        OR NOT list_bool_and(list_transform(vec, value -> isfinite(value)))
+        OR list_max(list_transform(vec, value -> abs(value)))=0`,
+    )
+  ) {
+    throw new Error("Cosine distance requires finite, non-null values and nonzero vectors.");
+  }
+  await connection.run(`WITH magnitudes AS MATERIALIZED (
+      SELECT vertex,vec,list_max(list_transform(vec, value -> abs(value))) AS magnitude
+      FROM ${rows}
+    ), scaled AS MATERIALIZED (
+      SELECT vertex,list_transform(vec, value -> value / magnitude) AS vec
+      FROM magnitudes
+      WHERE magnitude < 1e-75 OR magnitude > 1e75 / sqrt(len(vec))
+    ), lengths AS MATERIALIZED (
+      SELECT vertex,vec,sqrt(list_sum(list_transform(vec, value -> value * value))) AS norm
+      FROM scaled
+    )
+    UPDATE ${rows} AS target SET vec=cast_to_type(
+      list_transform(lengths.vec, value -> value / lengths.norm),target.vec)
+    FROM lengths WHERE target.vertex=lengths.vertex`);
+}

package/src/helpers/statCanCache.ts

@@ -0,0 +1,57 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+import fileCache, { createFileCacheId } from "./fileCache.ts";
+import mergeOptions from "./mergeOptions.ts";
+import parseValue from "./parseValue.ts";
+import queryDB from "./queryDB.ts";
+import quoteIdentifier from "./quoteIdentifier.ts";
+
+const CACHE_DIRECTORY = ".sda-cache/statcan";
+const CACHE_FORMAT_VERSION = "statcan-parquet-v1";
+
+export function createStatCanCacheId(pid: string, lang: "en" | "fr"): string {
+  return createFileCacheId("statcan", [CACHE_FORMAT_VERSION, pid, lang]);
+}
+
+export async function useStatCanCache(
+  table: SimpleTable,
+  id: string,
+  enabled: boolean,
+  ttl: number | undefined,
+  parameters: { [key: string]: unknown },
+  compute: () => Promise<void>,
+): Promise<void> {
+  await fileCache(table, {
+    directory: CACHE_DIRECTORY,
+    enabled,
+    extension: "parquet",
+    formatVersion: CACHE_FORMAT_VERSION,
+    id,
+    method: "loadStatCanData()",
+    ttl,
+    load: async (file) => {
+      await queryDB(
+        table,
+        `CREATE OR REPLACE TABLE ${quoteIdentifier(table.name)} AS
+        SELECT * FROM read_parquet(${parseValue(file)});`,
+        mergeOptions(table, {
+          table: table.name,
+          method: "loadStatCanData()",
+          parameters,
+        }),
+      );
+    },
+    compute,
+    write: async (file) => {
+      await queryDB(
+        table,
+        `COPY ${quoteIdentifier(table.name)} TO ${parseValue(file)}
+        WITH (FORMAT PARQUET);`,
+        mergeOptions(table, {
+          table: table.name,
+          method: "loadStatCanData()",
+          parameters,
+        }),
+      );
+    },
+  });
+}

package/src/helpers/stringToArray.ts

@@ -0,0 +1,24 @@
+/**
+ * Normalizes a string or array of strings to a new array.
+ * Existing arrays are copied so callers can change the result independently.
+ *
+ * @example
+ * ```ts
+ * import { stringToArray } from "@nshiab/simple-data-analysis-core/helpers";
+ * console.log(stringToArray("name")); // ["name"]
+ * console.log(stringToArray(["name", "age"])); // ["name", "age"]
+ * ```
+ *
+ * @param argument - One string or an array of strings.
+ *
+ * @returns A new array containing the supplied strings.
+ */
+export default function stringToArray(argument: string | string[]): string[] {
+  if (Array.isArray(argument)) {
+    return [...argument];
+  } else if (typeof argument === "string") {
+    return [argument];
+  } else {
+    throw new Error(`argument should be a string or an array of strings.`);
+  }
+}

package/src/helpers/stringifyDates.ts

@@ -0,0 +1,19 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default async function stringifyDates(
+  simpleTable: SimpleTable,
+  types: { [key: string]: string },
+) {
+  const typesKeys = Object.keys(types);
+  const typesValues = Object.values(types);
+  const toConvert: { [key: string]: "string" } = {};
+  for (let i = 0; i < typesKeys.length; i++) {
+    if (typesValues[i] === "TIMESTAMP" || typesValues[i] === "DATE") {
+      toConvert[typesKeys[i]] = "string";
+    }
+  }
+  if (Object.keys(toConvert).length > 0) {
+    simpleTable.convert(toConvert, { datetimeFormat: "%xT%T.%g%z" });
+    simpleTable.replace(Object.keys(toConvert), { "+00": "Z" });
+  }
+}

package/src/helpers/stringifyDatesInvert.ts

@@ -0,0 +1,19 @@
+import type SimpleTable from "../class/SimpleTable.ts";
+
+export default async function stringifyDatesInvert(
+  simpleTable: SimpleTable,
+  types: { [key: string]: string },
+) {
+  const typesKeys = Object.keys(types);
+  const typesValues = Object.values(types);
+  const toConvertBack: { [key: string]: "timestamp" | "date" } = {};
+  for (let i = 0; i < typesKeys.length; i++) {
+    if (typesValues[i] === "TIMESTAMP" || typesValues[i] === "DATE") {
+      toConvertBack[typesKeys[i]] = typesValues[i].toLowerCase() as "timestamp" | "date";
+    }
+  }
+  if (Object.keys(toConvertBack).length > 0) {
+    simpleTable.replace(Object.keys(toConvertBack), { Z: "+00" });
+    simpleTable.convert(toConvertBack);
+  }
+}

package/src/helpers/stringifyUnknown.ts

@@ -0,0 +1,56 @@
+/**
+ * Converts an unknown value to text with JavaScript's primitive-conversion
+ * semantics. The default hint matches string concatenation; the string hint
+ * matches String(value). Template mode uses the string hint but rejects symbols,
+ * matching template interpolation. Explicit object conversion also preserves
+ * custom valueOf, toString, and Symbol.toPrimitive implementations.
+ */
+export default function stringifyUnknown(
+  value: unknown,
+  mode: "string" | "default" | "template" = "string",
+): string {
+  const hint = mode === "default" ? "default" : "string";
+  if (value !== null && (typeof value === "object" || typeof value === "function")) {
+    const convert: unknown = Reflect.get(value, Symbol.toPrimitive);
+    if (convert !== null && convert !== undefined) {
+      if (typeof convert !== "function")
+        throw new TypeError("Cannot convert object to primitive value");
+      const primitive: unknown = Reflect.apply(convert, value, [hint]);
+      return stringifyPrimitive(primitive, false);
+    }
+    const methods = hint === "string" ? ["toString", "valueOf"] : ["valueOf", "toString"];
+    for (const name of methods) {
+      const method: unknown = Reflect.get(value, name);
+      if (typeof method === "function") {
+        const primitive: unknown = Reflect.apply(method, value, []);
+        if (
+          primitive === null ||
+          (typeof primitive !== "object" && typeof primitive !== "function")
+        ) {
+          return stringifyPrimitive(primitive, false);
+        }
+      }
+    }
+    throw new TypeError("Cannot convert object to primitive value");
+  }
+  return stringifyPrimitive(value, mode === "string");
+}
+
+function stringifyPrimitive(value: unknown, allowSymbol: boolean): string {
+  if (value === null) return "null";
+  switch (typeof value) {
+    case "string":
+      return value;
+    case "undefined":
+      return "undefined";
+    case "number":
+    case "bigint":
+    case "boolean":
+      return String(value);
+    case "symbol":
+      if (allowSymbol) return String(value);
+      throw new TypeError("Cannot convert a Symbol value to a string");
+    default:
+      throw new TypeError("Cannot convert object to primitive value");
+  }
+}

package/src/helpers/tableGeneration.ts

@@ -0,0 +1,43 @@
+import crypto from "node:crypto";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+declare const tableGenerationIdBrand: unique symbol;
+
+/** An opaque identifier for one in-process or cached table generation. */
+export type TableGenerationId = string & {
+  readonly [tableGenerationIdBrand]: true;
+};
+
+const generations = new WeakMap<SimpleTable, TableGenerationId>();
+
+/** Returns an established generation without creating one for a new table. */
+export function peekTableGeneration(table: SimpleTable): TableGenerationId | undefined {
+  return generations.get(table);
+}
+
+/** Creates a new table generation identifier. */
+export function createTableGenerationId(): TableGenerationId {
+  return crypto.randomUUID() as TableGenerationId;
+}
+
+/** Returns the current in-process generation identifier for a table. */
+export function getTableGeneration(table: SimpleTable): TableGenerationId {
+  let generation = generations.get(table);
+  if (generation === undefined) {
+    generation = createTableGenerationId();
+    generations.set(table, generation);
+  }
+  return generation;
+}
+
+/** Gives a table a new generation after it may have changed. */
+export function markTableChanged(table: SimpleTable): TableGenerationId {
+  const generation = createTableGenerationId();
+  generations.set(table, generation);
+  return generation;
+}
+
+/** Restores the generation persisted with a cached table artifact. */
+export function restoreTableGeneration(table: SimpleTable, generation: TableGenerationId): void {
+  generations.set(table, generation);
+}

package/src/helpers/tableRegistry.ts

@@ -0,0 +1,89 @@
+import type SimpleDB from "../class/SimpleDB.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+import { markTableChanged } from "./tableGeneration.ts";
+import { recordCacheTableCreation } from "./cacheTableDependencies.ts";
+import foldIdentifier from "./foldIdentifier.ts";
+
+const registries = new WeakMap<object, SimpleTable[]>();
+
+function getMutableTables<Table extends SimpleTable>(simpleDB: SimpleDB<Table>): Table[] {
+  const tables = registries.get(simpleDB);
+  if (tables === undefined) {
+    throw new Error("The table registry has not been initialized.");
+  }
+  return tables as Table[];
+}
+
+export function initializeTableRegistry<Table extends SimpleTable>(
+  simpleDB: SimpleDB<Table>,
+): void {
+  registries.set(simpleDB, []);
+}
+
+export function getRegisteredTables<Table extends SimpleTable>(
+  simpleDB: SimpleDB<Table>,
+): readonly Table[] {
+  return getMutableTables(simpleDB);
+}
+
+export function listRegisteredTables<Table extends SimpleTable>(
+  simpleDB: SimpleDB<Table>,
+): readonly Table[] {
+  return [...getMutableTables(simpleDB)];
+}
+
+export function registerTable<Table extends SimpleTable>(
+  simpleDB: SimpleDB<Table>,
+  table: Table,
+): void {
+  const TableClass = simpleDB.tableClass;
+  if (!(table instanceof TableClass)) {
+    throw new Error(`The table must be an instance of ${TableClass.name}.`);
+  }
+
+  const tables = getMutableTables(simpleDB);
+  assertTableNameAvailable(simpleDB, table.name);
+  tables.push(table);
+  recordCacheTableCreation(table);
+}
+
+export function ensureTableRegistered<Table extends SimpleTable>(
+  simpleDB: SimpleDB<Table>,
+  table: Table,
+): void {
+  const tables = getMutableTables(simpleDB);
+  if (!tables.includes(table)) {
+    registerTable(simpleDB, table);
+  }
+}
+
+/** Prevents distinct handles from owning the same DuckDB table name. */
+export function assertTableNameAvailable(
+  simpleDB: SimpleDB,
+  name: string,
+  renamedTable?: SimpleTable,
+): void {
+  const normalizedName = foldIdentifier(name);
+  if (
+    getRegisteredTables(simpleDB).some(
+      (table) => table !== renamedTable && foldIdentifier(table.name) === normalizedName,
+    )
+  ) {
+    throw new Error(`Table ${name} already exists.`);
+  }
+}
+
+export function retainRegisteredTables<Table extends SimpleTable>(
+  simpleDB: SimpleDB<Table>,
+  predicate: (table: Table) => boolean,
+): void {
+  const tables = getMutableTables(simpleDB);
+  const retained = tables.filter(predicate);
+  const retainedSet = new Set(retained);
+  for (const table of tables) {
+    if (!retainedSet.has(table)) {
+      markTableChanged(table);
+    }
+  }
+  tables.splice(0, tables.length, ...retained);
+}

package/src/helpers/toDuckDBValue.ts

@@ -0,0 +1,21 @@
+import stringifyUnknown from "./stringifyUnknown.ts";
+import { DuckDBTimestampValue, type DuckDBValue } from "@duckdb/node-api";
+
+/** Converts a supported JavaScript data value to its DuckDB binding value. */
+export default function toDuckDBValue(value: unknown): DuckDBValue {
+  if (value === null || value === undefined || Number.isNaN(value)) {
+    return null;
+  }
+  if (value instanceof Date) {
+    return new DuckDBTimestampValue(BigInt(value.getTime() * 1000));
+  }
+  if (
+    typeof value === "string" ||
+    typeof value === "number" ||
+    typeof value === "boolean" ||
+    typeof value === "bigint"
+  ) {
+    return value;
+  }
+  throw new Error(`Unknown bind value type ${typeof value} of ${stringifyUnknown(value)}`);
+}

package/src/helpers/umapRandom.ts

@@ -0,0 +1,10 @@
+// Mulberry32: separate initialization and sampling streams, stable per seed.
+export default function umapRandom(seed: number) {
+  let state = seed >>> 0;
+  return () => {
+    state = (state + 0x6d2b79f5) | 0;
+    let value = Math.imul(state ^ (state >>> 15), 1 | state);
+    value ^= value + Math.imul(value ^ (value >>> 7), 61 | value);
+    return ((value ^ (value >>> 14)) >>> 0) / 4294967296;
+  };
+}

package/src/helpers/unifyColumns.ts

@@ -0,0 +1,49 @@
+import quoteIdentifier from "./quoteIdentifier.ts";
+import mergeOptions from "./mergeOptions.ts";
+import queryDB from "./queryDB.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+/**
+ * Adds the columns missing from each table (filled with NULL) so all tables
+ * share the same columns. Runs immediately, without queueing, so it's safe
+ * inside a barrier's execute, which runs during a flush.
+ */
+export default async function unifyColumns(
+  allTables: SimpleTable[],
+  allTypes: { [key: string]: string },
+) {
+  const columnsAdded: {
+    [key: string]: string[];
+  } = {};
+
+  // Each table's columns are fetched once, instead of one hasColumn
+  // round-trip per column and table.
+  const existingColumns = new Map<SimpleTable, Set<string>>();
+  for (const table of allTables) {
+    existingColumns.set(table, new Set(await table.getColumns()));
+  }
+
+  for (const column in allTypes) {
+    for (const table of allTables) {
+      if (!existingColumns.get(table)!.has(column)) {
+        await queryDB(
+          table,
+          `ALTER TABLE ${quoteIdentifier(table.name)} ADD ${quoteIdentifier(
+            column,
+          )} ${allTypes[column]};`,
+          mergeOptions(table, {
+            table: table.name,
+            method: "insertTables()",
+            parameters: { column, type: allTypes[column] },
+          }),
+        );
+        if (!columnsAdded[table.name]) {
+          columnsAdded[table.name] = [];
+        }
+        columnsAdded[table.name].push(column);
+      }
+    }
+  }
+
+  return columnsAdded;
+}