@nshiab/simple-data-analysis-core 2.1.12 → 2.1.14

package/src/helpers/fileCache.ts

@@ -0,0 +1,199 @@
+import crypto from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
+import type SimpleTable from "../class/SimpleTable.ts";
+import { cacheEntryExpired, cacheLoadMessage, cacheTtlMessage } from "./cacheDiagnostics.ts";
+import prettyDuration from "./prettyDuration.ts";
+
+type FileCacheSource = {
+  file: string | null;
+  creation: number;
+  duration?: number;
+  formatVersion?: string;
+};
+
+type FileCacheSources = { [id: string]: FileCacheSource };
+
+export function createFileCacheId(prefix: string, identity: unknown): string {
+  const hash = crypto.createHash("sha256").update(JSON.stringify(identity)).digest("hex");
+  return `${prefix}.${hash}`;
+}
+
+export default async function fileCache(
+  table: SimpleTable,
+  options: {
+    directory: string;
+    enabled: boolean;
+    extension: string;
+    formatVersion: string;
+    id: string;
+    method: string;
+    ttl?: number;
+    load: (file: string) => Promise<void>;
+    compute: () => Promise<void>;
+    write: (file: string) => Promise<void>;
+  },
+): Promise<void> {
+  if (!options.enabled) {
+    logFileCacheDisabled(table, options.method);
+  } else if (await loadFileCache(table, options)) {
+    return;
+  }
+
+  const start = Date.now();
+  await options.compute();
+  if (options.enabled) {
+    await writeFileCache(table, {
+      ...options,
+      duration: Date.now() - start,
+    });
+  }
+}
+
+async function loadFileCache(
+  table: SimpleTable,
+  options: {
+    directory: string;
+    formatVersion: string;
+    id: string;
+    method: string;
+    ttl?: number;
+    load: (file: string) => Promise<void>;
+  },
+): Promise<boolean> {
+  const sourcesFile = `${options.directory}/sources.json`;
+  const sources = readFileCacheSources(sourcesFile);
+  const source = sources[options.id];
+  const now = Date.now();
+
+  if (table.sdb.cacheVerbose) {
+    console.log(`\n${options.method} cache for ${table.name}`);
+  }
+  if (source === undefined) {
+    logCacheMiss(table, "No matching cache entry exists.");
+    return false;
+  }
+  if (source.formatVersion !== options.formatVersion) {
+    logCacheMiss(table, "The cache format changed.");
+    return false;
+  }
+  if (source.file === null || !existsSync(source.file)) {
+    logCacheMiss(table, "The cached file is missing.");
+    return false;
+  }
+  if (options.ttl !== undefined && cacheEntryExpired(source.creation, options.ttl, now)) {
+    if (table.sdb.cacheVerbose) {
+      console.log(
+        `Cache entry is stale.\n${cacheTtlMessage(
+          source.creation,
+          options.ttl,
+          now,
+          true,
+        )}\nFetching fresh data and refreshing the cache entry.`,
+      );
+    }
+    return false;
+  }
+
+  if (table.sdb.cacheVerbose) {
+    console.log("Cache hit.");
+    if (options.ttl !== undefined) {
+      console.log(cacheTtlMessage(source.creation, options.ttl, now, false));
+    }
+  }
+
+  const start = Date.now();
+  try {
+    await options.load(source.file);
+  } catch {
+    rmSync(source.file, { force: true });
+    delete sources[options.id];
+    writeFileCacheSources(options.directory, sourcesFile, sources);
+    if (table.sdb.cacheVerbose) {
+      console.log("Could not load cached data. Fetching fresh data...");
+    }
+    return false;
+  }
+  const end = Date.now();
+  if (table.sdb.cacheVerbose) {
+    console.log(`${cacheLoadMessage(start, end, source.duration)}\n`);
+    if (source.duration !== undefined) {
+      table.sdb.cacheTimeSaved += source.duration - (end - start);
+    }
+  }
+  return true;
+}
+
+async function writeFileCache(
+  table: SimpleTable,
+  options: {
+    directory: string;
+    extension: string;
+    formatVersion: string;
+    id: string;
+    duration: number;
+    write: (file: string) => Promise<void>;
+  },
+): Promise<void> {
+  mkdirSync(options.directory, { recursive: true });
+  const file = `${options.directory}/${options.id}.${options.extension}`;
+  const temporaryFile = `${file}.${crypto.randomUUID()}.tmp.${options.extension}`;
+
+  if (table.sdb.cacheVerbose) {
+    console.log(`Computations done in ${prettyDuration(0, { end: options.duration })}.`);
+  }
+  const start = Date.now();
+  try {
+    await options.write(temporaryFile);
+    rmSync(file, { force: true });
+    renameSync(temporaryFile, file);
+  } catch (error) {
+    rmSync(temporaryFile, { force: true });
+    throw error;
+  }
+  const end = Date.now();
+
+  const sourcesFile = `${options.directory}/sources.json`;
+  const sources = readFileCacheSources(sourcesFile);
+  sources[options.id] = {
+    file,
+    creation: Date.now(),
+    duration: options.duration,
+    formatVersion: options.formatVersion,
+  };
+  writeFileCacheSources(options.directory, sourcesFile, sources);
+
+  if (table.sdb.cacheVerbose) {
+    console.log(`Wrote in cache in ${prettyDuration(start, { end })}.\n`);
+    table.sdb.cacheTimeWriting += end - start;
+  }
+}
+
+function logFileCacheDisabled(table: SimpleTable, method: string): void {
+  if (table.sdb.cacheVerbose) {
+    console.log(
+      `\n${method} cache for ${table.name}\nCache disabled.\nFetching fresh data without storing a cache entry.`,
+    );
+  }
+}
+
+function logCacheMiss(table: SimpleTable, reason: string): void {
+  if (table.sdb.cacheVerbose) {
+    console.log(`Cache miss.\n${reason}\nFetching fresh data and storing a new cache entry.`);
+  }
+}
+
+function readFileCacheSources(sourcesFile: string): FileCacheSources {
+  if (!existsSync(sourcesFile)) {
+    return {};
+  }
+  return JSON.parse(readFileSync(sourcesFile, "utf-8")) as FileCacheSources;
+}
+
+function writeFileCacheSources(
+  directory: string,
+  sourcesFile: string,
+  sources: FileCacheSources,
+): void {
+  mkdirSync(directory, { recursive: true });
+  writeFileSync(sourcesFile, JSON.stringify(sources));
+}

package/src/helpers/findGeoColumn.ts

@@ -0,0 +1,23 @@
+import findGeoColumnFromSchema from "./findGeoColumnFromSchema.ts";
+import type SimpleTable from "../class/SimpleTable.ts";
+
+/**
+ * Returns the only geometry column in a table.
+ *
+ * @example
+ * ```ts
+ * const column = await findGeoColumn(table, "getGeoData()");
+ * ```
+ *
+ * @param simpleTable - The table to inspect.
+ *
+ * @param method - The SDA method requesting a geometry column.
+ *
+ * @returns A promise resolving to the name of the only geometry column.
+ */
+export default async function findGeoColumn(
+  simpleTable: SimpleTable,
+  method: string,
+): Promise<string> {
+  return findGeoColumnFromSchema(await simpleTable.getTypes(), method, simpleTable.name);
+}

package/src/helpers/findGeoColumnFromSchema.ts

@@ -0,0 +1,46 @@
+import quoteIdentifier from "./quoteIdentifier.ts";
+import type { TableSchema } from "./pendingOps.ts";
+
+/**
+ * Returns the only geometry column in a table schema.
+ *
+ * @example
+ * ```ts
+ * const column = findGeoColumnFromSchema(
+ *   { geom: "GEOMETRY('EPSG:4326')" },
+ *   "area()",
+ *   "places",
+ * );
+ * ```
+ *
+ * @param types - The table schema to inspect.
+ *
+ * @param method - The SDA method requesting a geometry column.
+ *
+ * @param tableName - The name of the table represented by the schema.
+ *
+ * @returns The name of the only geometry column.
+ */
+export default function findGeoColumnFromSchema(
+  types: TableSchema,
+  method: string,
+  tableName: string,
+): string {
+  const geometryColumns = Object.keys(types)
+    .filter((column) => types[column].toLowerCase().includes("geometry"))
+    .sort();
+  if (geometryColumns.length === 0) {
+    throw new Error(
+      `${method} could not find a geometry column in table ${quoteIdentifier(
+        tableName,
+      )}. Specify a geometry column explicitly, or add one to the table first.`,
+    );
+  } else if (geometryColumns.length > 1) {
+    throw new Error(
+      `${method} found ${geometryColumns.length} geometry columns in table ${quoteIdentifier(
+        tableName,
+      )}: ${geometryColumns.map(quoteIdentifier).join(", ")}. Specify one explicitly.`,
+    );
+  }
+  return geometryColumns[0];
+}

package/src/helpers/fitUmapCurve.ts

@@ -0,0 +1,68 @@
+// Least-squares fit of 1/(1+a*x^(2*b)) to the reference UMAP target curve.
+// Spread is fixed to one. See umap-learn's find_ab_params (0.5.9.post2).
+// A two-parameter damped Gauss-Newton solve avoids a runtime fitting dependency.
+export default function fitUmapCurve(minDistance: number): [number, number] {
+  if (!Number.isFinite(minDistance) || minDistance < 0 || minDistance > 1) {
+    throw new Error("minDistance must be between zero and one.");
+  }
+  const samples = Array.from({ length: 299 }, (_, i) => {
+    const x = ((i + 1) * 3) / 299;
+    return {
+      logX: Math.log(x),
+      target: Math.exp(-Math.max(0, x - minDistance)),
+    };
+  });
+  const loss = (logA: number, logB: number) => {
+    let sum = 0;
+    for (const { logX, target } of samples) {
+      const residual = 1 / (1 + Math.exp(logA + 2 * Math.exp(logB) * logX)) - target;
+      sum += residual * residual;
+    }
+    return sum;
+  };
+  let logA = 0,
+    logB = 0,
+    damping = 1e-3;
+  let error = loss(logA, logB);
+  for (let iteration = 0; iteration < 100; iteration++) {
+    let aa = 0,
+      ab = 0,
+      bb = 0,
+      ga = 0,
+      gb = 0;
+    for (const { logX, target } of samples) {
+      const power = Math.exp(logA + 2 * Math.exp(logB) * logX);
+      const predicted = 1 / (1 + power);
+      const da = -power * predicted * predicted;
+      const db = da * 2 * Math.exp(logB) * logX;
+      aa += da * da;
+      ab += da * db;
+      bb += db * db;
+      ga += da * (predicted - target);
+      gb += db * (predicted - target);
+    }
+    const aDiagonal = aa + damping * Math.max(aa, 1e-12);
+    const bDiagonal = bb + damping * Math.max(bb, 1e-12);
+    const determinant = aDiagonal * bDiagonal - ab * ab;
+    if (!(determinant > 0)) throw new Error("UMAP curve fit became singular.");
+    const stepA = (ab * gb - bDiagonal * ga) / determinant;
+    const stepB = (ab * ga - aDiagonal * gb) / determinant;
+    const nextA = Math.max(-20, Math.min(20, logA + stepA));
+    const nextB = Math.max(-20, Math.min(20, logB + stepB));
+    const nextError = loss(nextA, nextB);
+    if (nextError < error) {
+      const improvement = error - nextError;
+      logA = nextA;
+      logB = nextB;
+      error = nextError;
+      damping = Math.max(damping / 3, 1e-12);
+      if (improvement < 1e-13) return [Math.exp(logA), Math.exp(logB)];
+    } else {
+      damping *= 10;
+      if (Math.max(Math.abs(ga), Math.abs(gb)) < 1e-7) {
+        return [Math.exp(logA), Math.exp(logB)];
+      }
+    }
+  }
+  throw new Error("UMAP curve fit did not converge.");
+}