@nshiab/simple-data-analysis-core 2.0.8 → 2.1.0

package/esm/methods/topologicalSort.js

@@ -0,0 +1,164 @@
+import buildWeakGraphComponentsSql from "../helpers/buildWeakGraphComponentsSql.js";
+import getGraphEndpointColumns from "../helpers/getGraphEndpointColumns.js";
+import { prepareGraphSql } from "../helpers/prepareGraphTraversal.js";
+import queueGraphResult from "../helpers/queueGraphResult.js";
+import queueOp from "../helpers/queueOp.js";
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+export default function topologicalSort(simpleTable, sourceColumn, targetColumn, options = {}) {
+    if (typeof sourceColumn !== "string") {
+        throw new TypeError("topologicalSort() sourceColumn must be a string.");
+    }
+    if (typeof targetColumn !== "string") {
+        throw new TypeError("topologicalSort() targetColumn must be a string.");
+    }
+    if (options === null || typeof options !== "object" || Array.isArray(options)) {
+        throw new TypeError("topologicalSort() options must be an object.");
+    }
+    if (options.outputTable !== undefined &&
+        typeof options.outputTable !== "string" &&
+        typeof options.outputTable !== "boolean") {
+        throw new TypeError("topologicalSort() options.outputTable must be a string or boolean.");
+    }
+    options = structuredClone(options);
+    const parameters = { sourceColumn, targetColumn, options };
+    const result = queueGraphResult(simpleTable, {
+        method: "topologicalSort()",
+        parameters,
+        outputTable: options.outputTable,
+        values: (schema) => {
+            validateEndpoints(schema, sourceColumn, targetColumn);
+            return [];
+        },
+        buildSelect: (input, schema) => topologicalSortSelect(input, schema, sourceColumn, targetColumn),
+        outputSchema: (schema) => ({
+            node: validateEndpoints(schema, sourceColumn, targetColumn).idType,
+            componentId: "BIGINT",
+            order: "BIGINT",
+        }),
+    });
+    // Materialize the sorting computation before downstream operations: DuckDB
+    // can otherwise discard even its terminal cycle check for LIMIT 0 or FALSE.
+    // This barrier adds no query or separate validation traversal, and upstream
+    // preparation still fuses into the sorting statement.
+    queueOp(result, {
+        kind: "barrier",
+        method: "topologicalSort()",
+        parameters,
+        execute: () => Promise.resolve(),
+    });
+    return result;
+}
+function validateEndpoints(schema, source, target) {
+    return getGraphEndpointColumns(schema, source, target, "topologicalSort()");
+}
+function topologicalSortSelect(input, schema, source, target) {
+    const prepared = prepareGraphSql(input, schema, source, target, "topologicalSort()");
+    const relations = prepared.relationNames([
+        "graph_edges",
+        "graph_nodes",
+        "graph_topological_order",
+        "graph_topological_result",
+        "graph_component_edges",
+        "graph_component_labels",
+        "graph_component_roots",
+        "graph_ordered_nodes",
+    ]);
+    const edges = relations.graph_edges;
+    const nodes = relations.graph_nodes;
+    const ordering = relations.graph_topological_order;
+    const result = relations.graph_topological_result;
+    const componentEdges = relations.graph_component_edges;
+    const labels = relations.graph_component_labels;
+    const roots = relations.graph_component_roots;
+    const orderedNodes = relations.graph_ordered_nodes;
+    const q = quoteIdentifier;
+    return `WITH RECURSIVE ${edges} AS (
+      ${prepared.edges("outgoing")}
+    ), ${nodes} AS (
+      SELECT ${q("__from")} AS ${q("node")},
+        ${q("__from_key")} AS ${q("__key")}
+      FROM ${edges}
+      UNION
+      SELECT ${q("__to")} AS ${q("node")},
+        ${q("__to_key")} AS ${q("__key")}
+      FROM ${edges}
+    ), ${componentEdges} AS (
+      SELECT * FROM ${edges}
+      UNION ALL
+      SELECT ${q("__to")}, ${q("__from")},
+        ${q("__to_key")}, ${q("__from_key")}
+      FROM ${edges}
+    ), ${buildWeakGraphComponentsSql(componentEdges, nodes, labels, roots)},
+    ${ordering}(
+      ${q("node")}, ${q("__key")}, ${q("order")}, ${q("__ordered")}
+    ) AS (
+      SELECT ${q("eligible")}.${q("node")},
+        ${q("eligible")}.${q("__key")}, CAST(1 AS BIGINT),
+        list_value(${q("eligible")}.${q("__key")})
+      FROM (
+        SELECT ${q("candidate")}.${q("node")},
+          ${q("candidate")}.${q("__key")}
+        FROM ${nodes} AS ${q("candidate")}
+        WHERE NOT EXISTS (
+          SELECT 1 FROM ${edges} AS ${q("edge")}
+          WHERE ${q("edge")}.${q("__to_key")} =
+            ${q("candidate")}.${q("__key")}
+        )
+        ORDER BY ${q("candidate")}.${q("__key")}
+        LIMIT 1
+      ) AS ${q("eligible")}
+      UNION ALL
+      SELECT ${q("eligible")}.${q("node")},
+        ${q("eligible")}.${q("__key")},
+        ${q("ordered")}.${q("order")} + 1,
+        list_append(
+          ${q("ordered")}.${q("__ordered")},
+          ${q("eligible")}.${q("__key")}
+        )
+      FROM ${ordering} AS ${q("ordered")}
+      CROSS JOIN LATERAL (
+        SELECT ${q("candidate")}.${q("node")},
+          ${q("candidate")}.${q("__key")}
+        FROM ${nodes} AS ${q("candidate")}
+        WHERE NOT list_contains(
+          ${q("ordered")}.${q("__ordered")},
+          ${q("candidate")}.${q("__key")}
+        )
+          AND NOT EXISTS (
+            SELECT 1 FROM ${edges} AS ${q("edge")}
+            WHERE ${q("edge")}.${q("__to_key")} =
+                ${q("candidate")}.${q("__key")}
+              AND NOT list_contains(
+                ${q("ordered")}.${q("__ordered")},
+                ${q("edge")}.${q("__from_key")}
+              )
+          )
+        ORDER BY ${q("candidate")}.${q("__key")}
+        LIMIT 1
+      ) AS ${q("eligible")}
+    ), ${result} AS (
+      SELECT CASE WHEN count(*) < (SELECT count(*) FROM ${nodes})
+        THEN error(
+          'topologicalSort() cannot order the graph because it contains a directed cycle.'
+        )
+        ELSE list(${q("node")} ORDER BY ${q("order")})
+      END AS ${q("__nodes")}
+      FROM ${ordering}
+    ), ${orderedNodes} AS (
+      SELECT unnest(${q("__nodes")}) AS ${q("node")},
+        generate_subscripts(${q("__nodes")}, 1) AS ${q("__order")}
+      FROM ${result}
+    )
+    SELECT ${q("ordered")}.${q("node")}, ${q("roots")}.${q("componentId")},
+      row_number() OVER (
+        PARTITION BY ${q("roots")}.${q("componentId")}
+        ORDER BY ${q("ordered")}.${q("__order")}
+      ) AS ${q("order")}
+    FROM ${orderedNodes} AS ${q("ordered")}
+    INNER JOIN ${labels} AS ${q("labels")}
+      ON ${prepared.key(`${q("ordered")}.${q("node")}`)} =
+        ${q("labels")}.${q("__node_key")}
+    INNER JOIN ${roots} AS ${q("roots")}
+      ON ${q("labels")}.${q("__root_key")} = ${q("roots")}.${q("__root_key")}
+    ORDER BY ${q("componentId")}, ${q("order")}`;
+}

package/esm/methods/umap.d.ts

@@ -0,0 +1,5 @@
+import type SimpleTable from "../class/SimpleTable.js";
+type Options = NonNullable<Parameters<SimpleTable["umap"]>[1]>;
+export default function umap(table: SimpleTable, column: string, options?: Options): void;
+export {};
+//# sourceMappingURL=umap.d.ts.map

package/esm/methods/umap.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"umap.d.ts","sourceRoot":"","sources":["../../src/methods/umap.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AAavD,KAAK,OAAO,GAAG,WAAW,CAAC,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAE/D,MAAM,CAAC,OAAO,UAAU,IAAI,CAC1B,KAAK,EAAE,WAAW,EAClB,MAAM,EAAE,MAAM,EACd,OAAO,GAAE,OAAY,QAStB"}

package/esm/methods/umap.js

@@ -0,0 +1,156 @@
+import queueOp from "../helpers/queueOp.js";
+import quoteIdentifier from "../helpers/quoteIdentifier.js";
+import queryDB from "../helpers/queryDB.js";
+import mergeOptions from "../helpers/mergeOptions.js";
+import buildUmapGraph from "../helpers/buildUmapGraph.js";
+import prepareUmapLayoutOptions from "../helpers/prepareUmapLayoutOptions.js";
+import readScalarNumber from "../helpers/readScalarNumber.js";
+import optimizeUmapLayout from "../helpers/optimizeUmapLayout.js";
+import initialUmapCoordinates from "../helpers/initialUmapCoordinates.js";
+import appendColumnBatches from "../helpers/appendColumnBatches.js";
+import { DOUBLE, INTEGER } from "@duckdb/node-api";
+export default function umap(table, column, options = {}) {
+    const settings = structuredClone(options);
+    queueOp(table, {
+        kind: "barrier",
+        method: "umap()",
+        parameters: { column, options: settings },
+        execute: () => execute(table, column, settings),
+    });
+}
+async function execute(table, column, options) {
+    const { neighbors = 15, metric = "euclidean" } = options;
+    if (!Number.isSafeInteger(neighbors) || neighbors < 2) {
+        throw new Error("neighbors must be a safe integer of at least 2.");
+    }
+    const layoutOptions = prepareUmapLayoutOptions(options);
+    if (!["euclidean", "cosine"].includes(metric)) {
+        throw new Error("metric must be euclidean or cosine.");
+    }
+    const types = await table.getTypes();
+    const columns = Object.keys(types);
+    const resolve = (name) => {
+        const found = columns.find((c) => c.toLowerCase() === name.toLowerCase());
+        if (found === undefined)
+            throw new Error(`Column ${name} does not exist.`);
+        return found;
+    };
+    const vector = resolve(column);
+    if (columns.some((c) => ["umapx", "umapy"].includes(c.toLowerCase()))) {
+        throw new Error("UMAP output columns umapX and umapY must not already exist.");
+    }
+    if (!/^(FLOAT|DOUBLE|TINYINT|SMALLINT|INTEGER|BIGINT|HUGEINT|UTINYINT|USMALLINT|UINTEGER|UBIGINT|UHUGEINT|DECIMAL\(\d+,\d+\))\[\d*\]$/
+        .test(types[vector])) {
+        throw new Error("UMAP requires a one-dimensional numeric LIST or ARRAY column.");
+    }
+    const connection = table.connection;
+    const [sourceOid, sourceTemporary] = (await connection.runAndReadAll(`SELECT table_oid,temporary FROM duckdb_tables()
+      WHERE lower(table_name)=lower($1) AND schema_name=current_schema()
+        AND database_name IN (current_database(),'temp')
+      ORDER BY temporary DESC LIMIT 1`, [table.name])).getRowsJS()[0] ?? [];
+    if (typeof sourceOid !== "number" && typeof sourceOid !== "bigint") {
+        throw new Error("UMAP requires a materialized source table.");
+    }
+    const q = quoteIdentifier;
+    const suffix = crypto.randomUUID().replaceAll("-", "");
+    const name = (part) => `__sda_umap_${suffix}_${part}`;
+    const ordinal = q(name("ordinal"));
+    const source = q(table.name), snapshot = q(name("snapshot"));
+    const layoutName = name("layout"), layout = q(layoutName);
+    const names = {
+        rows: q(name("rows")),
+        knn: q(name("knn")),
+        search: q(name("search")),
+        hnsw: q(name("hnsw")),
+        candidates: q(name("candidates")),
+        scales: q(name("scales")),
+        directed: q(name("directed")),
+        graph: q(name("graph")),
+    };
+    const scratch = [
+        snapshot,
+        layout,
+        ...Object.entries(names)
+            .filter(([key]) => key !== "hnsw").map(([, value]) => value),
+    ];
+    try {
+        // Retain payloads in native DuckDB types. A private ordinal distinguishes
+        // duplicate rows and is independent of any user column named rowid.
+        await connection.run(`CREATE TEMP TABLE ${snapshot} AS
+      SELECT *, row_number() OVER () - 1 AS ${ordinal} FROM ${source}`);
+        const count = await readScalarNumber(connection, `SELECT count(*) FROM ${snapshot}`);
+        if (count < 3 || count > 0x7fffffff) {
+            throw new Error("UMAP requires between 3 and 2147483647 rows.");
+        }
+        if (await readScalarNumber(connection, `SELECT count(*) FROM ${snapshot}
+      WHERE ${q(vector)} IS NULL OR len(${q(vector)})=0
+        OR list_count(${q(vector)}) != len(${q(vector)})
+        OR NOT list_bool_and(list_transform(${q(vector)}, x -> isfinite(x::DOUBLE)))`)) {
+            throw new Error("Vectors must be nonempty, non-null and contain finite numbers only.");
+        }
+        if (await readScalarNumber(connection, `SELECT count(DISTINCT len(${q(vector)})) FROM ${snapshot}`) !== 1) {
+            throw new Error("Vectors must have equal dimensions.");
+        }
+        const dimensions = await readScalarNumber(connection, `SELECT len(${q(vector)}) FROM ${snapshot} LIMIT 1`);
+        await connection.run(`CREATE TEMP TABLE ${names.rows} AS
+      SELECT (row_number() OVER (ORDER BY ${ordinal})-1)::INTEGER AS vertex,
+        ${ordinal} AS ordinal, ${q(vector)}::DOUBLE[${dimensions}] AS vec FROM ${snapshot}`);
+        if (await readScalarNumber(connection, `SELECT count(*) FROM ${names.rows}
+      WHERE NOT isfinite(array_inner_product(vec,vec)) OR array_inner_product(vec,vec)>1e300
+      ${metric === "cosine" ? "OR array_inner_product(vec,vec)=0" : ""}`)) {
+            throw new Error("Vector norms overflow, or cosine distance received a zero vector.");
+        }
+        const graph = await buildUmapGraph(connection, {
+            count,
+            dimensions,
+            neighbors: Math.min(neighbors, count - 1),
+        }, {
+            metric,
+            search: count <= 1000 ? "exact" : "hnsw",
+        }, names);
+        // Release index and vector scratch before allocating optimizer state.
+        for (const relation of [
+            names.search,
+            names.candidates,
+            names.knn,
+            names.directed,
+            names.scales,
+            names.graph,
+        ]) {
+            await connection.run(`DROP TABLE IF EXISTS ${relation}`);
+        }
+        await connection.run(`ALTER TABLE ${names.rows} DROP COLUMN vec`);
+        const coordinates = optimizeUmapLayout(graph, initialUmapCoordinates(count, layoutOptions.seed), layoutOptions);
+        await connection.run(`CREATE TEMP TABLE ${layout} (vertex INTEGER,x DOUBLE,y DOUBLE)`);
+        await appendColumnBatches(connection, layoutName, [INTEGER, DOUBLE, DOUBLE], count, (column, start, end) => Array.from({ length: end - start }, (_, row) => column === 0
+            ? start + row
+            : coordinates[2 * (start + row) + column - 1]));
+        // Publish in one statement: failed fits never add partial columns. Existing
+        // native indexes must be restored because CREATE OR REPLACE drops them.
+        const indexes = (await connection.runAndReadAll("SELECT sql FROM duckdb_indexes() WHERE table_oid=$1", [sourceOid])).getRowsJS().map((row) => String(row[0]));
+        await connection.run("BEGIN TRANSACTION");
+        try {
+            await queryDB(table, `CREATE OR REPLACE ${sourceTemporary ? "TEMP " : ""}TABLE ${source} AS
+        SELECT ${columns.map((c) => `s.${q(c)}`).join(",")}, l.x AS "umapX", l.y AS "umapY"
+        FROM ${snapshot} s JOIN ${names.rows} r ON s.${ordinal}=r.ordinal
+        JOIN ${layout} l ON r.vertex=l.vertex ORDER BY s.${ordinal}`, mergeOptions(table, {
+                table: table.name,
+                method: "umap()",
+                parameters: { column, options },
+                noClean: true,
+            }));
+            for (const sql of indexes)
+                await connection.run(sql);
+            await connection.run("COMMIT");
+        }
+        catch (error) {
+            await connection.run("ROLLBACK");
+            throw error;
+        }
+    }
+    finally {
+        for (const relation of scratch) {
+            await connection.run(`DROP TABLE IF EXISTS ${relation}`);
+        }
+    }
+}

package/esm/methods/updateWithJS.js

@@ -4,7 +4,7 @@ import quoteIdentifier from "../helpers/quoteIdentifier.js";
 import { retainRegisteredTables } from "../helpers/tableRegistry.js";
 import queueOp from "../helpers/queueOp.js";
 import { prepareArray } from "./loadArray.js";
-import { DuckDBDataChunk } from "@duckdb/node-api";
+import appendColumnBatches from "../helpers/appendColumnBatches.js";
 import readMutationRows from "../helpers/readMutationRows.js";
 import parseDuckDBType from "../helpers/parseDuckDBType.js";
 export default function updateWithJS(simpleTable, dataModifier, options = {}) {
@@ -133,26 +133,14 @@ async function executeUpdateWithJS(simpleTable, dataModifier, options) {
                     parseDuckDBType(added.types[newKeys.indexOf(key)]));
             await simpleTable.sdb.customQuery(`CREATE OR REPLACE TABLE ${quoteIdentifier(scratch.name)} (${keys.map((key, i) => `${quoteIdentifier(key)} ${columnTypes[i]}`)
                 .join(", ")})`);
-            const appender = await simpleTable.connection.createAppender(scratch.name);
-            try {
-                for (let start = 0; start < modified.length; start += 2000) {
-                    const end = Math.min(start + 2000, modified.length);
-                    const chunk = DuckDBDataChunk.create(columnTypes, end - start);
-                    for (let i = 0; i < keys.length; i++) {
-                        const key = keys[i];
-                        chunk.setColumnValues(i, outputGeometry.includes(key)
-                            ? geometryData.get(key).slice(start, end)
-                            : source.types.has(key)
-                                ? modified.slice(start, end).map((row) => source.toNative(row[key], columnTypes[i]))
-                                : added.columnsData[newKeys.indexOf(key)].slice(start, end));
-                    }
-                    appender.appendDataChunk(chunk);
-                }
-                appender.flushSync();
-            }
-            finally {
-                appender.closeSync();
-            }
+            await appendColumnBatches(simpleTable.connection, scratch.name, columnTypes, modified.length, (column, start, end) => {
+                const key = keys[column];
+                return outputGeometry.includes(key)
+                    ? geometryData.get(key).slice(start, end)
+                    : source.types.has(key)
+                        ? modified.slice(start, end).map((row) => source.toNative(row[key], columnTypes[column]))
+                        : added.columnsData[newKeys.indexOf(key)].slice(start, end);
+            });
             const projection = keys.map((key) => outputGeometry.includes(key)
                 ? `${geometryFromJSON(quoteIdentifier(key))} AS ${quoteIdentifier(key)}`
                 : outputJSON.includes(key)