@nshiab/simple-data-analysis-core 2.0.9 → 2.1.0

package/script/helpers/buildGraphTemporalCostStateSql.js

@@ -0,0 +1,92 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildGraphTemporalCostStateSql;
+const quoteIdentifier_js_1 = __importDefault(require("./quoteIdentifier.js"));
+/** Builds keyed best-cost chronological states from an arbitrary start relation. */
+function buildGraphTemporalCostStateSql(prepared, startsSelect, direction, temporal, distanceType, edgeWeight, eventSelections = []) {
+    const q = quoteIdentifier_js_1.default;
+    const relations = prepared.relationNames([
+        "graph_starts",
+        "graph_temporal_settings",
+        "graph_event_rows",
+        "graph_edges",
+        "graph_cost_states",
+    ]);
+    const startsRelation = relations.graph_starts;
+    const settingsRelation = relations.graph_temporal_settings;
+    const eventRowsRelation = relations.graph_event_rows;
+    const edgesRelation = relations.graph_edges;
+    const costRelation = relations.graph_cost_states;
+    const start = `${q("starts")}.${q("start")}`;
+    const startKey = `${q("starts")}.${q("__key")}`;
+    const reachedStart = `${q("reached")}.${q("start")}`;
+    const reachedStartKey = `${q("reached")}.${q("__start_key")}`;
+    const reachedNodeKey = `${q("reached")}.${q("__node_key")}`;
+    const reachedDistance = `${q("reached")}.${q("distance")}`;
+    const edgeTo = `${q("edges")}.${q("__to")}`;
+    const edgeToKey = `${q("edges")}.${q("__to_key")}`;
+    const edgeFromKey = `${q("edges")}.${q("__from_key")}`;
+    const eventId = `${q("edges")}.${q("__event_id")}`;
+    const eventStart = `${q("edges")}.${q("__event_start")}`;
+    const eventEnd = `${q("edges")}.${q("__event_end")}`;
+    const edgeCost = `${q("edges")}.${q("__weight")}`;
+    const bestDistance = `${q("best")}.${q("distance")}`;
+    const gap = `${q("settings")}.${q("__gap")}`;
+    const candidateDistance = `CAST(${reachedDistance} + ${edgeCost} AS ${distanceType})`;
+    const resolvedStartsSelect = typeof startsSelect === "string"
+        ? startsSelect
+        : startsSelect(edgesRelation);
+    // Only costs for the same start and physical event can dominate each other.
+    // Strict improvements and non-negative weights terminate even zero-cost cycles.
+    return {
+        costRelation,
+        edgesRelation,
+        settingsRelation,
+        startsRelation,
+        withClause: `WITH RECURSIVE ${settingsRelation} AS MATERIALIZED (
+      SELECT CAST(? AS HUGEINT) AS ${q("__gap")}
+    ), ${eventRowsRelation} AS MATERIALIZED (
+      ${prepared.edges(direction, [
+            `${edgeWeight} AS ${q("__weight")}`,
+            ...eventSelections,
+            ...temporal.eventSelections("edges"),
+        ])}
+    ), ${edgesRelation} AS MATERIALIZED (
+      SELECT *
+      FROM ${eventRowsRelation} AS ${q("events")}
+      WHERE ${temporal.eventValidity("events")}
+    ), ${startsRelation} AS MATERIALIZED (
+      ${resolvedStartsSelect}
+    ), ${costRelation}(
+      ${q("start")}, ${q("node")},
+      ${q("__start_key")}, ${q("__node_key")},
+      ${q("__event_id")}, ${q("__event_start")}, ${q("__event_end")},
+      ${q("distance")}
+    ) USING KEY(${q("__start_key")}, ${q("__event_id")}) AS (
+      SELECT ${start}, ${edgeTo}, ${startKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd}, ${edgeCost}
+      FROM ${startsRelation} AS ${q("starts")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${startKey} = ${edgeFromKey}
+      UNION
+      SELECT ${reachedStart}, ${edgeTo}, ${reachedStartKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd},
+        MIN(${candidateDistance}) AS ${q("distance")}
+      FROM ${costRelation} AS ${q("reached")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${reachedNodeKey} = ${edgeFromKey}
+      CROSS JOIN ${settingsRelation} AS ${q("settings")}
+      LEFT JOIN recurring.${costRelation} AS ${q("best")}
+        ON ${reachedStartKey} = ${q("best")}.${q("__start_key")}
+        AND ${eventId} = ${q("best")}.${q("__event_id")}
+      WHERE ${temporal.transition("reached", "edges", direction, gap)}
+      GROUP BY ${reachedStart}, ${edgeTo}, ${reachedStartKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd}, ${bestDistance}
+      HAVING ${bestDistance} IS NULL OR
+        MIN(${candidateDistance}) < ${bestDistance}
+    )`,
+    };
+}

package/script/helpers/buildGraphTemporalReachabilitySql.d.ts

@@ -0,0 +1,13 @@
+import type { PreparedGraphTemporalSql } from "./prepareGraphTemporalSql.js";
+import type { GraphDirection, PreparedGraphSql } from "./prepareGraphTraversal.js";
+export type GraphTemporalReachabilitySql = {
+    edgesRelation: string;
+    reachableRelation: string;
+    withClause: string;
+};
+/**
+ * Builds reusable chronological reachability CTEs from a start relation.
+ * A callback may derive all-node starts from the materialized valid edges.
+ */
+export default function buildGraphTemporalReachabilitySql(prepared: PreparedGraphSql, startsSelect: string | ((validEdgesRelation: string) => string), direction: Exclude<GraphDirection, "both">, temporal: PreparedGraphTemporalSql): GraphTemporalReachabilitySql;
+//# sourceMappingURL=buildGraphTemporalReachabilitySql.d.ts.map

package/script/helpers/buildGraphTemporalReachabilitySql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildGraphTemporalReachabilitySql.d.ts","sourceRoot":"","sources":["../../src/helpers/buildGraphTemporalReachabilitySql.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,KAAK,EACV,cAAc,EACd,gBAAgB,EACjB,MAAM,4BAA4B,CAAC;AAGpC,MAAM,MAAM,4BAA4B,GAAG;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,OAAO,UAAU,iCAAiC,CACvD,QAAQ,EAAE,gBAAgB,EAC1B,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC,kBAAkB,EAAE,MAAM,KAAK,MAAM,CAAC,EAC/D,SAAS,EAAE,OAAO,CAAC,cAAc,EAAE,MAAM,CAAC,EAC1C,QAAQ,EAAE,wBAAwB,GACjC,4BAA4B,CAiE9B"}

package/script/helpers/buildGraphTemporalReachabilitySql.js

@@ -0,0 +1,76 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildGraphTemporalReachabilitySql;
+const quoteIdentifier_js_1 = __importDefault(require("./quoteIdentifier.js"));
+/**
+ * Builds reusable chronological reachability CTEs from a start relation.
+ * A callback may derive all-node starts from the materialized valid edges.
+ */
+function buildGraphTemporalReachabilitySql(prepared, startsSelect, direction, temporal) {
+    const q = quoteIdentifier_js_1.default;
+    const relations = prepared.relationNames([
+        "graph_starts",
+        "graph_temporal_settings",
+        "graph_event_rows",
+        "graph_edges",
+        "graph_reachable_states",
+    ]);
+    const startsRelation = relations.graph_starts;
+    const settingsRelation = relations.graph_temporal_settings;
+    const eventRowsRelation = relations.graph_event_rows;
+    const edgesRelation = relations.graph_edges;
+    const reachableRelation = relations.graph_reachable_states;
+    const start = `${q("starts")}.${q("start")}`;
+    const startKey = `${q("starts")}.${q("__key")}`;
+    const reachedStart = `${q("reached")}.${q("start")}`;
+    const reachedStartKey = `${q("reached")}.${q("__start_key")}`;
+    const reachedNodeKey = `${q("reached")}.${q("__node_key")}`;
+    const edgeTo = `${q("edges")}.${q("__to")}`;
+    const edgeToKey = `${q("edges")}.${q("__to_key")}`;
+    const edgeFromKey = `${q("edges")}.${q("__from_key")}`;
+    const eventId = `${q("edges")}.${q("__event_id")}`;
+    const eventStart = `${q("edges")}.${q("__event_start")}`;
+    const eventEnd = `${q("edges")}.${q("__event_end")}`;
+    const gap = `${q("settings")}.${q("__gap")}`;
+    const resolvedStartsSelect = typeof startsSelect === "string"
+        ? startsSelect
+        : startsSelect(edgesRelation);
+    // Materialized event IDs stay fixed; UNION bounds states to starts × events,
+    // including equal-time cycles, because the last event determines all next steps.
+    return {
+        edgesRelation,
+        reachableRelation,
+        withClause: `WITH RECURSIVE ${settingsRelation} AS MATERIALIZED (
+      SELECT CAST(? AS HUGEINT) AS ${q("__gap")}
+    ), ${eventRowsRelation} AS MATERIALIZED (
+      ${prepared.edges(direction, temporal.eventSelections("edges"))}
+    ), ${edgesRelation} AS MATERIALIZED (
+      SELECT *
+      FROM ${eventRowsRelation} AS ${q("events")}
+      WHERE ${temporal.eventValidity("events")}
+    ), ${startsRelation} AS MATERIALIZED (
+      ${resolvedStartsSelect}
+    ), ${reachableRelation}(
+      ${q("start")}, ${q("node")},
+      ${q("__start_key")}, ${q("__node_key")},
+      ${q("__event_id")}, ${q("__event_start")}, ${q("__event_end")}
+    ) AS (
+      SELECT ${start}, ${edgeTo}, ${startKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd}
+      FROM ${startsRelation} AS ${q("starts")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${startKey} = ${edgeFromKey}
+      UNION
+      SELECT ${reachedStart}, ${edgeTo}, ${reachedStartKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd}
+      FROM ${reachableRelation} AS ${q("reached")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${reachedNodeKey} = ${edgeFromKey}
+      CROSS JOIN ${settingsRelation} AS ${q("settings")}
+      WHERE ${temporal.transition("reached", "edges", direction, gap)}
+    )`,
+    };
+}

package/script/helpers/buildSparseMutualReachabilityMst.d.ts

@@ -0,0 +1,10 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+type WeightedEdge = {
+    source: number;
+    target: number;
+    distance: number;
+};
+/** Select a deterministically ordered MST from a connected sparse candidate graph. */
+export default function buildSparseMutualReachabilityMst(connection: DuckDBConnection, edges: string, count: number): Promise<WeightedEdge[]>;
+export {};
+//# sourceMappingURL=buildSparseMutualReachabilityMst.d.ts.map

package/script/helpers/buildSparseMutualReachabilityMst.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildSparseMutualReachabilityMst.d.ts","sourceRoot":"","sources":["../../src/helpers/buildSparseMutualReachabilityMst.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,KAAK,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,sFAAsF;AACtF,wBAA8B,gCAAgC,CAC5D,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,YAAY,EAAE,CAAC,CA4CzB"}

package/script/helpers/buildSparseMutualReachabilityMst.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildSparseMutualReachabilityMst;
+/** Select a deterministically ordered MST from a connected sparse candidate graph. */
+async function buildSparseMutualReachabilityMst(connection, edges, count) {
+    const parent = new Int32Array(count);
+    const size = new Int32Array(count).fill(1);
+    for (let vertex = 0; vertex < count; vertex++)
+        parent[vertex] = vertex;
+    const find = (start) => {
+        let root = start;
+        while (parent[root] !== root)
+            root = parent[root];
+        let current = start;
+        while (parent[current] !== current) {
+            const next = parent[current];
+            parent[current] = root;
+            current = next;
+        }
+        return root;
+    };
+    const mst = [];
+    const result = await connection.stream(`SELECT source,target,distance FROM ${edges}
+     ORDER BY distance,source,target`);
+    while (mst.length < count - 1) {
+        const chunk = await result.fetchChunk();
+        if (!chunk || chunk.rowCount === 0)
+            break;
+        const sources = chunk.getColumnVector(0);
+        const targets = chunk.getColumnVector(1);
+        const distances = chunk.getColumnVector(2);
+        for (let row = 0; row < chunk.rowCount && mst.length < count - 1; row++) {
+            const source = Number(sources.getItem(row));
+            const target = Number(targets.getItem(row));
+            const distance = Number(distances.getItem(row));
+            let left = find(source), right = find(target);
+            if (left === right)
+                continue;
+            if (size[left] < size[right])
+                [left, right] = [right, left];
+            parent[right] = left;
+            size[left] += size[right];
+            mst.push({ source, target, distance });
+        }
+    }
+    if (mst.length !== count - 1) {
+        throw new Error("Approximate HDBSCAN candidate graph has no spanning tree.");
+    }
+    return mst;
+}

package/script/helpers/buildUmapGraph.d.ts

@@ -1,10 +1,7 @@
 import type { DuckDBConnection } from "@duckdb/node-api";
-export type UmapTables = {
-    rows: string;
+import { type VectorNeighborTables } from "./buildVectorNeighbors.js";
+export type UmapTables = Omit<VectorNeighborTables, "neighbors"> & {
     knn: string;
-    search: string;
-    hnsw: string;
-    candidates: string;
     scales: string;
     directed: string;
     graph: string;

package/script/helpers/buildUmapGraph.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"buildUmapGraph.d.ts","sourceRoot":"","sources":["../../src/helpers/buildUmapGraph.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAKzD,MAAM,MAAM,UAAU,GAAG;IACvB,IAAI,EAAE,MAAM,CAAC;IACb,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wBAA8B,cAAc,CAC1C,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,EAC/D,OAAO,EAAE;IAAE,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAA;CAAE,EACrE,KAAK,EAAE,UAAU;;;;GAkClB"}
+{"version":3,"file":"buildUmapGraph.d.ts","sourceRoot":"","sources":["../../src/helpers/buildUmapGraph.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAA6B,EAC3B,KAAK,oBAAoB,EAC1B,MAAM,2BAA2B,CAAC;AAInC,MAAM,MAAM,UAAU,GAAG,IAAI,CAAC,oBAAoB,EAAE,WAAW,CAAC,GAAG;IACjE,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wBAA8B,cAAc,CAC1C,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,EAC/D,OAAO,EAAE;IAAE,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAA;CAAE,EACrE,KAAK,EAAE,UAAU;;;;GA2ClB"}

package/script/helpers/buildUmapGraph.js

@@ -5,8 +5,13 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.default = buildUmapGraph;
 const readScalarNumber_js_1 = __importDefault(require("./readScalarNumber.js"));
+const buildVectorNeighbors_js_1 = __importDefault(require("./buildVectorNeighbors.js"));
 async function buildUmapGraph(connection, input, options, names) {
-    await buildNeighbors(connection, input, options, names);
+    await (0, buildVectorNeighbors_js_1.default)(connection, {
+        count: input.count,
+        dimensions: input.dimensions,
+        neighborCount: input.neighbors,
+    }, { ...options, includeSelf: true }, { ...names, neighbors: names.knn });
     await buildFuzzyGraph(connection, input.neighbors, names);
     const edges = await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.graph}`);
     const source = new Uint32Array(edges), target = new Uint32Array(edges), weight = new Float64Array(edges);
@@ -25,105 +30,6 @@ async function buildUmapGraph(connection, input, options, names) {
         throw new Error("Graph changed during transfer.");
     return { source, target, weight };
 }
-async function buildNeighbors(connection, input, options, names) {
-    const { count, dimensions, neighbors } = input;
-    const distance = options.metric === "cosine"
-        ? "array_cosine_distance"
-        : "array_distance";
-    // Exact duplicates must have zero cosine distance. Positive roundoff would
-    // otherwise change rho to that error instead of the next distinct neighbor.
-    const exactDistance = options.metric === "cosine"
-        ? "CASE WHEN a.vec=b.vec THEN 0 ELSE greatest(0,array_cosine_distance(a.vec,b.vec)) END"
-        : "array_distance(a.vec,b.vec)";
-    await connection.run(`CREATE OR REPLACE TEMP TABLE ${names.knn}
-    (source INTEGER, target INTEGER, distance DOUBLE);
-    INSERT INTO ${names.knn} SELECT vertex, vertex, 0 FROM ${names.rows}`);
-    if (options.search === "exact") {
-        // O(n²d) work, but only a bounded source batch and top-k heaps, never a
-        // materialized n×n matrix. Include target id in the key for stable ties.
-        for (let offset = 0; offset < count; offset += 32) {
-            const query = `SELECT source, item.target, item.distance FROM (
-        SELECT a.vertex AS source,
-          min_by(struct_pack(target := b.vertex, distance := ${exactDistance}),
-            struct_pack(distance := ${exactDistance}, target := b.vertex),
-            ${neighbors - 1}) AS closest
-        FROM ${names.rows} a CROSS JOIN ${names.rows} b
-        WHERE a.vertex >= ${offset} AND a.vertex < ${offset + 32}
-          AND a.vertex != b.vertex GROUP BY a.vertex
-      ), UNNEST(closest) AS t(item)`;
-            await connection.run(`INSERT INTO ${names.knn} ${query}`);
-        }
-    }
-    else {
-        await connection.run(`INSTALL vss; LOAD vss;
-      CREATE OR REPLACE TEMP TABLE ${names.search} AS
-      SELECT vertex, vec::FLOAT[${dimensions}] AS vec FROM ${names.rows} ORDER BY vertex`);
-        if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.search} WHERE
-        NOT isfinite(array_inner_product(vec,vec))
-        ${options.metric === "cosine"
-            ? "OR array_inner_product(vec,vec)=0"
-            : ""}`)) {
-            throw new Error("HNSW requires vectors with finite FLOAT norms (and nonzero cosine norms).");
-        }
-        await connection.run(`CREATE INDEX ${names.hnsw} ON ${names.search} USING HNSW(vec)
-      WITH (metric='${options.metric === "cosine" ? "cosine" : "l2sq"}',
-        ef_construction=128, ef_search=128, M=16)`);
-        const previous = String((await connection.runAndReadAll("SELECT current_setting('disabled_optimizers')")).getRowsJS()[0][0]);
-        // The shipped VSS optimizer recognizes the window form of a lateral top-k.
-        // DuckDB 1.5.5's top_n_window_elimination otherwise hides that pattern.
-        const disabled = [
-            ...new Set([
-                ...previous.split(",").filter(Boolean),
-                "top_n_window_elimination",
-            ]),
-        ].join(",");
-        try {
-            await connection.run(`SET disabled_optimizers='${disabled.replaceAll("'", "''")}'`);
-            const query = `SELECT a.vertex AS source, b.vertex AS target, b.distance
-        FROM ${names.search} a, LATERAL (
-          SELECT vertex, ${distance}(vec,a.vec) AS distance FROM ${names.search}
-          ORDER BY distance LIMIT ${Math.min(neighbors + 1, count - 1)}
-        ) b`;
-            // Materialize ONLY ids and scalar distances before ranking. Keeping the
-            // vector joins inside the window query makes DuckDB carry two full DOUBLE
-            // vectors per candidate through the sort (O(n*k*d) scratch memory).
-            const candidates = `SELECT q.source,q.target,${exactDistance} AS distance
-        FROM (${query}) q JOIN ${names.rows} a ON q.source=a.vertex
-        JOIN ${names.rows} b ON q.target=b.vertex WHERE q.source != q.target`;
-            const plan = JSON.stringify((await connection.runAndReadAll(`EXPLAIN ${candidates}`)).getRowsJS());
-            if (!plan.includes("HNSW_INDEX_JOIN")) {
-                // The optimizer can retain a correlated cross product for tiny or
-                // duplicate-heavy inputs. Only small inputs may use our bounded exact
-                // implementation; never execute the unbounded candidate query.
-                if (count <= 1000) {
-                    return await buildNeighbors(connection, input, {
-                        ...options,
-                        search: "exact",
-                    }, names);
-                }
-                throw new Error("UMAP could not use DuckDB's vector index for neighbor search.");
-            }
-            // Exclude self AFTER index search, then stable-sort candidates and keep k-1.
-            // Recompute candidate distances in DOUBLE to isolate ANN candidate error.
-            await connection.run(`CREATE OR REPLACE TEMP TABLE ${names.candidates} AS ${candidates};
-        INSERT INTO ${names.knn} SELECT source,target,distance FROM ${names.candidates}
-        QUALIFY row_number() OVER (PARTITION BY source
-          ORDER BY distance,target) <= ${neighbors - 1}`);
-        }
-        finally {
-            await connection.run(`SET disabled_optimizers='${previous.replaceAll("'", "''")}'`);
-        }
-    }
-    if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.knn}`) !==
-        count * neighbors) {
-        throw new Error("Neighbor search did not return the required number of neighbors.");
-    }
-    if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.knn} WHERE NOT isfinite(distance)`)) {
-        throw new Error("Distance computation returned a non-finite value.");
-    }
-    // Cosine roundoff can produce a tiny negative distance.
-    await connection.run(`UPDATE ${names.knn} SET distance=greatest(0,distance)`);
-}
 async function buildFuzzyGraph(connection, neighbors, names) {
     // Smooth-kNN with local_connectivity=1, bandwidth=1, 64 bisection steps,
     // tolerance 1e-5 and the reference's 1e-3 minimum-distance scale.

package/script/helpers/buildVectorNeighbors.d.ts

@@ -0,0 +1,45 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+/** Private, already-quoted relation names owned and cleaned up by the caller. */
+export type VectorNeighborTables = {
+    rows: string;
+    neighbors: string;
+    search: string;
+    hnsw: string;
+    candidates: string;
+};
+type NeighborInput = {
+    count: number;
+    dimensions: number;
+    neighborCount: number;
+};
+type NeighborOptions = {
+    metric: "euclidean" | "cosine";
+    search: "exact" | "hnsw";
+    includeSelf: boolean;
+    hnsw?: {
+        efConstruction: number;
+        efSearch: number;
+        connectivity: number;
+        candidateCount: number;
+        singleThreaded: boolean;
+        cosineAsL2: boolean;
+    };
+};
+/**
+ * Materialize ranked vector neighbors without moving source vectors into JS.
+ *
+ * `neighborCount` is the number of rows produced per source. When `includeSelf`
+ * is true, the source is rank 0 and consumes one of those rows. Otherwise all
+ * rows are other points. Ranks are zero-based and ties use target id order.
+ * Input rows have unique contiguous `vertex` ids from 0 through count-1 and
+ * validated, non-null, fixed-size DOUBLE `vec` values. Cosine preparation may
+ * rescale this private vec column, preserving direction.
+ *
+ * The caller owns every supplied scratch relation and must drop them after
+ * success or failure. Approximate search only approximates candidate retrieval;
+ * candidate distances are recomputed from the DOUBLE source vectors before
+ * stable final ranking.
+ */
+export default function buildVectorNeighbors(connection: DuckDBConnection, input: NeighborInput, options: NeighborOptions, names: VectorNeighborTables): Promise<void>;
+export {};
+//# sourceMappingURL=buildVectorNeighbors.d.ts.map

package/script/helpers/buildVectorNeighbors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildVectorNeighbors.d.ts","sourceRoot":"","sources":["../../src/helpers/buildVectorNeighbors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMzD,iFAAiF;AACjF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC;IACzB,WAAW,EAAE,OAAO,CAAC;IACrB,IAAI,CAAC,EAAE;QACL,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,OAAO,CAAC;QACxB,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC;CACH,CAAC;AAEF;;;;;;;;;;;;;;GAcG;AACH,wBAA8B,oBAAoB,CAChD,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE,aAAa,EACpB,OAAO,EAAE,eAAe,EACxB,KAAK,EAAE,oBAAoB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAqHf"}

package/script/helpers/buildVectorNeighbors.js

@@ -0,0 +1,176 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildVectorNeighbors;
+const readScalarNumber_js_1 = __importDefault(require("./readScalarNumber.js"));
+const vectorDistanceExpression_js_1 = __importDefault(require("./vectorDistanceExpression.js"));
+const stabilizeCosineVectors_js_1 = __importDefault(require("./stabilizeCosineVectors.js"));
+const validateVectorRowIds_js_1 = __importDefault(require("./validateVectorRowIds.js"));
+/**
+ * Materialize ranked vector neighbors without moving source vectors into JS.
+ *
+ * `neighborCount` is the number of rows produced per source. When `includeSelf`
+ * is true, the source is rank 0 and consumes one of those rows. Otherwise all
+ * rows are other points. Ranks are zero-based and ties use target id order.
+ * Input rows have unique contiguous `vertex` ids from 0 through count-1 and
+ * validated, non-null, fixed-size DOUBLE `vec` values. Cosine preparation may
+ * rescale this private vec column, preserving direction.
+ *
+ * The caller owns every supplied scratch relation and must drop them after
+ * success or failure. Approximate search only approximates candidate retrieval;
+ * candidate distances are recomputed from the DOUBLE source vectors before
+ * stable final ranking.
+ */
+async function buildVectorNeighbors(connection, input, options, names) {
+    const { count, dimensions, neighborCount } = input;
+    await (0, validateVectorRowIds_js_1.default)(connection, names.rows, count);
+    if (!Number.isSafeInteger(dimensions) || dimensions < 1) {
+        throw new Error("Vector dimensions must be a positive safe integer.");
+    }
+    const maximum = count - (options.includeSelf ? 0 : 1);
+    if (!Number.isSafeInteger(neighborCount) || neighborCount < 1 ||
+        neighborCount > maximum) {
+        throw new Error(`Neighbor count must be a safe integer between 1 and ${maximum}.`);
+    }
+    if (options.metric === "cosine") {
+        await (0, stabilizeCosineVectors_js_1.default)(connection, names.rows);
+    }
+    // FLOAT representation limits apply only to the HNSW search vectors.
+    // Exact Euclidean distance does not require representable squared norms.
+    if (options.search === "hnsw") {
+        const normExpression = `array_inner_product(vec::FLOAT[${dimensions}],vec::FLOAT[${dimensions}])`;
+        if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.rows} WHERE NOT isfinite(${normExpression})
+          ${options.metric === "cosine" ? `OR ${normExpression}=0` : ""}`)) {
+            throw new Error("HNSW requires vectors with finite FLOAT norms (and nonzero cosine norms).");
+        }
+    }
+    const distanceFunction = options.metric === "cosine"
+        ? "array_cosine_distance"
+        : "array_distance";
+    // Equal vectors must have exactly zero cosine distance. Positive roundoff can
+    // otherwise alter downstream density estimates. Clamp negative roundoff too.
+    const exactDistance = (0, vectorDistanceExpression_js_1.default)("a.vec", "b.vec", options.metric);
+    const otherCount = neighborCount - (options.includeSelf ? 1 : 0);
+    await connection.run(`CREATE OR REPLACE TEMP TABLE ${names.candidates}
+    (source INTEGER, target INTEGER, distance DOUBLE)`);
+    if (otherCount > 0) {
+        if (options.search === "exact") {
+            // O(n²d) work with a bounded source batch and top-k heaps. The full
+            // distance matrix is never materialized.
+            await buildExactCandidates(connection, input, names, exactDistance, otherCount);
+        }
+        else {
+            await buildHnswCandidates(connection, input, options, names, distanceFunction, exactDistance, otherCount);
+        }
+    }
+    if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.candidates} WHERE distance IS NULL OR NOT isfinite(distance)`)) {
+        throw new Error("Distance computation returned a non-finite value.");
+    }
+    const selfRows = options.includeSelf
+        ? `SELECT vertex AS source,vertex AS target,0::DOUBLE AS distance FROM ${names.rows}`
+        : "SELECT NULL::INTEGER AS source,NULL::INTEGER AS target,NULL::DOUBLE AS distance WHERE false";
+    const rankOrder = options.includeSelf
+        ? "CASE WHEN source=target THEN 0 ELSE 1 END,distance,target"
+        : "distance,target";
+    await connection.run(`CREATE OR REPLACE TEMP TABLE ${names.neighbors} AS
+    SELECT source,target,
+      (row_number() OVER (PARTITION BY source ORDER BY ${rankOrder})-1)::INTEGER AS rank,
+      greatest(0,distance)::DOUBLE AS distance
+    FROM (${selfRows} UNION ALL
+      SELECT source,target,distance FROM ${names.candidates})
+    QUALIFY row_number() OVER (PARTITION BY source ORDER BY ${rankOrder})
+      <= ${neighborCount}`);
+    const produced = await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.neighbors}`);
+    if (produced !== count * neighborCount) {
+        throw new Error(`Neighbor search returned ${produced} rows; ${count * neighborCount} were required.`);
+    }
+    if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.neighbors} WHERE NOT isfinite(distance)`)) {
+        throw new Error("Distance computation returned a non-finite value.");
+    }
+}
+async function buildHnswCandidates(connection, input, options, names, distanceFunction, exactDistance, otherCount) {
+    const { count, dimensions, neighborCount } = input;
+    const settings = options.hnsw;
+    const cosineAsL2 = options.metric === "cosine" && settings?.cosineAsL2;
+    // Euclidean distance on unit vectors has the same mathematical ordering as
+    // cosine. Computing differences avoids FLOAT dot-product cancellation for
+    // near-collinear vectors; final metric distances still use source DOUBLEs.
+    const searchVector = cosineAsL2
+        ? "list_transform(vec,x -> x/sqrt(array_inner_product(vec,vec)))"
+        : "vec";
+    const previousThreads = Number((await connection.runAndReadAll("SELECT current_setting('threads')")).getRowsJS()[0][0]);
+    try {
+        if (settings?.singleThreaded)
+            await connection.run("SET threads=1");
+        await connection.run(`INSTALL vss; LOAD vss;
+      CREATE OR REPLACE TEMP TABLE ${names.search} AS
+      SELECT vertex,${searchVector}::FLOAT[${dimensions}] AS vec
+      FROM ${names.rows} ORDER BY vertex;
+      CREATE INDEX ${names.hnsw} ON ${names.search} USING HNSW(vec)
+      WITH (metric='${options.metric === "cosine" && !cosineAsL2 ? "cosine" : "l2sq"}',
+        ef_construction=${settings?.efConstruction ?? 128},
+        ef_search=${settings?.efSearch ?? 128},M=${settings?.connectivity ?? 16})`);
+    }
+    finally {
+        if (settings?.singleThreaded) {
+            await connection.run(`SET threads=${previousThreads}`);
+        }
+    }
+    const previous = String((await connection.runAndReadAll("SELECT current_setting('disabled_optimizers')")).getRowsJS()[0][0]);
+    const disabled = [
+        ...new Set([
+            ...previous.split(",").filter(Boolean),
+            "top_n_window_elimination",
+        ]),
+    ].join(",");
+    try {
+        await connection.run(`SET disabled_optimizers='${disabled.replaceAll("'", "''")}'`);
+        // Preserve UMAP's established candidate budget when self is included.
+        // Excluding self needs one extra result because HNSW normally returns it.
+        const candidateCount = settings?.candidateCount ??
+            (options.includeSelf
+                ? Math.max(otherCount + 1, Math.min(neighborCount + 1, count - 1))
+                : Math.min(otherCount + 1, count));
+        const candidateDistanceFunction = cosineAsL2
+            ? "array_distance"
+            : distanceFunction;
+        const query = `SELECT a.vertex AS source,b.vertex AS target,b.distance
+      FROM ${names.search} a,LATERAL (
+        SELECT vertex,${candidateDistanceFunction}(vec,a.vec) AS distance
+        FROM ${names.search} ORDER BY distance LIMIT ${candidateCount}
+      ) b`;
+        // Carry ids and scalar distances through the candidate relation. Recompute
+        // in DOUBLE against source rows to isolate ANN retrieval from precision.
+        const candidates = `SELECT q.source,q.target,${exactDistance} AS distance
+      FROM (${query}) q JOIN ${names.rows} a ON q.source=a.vertex
+      JOIN ${names.rows} b ON q.target=b.vertex WHERE q.source != q.target`;
+        const plan = JSON.stringify((await connection.runAndReadAll(`EXPLAIN ${candidates}`)).getRowsJS());
+        if (!plan.includes("HNSW_INDEX_JOIN")) {
+            if (count <= 1000) {
+                return await buildExactCandidates(connection, input, names, exactDistance, otherCount);
+            }
+            throw new Error("DuckDB could not use its vector index for neighbor search.");
+        }
+        await connection.run(`INSERT INTO ${names.candidates} ${candidates}`);
+    }
+    finally {
+        await connection.run(`SET disabled_optimizers='${previous.replaceAll("'", "''")}'`);
+    }
+}
+async function buildExactCandidates(connection, input, names, exactDistance, otherCount) {
+    for (let offset = 0; offset < input.count; offset += 32) {
+        const query = `SELECT source,item.target,item.distance FROM (
+      SELECT a.vertex AS source,
+        CASE WHEN bool_and(isfinite(${exactDistance})) THEN
+        min_by(struct_pack(target := b.vertex,distance := ${exactDistance}),
+          struct_pack(distance := ${exactDistance},target := b.vertex),
+          ${otherCount}) ELSE error('Distance computation returned a non-finite value.') END AS closest
+      FROM ${names.rows} a CROSS JOIN ${names.rows} b
+      WHERE a.vertex >= ${offset} AND a.vertex < ${offset + 32}
+        AND a.vertex != b.vertex GROUP BY a.vertex
+      ),UNNEST(closest) AS t(item)`;
+        await connection.run(`INSERT INTO ${names.candidates} ${query}`);
+    }
+}

package/script/helpers/buildWeakGraphComponentsSql.d.ts

@@ -0,0 +1,3 @@
+/** Builds weak component labels and zero-based IDs from bidirectional edges. */
+export default function buildWeakGraphComponentsSql(edges: string, nodes: string, labels: string, roots: string): string;
+//# sourceMappingURL=buildWeakGraphComponentsSql.d.ts.map

package/script/helpers/buildWeakGraphComponentsSql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildWeakGraphComponentsSql.d.ts","sourceRoot":"","sources":["../../src/helpers/buildWeakGraphComponentsSql.ts"],"names":[],"mappings":"AAEA,gFAAgF;AAChF,MAAM,CAAC,OAAO,UAAU,2BAA2B,CACjD,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACZ,MAAM,CA6BR"}