@nshiab/simple-data-analysis-core 2.0.8 → 2.1.0

package/script/helpers/buildGraphTemporalCostStateSql.js

@@ -0,0 +1,92 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildGraphTemporalCostStateSql;
+const quoteIdentifier_js_1 = __importDefault(require("./quoteIdentifier.js"));
+/** Builds keyed best-cost chronological states from an arbitrary start relation. */
+function buildGraphTemporalCostStateSql(prepared, startsSelect, direction, temporal, distanceType, edgeWeight, eventSelections = []) {
+    const q = quoteIdentifier_js_1.default;
+    const relations = prepared.relationNames([
+        "graph_starts",
+        "graph_temporal_settings",
+        "graph_event_rows",
+        "graph_edges",
+        "graph_cost_states",
+    ]);
+    const startsRelation = relations.graph_starts;
+    const settingsRelation = relations.graph_temporal_settings;
+    const eventRowsRelation = relations.graph_event_rows;
+    const edgesRelation = relations.graph_edges;
+    const costRelation = relations.graph_cost_states;
+    const start = `${q("starts")}.${q("start")}`;
+    const startKey = `${q("starts")}.${q("__key")}`;
+    const reachedStart = `${q("reached")}.${q("start")}`;
+    const reachedStartKey = `${q("reached")}.${q("__start_key")}`;
+    const reachedNodeKey = `${q("reached")}.${q("__node_key")}`;
+    const reachedDistance = `${q("reached")}.${q("distance")}`;
+    const edgeTo = `${q("edges")}.${q("__to")}`;
+    const edgeToKey = `${q("edges")}.${q("__to_key")}`;
+    const edgeFromKey = `${q("edges")}.${q("__from_key")}`;
+    const eventId = `${q("edges")}.${q("__event_id")}`;
+    const eventStart = `${q("edges")}.${q("__event_start")}`;
+    const eventEnd = `${q("edges")}.${q("__event_end")}`;
+    const edgeCost = `${q("edges")}.${q("__weight")}`;
+    const bestDistance = `${q("best")}.${q("distance")}`;
+    const gap = `${q("settings")}.${q("__gap")}`;
+    const candidateDistance = `CAST(${reachedDistance} + ${edgeCost} AS ${distanceType})`;
+    const resolvedStartsSelect = typeof startsSelect === "string"
+        ? startsSelect
+        : startsSelect(edgesRelation);
+    // Only costs for the same start and physical event can dominate each other.
+    // Strict improvements and non-negative weights terminate even zero-cost cycles.
+    return {
+        costRelation,
+        edgesRelation,
+        settingsRelation,
+        startsRelation,
+        withClause: `WITH RECURSIVE ${settingsRelation} AS MATERIALIZED (
+      SELECT CAST(? AS HUGEINT) AS ${q("__gap")}
+    ), ${eventRowsRelation} AS MATERIALIZED (
+      ${prepared.edges(direction, [
+            `${edgeWeight} AS ${q("__weight")}`,
+            ...eventSelections,
+            ...temporal.eventSelections("edges"),
+        ])}
+    ), ${edgesRelation} AS MATERIALIZED (
+      SELECT *
+      FROM ${eventRowsRelation} AS ${q("events")}
+      WHERE ${temporal.eventValidity("events")}
+    ), ${startsRelation} AS MATERIALIZED (
+      ${resolvedStartsSelect}
+    ), ${costRelation}(
+      ${q("start")}, ${q("node")},
+      ${q("__start_key")}, ${q("__node_key")},
+      ${q("__event_id")}, ${q("__event_start")}, ${q("__event_end")},
+      ${q("distance")}
+    ) USING KEY(${q("__start_key")}, ${q("__event_id")}) AS (
+      SELECT ${start}, ${edgeTo}, ${startKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd}, ${edgeCost}
+      FROM ${startsRelation} AS ${q("starts")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${startKey} = ${edgeFromKey}
+      UNION
+      SELECT ${reachedStart}, ${edgeTo}, ${reachedStartKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd},
+        MIN(${candidateDistance}) AS ${q("distance")}
+      FROM ${costRelation} AS ${q("reached")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${reachedNodeKey} = ${edgeFromKey}
+      CROSS JOIN ${settingsRelation} AS ${q("settings")}
+      LEFT JOIN recurring.${costRelation} AS ${q("best")}
+        ON ${reachedStartKey} = ${q("best")}.${q("__start_key")}
+        AND ${eventId} = ${q("best")}.${q("__event_id")}
+      WHERE ${temporal.transition("reached", "edges", direction, gap)}
+      GROUP BY ${reachedStart}, ${edgeTo}, ${reachedStartKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd}, ${bestDistance}
+      HAVING ${bestDistance} IS NULL OR
+        MIN(${candidateDistance}) < ${bestDistance}
+    )`,
+    };
+}

package/script/helpers/buildGraphTemporalReachabilitySql.d.ts

@@ -0,0 +1,13 @@
+import type { PreparedGraphTemporalSql } from "./prepareGraphTemporalSql.js";
+import type { GraphDirection, PreparedGraphSql } from "./prepareGraphTraversal.js";
+export type GraphTemporalReachabilitySql = {
+    edgesRelation: string;
+    reachableRelation: string;
+    withClause: string;
+};
+/**
+ * Builds reusable chronological reachability CTEs from a start relation.
+ * A callback may derive all-node starts from the materialized valid edges.
+ */
+export default function buildGraphTemporalReachabilitySql(prepared: PreparedGraphSql, startsSelect: string | ((validEdgesRelation: string) => string), direction: Exclude<GraphDirection, "both">, temporal: PreparedGraphTemporalSql): GraphTemporalReachabilitySql;
+//# sourceMappingURL=buildGraphTemporalReachabilitySql.d.ts.map

package/script/helpers/buildGraphTemporalReachabilitySql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildGraphTemporalReachabilitySql.d.ts","sourceRoot":"","sources":["../../src/helpers/buildGraphTemporalReachabilitySql.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,8BAA8B,CAAC;AAC7E,OAAO,KAAK,EACV,cAAc,EACd,gBAAgB,EACjB,MAAM,4BAA4B,CAAC;AAGpC,MAAM,MAAM,4BAA4B,GAAG;IACzC,aAAa,EAAE,MAAM,CAAC;IACtB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;GAGG;AACH,MAAM,CAAC,OAAO,UAAU,iCAAiC,CACvD,QAAQ,EAAE,gBAAgB,EAC1B,YAAY,EAAE,MAAM,GAAG,CAAC,CAAC,kBAAkB,EAAE,MAAM,KAAK,MAAM,CAAC,EAC/D,SAAS,EAAE,OAAO,CAAC,cAAc,EAAE,MAAM,CAAC,EAC1C,QAAQ,EAAE,wBAAwB,GACjC,4BAA4B,CAiE9B"}

package/script/helpers/buildGraphTemporalReachabilitySql.js

@@ -0,0 +1,76 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildGraphTemporalReachabilitySql;
+const quoteIdentifier_js_1 = __importDefault(require("./quoteIdentifier.js"));
+/**
+ * Builds reusable chronological reachability CTEs from a start relation.
+ * A callback may derive all-node starts from the materialized valid edges.
+ */
+function buildGraphTemporalReachabilitySql(prepared, startsSelect, direction, temporal) {
+    const q = quoteIdentifier_js_1.default;
+    const relations = prepared.relationNames([
+        "graph_starts",
+        "graph_temporal_settings",
+        "graph_event_rows",
+        "graph_edges",
+        "graph_reachable_states",
+    ]);
+    const startsRelation = relations.graph_starts;
+    const settingsRelation = relations.graph_temporal_settings;
+    const eventRowsRelation = relations.graph_event_rows;
+    const edgesRelation = relations.graph_edges;
+    const reachableRelation = relations.graph_reachable_states;
+    const start = `${q("starts")}.${q("start")}`;
+    const startKey = `${q("starts")}.${q("__key")}`;
+    const reachedStart = `${q("reached")}.${q("start")}`;
+    const reachedStartKey = `${q("reached")}.${q("__start_key")}`;
+    const reachedNodeKey = `${q("reached")}.${q("__node_key")}`;
+    const edgeTo = `${q("edges")}.${q("__to")}`;
+    const edgeToKey = `${q("edges")}.${q("__to_key")}`;
+    const edgeFromKey = `${q("edges")}.${q("__from_key")}`;
+    const eventId = `${q("edges")}.${q("__event_id")}`;
+    const eventStart = `${q("edges")}.${q("__event_start")}`;
+    const eventEnd = `${q("edges")}.${q("__event_end")}`;
+    const gap = `${q("settings")}.${q("__gap")}`;
+    const resolvedStartsSelect = typeof startsSelect === "string"
+        ? startsSelect
+        : startsSelect(edgesRelation);
+    // Materialized event IDs stay fixed; UNION bounds states to starts × events,
+    // including equal-time cycles, because the last event determines all next steps.
+    return {
+        edgesRelation,
+        reachableRelation,
+        withClause: `WITH RECURSIVE ${settingsRelation} AS MATERIALIZED (
+      SELECT CAST(? AS HUGEINT) AS ${q("__gap")}
+    ), ${eventRowsRelation} AS MATERIALIZED (
+      ${prepared.edges(direction, temporal.eventSelections("edges"))}
+    ), ${edgesRelation} AS MATERIALIZED (
+      SELECT *
+      FROM ${eventRowsRelation} AS ${q("events")}
+      WHERE ${temporal.eventValidity("events")}
+    ), ${startsRelation} AS MATERIALIZED (
+      ${resolvedStartsSelect}
+    ), ${reachableRelation}(
+      ${q("start")}, ${q("node")},
+      ${q("__start_key")}, ${q("__node_key")},
+      ${q("__event_id")}, ${q("__event_start")}, ${q("__event_end")}
+    ) AS (
+      SELECT ${start}, ${edgeTo}, ${startKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd}
+      FROM ${startsRelation} AS ${q("starts")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${startKey} = ${edgeFromKey}
+      UNION
+      SELECT ${reachedStart}, ${edgeTo}, ${reachedStartKey}, ${edgeToKey},
+        ${eventId}, ${eventStart}, ${eventEnd}
+      FROM ${reachableRelation} AS ${q("reached")}
+      INNER JOIN ${edgesRelation} AS ${q("edges")}
+        ON ${reachedNodeKey} = ${edgeFromKey}
+      CROSS JOIN ${settingsRelation} AS ${q("settings")}
+      WHERE ${temporal.transition("reached", "edges", direction, gap)}
+    )`,
+    };
+}

package/script/helpers/buildSparseMutualReachabilityMst.d.ts

@@ -0,0 +1,10 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+type WeightedEdge = {
+    source: number;
+    target: number;
+    distance: number;
+};
+/** Select a deterministically ordered MST from a connected sparse candidate graph. */
+export default function buildSparseMutualReachabilityMst(connection: DuckDBConnection, edges: string, count: number): Promise<WeightedEdge[]>;
+export {};
+//# sourceMappingURL=buildSparseMutualReachabilityMst.d.ts.map

package/script/helpers/buildSparseMutualReachabilityMst.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildSparseMutualReachabilityMst.d.ts","sourceRoot":"","sources":["../../src/helpers/buildSparseMutualReachabilityMst.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,KAAK,YAAY,GAAG;IAAE,MAAM,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,sFAAsF;AACtF,wBAA8B,gCAAgC,CAC5D,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,YAAY,EAAE,CAAC,CA4CzB"}

package/script/helpers/buildSparseMutualReachabilityMst.js

@@ -0,0 +1,50 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildSparseMutualReachabilityMst;
+/** Select a deterministically ordered MST from a connected sparse candidate graph. */
+async function buildSparseMutualReachabilityMst(connection, edges, count) {
+    const parent = new Int32Array(count);
+    const size = new Int32Array(count).fill(1);
+    for (let vertex = 0; vertex < count; vertex++)
+        parent[vertex] = vertex;
+    const find = (start) => {
+        let root = start;
+        while (parent[root] !== root)
+            root = parent[root];
+        let current = start;
+        while (parent[current] !== current) {
+            const next = parent[current];
+            parent[current] = root;
+            current = next;
+        }
+        return root;
+    };
+    const mst = [];
+    const result = await connection.stream(`SELECT source,target,distance FROM ${edges}
+     ORDER BY distance,source,target`);
+    while (mst.length < count - 1) {
+        const chunk = await result.fetchChunk();
+        if (!chunk || chunk.rowCount === 0)
+            break;
+        const sources = chunk.getColumnVector(0);
+        const targets = chunk.getColumnVector(1);
+        const distances = chunk.getColumnVector(2);
+        for (let row = 0; row < chunk.rowCount && mst.length < count - 1; row++) {
+            const source = Number(sources.getItem(row));
+            const target = Number(targets.getItem(row));
+            const distance = Number(distances.getItem(row));
+            let left = find(source), right = find(target);
+            if (left === right)
+                continue;
+            if (size[left] < size[right])
+                [left, right] = [right, left];
+            parent[right] = left;
+            size[left] += size[right];
+            mst.push({ source, target, distance });
+        }
+    }
+    if (mst.length !== count - 1) {
+        throw new Error("Approximate HDBSCAN candidate graph has no spanning tree.");
+    }
+    return mst;
+}

package/script/helpers/buildUmapGraph.d.ts

@@ -0,0 +1,21 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+import { type VectorNeighborTables } from "./buildVectorNeighbors.js";
+export type UmapTables = Omit<VectorNeighborTables, "neighbors"> & {
+    knn: string;
+    scales: string;
+    directed: string;
+    graph: string;
+};
+export default function buildUmapGraph(connection: DuckDBConnection, input: {
+    count: number;
+    dimensions: number;
+    neighbors: number;
+}, options: {
+    metric: "euclidean" | "cosine";
+    search: "exact" | "hnsw";
+}, names: UmapTables): Promise<{
+    source: Uint32Array<ArrayBuffer>;
+    target: Uint32Array<ArrayBuffer>;
+    weight: Float64Array<ArrayBuffer>;
+}>;
+//# sourceMappingURL=buildUmapGraph.d.ts.map

package/script/helpers/buildUmapGraph.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildUmapGraph.d.ts","sourceRoot":"","sources":["../../src/helpers/buildUmapGraph.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAEzD,OAA6B,EAC3B,KAAK,oBAAoB,EAC1B,MAAM,2BAA2B,CAAC;AAInC,MAAM,MAAM,UAAU,GAAG,IAAI,CAAC,oBAAoB,EAAE,WAAW,CAAC,GAAG;IACjE,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,wBAA8B,cAAc,CAC1C,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,SAAS,EAAE,MAAM,CAAA;CAAE,EAC/D,OAAO,EAAE;IAAE,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAAC,MAAM,EAAE,OAAO,GAAG,MAAM,CAAA;CAAE,EACrE,KAAK,EAAE,UAAU;;;;GA2ClB"}

package/script/helpers/buildUmapGraph.js

@@ -0,0 +1,67 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildUmapGraph;
+const readScalarNumber_js_1 = __importDefault(require("./readScalarNumber.js"));
+const buildVectorNeighbors_js_1 = __importDefault(require("./buildVectorNeighbors.js"));
+async function buildUmapGraph(connection, input, options, names) {
+    await (0, buildVectorNeighbors_js_1.default)(connection, {
+        count: input.count,
+        dimensions: input.dimensions,
+        neighborCount: input.neighbors,
+    }, { ...options, includeSelf: true }, { ...names, neighbors: names.knn });
+    await buildFuzzyGraph(connection, input.neighbors, names);
+    const edges = await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.graph}`);
+    const source = new Uint32Array(edges), target = new Uint32Array(edges), weight = new Float64Array(edges);
+    const rows = await connection.stream(`SELECT source,target,weight FROM ${names.graph} ORDER BY source,target`);
+    let edge = 0;
+    while (true) {
+        const chunk = await rows.fetchChunk();
+        if (!chunk || chunk.rowCount === 0)
+            break;
+        chunk.visitColumnValues(0, (value, row) => source[edge + row] = Number(value));
+        chunk.visitColumnValues(1, (value, row) => target[edge + row] = Number(value));
+        chunk.visitColumnValues(2, (value, row) => weight[edge + row] = Number(value));
+        edge += chunk.rowCount;
+    }
+    if (edge !== edges)
+        throw new Error("Graph changed during transfer.");
+    return { source, target, weight };
+}
+async function buildFuzzyGraph(connection, neighbors, names) {
+    // Smooth-kNN with local_connectivity=1, bandwidth=1, 64 bisection steps,
+    // tolerance 1e-5 and the reference's 1e-3 minimum-distance scale.
+    await connection.run(`CREATE OR REPLACE TEMP TABLE ${names.scales} AS
+    SELECT source, coalesce(min(distance) FILTER(WHERE distance>0),0) AS rho,
+      avg(distance) AS mean_distance, 0::DOUBLE AS lo, 'Infinity'::DOUBLE AS hi,
+      1::DOUBLE AS sigma, false AS done FROM ${names.knn} GROUP BY source`);
+    for (let iteration = 0; iteration < 64; iteration++) {
+        await connection.run(`CREATE OR REPLACE TEMP TABLE ${names.scales} AS
+      WITH sums AS (
+        SELECT s.*, (SELECT sum(CASE WHEN n.distance <= s.rho THEN 1
+          ELSE exp(-(n.distance-s.rho)/s.sigma) END)
+          FROM ${names.knn} n WHERE n.source=s.source AND n.target != n.source) AS mass
+        FROM ${names.scales} s
+      ), bounds AS (
+        SELECT * EXCLUDE(lo,hi,done),
+          CASE WHEN NOT done AND mass < log2(${neighbors}) THEN sigma ELSE lo END AS lo,
+          CASE WHEN NOT done AND mass > log2(${neighbors}) THEN sigma ELSE hi END AS hi,
+          done OR abs(mass-log2(${neighbors})) < 1e-5 AS done FROM sums
+      ) SELECT source,rho,mean_distance,lo,hi,
+        CASE WHEN done THEN sigma WHEN isinf(hi) THEN sigma*2 ELSE (lo+hi)/2 END AS sigma,
+        done FROM bounds`);
+    }
+    await connection.run(`UPDATE ${names.scales} SET sigma=greatest(sigma,0.001 *
+    CASE WHEN rho>0 THEN mean_distance ELSE (SELECT avg(distance) FROM ${names.knn}) END);
+    CREATE OR REPLACE TEMP TABLE ${names.directed} AS
+    SELECT n.source,n.target,CASE WHEN n.distance<=s.rho OR s.sigma=0 THEN 1
+      ELSE exp(-(n.distance-s.rho)/s.sigma) END AS weight
+    FROM ${names.knn} n JOIN ${names.scales} s USING(source) WHERE n.source != n.target;
+    CREATE OR REPLACE TEMP TABLE ${names.graph} AS
+    SELECT coalesce(a.source,b.target) AS source, coalesce(a.target,b.source) AS target,
+      coalesce(a.weight,0)+coalesce(b.weight,0)-coalesce(a.weight*b.weight,0) AS weight
+    FROM ${names.directed} a FULL OUTER JOIN ${names.directed} b
+      ON a.source=b.target AND a.target=b.source`);
+}

package/script/helpers/buildVectorNeighbors.d.ts

@@ -0,0 +1,45 @@
+import type { DuckDBConnection } from "@duckdb/node-api";
+/** Private, already-quoted relation names owned and cleaned up by the caller. */
+export type VectorNeighborTables = {
+    rows: string;
+    neighbors: string;
+    search: string;
+    hnsw: string;
+    candidates: string;
+};
+type NeighborInput = {
+    count: number;
+    dimensions: number;
+    neighborCount: number;
+};
+type NeighborOptions = {
+    metric: "euclidean" | "cosine";
+    search: "exact" | "hnsw";
+    includeSelf: boolean;
+    hnsw?: {
+        efConstruction: number;
+        efSearch: number;
+        connectivity: number;
+        candidateCount: number;
+        singleThreaded: boolean;
+        cosineAsL2: boolean;
+    };
+};
+/**
+ * Materialize ranked vector neighbors without moving source vectors into JS.
+ *
+ * `neighborCount` is the number of rows produced per source. When `includeSelf`
+ * is true, the source is rank 0 and consumes one of those rows. Otherwise all
+ * rows are other points. Ranks are zero-based and ties use target id order.
+ * Input rows have unique contiguous `vertex` ids from 0 through count-1 and
+ * validated, non-null, fixed-size DOUBLE `vec` values. Cosine preparation may
+ * rescale this private vec column, preserving direction.
+ *
+ * The caller owns every supplied scratch relation and must drop them after
+ * success or failure. Approximate search only approximates candidate retrieval;
+ * candidate distances are recomputed from the DOUBLE source vectors before
+ * stable final ranking.
+ */
+export default function buildVectorNeighbors(connection: DuckDBConnection, input: NeighborInput, options: NeighborOptions, names: VectorNeighborTables): Promise<void>;
+export {};
+//# sourceMappingURL=buildVectorNeighbors.d.ts.map

package/script/helpers/buildVectorNeighbors.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildVectorNeighbors.d.ts","sourceRoot":"","sources":["../../src/helpers/buildVectorNeighbors.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,kBAAkB,CAAC;AAMzD,iFAAiF;AACjF,MAAM,MAAM,oBAAoB,GAAG;IACjC,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF,KAAK,aAAa,GAAG;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,EAAE,MAAM,CAAC;CACvB,CAAC;AAEF,KAAK,eAAe,GAAG;IACrB,MAAM,EAAE,WAAW,GAAG,QAAQ,CAAC;IAC/B,MAAM,EAAE,OAAO,GAAG,MAAM,CAAC;IACzB,WAAW,EAAE,OAAO,CAAC;IACrB,IAAI,CAAC,EAAE;QACL,cAAc,EAAE,MAAM,CAAC;QACvB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,cAAc,EAAE,MAAM,CAAC;QACvB,cAAc,EAAE,OAAO,CAAC;QACxB,UAAU,EAAE,OAAO,CAAC;KACrB,CAAC;CACH,CAAC;AAEF;;;;;;;;;;;;;;GAcG;AACH,wBAA8B,oBAAoB,CAChD,UAAU,EAAE,gBAAgB,EAC5B,KAAK,EAAE,aAAa,EACpB,OAAO,EAAE,eAAe,EACxB,KAAK,EAAE,oBAAoB,GAC1B,OAAO,CAAC,IAAI,CAAC,CAqHf"}

package/script/helpers/buildVectorNeighbors.js

@@ -0,0 +1,176 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildVectorNeighbors;
+const readScalarNumber_js_1 = __importDefault(require("./readScalarNumber.js"));
+const vectorDistanceExpression_js_1 = __importDefault(require("./vectorDistanceExpression.js"));
+const stabilizeCosineVectors_js_1 = __importDefault(require("./stabilizeCosineVectors.js"));
+const validateVectorRowIds_js_1 = __importDefault(require("./validateVectorRowIds.js"));
+/**
+ * Materialize ranked vector neighbors without moving source vectors into JS.
+ *
+ * `neighborCount` is the number of rows produced per source. When `includeSelf`
+ * is true, the source is rank 0 and consumes one of those rows. Otherwise all
+ * rows are other points. Ranks are zero-based and ties use target id order.
+ * Input rows have unique contiguous `vertex` ids from 0 through count-1 and
+ * validated, non-null, fixed-size DOUBLE `vec` values. Cosine preparation may
+ * rescale this private vec column, preserving direction.
+ *
+ * The caller owns every supplied scratch relation and must drop them after
+ * success or failure. Approximate search only approximates candidate retrieval;
+ * candidate distances are recomputed from the DOUBLE source vectors before
+ * stable final ranking.
+ */
+async function buildVectorNeighbors(connection, input, options, names) {
+    const { count, dimensions, neighborCount } = input;
+    await (0, validateVectorRowIds_js_1.default)(connection, names.rows, count);
+    if (!Number.isSafeInteger(dimensions) || dimensions < 1) {
+        throw new Error("Vector dimensions must be a positive safe integer.");
+    }
+    const maximum = count - (options.includeSelf ? 0 : 1);
+    if (!Number.isSafeInteger(neighborCount) || neighborCount < 1 ||
+        neighborCount > maximum) {
+        throw new Error(`Neighbor count must be a safe integer between 1 and ${maximum}.`);
+    }
+    if (options.metric === "cosine") {
+        await (0, stabilizeCosineVectors_js_1.default)(connection, names.rows);
+    }
+    // FLOAT representation limits apply only to the HNSW search vectors.
+    // Exact Euclidean distance does not require representable squared norms.
+    if (options.search === "hnsw") {
+        const normExpression = `array_inner_product(vec::FLOAT[${dimensions}],vec::FLOAT[${dimensions}])`;
+        if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.rows} WHERE NOT isfinite(${normExpression})
+          ${options.metric === "cosine" ? `OR ${normExpression}=0` : ""}`)) {
+            throw new Error("HNSW requires vectors with finite FLOAT norms (and nonzero cosine norms).");
+        }
+    }
+    const distanceFunction = options.metric === "cosine"
+        ? "array_cosine_distance"
+        : "array_distance";
+    // Equal vectors must have exactly zero cosine distance. Positive roundoff can
+    // otherwise alter downstream density estimates. Clamp negative roundoff too.
+    const exactDistance = (0, vectorDistanceExpression_js_1.default)("a.vec", "b.vec", options.metric);
+    const otherCount = neighborCount - (options.includeSelf ? 1 : 0);
+    await connection.run(`CREATE OR REPLACE TEMP TABLE ${names.candidates}
+    (source INTEGER, target INTEGER, distance DOUBLE)`);
+    if (otherCount > 0) {
+        if (options.search === "exact") {
+            // O(n²d) work with a bounded source batch and top-k heaps. The full
+            // distance matrix is never materialized.
+            await buildExactCandidates(connection, input, names, exactDistance, otherCount);
+        }
+        else {
+            await buildHnswCandidates(connection, input, options, names, distanceFunction, exactDistance, otherCount);
+        }
+    }
+    if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.candidates} WHERE distance IS NULL OR NOT isfinite(distance)`)) {
+        throw new Error("Distance computation returned a non-finite value.");
+    }
+    const selfRows = options.includeSelf
+        ? `SELECT vertex AS source,vertex AS target,0::DOUBLE AS distance FROM ${names.rows}`
+        : "SELECT NULL::INTEGER AS source,NULL::INTEGER AS target,NULL::DOUBLE AS distance WHERE false";
+    const rankOrder = options.includeSelf
+        ? "CASE WHEN source=target THEN 0 ELSE 1 END,distance,target"
+        : "distance,target";
+    await connection.run(`CREATE OR REPLACE TEMP TABLE ${names.neighbors} AS
+    SELECT source,target,
+      (row_number() OVER (PARTITION BY source ORDER BY ${rankOrder})-1)::INTEGER AS rank,
+      greatest(0,distance)::DOUBLE AS distance
+    FROM (${selfRows} UNION ALL
+      SELECT source,target,distance FROM ${names.candidates})
+    QUALIFY row_number() OVER (PARTITION BY source ORDER BY ${rankOrder})
+      <= ${neighborCount}`);
+    const produced = await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.neighbors}`);
+    if (produced !== count * neighborCount) {
+        throw new Error(`Neighbor search returned ${produced} rows; ${count * neighborCount} were required.`);
+    }
+    if (await (0, readScalarNumber_js_1.default)(connection, `SELECT count(*) FROM ${names.neighbors} WHERE NOT isfinite(distance)`)) {
+        throw new Error("Distance computation returned a non-finite value.");
+    }
+}
+async function buildHnswCandidates(connection, input, options, names, distanceFunction, exactDistance, otherCount) {
+    const { count, dimensions, neighborCount } = input;
+    const settings = options.hnsw;
+    const cosineAsL2 = options.metric === "cosine" && settings?.cosineAsL2;
+    // Euclidean distance on unit vectors has the same mathematical ordering as
+    // cosine. Computing differences avoids FLOAT dot-product cancellation for
+    // near-collinear vectors; final metric distances still use source DOUBLEs.
+    const searchVector = cosineAsL2
+        ? "list_transform(vec,x -> x/sqrt(array_inner_product(vec,vec)))"
+        : "vec";
+    const previousThreads = Number((await connection.runAndReadAll("SELECT current_setting('threads')")).getRowsJS()[0][0]);
+    try {
+        if (settings?.singleThreaded)
+            await connection.run("SET threads=1");
+        await connection.run(`INSTALL vss; LOAD vss;
+      CREATE OR REPLACE TEMP TABLE ${names.search} AS
+      SELECT vertex,${searchVector}::FLOAT[${dimensions}] AS vec
+      FROM ${names.rows} ORDER BY vertex;
+      CREATE INDEX ${names.hnsw} ON ${names.search} USING HNSW(vec)
+      WITH (metric='${options.metric === "cosine" && !cosineAsL2 ? "cosine" : "l2sq"}',
+        ef_construction=${settings?.efConstruction ?? 128},
+        ef_search=${settings?.efSearch ?? 128},M=${settings?.connectivity ?? 16})`);
+    }
+    finally {
+        if (settings?.singleThreaded) {
+            await connection.run(`SET threads=${previousThreads}`);
+        }
+    }
+    const previous = String((await connection.runAndReadAll("SELECT current_setting('disabled_optimizers')")).getRowsJS()[0][0]);
+    const disabled = [
+        ...new Set([
+            ...previous.split(",").filter(Boolean),
+            "top_n_window_elimination",
+        ]),
+    ].join(",");
+    try {
+        await connection.run(`SET disabled_optimizers='${disabled.replaceAll("'", "''")}'`);
+        // Preserve UMAP's established candidate budget when self is included.
+        // Excluding self needs one extra result because HNSW normally returns it.
+        const candidateCount = settings?.candidateCount ??
+            (options.includeSelf
+                ? Math.max(otherCount + 1, Math.min(neighborCount + 1, count - 1))
+                : Math.min(otherCount + 1, count));
+        const candidateDistanceFunction = cosineAsL2
+            ? "array_distance"
+            : distanceFunction;
+        const query = `SELECT a.vertex AS source,b.vertex AS target,b.distance
+      FROM ${names.search} a,LATERAL (
+        SELECT vertex,${candidateDistanceFunction}(vec,a.vec) AS distance
+        FROM ${names.search} ORDER BY distance LIMIT ${candidateCount}
+      ) b`;
+        // Carry ids and scalar distances through the candidate relation. Recompute
+        // in DOUBLE against source rows to isolate ANN retrieval from precision.
+        const candidates = `SELECT q.source,q.target,${exactDistance} AS distance
+      FROM (${query}) q JOIN ${names.rows} a ON q.source=a.vertex
+      JOIN ${names.rows} b ON q.target=b.vertex WHERE q.source != q.target`;
+        const plan = JSON.stringify((await connection.runAndReadAll(`EXPLAIN ${candidates}`)).getRowsJS());
+        if (!plan.includes("HNSW_INDEX_JOIN")) {
+            if (count <= 1000) {
+                return await buildExactCandidates(connection, input, names, exactDistance, otherCount);
+            }
+            throw new Error("DuckDB could not use its vector index for neighbor search.");
+        }
+        await connection.run(`INSERT INTO ${names.candidates} ${candidates}`);
+    }
+    finally {
+        await connection.run(`SET disabled_optimizers='${previous.replaceAll("'", "''")}'`);
+    }
+}
+async function buildExactCandidates(connection, input, names, exactDistance, otherCount) {
+    for (let offset = 0; offset < input.count; offset += 32) {
+        const query = `SELECT source,item.target,item.distance FROM (
+      SELECT a.vertex AS source,
+        CASE WHEN bool_and(isfinite(${exactDistance})) THEN
+        min_by(struct_pack(target := b.vertex,distance := ${exactDistance}),
+          struct_pack(distance := ${exactDistance},target := b.vertex),
+          ${otherCount}) ELSE error('Distance computation returned a non-finite value.') END AS closest
+      FROM ${names.rows} a CROSS JOIN ${names.rows} b
+      WHERE a.vertex >= ${offset} AND a.vertex < ${offset + 32}
+        AND a.vertex != b.vertex GROUP BY a.vertex
+      ),UNNEST(closest) AS t(item)`;
+        await connection.run(`INSERT INTO ${names.candidates} ${query}`);
+    }
+}

package/script/helpers/buildWeakGraphComponentsSql.d.ts

@@ -0,0 +1,3 @@
+/** Builds weak component labels and zero-based IDs from bidirectional edges. */
+export default function buildWeakGraphComponentsSql(edges: string, nodes: string, labels: string, roots: string): string;
+//# sourceMappingURL=buildWeakGraphComponentsSql.d.ts.map

package/script/helpers/buildWeakGraphComponentsSql.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"buildWeakGraphComponentsSql.d.ts","sourceRoot":"","sources":["../../src/helpers/buildWeakGraphComponentsSql.ts"],"names":[],"mappings":"AAEA,gFAAgF;AAChF,MAAM,CAAC,OAAO,UAAU,2BAA2B,CACjD,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACZ,MAAM,CA6BR"}

package/script/helpers/buildWeakGraphComponentsSql.js

@@ -0,0 +1,38 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.default = buildWeakGraphComponentsSql;
+const quoteIdentifier_js_1 = __importDefault(require("./quoteIdentifier.js"));
+/** Builds weak component labels and zero-based IDs from bidirectional edges. */
+function buildWeakGraphComponentsSql(edges, nodes, labels, roots) {
+    const q = quoteIdentifier_js_1.default;
+    return `${labels}(
+      ${q("node")}, ${q("__node_key")}, ${q("__root_key")}
+    ) USING KEY(${q("__node_key")}) AS (
+      SELECT ${q("node")}, ${q("__key")}, ${q("__key")}
+      FROM ${nodes}
+      UNION
+      SELECT ${q("edges")}.${q("__to")},
+        ${q("edges")}.${q("__to_key")},
+        MIN(${q("reached")}.${q("__root_key")})
+      FROM ${labels} AS ${q("reached")}
+      INNER JOIN ${edges} AS ${q("edges")}
+        ON ${q("reached")}.${q("__node_key")} =
+          ${q("edges")}.${q("__from_key")}
+      INNER JOIN recurring.${labels} AS ${q("best")}
+        ON ${q("edges")}.${q("__to_key")} =
+          ${q("best")}.${q("__node_key")}
+      GROUP BY ${q("edges")}.${q("__to")},
+        ${q("edges")}.${q("__to_key")}, ${q("best")}.${q("__root_key")}
+      HAVING MIN(${q("reached")}.${q("__root_key")}) <
+        ${q("best")}.${q("__root_key")}
+    ), ${roots} AS (
+      SELECT ${q("__root_key")},
+        CAST(row_number() OVER (ORDER BY ${q("__root_key")}) - 1 AS BIGINT)
+          AS ${q("componentId")}
+      FROM (SELECT DISTINCT ${q("__root_key")} FROM ${labels})
+        AS ${q("distinct_roots")}
+    )`;
+}

package/script/helpers/cacheDatabase.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"cacheDatabase.d.ts","sourceRoot":"","sources":["../../src/helpers/cacheDatabase.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAU7D,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,eAAe,EAAE,GACzB,OAAO,CAAC,IAAI,CAAC,CAyCf;AASD,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,eAAe,EAAE,EAC1B,GAAG,EAAE,OAAO,GACX,OAAO,CAAC,IAAI,CAAC,CAiCf"}
+{"version":3,"file":"cacheDatabase.d.ts","sourceRoot":"","sources":["../../src/helpers/cacheDatabase.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,WAAW,MAAM,yBAAyB,CAAC;AACvD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAU7D,wBAAsB,kBAAkB,CACtC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,eAAe,EAAE,GACzB,OAAO,CAAC,IAAI,CAAC,CA0Cf;AASD,wBAAsB,iBAAiB,CACrC,KAAK,EAAE,WAAW,EAClB,IAAI,EAAE,MAAM,EACZ,OAAO,EAAE,eAAe,EAAE,EAC1B,GAAG,EAAE,OAAO,GACX,OAAO,CAAC,IAAI,CAAC,CAiCf"}

package/script/helpers/cacheDatabase.js

@@ -18,7 +18,8 @@ async function writeCacheDatabase(table, file, indexes) {
     const persistedIndexes = indexes.filter((index) => (0, indexDefinitions_js_1.getIndexCacheStrategy)(index) === "persist");
     await loadIndexExtensions(table, persistedIndexes);
     try {
-        await table.sdb.customQuery(`ATTACH '${(0, cleanPath_js_1.default)(temporaryFile)}' AS ${cacheDatabase};
+        await table.sdb.customQuery(`ATTACH '${(0, cleanPath_js_1.default)(temporaryFile)}' AS ${cacheDatabase}
+        (STORAGE_VERSION 'latest');
       CREATE TABLE ${cacheDatabase}.${(0, quoteIdentifier_js_1.default)(table.name)} AS
         SELECT * FROM ${(0, quoteIdentifier_js_1.default)(table.name)};`);
         if (persistedIndexes.length > 0) {