memhtml 0.2.3 → 0.2.5

package/dist/{dist-dfrcDFad.mjs → dist-Uj47oBRC.mjs}

@@ -644,6 +644,139 @@ const applyMmr = (candidates, limit, lambda = MMR_LAMBDA) => {
 	return selected;
 };
 
+//#endregion
+//#region packages/domain/dist/neighbors.js
+/**
+* A `Float32Array` over stored bytes, copying only when it must.
+*
+* `Float32Array` requires a 4-byte-aligned `byteOffset`, and a driver row's `Uint8Array` may be
+* a view into a pooled buffer at any offset. Viewing in place is the common case and costs
+* nothing. A misaligned or ragged blob is copied rather than rejected, because the vector arm's
+* job is to rank and a throw here would fail a whole search over one row.
+*/
+const float32View = (bytes) => {
+	if (bytes.byteLength === 0 || bytes.byteLength % 4 !== 0) return void 0;
+	const aligned = bytes.byteOffset % 4 === 0 ? bytes : Uint8Array.from(bytes);
+	return new Float32Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 4);
+};
+/** `sqrt(Σ x²)` accumulated in index order — the same order {@link cosine} accumulates its norms. */
+const sqrtNorm = (vec) => {
+	let sum = 0;
+	for (let index = 0; index < vec.length; index += 1) {
+		const x = vec[index];
+		sum += x * x;
+	}
+	return Math.sqrt(sum);
+};
+/**
+* Cosine similarity from precomputed norms: {@link cosine}'s operations with the square roots
+* hoisted out of the pair loop, so the result is bit-identical for equal-length vectors. The
+* zero-norm rule and the `[-1, 1]` clamp are the same ones, for the same reasons. Mismatched
+* lengths fall back to {@link cosine}, whose min-length walk defines that case; equal lengths are
+* what the `embed_model` watermark guarantees for stored vectors.
+*/
+const pairSimilarity = (a, aNorm, b, bNorm) => {
+	if (a.length !== b.length) return cosine(a, b);
+	if (aNorm === 0 || bNorm === 0) return 0;
+	let dot = 0;
+	for (let index = 0; index < a.length; index += 1) dot += a[index] * b[index];
+	return Math.max(-1, Math.min(1, dot / (aNorm * bNorm)));
+};
+/**
+* Insert into a per-source list ordered `sim` DESC then `dst` ASC, bounded at `k`. Linear
+* insertion, because `k` is single-digit everywhere this runs and a heap's constant factors lose
+* at that size. Memory across the whole selection is O(n·k), never the pair space.
+*/
+const insertBounded = (list, k, dst, sim) => {
+	let at = list.length;
+	for (let index = 0; index < list.length; index += 1) {
+		const held = list[index];
+		if (sim > held.sim || sim === held.sim && dst < held.dst) {
+			at = index;
+			break;
+		}
+	}
+	if (at >= k) return;
+	list.splice(at, 0, {
+		dst,
+		sim
+	});
+	if (list.length > k) list.pop();
+};
+/** The final ordering every consumer sees: `sim` DESC, then `src` ASC, then `dst` ASC, then cap. */
+const collectRanked = (bySource, limit) => {
+	const rows = [];
+	for (const [src, list] of bySource) for (const held of list) rows.push({
+		src,
+		dst: held.dst,
+		sim: held.sim
+	});
+	rows.sort((left, right) => {
+		if (left.sim !== right.sim) return left.sim < right.sim ? 1 : -1;
+		if (left.src !== right.src) return left.src < right.src ? -1 : 1;
+		return left.dst < right.dst ? -1 : left.dst > right.dst ? 1 : 0;
+	});
+	return rows.slice(0, limit);
+};
+/**
+* Per-source top-`k` nearest neighbors above a similarity floor, over every unordered pair.
+*
+* Each pair's similarity is computed ONCE and offered to BOTH endpoints' neighborhoods, so the
+* output can hold `(a, b)` and `(b, a)` — each is a fact about a different source's neighborhood,
+* and a consumer folding pairs must dedup the mirror itself (dedup-merge does, with its `seen`
+* set). A floor comparison a NaN similarity cannot pass keeps a vector carrying NaN bytes out of
+* every neighborhood rather than poisoning an ordering.
+*/
+const topNeighborPairs = (vectors, options) => {
+	const norms = vectors.map((entry) => sqrtNorm(entry.vec));
+	const bySource = /* @__PURE__ */ new Map();
+	for (const entry of vectors) bySource.set(entry.key, []);
+	for (let i = 0; i < vectors.length; i += 1) {
+		const left = vectors[i];
+		const leftNorm = norms[i];
+		const leftList = bySource.get(left.key);
+		for (let j = i + 1; j < vectors.length; j += 1) {
+			const right = vectors[j];
+			const sim = pairSimilarity(left.vec, leftNorm, right.vec, norms[j]);
+			if (!(sim >= options.floor)) continue;
+			insertBounded(leftList, options.perSourceK, right.key, sim);
+			insertBounded(bySource.get(right.key), options.perSourceK, left.key, sim);
+		}
+	}
+	return collectRanked(bySource, options.limit);
+};
+/**
+* Rank an ENUMERATED pair set: similarity, floor, per-source top-`k`, final ordering, cap.
+*
+* This is the shape for a consumer whose candidate pairs come from a selective predicate — the
+* conflict scan's shared-entity join — rather than from the whole pair space. The predicate runs
+* BEFORE ranking, exactly as a `WHERE` inside the ranking CTE would, so per-source top-`k` is
+* computed over passing pairs only. A pair naming a key with no vector contributes nothing, the
+* same outcome the SQL join's missing-embedding row produces.
+*/
+const rankCandidatePairs = (pairs, vectors, options) => {
+	const byKey = /* @__PURE__ */ new Map();
+	for (const entry of vectors) byKey.set(entry.key, {
+		vec: entry.vec,
+		norm: sqrtNorm(entry.vec)
+	});
+	const bySource = /* @__PURE__ */ new Map();
+	for (const pair of pairs) {
+		const left = byKey.get(pair.src);
+		const right = byKey.get(pair.dst);
+		if (left === void 0 || right === void 0) continue;
+		const sim = pairSimilarity(left.vec, left.norm, right.vec, right.norm);
+		if (!(sim >= options.floor)) continue;
+		let list = bySource.get(pair.src);
+		if (list === void 0) {
+			list = [];
+			bySource.set(pair.src, list);
+		}
+		insertBounded(list, options.perSourceK, pair.dst, sim);
+	}
+	return collectRanked(bySource, options.limit);
+};
+
 //#endregion
 //#region packages/domain/dist/reinforce.js
 /** The signal a reinforcement carries. `negative` is what drives the outcome EWMA down. */
@@ -5487,19 +5620,6 @@ const packSentences = (units, maxChars) => {
 */
 const BUSY_TIMEOUT_MS = 5e3;
 /**
-* A `Float32Array` over stored bytes, copying only when it must.
-*
-* `Float32Array` requires a 4-byte-aligned `byteOffset`, and a driver row's `Uint8Array` may be
-* a view into a pooled buffer at any offset. Viewing in place is the common case and costs
-* nothing. A misaligned or ragged blob is copied rather than rejected, because the vector arm's
-* job is to rank and a throw here would fail a whole search over one row.
-*/
-const float32View = (bytes) => {
-	if (bytes.byteLength === 0 || bytes.byteLength % 4 !== 0) return void 0;
-	const aligned = bytes.byteOffset % 4 === 0 ? bytes : Uint8Array.from(bytes);
-	return new Float32Array(aligned.buffer, aligned.byteOffset, aligned.byteLength / 4);
-};
-/**
 * Register `vector_distance_cos(a, b)`, the cosine distance over two float32 blobs.
 *
 * SQLite ships no vector functions, so the vector retrieval arm's distance is this. It calls
@@ -5513,6 +5633,12 @@ const float32View = (bytes) => {
 * vectors at top-40 (probed 2026-08-12 on node 24.19.0), against a Bedrock query-embedding round
 * trip of a few hundred milliseconds that every vector search pays first. An approximate index buys
 * nothing until the corpus is an order of magnitude larger.
+*
+* That measurement is the 1×n shape — one bound query vector against the table, n calls, n blob
+* copies — and it is the ONLY shape this function serves. Each invocation materializes a fresh
+* `Uint8Array` per blob argument, so an n×n consumer pays the corpus re-copied n times (probed
+* 2026-08-18, issue #40: 8.45M calls and an OOM at n = 2,907). The sleep pair scans decode once
+* and rank in `@memhtml/domain`'s neighbors module instead.
 */
 const registerVectorDistance = (db) => {
 	db.function("vector_distance_cos", { deterministic: true }, (a, b) => {
@@ -9080,8 +9206,10 @@ const isSleepExcluded = (memoryType) => SLEEP_EXCLUDED_TYPES.includes(memoryType
 const activeCorpus = (db) => db.all(`SELECT path, memory_type, title, gist, body_text, content_hash, confidence, importance,
             word_count, created_at, updated_at, valid_until, reprieves
      FROM files WHERE archived = 0 ORDER BY created_at ASC, path ASC`);
+/** A `memory_type NOT IN (…)` clause against alias `f`, or nothing when nothing is excluded. */
+const typeFilterFor = (alias, excluded) => excluded.length === 0 ? "" : ` AND ${alias}.memory_type NOT IN (${excluded.map(() => "?").join(", ")})`;
 /**
-* Per-source top-`k` nearest neighbors above a similarity floor, over first-chunk vectors.
+* Every active file's first-chunk vector, decoded ONCE into the shape the pair kernel ranks.
 *
 * `ordinal = 0` collapses a file to its first chunk, not its best chunk. The format is one
 * fact per file, so almost every file is a single chunk. Taking the first keeps the pair set
@@ -9089,36 +9217,35 @@ const activeCorpus = (db) => db.all(`SELECT path, memory_type, title, gist, body
 * would make `(a, b)` a candidate while `(b, a)` is not, so which of two files was read first
 * would decide whether they merge.
 *
-* `ROW_NUMBER() OVER (PARTITION BY src ...)` is the per-source cap. `vector_distance_cos` takes two
-* STORED blobs here instead of a blob and a bound parameter. It can, because it is a registered
-* SQL function over two `Uint8Array` arguments (`packages/index/src/database.ts`) and not a driver
-* builtin with a fixed calling shape.
+* A row whose blob does not decode (empty or ragged) is dropped, the same exclusion the SQL
+* UDF's NULL produces for it in the retrieval arm.
 */
-const neighborPairs = (db, options) => {
-	const excluded = options.excludeTypes ?? [];
-	const typeFilter = excluded.length === 0 ? "" : ` AND f.memory_type NOT IN (${excluded.map(() => "?").join(", ")})`;
-	return db.all(`WITH vecs AS (
-       SELECT f.path AS path, e.vec AS vec
+const firstChunkVectors = (db, excluded) => db.all(`SELECT f.path AS path, e.vec AS vec
        FROM files f
        JOIN chunks c ON c.path = f.path AND c.ordinal = 0
        JOIN embeddings e ON e.chunk_id = c.chunk_id
-       WHERE f.archived = 0${typeFilter}
-     ),
-     pairs AS (
-       SELECT l.path AS src, r.path AS dst, 1 - vector_distance_cos(l.vec, r.vec) AS sim
-       FROM vecs l JOIN vecs r ON r.path <> l.path
-     ),
-     ranked AS (
-       SELECT src, dst, sim, ROW_NUMBER() OVER (PARTITION BY src ORDER BY sim DESC, dst ASC) AS k
-       FROM pairs WHERE sim >= ?
-     )
-     SELECT src, dst, sim FROM ranked WHERE k <= ? ORDER BY sim DESC, src ASC, dst ASC LIMIT ?`, [
-		...excluded,
-		options.floor,
-		options.perSourceK,
-		options.limit
-	]);
-};
+       WHERE f.archived = 0${typeFilterFor("f", excluded)}`, [...excluded]).pipe(Effect.map((rows) => rows.flatMap((row) => {
+	const vec = float32View(row.vec);
+	return vec === void 0 ? [] : [{
+		key: row.path,
+		vec
+	}];
+})));
+/**
+* Per-source top-`k` nearest neighbors above a similarity floor, over first-chunk vectors.
+*
+* The corpus filter is SQL, because that is where the index's reading semantics live. The pair
+* space is n² and ranks in TypeScript (`topNeighborPairs`), because a pair routed through the
+* `vector_distance_cos` UDF pays a fresh decode of BOTH 4 KB blobs per call — at a ~3k corpus
+* that is 8.45M calls and an OOM before the first phase records (issue #40), against ~12 MB
+* decoded once. The kernel reproduces this ordering exactly: floor, then per-source `sim` DESC /
+* `dst` ASC, then global `sim` DESC / `src` ASC / `dst` ASC, then the cap.
+*/
+const neighborPairs = (db, options) => firstChunkVectors(db, options.excludeTypes ?? []).pipe(Effect.map((vectors) => topNeighborPairs(vectors, {
+	floor: options.floor,
+	perSourceK: options.perSourceK,
+	limit: options.limit
+})));
 /**
 * Candidate pairs for conflict detection: embedding-near, sharing an entity, and carrying no
 * AUTHORED edge between them in either direction.
@@ -9133,42 +9260,32 @@ const neighborPairs = (db, options) => {
 * pairs above the 0.80 conflict floor. An anti-join over ALL edges therefore excludes every candidate
 * this phase exists to find, and the phase reports `candidates: 0` forever with no error anywhere.
 * A mined edge is a machine suspicion, not a settled relationship; only an authored one closes a pair.
+*
+* The statement ENUMERATES pairs from the shared-entity join instead of filtering an n×n vector
+* self-join, so its cost follows the entity sharing that actually exists. Similarity then ranks in
+* TypeScript over vectors decoded once (`rankCandidatePairs`), with the enumerated set standing
+* where the ranking CTE's `WHERE` stood: the predicates run BEFORE per-source top-`k`. `re.path <
+* le.path` orients each pair once, dst below src.
 */
 const conflictCandidates = (db, options) => {
 	const excluded = options.excludeTypes ?? [];
-	const typeFilter = excluded.length === 0 ? "" : ` AND f.memory_type NOT IN (${excluded.map(() => "?").join(", ")})`;
-	return db.all(`WITH vecs AS (
-       SELECT f.path AS path, e.vec AS vec
-       FROM files f
-       JOIN chunks c ON c.path = f.path AND c.ordinal = 0
-       JOIN embeddings e ON e.chunk_id = c.chunk_id
-       WHERE f.archived = 0${typeFilter}
-     ),
-     pairs AS (
-       SELECT l.path AS src, r.path AS dst, 1 - vector_distance_cos(l.vec, r.vec) AS sim
-       FROM vecs l JOIN vecs r ON r.path < l.path
-       WHERE EXISTS (
-         SELECT 1 FROM file_entities le
-         JOIN file_entities re ON re.entity_type = le.entity_type AND re.entity_name = le.entity_name
-         WHERE le.path = l.path AND re.path = r.path
-       )
-       AND NOT EXISTS (
-         SELECT 1 FROM edges e
-         WHERE e.derived = 0
-           AND ((e.src_path = l.path AND e.dst_path = r.path)
-             OR (e.src_path = r.path AND e.dst_path = l.path))
-       )
-     ),
-     ranked AS (
-       SELECT src, dst, sim, ROW_NUMBER() OVER (PARTITION BY src ORDER BY sim DESC, dst ASC) AS k
-       FROM pairs WHERE sim >= ?
-     )
-     SELECT src, dst, sim FROM ranked WHERE k <= ? ORDER BY sim DESC, src ASC, dst ASC LIMIT ?`, [
-		...excluded,
-		options.floor,
-		options.perSourceK,
-		options.limit
-	]);
+	const pairs = db.all(`SELECT DISTINCT le.path AS src, re.path AS dst
+     FROM file_entities le
+     JOIN file_entities re ON re.entity_type = le.entity_type
+       AND re.entity_name = le.entity_name AND re.path < le.path
+     JOIN files fl ON fl.path = le.path AND fl.archived = 0${typeFilterFor("fl", excluded)}
+     JOIN files fr ON fr.path = re.path AND fr.archived = 0${typeFilterFor("fr", excluded)}
+     WHERE NOT EXISTS (
+       SELECT 1 FROM edges e
+       WHERE e.derived = 0
+         AND ((e.src_path = le.path AND e.dst_path = re.path)
+           OR (e.src_path = re.path AND e.dst_path = le.path))
+     )`, [...excluded, ...excluded]);
+	return Effect.all([pairs, firstChunkVectors(db, excluded)]).pipe(Effect.map(([candidatePairs, vectors]) => rankCandidatePairs(candidatePairs, vectors, {
+		floor: options.floor,
+		perSourceK: options.perSourceK,
+		limit: options.limit
+	})));
 };
 /**
 * Every entity on an active NON-TASK file, with its file count. The union-find's input.
@@ -10716,7 +10833,11 @@ const preflight = (env) => Effect.gen(function* () {
 const MINING_COSINE_FLOOR = .85;
 /** Nearest neighbors considered per source file. */
 const MINING_PER_SOURCE_K = 5;
-/** Pairs mined per cycle. The cost guard on a corpus whose pair space is quadratic. */
+/**
+* Pairs mined per cycle: a cap on what {@link replaceMinedEdges} writes, not on the scan — the
+* kernel's arithmetic is O(n²·d) whatever this says, and it bounds the edge table so one dense
+* neighborhood cannot flood the graph the lateral arm and PageRank read.
+*/
 const MINING_SAMPLE_LIMIT = 2e3;
 const relationshipMining = (env) => Effect.gen(function* () {
 	/**
@@ -12847,4 +12968,4 @@ const latest = (left, right) => left === null ? right : right === null ? left :
 
 //#endregion
 export { STATE_SIDECAR_PATH as $, makeIndexRecorder as A, makeGitPort as B, ModelClientLive as C, EMBED_DIM as D, EmbeddingsLive as E, reinforce as F, STATE_MIGRATIONS_DIR as G, DatabaseService as H, Indexer as I, expandRoot as J, STATE_SCHEMA as K, makeIndexer as L, readWatermark as M, Retrieval as N, EMBED_WATERMARK as O, makeRetrieval as P, STATE_DB_PATH as Q, readIndexState as R, ModelClient as S, Embeddings as T, makeDatabase as U, sanitizeFtsQuery as V, MIGRATIONS_DIR as W, INDEX_DB_PATH as X, makeStore as Y, SLEEP_REPORTS_DIR as Z, unlink as _, parseSidecar as a, commitSubject as at, discriminationGate as b, generateArtifacts as c, isValidDatetime as ct, danglingEdges as d, REINFORCE_SIGNALS as dt, attemptIo as et, publishRows as f, frameKeyOf as ft, meta as g, link as h, makeSleep as i, makeGit as it, persistScanned as j, IndexRecorder as k, accessRows as l, closesFence as lt, hrefFor as m, scanTraceRoot as n, readFileOrNull as nt, renderSidecar as o, checkMemory as ot, applyHeadEdits as p, Store as q, Sleep as r, Git as rt, archivedFormOf as s, setMeta as st, mergeTailExtract as t, initRepo as tt, allPaths as u, fenceOpeningOf as ut, SLEEP_PHASES as v, wrapAsData as w, runDiscrimination as x, isSleepPhase as y, IndexGit as z };
-//# sourceMappingURL=dist-dfrcDFad.mjs.map
+//# sourceMappingURL=dist-Uj47oBRC.mjs.map