@remnic/coding-graph 9.3.759

package/src/semantic/index.ts

@@ -0,0 +1,78 @@
+/**
+ * Semantic layer barrel (issue #1556).
+ *
+ * One import surface for the optional semantic layer:
+ *   - config (SemanticConfig + resolveSemanticConfig)
+ *   - canonical-text (buildCanonicalText + hash)
+ *   - minhash (MinHasher + cosineSimilarity)
+ *   - vectors (indexSymbolVectors)
+ *   - similarity (computeSimilarTo + similarEdgesToEdgeIR)
+ *   - semantic-query (semanticQuery)
+ *   - types (tagged results)
+ */
+export {
+  DEFAULT_SIMILAR_TO_THRESHOLD,
+  DEFAULT_MAX_SYMBOLS_PER_RUN,
+  MINHASH_ONLY_CONFIDENCE,
+  SIMILAR_TO_EDGE_TYPE,
+  SEMANTIC_PROVENANCE,
+  DEFAULT_CANONICAL_BODY_LINES,
+  resolveSemanticConfig,
+  type SemanticConfig,
+} from "./config.js";
+
+export {
+  buildCanonicalText,
+  buildCanonicalTextAndHash,
+  canonicalTextHash,
+  collapseWhitespace,
+  extractSignatureLine,
+  extractBodyText,
+  type CanonicalTextInput,
+} from "./canonical-text.js";
+
+export {
+  MinHasher,
+  createMinHasher,
+  minHashSignature,
+  lshBandKeys,
+  shingleSet,
+  tokenizeForShingling,
+  cosineSimilarity,
+  MINHASH_SEEDS,
+  type LshIndexEntry,
+} from "./minhash.js";
+
+export {
+  indexSymbolVectors,
+  modelIdFor,
+  type IndexVectorsInput,
+} from "./vectors.js";
+
+export {
+  computeSimilarTo,
+  similarEdgesToEdgeIR,
+  estimateJaccard,
+  CONFIRM_OPERATOR,
+  type SimilarToInput,
+} from "./similarity.js";
+
+export {
+  semanticQuery,
+  DEFAULT_SEMANTIC_QUERY_LIMIT,
+  type SemanticQueryInput,
+} from "./semantic-query.js";
+
+export type {
+  SymbolVector,
+  SymbolVectorRow,
+  SemanticFailure,
+  SemanticFailureCode,
+  IndexVectorsResult,
+  SimilarCandidate,
+  SimilarEdge,
+  SimilarToResult,
+  SemanticQueryHit,
+  SemanticQuerySuccess,
+  SemanticQueryOutcome,
+} from "./types.js";

package/src/semantic/minhash.test.ts

@@ -0,0 +1,197 @@
+/**
+ * MinHash/LSH determinism + clone/hard-negative tests (issue #1556 steps 3–4).
+ *
+ * Rule 38: candidate set is a pure function of (seeds, inputs). Two runs
+ * over the same fixture produce an identical candidate set.
+ *
+ * Fixtures:
+ *   - TRUE CLONE: copy-pasted function with a renamed variable MUST pair.
+ *   - HARD NEGATIVE: similar length, different logic — must NOT pair.
+ *
+ * Rule 35 spirit: cosine confirmation boundary at exactly similarToThreshold.
+ */
+import assert from "node:assert/strict";
+import test from "node:test";
+
+import {
+  MinHasher,
+  createMinHasher,
+  cosineSimilarity,
+} from "./minhash.js";
+import {
+  estimateJaccard,
+} from "./similarity.js";
+import {
+  tokenizeForShingling,
+  shingleSet,
+  MINHASH_SEEDS,
+} from "./minhash.js";
+import {
+  MINHASH_NUM_PERMUTATIONS,
+  LSH_NUM_BANDS,
+  LSH_ROWS_PER_BAND,
+} from "./config.js";
+
+// ──────────────────────────────────────────────────────────────────────────
+// Rule 38: determinism — same fixture, two runs, identical candidate set
+// ──────────────────────────────────────────────────────────────────────────
+
+const CLONE_A = `function processPayment(amount, currency) {
+  const fee = amount * 0.029;
+  const total = amount + fee;
+  return { total, currency };
+}`;
+
+const CLONE_B = `function processPayment(amount, currency) {
+  const fee = amount * 0.029;
+  const totalAmount = amount + fee;
+  return { total: totalAmount, currency };
+}`;
+
+const HARD_NEG = `function validateSchema(data, schema) {
+  for (const key of Object.keys(schema)) {
+    if (!(key in data)) return false;
+  }
+  return true;
+}`;
+
+test("MinHash: determinism — two runs produce identical candidate sets", () => {
+  const run1 = createMinHasher();
+  const run2 = createMinHasher();
+  const entries = [
+    { nodeId: "n1", qualifiedName: "mod.cloneA", body: CLONE_A },
+    { nodeId: "n2", qualifiedName: "mod.cloneB", body: CLONE_B },
+    { nodeId: "n3", qualifiedName: "mod.hardNeg", body: HARD_NEG },
+  ];
+  for (const e of entries) {
+    run1.add(e);
+    run2.add(e);
+  }
+  const c1 = run1.findCandidates();
+  const c2 = run2.findCandidates();
+  assert.equal(c1.length, c2.length);
+  for (let i = 0; i < c1.length; i++) {
+    assert.equal(c1[i]!.aNodeId, c2[i]!.aNodeId);
+    assert.equal(c1[i]!.bNodeId, c2[i]!.bNodeId);
+    assert.equal(c1[i]!.jaccard, c2[i]!.jaccard);
+  }
+});
+
+test("MinHash: true-clone fixtures MUST pair as candidates", () => {
+  const hasher = createMinHasher();
+  hasher.add({ nodeId: "n1", qualifiedName: "mod.cloneA", body: CLONE_A });
+  hasher.add({ nodeId: "n2", qualifiedName: "mod.cloneB", body: CLONE_B });
+  hasher.add({ nodeId: "n3", qualifiedName: "mod.hardNeg", body: HARD_NEG });
+  const candidates = hasher.findCandidates();
+  // The clone pair must appear in the candidate set.
+  const clonePair = candidates.find(
+    (c) =>
+      (c.aNodeId === "n1" && c.bNodeId === "n2") ||
+      (c.aNodeId === "n2" && c.bNodeId === "n1"),
+  );
+  assert.ok(clonePair, "true-clone fixtures must be LSH candidates");
+  assert.ok(clonePair!.jaccard > 0.3, `clone Jaccard should be high, got ${clonePair!.jaccard}`);
+});
+
+test("MinHash: hard-negative fixtures must NOT pair as candidates", () => {
+  const hasher = createMinHasher();
+  hasher.add({ nodeId: "n1", qualifiedName: "mod.cloneA", body: CLONE_A });
+  hasher.add({ nodeId: "n3", qualifiedName: "mod.hardNeg", body: HARD_NEG });
+  const candidates = hasher.findCandidates();
+  const hardPair = candidates.find(
+    (c) =>
+      (c.aNodeId === "n1" && c.bNodeId === "n3") ||
+      (c.aNodeId === "n3" && c.bNodeId === "n1"),
+  );
+  // The hard negative must NOT be a candidate (or if it is, Jaccard must
+  // be low enough that the cosine/SIMILAR_TO gate rejects it).
+  if (hardPair) {
+    assert.ok(
+      hardPair.jaccard < 0.5,
+      `hard-negative Jaccard must be low, got ${hardPair.jaccard}`,
+    );
+  }
+  // Direct Jaccard estimate for clarity.
+  const direct = estimateJaccard(CLONE_A, HARD_NEG);
+  assert.ok(direct < 0.4, `clone vs hard-negative Jaccard should be < 0.4, got ${direct}`);
+});
+
+test("MinHash: estimateJaccard is symmetric and bounded [0,1]", () => {
+  const j1 = estimateJaccard(CLONE_A, CLONE_B);
+  const j2 = estimateJaccard(CLONE_B, CLONE_A);
+  assert.equal(j1, j2, "Jaccard must be symmetric");
+  assert.ok(j1 >= 0 && j1 <= 1);
+  assert.ok(j1 > 0.3, `clone Jaccard > 0.3 expected, got ${j1}`);
+});
+
+// ──────────────────────────────────────────────────────────────────────────
+// Rule 35 spirit: cosine boundary
+// ──────────────────────────────────────────────────────────────────────────
+
+test("cosineSimilarity: identical vectors → 1.0, orthogonal → 0.0", () => {
+  const v = new Float32Array([1, 0, 0]);
+  assert.equal(cosineSimilarity(v, v), 1.0);
+  const w = new Float32Array([0, 1, 0]);
+  assert.equal(cosineSimilarity(v, w), 0.0);
+});
+
+test("cosineSimilarity: boundary ≥ threshold confirms (decided once)", () => {
+  // Two vectors at exactly cosine 0.92 should confirm (≥, not >).
+  // Construct a pair at exactly the default threshold.
+  const a = new Float32Array([1, 0]);
+  // cos(a, b) = b[0] / |b|. For cos = 0.92: b = [0.92, sqrt(1-0.92^2)].
+  const cos = 0.92;
+  const b = new Float32Array([cos, Math.sqrt(1 - cos * cos)]);
+  const actual = cosineSimilarity(a, b);
+  assert.ok(Math.abs(actual - cos) < 1e-6, `cosine should be ~0.92, got ${actual}`);
+  // The similarity pipeline uses >= threshold (CONFIRM_OPERATOR), so at
+  // exactly threshold it confirms. The similarity.test.ts covers the
+  // full pipeline; here we verify the primitive.
+  assert.ok(actual >= 0.92, "at exactly 0.92, >= confirms");
+});
+
+test("cosineSimilarity: zero-norm vector → 0 (no NaN)", () => {
+  const zero = new Float32Array([0, 0, 0]);
+  const v = new Float32Array([1, 2, 3]);
+  assert.equal(cosineSimilarity(zero, v), 0);
+  assert.equal(cosineSimilarity(v, zero), 0);
+  assert.ok(Number.isFinite(cosineSimilarity(zero, zero)));
+});
+
+// ──────────────────────────────────────────────────────────────────────────
+// Seeds are frozen named constants (rule 38)
+// ──────────────────────────────────────────────────────────────────────────
+
+test("MINHASH_SEEDS: frozen, named, deterministic", () => {
+  assert.equal(MINHASH_SEEDS.length, MINHASH_NUM_PERMUTATIONS);
+  assert.ok(Object.isFrozen(MINHASH_SEEDS));
+  // Same seed values every run (determinism).
+  const first = MINHASH_SEEDS[0]!;
+  const tenth = MINHASH_SEEDS[10]!;
+  assert.ok(typeof first === "bigint");
+  assert.ok(first > 0n);
+  assert.ok(tenth > 0n);
+  assert.notEqual(first, tenth);
+});
+
+test("LSH banding: bands * rows = permutations", () => {
+  assert.equal(LSH_NUM_BANDS * LSH_ROWS_PER_BAND, MINHASH_NUM_PERMUTATIONS);
+});
+
+// ──────────────────────────────────────────────────────────────────────────
+// Tokenizer/shingle unit tests
+// ──────────────────────────────────────────────────────────────────────────
+
+test("tokenizeForShingling: lowercase + split on non-alphanumeric", () => {
+  assert.deepEqual(tokenizeForShingling("Hello, World! 42"), ["hello", "world", "42"]);
+  assert.deepEqual(tokenizeForShingling("foo_bar baz"), ["foo_bar", "baz"]);
+});
+
+test("shingleSet: width-2 shingles, deduped", () => {
+  const tokens = ["a", "b", "c", "d", "a", "b", "c"];
+  const shingles = shingleSet(tokens);
+  // width-2 shingles: "a b", "b c", "c d", "d a", "a b" (dup), "b c" (dup) → 4 unique
+  assert.ok(shingles.size >= 3);
+  assert.ok(shingles.has("a b"));
+  assert.ok(shingles.has("b c"));
+});

package/src/semantic/minhash.ts

@@ -0,0 +1,261 @@
+/**
+ * MinHash / LSH candidate generation for SIMILAR_TO near-clone detection
+ * (issue #1556).
+ *
+ * Design (per the issue): token-shingle MinHash/LSH over normalized symbol
+ * bodies (pure TS, deterministic given fixed seeds). This produces a
+ * candidate-pair set cheaply; the cosine-confirmation layer (similarity.ts)
+ * then filters candidates by embedding cosine ≥ threshold.
+ *
+ * Rule 38 — determinism. The seeds are NAMED CONSTANTS (not module-private
+ * state, not `Date.now()`/`Math.random()`). Two runs over the same fixture
+ * repo produce an identical candidate set. The determinism test asserts
+ * this byte-for-byte.
+ *
+ * Rule 11 — no module-level mutable state. The hash tables live on the
+ * MinHasher instance returned by `createMinHasher()`, never at module
+ * scope. Two indexers in one process are fully isolated.
+ */
+import { createHash } from "node:crypto";
+
+import { LSH_NUM_BANDS, LSH_ROWS_PER_BAND, MINHASH_NUM_PERMUTATIONS, MINHASH_SHINGLE_WIDTH } from "./config.js";
+
+/**
+ * Fixed, named 64-bit seeds for the MinHash permutations. These are the
+ * ONLY source of randomness in the candidate pipeline. Changing them
+ * changes every candidate set, so they are frozen constants — the
+ * determinism test pins their exact values.
+ *
+ * Generated as `sha256("remnic:minhash:seed:N").slice(0,16)` interpreted
+ * as a big-endian u64. Using a hash-of-a-string keeps them reproducible
+ * (no magic-looking literals) while still being arbitrary fixed values.
+ */
+function seedFor(index: number): bigint {
+  const hex = createHash("sha256")
+    .update(`remnic:minhash:seed:${index}`, "utf8")
+    .digest("hex")
+    .slice(0, 16);
+  return BigInt(`0x${hex}`);
+}
+
+export const MINHASH_SEEDS: readonly bigint[] = Object.freeze(
+  Array.from({ length: MINHASH_NUM_PERMUTATIONS }, (_, i) => seedFor(i)),
+);
+
+/**
+ * A 64-bit MurmurHash3 x64 finalizer — fast, well-distributed, and
+ * deterministic. Used as the base hash; MinHash permutations are
+ * (a*x + b) mod Mersenne over this base.
+ */
+function hash64(data: string): bigint {
+  const buf = createHash("sha256").update(data, "utf8").digest();
+  // Read the first 8 bytes as big-endian u64 — sha256 is already a strong
+  // mixing function, so we skip the MurmurHash3 body and use sha256 as
+  // the universal hash. Determinism is the only hard requirement; speed
+  // is secondary (the candidate set is small after LSH banding).
+  let h = 0n;
+  for (let i = 0; i < 8; i++) {
+    h = (h << 8n) | BigInt(buf[i]!);
+  }
+  return h;
+}
+
+// 2^61 - 1, a Mersenne prime — the MinHash modulus.
+const MINHASH_MODULUS = (1n << 61n) - 1n;
+
+/**
+ * Normalize symbol body text into a token stream for shingling.
+ *
+ * Normalization:
+ *   - lowercase (case-insensitive clone detection — `MyFunc` vs `myfunc`)
+ *   - split on non-alphanumeric (identifiers, numbers, operators become tokens)
+ *   - drop empty tokens
+ *
+ * This is deliberately coarse: the goal is Jaccard over token sets, not
+ * semantic parsing. A renamed variable changes exactly one token per
+ * occurrence, so Jaccard stays high for genuine clones.
+ */
+export function tokenizeForShingling(body: string): string[] {
+  return body
+    .toLowerCase()
+    .split(/[^a-z0-9_]+/)
+    .filter((t) => t.length > 0);
+}
+
+/**
+ * Build the set of shingles (n-grams of width MINHASH_SHINGLE_WIDTH) from
+ * a token stream. Returns a Set so duplicate shingles collapse (Jaccard
+ * is over the SET of shingles, not a multiset).
+ */
+export function shingleSet(tokens: readonly string[]): Set<string> {
+  if (tokens.length < MINHASH_SHINGLE_WIDTH) {
+    // Too few tokens for a full shingle — use the whole token sequence as
+    // a single shingle so short symbols still participate.
+    return new Set(tokens.length > 0 ? [tokens.join(" ")] : []);
+  }
+  const out = new Set<string>();
+  for (let i = 0; i <= tokens.length - MINHASH_SHINGLE_WIDTH; i++) {
+    out.add(tokens.slice(i, i + MINHASH_SHINGLE_WIDTH).join(" "));
+  }
+  return out;
+}
+
+/**
+ * Compute the MinHash signature (array of permutation minima) for a set
+ * of shingles. Signature length = MINHASH_NUM_PERMUTATIONS. The estimated
+ * Jaccard similarity between two signatures is the fraction of matching
+ * positions.
+ */
+export function minHashSignature(shingles: Set<string>): bigint[] {
+  const sig: bigint[] = [];
+  if (shingles.size === 0) {
+    // Empty body → max-value signature so it never collides with anything.
+    for (let i = 0; i < MINHASH_NUM_PERMUTATIONS; i++) sig.push(MINHASH_MODULUS);
+    return sig;
+  }
+  const shingleArr = Array.from(shingles);
+  for (let p = 0; p < MINHASH_NUM_PERMUTATIONS; p++) {
+    const a = MINHASH_SEEDS[p]!;
+    const b = MINHASH_SEEDS[(p * 2 + 1) % MINHASH_SEEDS.length]!;
+    let min = MINHASH_MODULUS;
+    for (const s of shingleArr) {
+      const h = (a * hash64(s) + b) % MINHASH_MODULUS;
+      if (h < min) min = h;
+    }
+    sig.push(min);
+  }
+  return sig;
+}
+
+/**
+ * LSH band key — the concatenation of one band's rows from the signature.
+ * Two signatures that share at least one band key are a candidate pair.
+ */
+export function lshBandKeys(signature: readonly bigint[]): string[] {
+  const keys: string[] = [];
+  for (let b = 0; b < LSH_NUM_BANDS; b++) {
+    const start = b * LSH_ROWS_PER_BAND;
+    const end = start + LSH_ROWS_PER_BAND;
+    keys.push(signature.slice(start, end).map((v) => v.toString(16)).join("|"));
+  }
+  return keys;
+}
+
+/**
+ * An indexed symbol body ready for LSH bucketing.
+ */
+export interface LshIndexEntry {
+  readonly nodeId: string;
+  readonly qualifiedName: string;
+  readonly body: string;
+}
+
+/**
+ * A MinHash/LSH indexer instance. Owns the band→node-id bucket map on the
+ * instance (rule 11). `findCandidates` returns the deduplicated candidate
+ * pair set with estimated Jaccard for each pair.
+ */
+export class MinHasher {
+  /** band key → set of node ids in that band. */
+  private readonly buckets: Map<string, Set<string>> = new Map();
+  /** node id → signature (for Jaccard estimation on candidate pairs). */
+  private readonly signatures: Map<string, bigint[]> = new Map();
+  /** node id → qualified name (for readable candidate output). */
+  private readonly qnames: Map<string, string> = new Map();
+
+  /**
+   * Add a symbol body to the LSH index. Idempotent — re-adding the same
+   * (nodeId, body) is a no-op.
+   */
+  add(entry: LshIndexEntry): void {
+    if (this.signatures.has(entry.nodeId)) return;
+    const tokens = tokenizeForShingling(entry.body);
+    const shingles = shingleSet(tokens);
+    // Skip empty bodies — they would all get the same all-max signature
+    // and flood the candidate set with false pairs
+    // (chatgpt-codex-connector: 'Keep empty bodies out of shared MinHash
+    // buckets').
+    if (shingles.size === 0) return;
+    const sig = minHashSignature(shingles);
+    this.signatures.set(entry.nodeId, sig);
+    this.qnames.set(entry.nodeId, entry.qualifiedName);
+    for (const key of lshBandKeys(sig)) {
+      let bucket = this.buckets.get(key);
+      if (!bucket) {
+        bucket = new Set();
+        this.buckets.set(key, bucket);
+      }
+      bucket.add(entry.nodeId);
+    }
+  }
+
+  /**
+   * Find all candidate pairs (pairs sharing at least one LSH band) with
+   * their estimated Jaccard similarity. Returns a stable-sorted array
+   * (by aNodeId then bNodeId) so the determinism test can compare runs
+   * byte-for-byte.
+   */
+  findCandidates(): { readonly aNodeId: string; readonly bNodeId: string; readonly aQualifiedName: string; readonly bQualifiedName: string; readonly jaccard: number }[] {
+    const pairSet = new Set<string>();
+    const pairs: { aNodeId: string; bNodeId: string; aQualifiedName: string; bQualifiedName: string; jaccard: number }[] = [];
+    for (const bucket of this.buckets.values()) {
+      if (bucket.size < 2) continue;
+      const ids = Array.from(bucket).sort();
+      for (let i = 0; i < ids.length; i++) {
+        for (let j = i + 1; j < ids.length; j++) {
+          const a = ids[i]!;
+          const b = ids[j]!;
+          const key = `${a}\0${b}`;
+          if (pairSet.has(key)) continue;
+          pairSet.add(key);
+          const sigA = this.signatures.get(a)!;
+          const sigB = this.signatures.get(b)!;
+          let matches = 0;
+          for (let k = 0; k < sigA.length; k++) {
+            if (sigA[k] === sigB[k]) matches++;
+          }
+          pairs.push({
+            aNodeId: a,
+            bNodeId: b,
+            aQualifiedName: this.qnames.get(a) ?? a,
+            bQualifiedName: this.qnames.get(b) ?? b,
+            jaccard: matches / sigA.length,
+          });
+        }
+      }
+    }
+    pairs.sort((x, y) => {
+      if (x.aNodeId !== y.aNodeId) return x.aNodeId < y.aNodeId ? -1 : 1;
+      return x.bNodeId < y.bNodeId ? -1 : x.bNodeId > y.bNodeId ? 1 : 0;
+    });
+    return pairs;
+  }
+}
+
+/**
+ * Construct a fresh MinHasher (rule 11 — state on the instance).
+ */
+export function createMinHasher(): MinHasher {
+  return new MinHasher();
+}
+
+/**
+ * Cosine similarity between two equal-length float vectors. Returns 0 for
+ * zero-norm vectors (no division-by-zero). This is the brute-force
+ * retrieval primitive shared by SIMILAR_TO confirmation and semantic_query.
+ */
+export function cosineSimilarity(a: Float32Array | number[], b: Float32Array | number[]): number {
+  const len = Math.min(a.length, b.length);
+  let dot = 0;
+  let normA = 0;
+  let normB = 0;
+  for (let i = 0; i < len; i++) {
+    const av = a[i]!;
+    const bv = b[i]!;
+    dot += av * bv;
+    normA += av * av;
+    normB += bv * bv;
+  }
+  if (normA === 0 || normB === 0) return 0;
+  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
+}