@dzhechkov/harness-core 0.3.150 → 0.4.2

package/src/agentdb-index.ts

@@ -16,6 +16,9 @@ import { existsSync, mkdirSync } from 'node:fs';
 import { join, dirname } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { createRequire } from 'node:module';
+// The backlog dedup embed form (PURE, zero-dep — no cycle): dz-backlog rows must be embedded in the
+// SAME bounded form the dedup query uses, including through the reindex path.
+import { BACKLOG_TASK_TYPE, dedupEmbedText } from './backlog-embed.js';
 import {
   currentEmbedManifest,
   guardEmbedSpace,
@@ -129,7 +132,13 @@ export async function indexPatternsToAgentdb(
       // ALL first, then commit the writes atomically (QE P2: a mid-loop failure must not leave a
       // partial batch reported as indexed:0).
       const prepared: Array<{ row: AgentdbRow; vec: Float32Array }> = [];
-      for (const row of rows) prepared.push({ row, vec: await emb.embed(`${row.taskType}: ${row.text}`) });
+      // dz-backlog rows use the BOUNDED dedup embed form (backlog-embed.ts) — the same form the
+      // dedup query and `mirrorIdeaVector` use, so a `dz vector reindex` lands backlog rows in the
+      // space they are queried in. Every other task type keeps the classic `${taskType}: ${text}`.
+      for (const row of rows) {
+        const embedText = row.taskType === BACKLOG_TASK_TYPE ? dedupEmbedText(row.text) : `${row.taskType}: ${row.text}`;
+        prepared.push({ row, vec: await emb.embed(embedText) });
+      }
       const commit = db.transaction(() => {
         for (const { row, vec } of prepared) {
           const r = insPattern.run(

package/src/backlog-embed.ts

@@ -0,0 +1,156 @@
+/**
+ * Backlog dedup EMBED FORM + lexical corroboration (the 2026-08-11 register-inflation fix).
+ *
+ * Zero-dep PURE module: imported by BOTH `backlog.ts` (query/mirror/harmonize) and
+ * `agentdb-index.ts` (the reindex/index write path), so every dz-backlog vector — fresh mirror,
+ * reindex, or query — is built from the SAME text form. A second definition of this form is exactly
+ * the query-vs-row space split this module exists to prevent.
+ *
+ * WHY the bounded excerpt (all numbers MEASURED 2026-08-11 on the real 105-idea store, reproducer:
+ * scratch pairwise-cosine harness over `EmbeddingService` / Xenova-paraphrase-multilingual-MiniLM-L12-v2,
+ * the same model + `${taskType}: ${text}` form the tool uses):
+ *
+ *   - Full-length embeddings INVERT the duplicate signal on long texts. Genuine reworded paraphrases
+ *     of 1073–1509-char ideas scored 0.35–0.61 against their own sources, while topically DISJOINT
+ *     long-RU pairs scored up to 0.9195 (genai-tweets x dz-cost) — i.e. above every genuine long
+ *     paraphrase. No threshold can separate classes that are ordered the wrong way round.
+ *   - The inflation is a register/length effect, not topic: long-RU x long-RU pairs mean cosine
+ *     0.6467 vs short-EN x short-EN 0.3929 (+0.25) over the same store; the 2026-08-05 incident
+ *     (an agentdb zombie-process idea absorbed as DUPLICATE of a dz-guard publish-gate rule at
+ *     cosine 0.941) shared only the register — правило/проверка/гейт/уровень/измерено.
+ *   - Bounded to the first 400 chars, the same pairs separate: paraphrase-vs-source 0.73–0.95 with
+ *     the source as top-1 in 8/8 probes, store-wide max non-duplicate pair 0.9181, and the
+ *     patient-values long-form scored 0.80 against its own short reword (was 0.26 at full length,
+ *     with an unrelated record at 0.89 on top).
+ *
+ * WHY the lexical corroboration: cosine alone still cannot tell a same-register pair from a true
+ * paraphrase near the threshold. A TRUE re-capture shares the idea's DISTINCTIVE vocabulary
+ * (subject nouns, identifiers); a register-only pair shares function words. Measured containment on
+ * the labeled set: true duplicates 0.33–0.97 (>= 0.538 for every pair that also cleared the cosine
+ * threshold); register-only false-positive pairs <= 0.171.
+ */
+
+/** The vector-store namespace that isolates ideas from lessons (ADR-001/005). */
+export const BACKLOG_TASK_TYPE = 'dz-backlog';
+
+/**
+ * Version of the dedup embed FORM (not the model). v1 = `dz-backlog: <full text>`; v2 = bounded
+ * excerpt (this module). Stored vectors written under an older form are in a DIFFERENT space than
+ * v2 queries for texts longer than the cap — `ensureBacklogEmbedForm` re-mirrors them once.
+ */
+export const DEDUP_EMBED_FORM_VERSION = 2;
+
+/**
+ * Embed-input cap in UTF-16 units. NOT config: the cap is part of the embed form — two stores (or a
+ * query and a row) built with different caps silently live in different spaces. 400 chosen over 300
+ * by measurement: same store-wide max (0.9181) with higher paraphrase-vs-source cosines (0.88 vs
+ * 0.84 on the hardest long-RU probe).
+ */
+export const DEDUP_EMBED_CAP = 400;
+
+/**
+ * The DISTINCTIVE excerpt of an idea for embedding: whitespace-collapsed, capped at
+ * {@link DEDUP_EMBED_CAP}. Short texts pass through untouched (byte-identical semantics to v1 for
+ * them). The cut never splits a surrogate pair.
+ */
+export function dedupExcerpt(text: string): string {
+  const collapsed = text.replace(/\s+/g, ' ').trim();
+  if (collapsed.length <= DEDUP_EMBED_CAP) return collapsed;
+  let cut = collapsed.slice(0, DEDUP_EMBED_CAP);
+  // A high surrogate at the cut would make the excerpt an invalid string — drop the half char.
+  const lastCode = cut.charCodeAt(cut.length - 1);
+  if (lastCode >= 0xd800 && lastCode <= 0xdbff) cut = cut.slice(0, -1);
+  return cut;
+}
+
+/** The ONE dedup embed form (v2) — used by the query, the mirror write, and the reindex write. */
+export function dedupEmbedText(text: string): string {
+  return `${BACKLOG_TASK_TYPE}: ${dedupExcerpt(text)}`;
+}
+
+/**
+ * Register/function words excluded from the DISTINCTIVE token set — the shared technical register
+ * that inflated the 0.941 false duplicate (правило/проверка/гейт/уровень/измерено + generic RU/EN
+ * function words). Deliberately small: the containment cut (0.30) sits far from both measured
+ * classes (<= 0.171 vs >= 0.538), so the list does not need to be complete to discriminate.
+ */
+const REGISTER_STOPWORDS = new Set(
+  (
+    'правило проверка гейт уровень измерено замерено идея задача контекст исследование урок вердикт ' +
+    'цель форма выход проблема источник материалы сегодня реально нужно нельзя должен должна должно ' +
+    'обязан обязана обязано который которая которое только через после перед против всего этого чтобы ' +
+    'если иначе когда где притом причем поэтому пока пусть надо есть нет был была было будут быть ' +
+    'может можно станет стало сразу самый самая самое наш наша наше свой своя своё туда сюда здесь ' +
+    'там опять снова очень давно потом чужой чужая every never always must should could would about ' +
+    'there their these those with from into over under this that what when where which while откуда ' +
+    'почему зачем такой такая такое весь вся всё они оный дать даёт дают взять берет берут'
+  ).split(/\s+/),
+);
+
+/**
+ * The distinctive-token set of an idea text: lowercased word-ish tokens (letters/digits, allowing
+ * inner `_ . - /` so identifiers and paths survive whole), length >= 4, register stopwords removed,
+ * Cyrillic words longer than 6 chars folded to their 6-char prefix (a crude but MEASURED-adequate
+ * stem that matches RU inflections: подкрепление/подкрепления → подкре).
+ */
+export function distinctiveTokens(text: string): Set<string> {
+  const out = new Set<string>();
+  for (const m of text.toLowerCase().matchAll(/[\p{L}\p{N}][\p{L}\p{N}_.\-/]*/gu)) {
+    let w = (m[0] ?? '').replace(/[.\-/]+$/, '');
+    if (w.length < 4 || REGISTER_STOPWORDS.has(w)) continue;
+    if (/[а-яё]/.test(w) && w.length > 6) w = w.slice(0, 6);
+    out.add(w);
+  }
+  return out;
+}
+
+/**
+ * Lexical containment in [0,1]: |A ∩ B| / min(|A|, |B|) over distinctive tokens. `min` (containment,
+ * not Jaccard) so a SHORT re-capture of a LONG idea — whose vocabulary is a subset — still scores
+ * high; that asymmetry is exactly the length-moved-the-verdict incident. Empty token sets score 0
+ * (no evidence is not corroboration).
+ */
+export function lexicalContainment(a: string, b: string): number {
+  const ta = distinctiveTokens(a);
+  const tb = distinctiveTokens(b);
+  if (ta.size === 0 || tb.size === 0) return 0;
+  let inter = 0;
+  for (const w of ta) if (tb.has(w)) inter++;
+  return inter / Math.min(ta.size, tb.size);
+}
+
+/** The corroboration knobs of the two-signal decision (defaults in `readBacklogConfig`). */
+export interface DedupBandConfig {
+  readonly duplicateThreshold: number;
+  readonly relatednessFloor: number;
+  /** DUPLICATE-by-cosine additionally requires containment >= this (undefined containment waives). */
+  readonly corroborationFloor: number;
+  /** Containment >= this + cosine >= subsetCosineFloor promotes to duplicate BELOW the threshold. */
+  readonly subsetContainment: number;
+  readonly subsetCosineFloor: number;
+}
+
+export type DedupPairBand = 'duplicate' | 'demoted' | 'subset-duplicate' | 'below';
+
+/**
+ * THE two-signal pair decision — shared by `classifyDedup` (capture) and `harmonizeBacklog` (batch)
+ * so the two paths can never disagree about what a duplicate is.
+ *
+ *   duplicate         cosine >= duplicateThreshold AND (containment unknown OR >= corroborationFloor)
+ *   demoted           cosine >= duplicateThreshold but containment < corroborationFloor — the
+ *                     register-only false positive (zombie x publish-gate @ 0.941, containment 0.077):
+ *                     high cosine with DISJOINT subject vocabulary is not a duplicate.
+ *   subset-duplicate  cosine in [subsetCosineFloor, duplicateThreshold) AND containment >=
+ *                     subsetContainment — the same idea re-captured at a different LENGTH (measured
+ *                     long-vs-short patient-values: cosine 0.8019, containment 0.971).
+ *   below             everything else (related/new banding is the caller's job).
+ */
+export function dedupPairBand(cosine: number, containment: number | undefined, cfg: DedupBandConfig): DedupPairBand {
+  if (cosine >= cfg.duplicateThreshold) {
+    return containment === undefined || containment >= cfg.corroborationFloor ? 'duplicate' : 'demoted';
+  }
+  if (containment !== undefined && containment >= cfg.subsetContainment && cosine >= cfg.subsetCosineFloor) {
+    return 'subset-duplicate';
+  }
+  return 'below';
+}