@dzhechkov/harness-core 0.3.150 → 0.4.2

package/src/backlog.ts

@@ -17,7 +17,7 @@
  */
 
 import { createHash } from 'node:crypto';
-import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs';
+import { appendFileSync, existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
 import { dirname, join } from 'node:path';
 
 // ── Outbound ports to the Brain engine (ADR-001): reuse VERBATIM, define none locally. ──
@@ -26,12 +26,26 @@ import { dirname, join } from 'node:path';
 // ALWAYS hit the SAME store regardless of `memory.vector.engine` (ADR-001/005 — the single-store
 // property; routing through the generic engine let `engine:'rvf'` split writes from reads).
 import { cosineSimilarity, deleteAgentdbByDzIds, importVectorsToAgentdb, resolveAgentdbEmbedder, searchAgentdbPatterns } from './agentdb-index.js';
+// The ONE dedup embed form + the two-signal pair decision (register-inflation fix, 2026-08-11) —
+// shared with agentdb-index's write path so query and row vectors can never live in different spaces.
+import { BACKLOG_TASK_TYPE, DEDUP_EMBED_FORM_VERSION, dedupEmbedText, dedupPairBand, lexicalContainment } from './backlog-embed.js';
 import { resolveEmbedModel } from './embedding-config.js';
 // Seeded PRNG — reuse the ONE mulberry32 already in the repo (no second RNG; ADR-004 determinism).
 import { mulberry32 } from './compounding.js';
 
-/** The vector-store namespace that isolates ideas from lessons (ADR-001/005). */
-export const BACKLOG_TASK_TYPE = 'dz-backlog';
+// Re-export the embed-form module through the backlog surface (index.ts stars this file).
+export {
+  BACKLOG_TASK_TYPE,
+  DEDUP_EMBED_CAP,
+  DEDUP_EMBED_FORM_VERSION,
+  dedupEmbedText,
+  dedupExcerpt,
+  dedupPairBand,
+  distinctiveTokens,
+  lexicalContainment,
+  type DedupBandConfig,
+  type DedupPairBand,
+} from './backlog-embed.js';
 
 /* ================================================================== */
 /*  DOMAIN TYPES (04 Domain Model)                                      */
@@ -54,6 +68,10 @@ export interface IdeaRecord {
   enrichedPath?: string; // features/<slug>/ once enriched (FR-5)
   jiraKey?: string; // IssueRef.key or the outbox ref (ADR-006)
   tags: string[];
+  /** ISO timestamp of the LAST explicit status transition (`dz backlog ship|drop|reopen`). */
+  statusTs?: string;
+  /** The `--reason` given at the LAST explicit transition — describes THAT transition only. */
+  statusReason?: string;
 }
 
 /** The compass (ADR-003) — user-owned, edited as a file. */
@@ -86,6 +104,20 @@ export interface DedupVerdict {
   readonly relatedIds: readonly string[];
   /** True when the embedder was unavailable and dedup degraded to exact-text (ADR-002 §degrade). */
   readonly exactTextOnly: boolean;
+  /** Lexical containment of the DECIDING candidate (matched for a duplicate, top-1 otherwise), when known. */
+  readonly containment?: number;
+  /**
+   * Set when a candidate cleared the cosine threshold but FAILED lexical corroboration and was
+   * demoted to the related band — the register-only false positive (zombie x publish-gate @ 0.941,
+   * containment 0.077, MEASURED 2026-08-05/11). Observability: the CLI prints it, so a demotion is
+   * never a silent judgment call.
+   */
+  readonly demoted?: { readonly id: string; readonly cosine: number; readonly containment: number };
+  /**
+   * True when the duplicate verdict came from the SUBSET band (containment >= subsetContainment at
+   * sub-threshold cosine) — the same idea re-captured at a different length.
+   */
+  readonly subsetMatch?: boolean;
 }
 
 /* ── Effort parsing (idea 86096d6d): a clamp the user cannot see is a silent surprise. ── */
@@ -126,13 +158,30 @@ export function parseEffort(raw: string | undefined, fallback: number): EffortPa
 
 export const DEFAULT_DUPLICATE_THRESHOLD = 0.92; // measured house constant (02 §R-D)
 export const DEFAULT_RELATEDNESS_FLOOR = 0.35; // GROUND_VECTOR_SIMILARITY_FLOOR (02 §R-D)
+/**
+ * Two-signal corroboration cuts (register-inflation fix, MEASURED 2026-08-11 on the real store —
+ * see backlog-embed.ts for the full numbers). Containment on the labeled classes: register-only
+ * false-positive pairs <= 0.171, true duplicates that also cleared the cosine threshold >= 0.538 —
+ * 0.30 sits between with real margin on both sides. Subset band: long-vs-short same-idea pair
+ * measured at containment 0.971 / cosine 0.8019; the nearest non-duplicate (a template-family pair
+ * differing in ONE entity) at containment 0.889 — 0.95 splits them.
+ */
+export const DEFAULT_CORROBORATION_FLOOR = 0.3;
+export const DEFAULT_SUBSET_CONTAINMENT = 0.95;
+export const DEFAULT_SUBSET_COSINE_FLOOR = 0.75;
 export const DEFAULT_ROULETTE_ALPHA = 1.5;
 export const DEFAULT_HALF_LIFE_DAYS = 30;
 export const DEFAULT_RECENCY_FLOOR = 0.3;
 export const DEFAULT_EFFORT = 3;
 
 export interface BacklogConfig {
-  readonly dedup: { readonly duplicateThreshold: number; readonly relatednessFloor: number };
+  readonly dedup: {
+    readonly duplicateThreshold: number;
+    readonly relatednessFloor: number;
+    readonly corroborationFloor: number;
+    readonly subsetContainment: number;
+    readonly subsetCosineFloor: number;
+  };
   readonly roulette: {
     readonly alpha: number;
     readonly halfLifeDays: number;
@@ -150,7 +199,13 @@ function clampNum(v: unknown, lo: number, hi: number, fallback: number, opts: {
 
 export function readBacklogConfig(projectRoot: string): BacklogConfig {
   const fallback: BacklogConfig = {
-    dedup: { duplicateThreshold: DEFAULT_DUPLICATE_THRESHOLD, relatednessFloor: DEFAULT_RELATEDNESS_FLOOR },
+    dedup: {
+      duplicateThreshold: DEFAULT_DUPLICATE_THRESHOLD,
+      relatednessFloor: DEFAULT_RELATEDNESS_FLOOR,
+      corroborationFloor: DEFAULT_CORROBORATION_FLOOR,
+      subsetContainment: DEFAULT_SUBSET_CONTAINMENT,
+      subsetCosineFloor: DEFAULT_SUBSET_COSINE_FLOOR,
+    },
     roulette: {
       alpha: DEFAULT_ROULETTE_ALPHA,
       halfLifeDays: DEFAULT_HALF_LIFE_DAYS,
@@ -164,7 +219,13 @@ export function readBacklogConfig(projectRoot: string): BacklogConfig {
   try {
     const parsed = JSON.parse(readFileSync(configPath, 'utf-8')) as {
       backlog?: {
-        dedup?: { duplicateThreshold?: unknown; relatednessFloor?: unknown };
+        dedup?: {
+          duplicateThreshold?: unknown;
+          relatednessFloor?: unknown;
+          corroborationFloor?: unknown;
+          subsetContainment?: unknown;
+          subsetCosineFloor?: unknown;
+        };
         roulette?: { alpha?: unknown; halfLifeDays?: unknown; recencyFloor?: unknown; defaultEffort?: unknown };
         jira?: { adapter?: unknown };
       };
@@ -180,7 +241,13 @@ export function readBacklogConfig(projectRoot: string): BacklogConfig {
     floor = Math.min(Math.max(floor, 0), dup);
     const adapter = isBacklogBackend(b.jira?.adapter) ? b.jira!.adapter : 'none'; // unknown ⇒ none (ADR-006 T-006d)
     return {
-      dedup: { duplicateThreshold: dup, relatednessFloor: floor },
+      dedup: {
+        duplicateThreshold: dup,
+        relatednessFloor: floor,
+        corroborationFloor: clampNum(b.dedup?.corroborationFloor, 0, 1, DEFAULT_CORROBORATION_FLOOR),
+        subsetContainment: clampNum(b.dedup?.subsetContainment, 0, 1, DEFAULT_SUBSET_CONTAINMENT),
+        subsetCosineFloor: clampNum(b.dedup?.subsetCosineFloor, 0, 1, DEFAULT_SUBSET_COSINE_FLOOR),
+      },
       roulette: {
         alpha: clampNum(b.roulette?.alpha, Number.MIN_VALUE, Number.MAX_VALUE, DEFAULT_ROULETTE_ALPHA),
         halfLifeDays: clampNum(b.roulette?.halfLifeDays, Number.MIN_VALUE, Number.MAX_VALUE, DEFAULT_HALF_LIFE_DAYS),
@@ -259,6 +326,8 @@ function normaliseIdea(raw: Record<string, unknown>): IdeaRecord | undefined {
   if (typeof raw.proposal === 'string') rec.proposal = raw.proposal;
   if (typeof raw.enrichedPath === 'string') rec.enrichedPath = raw.enrichedPath;
   if (typeof raw.jiraKey === 'string') rec.jiraKey = raw.jiraKey;
+  if (typeof raw.statusTs === 'string') rec.statusTs = raw.statusTs;
+  if (typeof raw.statusReason === 'string') rec.statusReason = raw.statusReason;
   return rec;
 }
 
@@ -292,8 +361,236 @@ export function writeIdeas(projectRoot: string, ideas: readonly IdeaRecord[]): v
   mkdirSync(dirname(path), { recursive: true });
   const body = ideas.map((i) => JSON.stringify(i)).join('\n') + (ideas.length > 0 ? '\n' : '');
   const tmp = `${path}.tmp-${process.pid}`;
-  writeFileSync(tmp, body);
-  renameSync(tmp, path);
+  try {
+    writeFileSync(tmp, body);
+    renameSync(tmp, path);
+  } catch (e) {
+    try { unlinkSync(tmp); } catch { /* best-effort litter cleanup */ }
+    throw e;
+  }
+}
+
+/* ================================================================== */
+/*  STATUS TRANSITIONS — dz backlog ship|drop|reopen                    */
+/*                                                                      */
+/*  The MEASURED defect (2026-08-10): the ONLY status mutation was      */
+/*  roulette --commit → in-progress, so work finished WITHOUT a commit  */
+/*  (the normal flow: spin, see the pick, do the work) stayed `new`     */
+/*  FOREVER and the roulette kept re-drawing already-shipped ideas      */
+/*  (21 of 100 records were hand-edited to `shipped` because no CLI     */
+/*  surface existed). This section is that missing surface.             */
+/* ================================================================== */
+
+export type TransitionVerb = 'ship' | 'drop' | 'reopen';
+
+/**
+ * THE transition table — the single source of truth for which statuses each verb may leave from.
+ * `target` is where the verb lands. For the IDEMPOTENT verbs (ship/drop) a record ALREADY at
+ * `target` is a no-op that SAYS so (not an error) — a cleanup batch may name already-marked ideas.
+ * `reopen` is deliberately NOT idempotent: reopening an already-`new` idea almost always means the
+ * user grabbed the wrong id (they believed it was shipped), so it is refused loudly. A status in
+ * neither set is an ILLEGAL transition — notably `dropped → shipped` and `shipped → dropped` both
+ * require an explicit `reopen` first; a terminal state never silently becomes the other terminal.
+ */
+export const IDEA_TRANSITIONS: Record<
+  TransitionVerb,
+  { readonly target: IdeaStatus; readonly from: readonly IdeaStatus[]; readonly idempotent: boolean }
+> = {
+  ship: { target: 'shipped', from: ['new', 'enriched', 'in-progress'], idempotent: true },
+  drop: { target: 'dropped', from: ['new', 'enriched', 'in-progress'], idempotent: true },
+  reopen: { target: 'new', from: ['shipped', 'dropped', 'in-progress'], idempotent: false },
+};
+
+export type TransitionCheck =
+  | { readonly kind: 'transition'; readonly to: IdeaStatus }
+  | { readonly kind: 'noop' }
+  | { readonly kind: 'illegal'; readonly message: string };
+
+/** PURE transition legality check against {@link IDEA_TRANSITIONS}. */
+export function checkTransition(verb: TransitionVerb, current: IdeaStatus): TransitionCheck {
+  const t = IDEA_TRANSITIONS[verb];
+  if (t.from.includes(current)) return { kind: 'transition', to: t.target };
+  if (current === t.target && t.idempotent) return { kind: 'noop' };
+  if (current === t.target) {
+    return { kind: 'illegal', message: `cannot ${verb} a ${current} idea — it is already ${t.target} (did you mean a different id?)` };
+  }
+  return {
+    kind: 'illegal',
+    message: `cannot ${verb} a ${current} idea (${verb}: ${t.from.join('|')} → ${t.target})`,
+  };
+}
+
+export type PrefixResolution =
+  | { readonly kind: 'ok'; readonly id: string }
+  | { readonly kind: 'not-found'; readonly prefix: string }
+  | { readonly kind: 'ambiguous'; readonly prefix: string; readonly matches: readonly string[] };
+
+/**
+ * Resolve a (possibly short) id prefix against the store's ids — the roulette prints 8-char ids, so
+ * the transition verbs accept them. An EXACT id always wins; otherwise a UNIQUE prefix resolves and
+ * anything else (no match / several matches) is a loud error, never a silent no-op (the
+ * inconclusive-≠-pass discipline from dz skills-verify).
+ */
+export function resolveIdPrefix(ids: readonly string[], prefix: string): PrefixResolution {
+  if (ids.includes(prefix)) return { kind: 'ok', id: prefix };
+  const matches = [...new Set(ids.filter((id) => id.startsWith(prefix)))];
+  if (matches.length === 1) return { kind: 'ok', id: matches[0]! };
+  if (matches.length === 0) return { kind: 'not-found', prefix };
+  return { kind: 'ambiguous', prefix, matches: matches.sort() };
+}
+
+export interface TransitionChange {
+  readonly id: string;
+  readonly from: IdeaStatus;
+  readonly to: IdeaStatus;
+  readonly action: 'transitioned' | 'noop';
+  /** First 70 chars of the idea text — for the human confirmation line. */
+  readonly text: string;
+}
+
+export interface TransitionReport {
+  /** True iff every requested id resolved and every transition was legal (no-ops count as ok). */
+  readonly ok: boolean;
+  readonly dryRun: boolean;
+  readonly changes: readonly TransitionChange[];
+  readonly errors: readonly string[];
+  /** True iff the store file was actually rewritten. */
+  readonly written: boolean;
+}
+
+/**
+ * Apply a batch of status transitions to `.dz/backlog/ideas.jsonl`.
+ *
+ * ALL-OR-NOTHING: any unresolved/ambiguous prefix or illegal transition fails the WHOLE batch with
+ * NO write (fail-closed — a partial batch that "mostly worked" is how shipped work silently stays
+ * eligible). LINE-PRESERVING: untouched lines (including corrupt ones readIdeas would skip) are
+ * emitted byte-for-byte; a transitioned line is re-serialized from ITS OWN parsed object, so
+ * `JSON.parse`/`stringify` key-order preservation keeps every non-status field byte-identical —
+ * unknown fields a future schema might add survive untouched. (Documented limit: exotic JSON
+ * scalars — lone surrogates, -0, >2^53 integers — would not round-trip byte-identically; no schema
+ * field has those shapes, ids are 16-hex.) ATOMIC: tmp + rename, same as {@link writeIdeas}, so a
+ * crash never leaves a torn store. (Documented limit: the read-modify-write is NOT locked against a
+ * concurrent writer — pre-existing across every backlog writer, same as writeIdeas.)
+ */
+export function transitionIdeas(
+  projectRoot: string,
+  verb: TransitionVerb,
+  prefixes: readonly string[],
+  opts: { reason?: string; dryRun?: boolean; nowIso?: string } = {},
+): TransitionReport {
+  const dryRun = opts.dryRun === true;
+  const path = ideasPath(projectRoot);
+  if (!existsSync(path)) {
+    return { ok: false, dryRun, changes: [], errors: ['no backlog store — nothing captured yet (dz backlog add "<idea>")'], written: false };
+  }
+  let text: string;
+  try {
+    text = readFileSync(path, 'utf-8');
+  } catch (e) {
+    return { ok: false, dryRun, changes: [], errors: [`cannot read ${path}: ${(e as Error).message}`], written: false };
+  }
+  // Preserve the file's exact line structure: split WITHOUT discarding anything; untouched entries
+  // (and unparseable lines) go back out verbatim.
+  const lines = text.split('\n');
+  interface ParsedLine {
+    readonly index: number;
+    readonly obj: Record<string, unknown>;
+    readonly id: string;
+    readonly status: string;
+  }
+  const parsed: ParsedLine[] = [];
+  for (let i = 0; i < lines.length; i += 1) {
+    const trimmed = (lines[i] ?? '').trim();
+    if (trimmed === '') continue;
+    try {
+      const obj = JSON.parse(trimmed) as Record<string, unknown>;
+      if (typeof obj.id === 'string' && obj.id !== '') {
+        parsed.push({ index: i, obj, id: obj.id, status: typeof obj.status === 'string' ? obj.status : 'new' });
+      }
+    } catch {
+      /* corrupt line — left byte-for-byte as-is */
+    }
+  }
+  const ids = parsed.map((p) => p.id);
+  const errors: string[] = [];
+  const changes: TransitionChange[] = [];
+  const resolvedIds: string[] = [];
+  const seen = new Set<string>();
+  for (const prefix of prefixes) {
+    if (!isSafeId(prefix)) {
+      errors.push(`refusing an unsafe idea id: ${JSON.stringify(prefix)}`);
+      continue;
+    }
+    const res = resolveIdPrefix(ids, prefix);
+    if (res.kind === 'not-found') {
+      errors.push(`no idea matches ${prefix} — run dz backlog list to see the ids`);
+      continue;
+    }
+    if (res.kind === 'ambiguous') {
+      errors.push(`ambiguous prefix ${prefix} — matches ${res.matches.join(', ')}; give more characters`);
+      continue;
+    }
+    if (seen.has(res.id)) continue; // the same idea named twice in one batch — count it once
+    seen.add(res.id);
+    resolvedIds.push(res.id);
+  }
+  // Legality pass over EVERY resolved id BEFORE any mutation (all-or-nothing).
+  //
+  // Legality and mutation are decided over the SAME set of lines: every line bearing the id
+  // (a hand-edited duplicate is a store-integrity problem, and deciding legality on the FIRST
+  // line while mutating ALL lines produced both failure modes — a first-`shipped` twin turned
+  // `ship` into a no-op that left the second `new` twin roulette-eligible forever, and a
+  // first-`new` twin let `ship` drag a second `dropped` twin through the dropped→shipped hop
+  // checkTransition refuses). The rule: ANY twin at an illegal source state refuses the WHOLE
+  // id loudly, naming the duplicate, and writes nothing; otherwise every not-yet-at-target twin
+  // transitions, so no stale twin remains. All twins already at target = a said no-op.
+  const nowIso = opts.nowIso ?? new Date().toISOString();
+  const toMutate: ParsedLine[] = [];
+  for (const id of resolvedIds) {
+    const entries = parsed.filter((p) => p.id === id);
+    const dupNote = entries.length > 1 ? ` — id ${id} appears ${entries.length}× in the store (duplicate lines; resolve the duplicate by hand)` : '';
+    const twins = entries.map((entry) => {
+      const current = (['new', 'enriched', 'in-progress', 'shipped', 'dropped'].includes(entry.status) ? entry.status : 'new') as IdeaStatus;
+      return { entry, current, check: checkTransition(verb, current) };
+    });
+    const textPreview = typeof entries[0]!.obj.text === 'string' ? (entries[0]!.obj.text as string).slice(0, 70) : '';
+    const illegal = twins.find((t) => t.check.kind === 'illegal');
+    if (illegal !== undefined) {
+      errors.push(`${id}: ${(illegal.check as { message: string }).message}${dupNote}`);
+      continue;
+    }
+    const transitioning = twins.filter((t) => t.check.kind === 'transition');
+    if (transitioning.length === 0) {
+      // EVERY twin is already at the target — a genuine, said no-op (nothing stale can remain).
+      changes.push({ id, from: twins[0]!.current, to: twins[0]!.current, action: 'noop', text: textPreview });
+      continue;
+    }
+    changes.push({ id, from: transitioning[0]!.current, to: (transitioning[0]!.check as { to: IdeaStatus }).to, action: 'transitioned', text: textPreview });
+    for (const t of transitioning) toMutate.push(t.entry);
+  }
+  if (errors.length > 0) {
+    return { ok: false, dryRun, changes: [], errors, written: false }; // fail-closed: NOTHING was written
+  }
+  const target = IDEA_TRANSITIONS[verb].target;
+  if (dryRun || toMutate.length === 0) {
+    return { ok: true, dryRun, changes, errors: [], written: false };
+  }
+  for (const p of toMutate) {
+    p.obj.status = target;
+    p.obj.statusTs = nowIso;
+    if (opts.reason !== undefined && opts.reason !== '') p.obj.statusReason = opts.reason;
+    else delete p.obj.statusReason; // a stale reason describes the PREVIOUS transition — never carry it
+    lines[p.index] = JSON.stringify(p.obj);
+  }
+  const tmp = `${path}.tmp-${process.pid}`;
+  try {
+    writeFileSync(tmp, lines.join('\n'));
+    renameSync(tmp, path);
+  } catch (e) {
+    try { unlinkSync(tmp); } catch { /* best-effort litter cleanup — never mask the original failure */ }
+    return { ok: false, dryRun, changes: [], errors: [`store write failed: ${(e as Error).message}`], written: false };
+  }
+  return { ok: true, dryRun, changes, errors: [], written: true };
 }
 
 /* ── Store privacy (idea ec4cd60d): raw ideas are prompt-class PRIVATE content, like recall-usage.jsonl. ── */
@@ -373,8 +670,13 @@ function dominantEol(text: string): '\r\n' | '\n' {
 /** Atomic write (tmp + rename in the SAME dir) — the ideas.jsonl discipline: a crash never truncates. */
 function writeFileAtomic(path: string, body: string): void {
   const tmp = `${path}.tmp-${process.pid}`;
-  writeFileSync(tmp, body);
-  renameSync(tmp, path);
+  try {
+    writeFileSync(tmp, body);
+    renameSync(tmp, path);
+  } catch (e) {
+    try { unlinkSync(tmp); } catch { /* best-effort litter cleanup */ }
+    throw e;
+  }
 }
 
 /**
@@ -429,18 +731,28 @@ export function snapshotIdeas(projectRoot: string, dest: string): SnapshotResult
 /*  DEDUP (AM-2 / ADR-002) — raw cosine bands, NOT RRF.                 */
 /* ================================================================== */
 
-/** One existing-idea comparison candidate: its id and the RAW cosine of the new idea against it. */
+/**
+ * One existing-idea comparison candidate: its id, the RAW cosine of the new idea against it, and
+ * (when the candidate's text is available) the lexical containment of the pair. `containment`
+ * undefined = corroboration unavailable (degraded search hit) ⇒ the cosine band decides alone,
+ * exactly the pre-fix behavior.
+ */
 export interface DedupCandidate {
   readonly id: string;
   readonly cosine: number;
+  readonly containment?: number;
 }
 
 /**
- * THE LOAD-BEARING CLASSIFIER (ADR-002 T-002a) — PURE. Bands the top-1 raw cosine:
- *   DUPLICATE  cosine ≥ duplicateThreshold        (default 0.92)
- *   RELATED    relatednessFloor ≤ cosine < dup    (default [0.35, 0.92))  ⇒ create + attach relatedIds
+ * THE LOAD-BEARING CLASSIFIER (ADR-002 T-002a, two-signal since the register-inflation fix) — PURE.
+ * Bands via {@link dedupPairBand} (shared with harmonize):
+ *   DUPLICATE  top-1 cosine ≥ duplicateThreshold AND corroborated (containment ≥ corroborationFloor
+ *              or unknown) — OR any candidate in the SUBSET band (containment ≥ subsetContainment,
+ *              cosine ≥ subsetCosineFloor): the same idea re-captured at a different length.
+ *   RELATED    relatednessFloor ≤ cosine < dup, PLUS any DEMOTED candidate (≥ threshold cosine that
+ *              failed corroboration — the register-only false positive) ⇒ create + attach relatedIds.
  *   NEW        cosine < relatednessFloor
- * Flip either cut and exactly one boundary fixture crosses a band — the test REDS.
+ * Flip any cut and exactly one boundary fixture crosses a band — the test REDS.
  */
 export function classifyDedup(
   candidates: readonly DedupCandidate[],
@@ -454,14 +766,55 @@ export function classifyDedup(
   const top = sorted[0];
   const exactTextOnly = opts.exactTextOnly === true;
   if (top === undefined) return { action: 'new', cosine: -1, matchedId: undefined, topMatchId: undefined, relatedIds: [], exactTextOnly };
-  if (top.cosine >= cfg.duplicateThreshold) {
-    return { action: 'duplicate', cosine: top.cosine, matchedId: top.id, topMatchId: top.id, relatedIds: [], exactTextOnly };
+  const bands = new Map(sorted.map((c) => [c.id, dedupPairBand(c.cosine, c.containment, cfg)]));
+  if (bands.get(top.id) === 'duplicate') {
+    return {
+      action: 'duplicate',
+      cosine: top.cosine,
+      matchedId: top.id,
+      topMatchId: top.id,
+      relatedIds: [],
+      exactTextOnly,
+      ...(top.containment !== undefined ? { containment: top.containment } : {}),
+    };
   }
-  const related = sorted.filter((c) => c.cosine >= cfg.relatednessFloor && c.cosine < cfg.duplicateThreshold);
+  // Demotion is OBSERVABLE, never silent: the highest-cosine candidate that cleared the threshold but
+  // failed corroboration is reported (the zombie x publish-gate incident surface).
+  const demotedTop = sorted.find((c) => bands.get(c.id) === 'demoted');
+  const demoted =
+    demotedTop !== undefined ? { demoted: { id: demotedTop.id, cosine: demotedTop.cosine, containment: demotedTop.containment ?? 0 } } : {};
+  // SUBSET promotion: highest-cosine candidate whose distinctive vocabulary contains (or is contained
+  // by) the new idea's — length alone must not move the verdict.
+  const subset = sorted.find((c) => bands.get(c.id) === 'subset-duplicate');
+  if (subset !== undefined) {
+    return {
+      action: 'duplicate',
+      cosine: subset.cosine,
+      matchedId: subset.id,
+      topMatchId: top.id,
+      relatedIds: [],
+      exactTextOnly,
+      subsetMatch: true,
+      ...(subset.containment !== undefined ? { containment: subset.containment } : {}),
+      ...demoted,
+    };
+  }
+  // A demoted candidate IS related (its cosine is above the whole related band by construction).
+  const related = sorted.filter((c) => bands.get(c.id) === 'demoted' || (c.cosine >= cfg.relatednessFloor && c.cosine < cfg.duplicateThreshold));
+  const topContainment = top.containment !== undefined ? { containment: top.containment } : {};
   if (related.length > 0) {
-    return { action: 'related', cosine: top.cosine, matchedId: undefined, topMatchId: top.id, relatedIds: related.map((c) => c.id), exactTextOnly };
+    return {
+      action: 'related',
+      cosine: top.cosine,
+      matchedId: undefined,
+      topMatchId: top.id,
+      relatedIds: related.map((c) => c.id),
+      exactTextOnly,
+      ...topContainment,
+      ...demoted,
+    };
   }
-  return { action: 'new', cosine: top.cosine, matchedId: undefined, topMatchId: top.id, relatedIds: [], exactTextOnly };
+  return { action: 'new', cosine: top.cosine, matchedId: undefined, topMatchId: top.id, relatedIds: [], exactTextOnly, ...topContainment };
 }
 
 /** Injectable deps so the production dedup path is testable without a live agentdb. */
@@ -486,8 +839,10 @@ export async function dedupIdea(projectRoot: string, text: string, cfg: BacklogC
   const search =
     deps.search ??
     ((root: string, query: string) => searchAgentdbPatterns(root, query, { taskTypes: [BACKLOG_TASK_TYPE], limit: 20 }));
-  // Match the stored embed form `${taskType}: ${text}` (02 §R-B) so query and row vectors co-locate.
-  const result = await search(projectRoot, `${BACKLOG_TASK_TYPE}: ${text}`);
+  // Match the stored embed form — the BOUNDED v2 excerpt (backlog-embed.ts) — so query and row
+  // vectors co-locate. Full-length embeds INVERTED the duplicate signal on long texts (MEASURED:
+  // genuine long-RU paraphrases 0.35–0.61 vs unrelated long-RU pairs up to 0.9195).
+  const result = await search(projectRoot, dedupEmbedText(text));
   // MED-5: build the VALID candidate set FIRST (a hit must carry a dzId AND a finite cosine). Only
   // then decide — a malformed hit (missing dzId / NaN cosine) must NOT bypass the exact-text net.
   // HIGH-A: a hit whose dzId is NOT a member of the CURRENT ideas.jsonl is an ORPHAN vector (its
@@ -495,9 +850,19 @@ export async function dedupIdea(projectRoot: string, text: string, cfg: BacklogC
   // of a nonexistent idea (a dead match). Drop orphans before deciding — defense in depth alongside the
   // prune-on-removal in `harmonizeBacklog`.
   const liveIds = new Set(ideas.map((i) => i.id));
+  const textById = new Map(ideas.map((i) => [i.id, i.text]));
   const candidates: DedupCandidate[] = [];
   for (const h of result.hits) {
-    if (typeof h.dzId === 'string' && liveIds.has(h.dzId) && Number.isFinite(h.similarity)) candidates.push({ id: h.dzId, cosine: h.similarity });
+    if (typeof h.dzId === 'string' && liveIds.has(h.dzId) && Number.isFinite(h.similarity)) {
+      // Lexical corroboration over the FULL texts (the excerpt bounds only the embedding): a member
+      // of liveIds always has a text, so containment is always attached on this path.
+      const candText = textById.get(h.dzId);
+      candidates.push({
+        id: h.dzId,
+        cosine: h.similarity,
+        ...(candText !== undefined ? { containment: lexicalContainment(text, candText) } : {}),
+      });
+    }
   }
   if (result.error !== undefined || candidates.length === 0) {
     // No usable semantic signal (embedder unavailable, nothing mirrored, or only malformed hits) ⇒
@@ -1011,7 +1376,7 @@ function declaredMcpAdapter(backend: 'jira-mcp' | 'copilot-mcp', instruction: st
   return {
     backend,
     async createIssue(draft, io) {
-      // v1: build + persist the same payload; NO live MCP call (FR-8.2). Honest stub — the wire is a TODO.
+      // v1: build + persist the same payload; NO live MCP call (FR-8.2). Honest stub — the wire is a TODO. no-stubs: pre-existing documented declared-not-wired seam (FR-8.2), untouched by the dedup fix
       const outboxPath = io.writeOutbox(draft.sourceIdeaId, { backend, draft, note: 'declared-not-wired seam (v1)' });
       return { backend, key: null, stub: true, outboxPath };
     },
@@ -1122,10 +1487,15 @@ export async function harmonizeBacklog(
   };
   if (embed !== null && ideas.length >= 2) {
     try {
-      const vecs = await Promise.all(ideas.map((i) => embed!(`${BACKLOG_TASK_TYPE}: ${i.text}`)));
+      // The SAME bounded embed form + two-signal pair decision as capture-time dedup (backlog-embed.ts):
+      // batch harmonize and `dz backlog add` must never disagree about what a duplicate is. The
+      // corroboration knobs come from config; the cosine threshold honors the `--threshold` override.
+      const bandCfg = { ...readBacklogConfig(projectRoot).dedup, duplicateThreshold: threshold };
+      const vecs = await Promise.all(ideas.map((i) => embed!(dedupEmbedText(i.text))));
       for (let a = 0; a < ideas.length; a += 1) {
         for (let b = a + 1; b < ideas.length; b += 1) {
-          if (cosineSimilarity(vecs[a]!, vecs[b]!) >= threshold) union(a, b);
+          const band = dedupPairBand(cosineSimilarity(vecs[a]!, vecs[b]!), lexicalContainment(ideas[a]!.text, ideas[b]!.text), bandCfg);
+          if (band === 'duplicate' || band === 'subset-duplicate') union(a, b);
         }
       }
     } catch {
@@ -1215,7 +1585,7 @@ export async function mirrorIdeaVector(projectRoot: string, idea: IdeaRecord): P
   if ('error' in emb) return { mirrored: 0, error: emb.error };
   let vector: Float32Array;
   try {
-    vector = await emb.embed(`${BACKLOG_TASK_TYPE}: ${idea.text}`); // SAME embed form the dedup query uses
+    vector = await emb.embed(dedupEmbedText(idea.text)); // SAME bounded embed form the dedup query uses (v2)
   } catch (err) {
     return { mirrored: 0, error: `embed failed: ${err instanceof Error ? err.message : String(err)}` };
   }
@@ -1231,3 +1601,141 @@ export async function mirrorIdeaVector(projectRoot: string, idea: IdeaRecord): P
   ]);
   return { mirrored: res.imported, ...(res.error !== undefined ? { error: res.error } : {}) };
 }
+
+/* ================================================================== */
+/*  ABSORPTION audit — a duplicate verdict must never DESTROY the text.  */
+/* ================================================================== */
+
+/** One absorbed capture: everything needed to audit — or reverse — a duplicate verdict later. */
+export interface AbsorptionEntry {
+  readonly ts: string;
+  readonly matchedId: string;
+  readonly cosine: number;
+  readonly containment?: number;
+  readonly subsetMatch?: boolean;
+  /** The FULL incoming text that was absorbed (uses++ on the match, no record created). */
+  readonly text: string;
+}
+
+export function absorbedLogPath(projectRoot: string): string {
+  return join(backlogDir(projectRoot), 'absorbed.jsonl');
+}
+
+/**
+ * Append the absorbed capture to `.dz/backlog/absorbed.jsonl`. Until this log existed, a DUPLICATE
+ * verdict was the only backlog path that DESTROYED user text: the 2026-08-05 zombie-process idea and
+ * the 2026-08-11 patient-values idea were both false absorptions whose original wording is gone
+ * forever (`uses++` keeps no copy). The log makes every absorption auditable and reversible
+ * (`dz backlog add` the logged text again after fixing the config). Best-effort: never throws, never
+ * blocks capture — an unwritable log returns `{error}` for the caller to surface.
+ */
+export function recordAbsorption(projectRoot: string, entry: AbsorptionEntry): { error?: string } {
+  try {
+    mkdirSync(backlogDir(projectRoot), { recursive: true });
+    appendFileSync(absorbedLogPath(projectRoot), `${JSON.stringify(entry)}\n`);
+    return {};
+  } catch (err) {
+    return { error: err instanceof Error ? err.message : String(err) };
+  }
+}
+
+/* ================================================================== */
+/*  EMBED-FORM migration — one-shot re-mirror when the form version bumps. */
+/* ================================================================== */
+
+/** Marker recording which dedup embed FORM the mirrored vectors were built with. */
+export function backlogEmbedFormPath(projectRoot: string): string {
+  return join(backlogDir(projectRoot), 'embed-form.json');
+}
+
+/** The recorded embed-form version, or 1 (the pre-marker full-text form) when absent/corrupt. */
+export function readBacklogEmbedFormVersion(projectRoot: string): number {
+  const p = backlogEmbedFormPath(projectRoot);
+  if (!existsSync(p)) return 1;
+  try {
+    const parsed = JSON.parse(readFileSync(p, 'utf-8')) as { version?: unknown };
+    return typeof parsed.version === 'number' && Number.isFinite(parsed.version) ? parsed.version : 1;
+  } catch {
+    return 1;
+  }
+}
+
+export interface EmbedFormReport {
+  readonly action: 'current' | 'migrated' | 'deferred' | 'empty';
+  /** The version the store is at AFTER this call. */
+  readonly version: number;
+  readonly remirrored?: number;
+  readonly error?: string;
+}
+
+/**
+ * Bring the mirrored dz-backlog vectors into the CURRENT embed form ({@link DEDUP_EMBED_FORM_VERSION}).
+ * v1 vectors were full-text embeds; v2 queries are bounded excerpts — comparing across the two forms
+ * is a query-vs-row space split for any idea longer than the cap (the exact "stale space rot" the
+ * ADR-001 comments warn about). Re-mirrors every idea through the ONE seam (upsert-by-dzId, so it is
+ * idempotent), and writes the marker ONLY after every re-mirror succeeded — a half-migrated store
+ * keeps claiming the old version and is retried next time. Best-effort: NEVER blocks capture; a
+ * `deferred` outcome must be surfaced by the caller (a silent stale space is the trap).
+ */
+export async function ensureBacklogEmbedForm(
+  projectRoot: string,
+  deps: { mirror?: (root: string, idea: IdeaRecord) => Promise<{ mirrored: number; error?: string | undefined }> } = {},
+): Promise<EmbedFormReport> {
+  const stored = readBacklogEmbedFormVersion(projectRoot);
+  if (stored >= DEDUP_EMBED_FORM_VERSION) return { action: 'current', version: stored };
+  const ideas = readIdeas(projectRoot);
+  const writeMarker = (): void => {
+    mkdirSync(backlogDir(projectRoot), { recursive: true });
+    writeFileAtomic(backlogEmbedFormPath(projectRoot), `${JSON.stringify({ version: DEDUP_EMBED_FORM_VERSION })}\n`);
+  };
+  if (ideas.length === 0) {
+    // Nothing to re-embed — stamp the marker so a store born under v2 never "migrates".
+    try {
+      writeMarker();
+      return { action: 'empty', version: DEDUP_EMBED_FORM_VERSION };
+    } catch (err) {
+      return { action: 'deferred', version: stored, error: `marker write failed: ${err instanceof Error ? err.message : String(err)}` };
+    }
+  }
+  let remirrored = 0;
+  if (deps.mirror !== undefined) {
+    for (const idea of ideas) {
+      const res = await deps.mirror(projectRoot, idea);
+      if (res.error !== undefined) {
+        // Abort WITHOUT the marker: the store stays honestly marked v1 and the migration retries later.
+        return { action: 'deferred', version: stored, remirrored, error: res.error };
+      }
+      remirrored += 1;
+    }
+  } else {
+    // BATCHED production path: ONE embedder init + ONE upsert transaction. A per-idea
+    // `mirrorIdeaVector` loop re-initializes the transformer model per idea — MEASURED 2026-08-11:
+    // 105 ideas = 105 model loads, minutes of pure init time.
+    const emb = await resolveAgentdbEmbedder(projectRoot);
+    if ('error' in emb) return { action: 'deferred', version: stored, remirrored: 0, error: emb.error };
+    const rows = [];
+    try {
+      for (const idea of ideas) {
+        rows.push({
+          dzId: idea.id,
+          vector: await emb.embed(dedupEmbedText(idea.text)),
+          text: idea.text,
+          taskType: BACKLOG_TASK_TYPE,
+          score: Math.max(0, Math.min(1, Number.isFinite(idea.goalAlignment) ? idea.goalAlignment : 0)),
+          metadata: { kind: 'dz-backlog-idea' },
+        });
+      }
+    } catch (err) {
+      return { action: 'deferred', version: stored, remirrored: 0, error: `embed failed: ${err instanceof Error ? err.message : String(err)}` };
+    }
+    const res = await importVectorsToAgentdb(projectRoot, rows);
+    if (res.error !== undefined) return { action: 'deferred', version: stored, remirrored: res.imported, error: res.error };
+    remirrored = res.imported;
+  }
+  try {
+    writeMarker();
+  } catch (err) {
+    return { action: 'deferred', version: stored, remirrored, error: `marker write failed: ${err instanceof Error ? err.message : String(err)}` };
+  }
+  return { action: 'migrated', version: DEDUP_EMBED_FORM_VERSION, remirrored };
+}