@dzhechkov/harness-core 0.6.1 → 0.7.0

package/src/sign.ts

@@ -246,6 +246,19 @@ export function verifyManifest(
   root: string,
   signed: SignedManifest | null | undefined,
   pubKeyPem: string,
+  /**
+   * The paths the pack actually SHIPS, when the caller can establish them (from `npm pack`). The
+   * added-file sweep is then scoped to those, because a working-tree file `files[]` excludes was
+   * never "added to the pack" — it simply is not part of it.
+   *
+   * Omit it and the sweep covers the whole tree, exactly as before. That is correct for the case that
+   * matters most: a consumer verifying an EXTRACTED tarball, where the tree IS the shipped set.
+   *
+   * This parameter exists because the signer and the sweep must never disagree about what a pack
+   * contains — the 10-false-TAMPERED lesson, recorded at `packFiles` and re-learned on 2026-08-21 the
+   * moment the signer started scoping and the sweep did not.
+   */
+  shippedPaths?: readonly string[],
 ): VerifyResult {
   const fail = (path: string, reason: string): VerifyResult => ({ ok: false, failures: [{ path, reason }] });
 
@@ -300,10 +313,14 @@ export function verifyManifest(
   }
 
   // Bidirectional, always: hashing only what the manifest lists lets an attacker ADD a file.
+  const shipped = shippedPaths === undefined ? null : new Set(shippedPaths);
   const present = listPackFiles(root);
   const listed = new Set(manifest.files.map((f) => f.path));
   for (const rel of present) {
     const p = rel.split(sep).join('/');
+    // A file the pack does not ship is not an ADDED file; scoping here is what keeps the sweep and the
+    // signer describing the same object.
+    if (shipped !== null && !shipped.has(p)) continue;
     if (!listed.has(p)) failures.push({ path: p, reason: 'present in the pack but not signed' });
   }
 
@@ -415,7 +432,7 @@ export function decidePublishGate(input: PublishGateInput): PublishGateDecision
 // A verifier nobody runs is a signature nobody checks. These two pure functions carry the whole
 // security content of `dz doctor` / `dz upgrade`; the CLI bodies only print and exit.
 
-export type PackVerdict = 'verified' | 'unsigned' | 'tampered' | 'no-trust-root';
+export type PackVerdict = 'verified' | 'unsigned' | 'tampered' | 'no-trust-root' | 'source-tree';
 
 export interface TrustRootCandidates {
   /** `--pubkey <path>`, if the caller passed one and it exists. */
@@ -458,11 +475,26 @@ export interface PolicyDecision {
  * | unsigned       | report             | fail |
  * | tampered       | FAIL               | FAIL |
  * | no-trust-root  | report             | fail |
+ * | source-tree    | report             | report |
+ *
+ * `source-tree` was added on 2026-08-21, when the manifest began describing the PUBLISHED TARBALL
+ * rather than the working tree. Those are two different objects — `pnpm publish` re-serialises
+ * package.json and rewrites `workspace:*` — so a source checkout CANNOT be hash-verified against a
+ * tarball-scoped manifest, and calling that TAMPERED is a false alarm. It is not `verified` either:
+ * nothing was established. It stays `report` even under `--require-signing`, because the honest
+ * remedy is to verify the artifact (`dz verify-pack` packs and checks the tarball), not to fail a
+ * developer's checkout for being a checkout.
  *
  * `tampered` is fatal in both columns; `no-trust-root` is never success. That is the load-bearing
  * property. Today every pack is `unsigned` or `no-trust-root` — the leg is wired, not armed.
  */
 export function decideVerifyPolicy(verdict: PackVerdict, requireSigning: boolean): PolicyDecision {
+  if (verdict === 'source-tree') {
+    return {
+      action: 'report',
+      reason: 'source checkout — its manifest describes the published tarball, which this tree is not; run `dz verify-pack` to check the artifact',
+    };
+  }
   if (verdict === 'tampered') {
     return { action: 'fail', reason: 'pack does not match its signed manifest' };
   }

package/src/vector-tier.ts

@@ -61,6 +61,7 @@ import {
   indexPatternsToAgentdb,
   searchAgentdbPatterns,
   listAgentdbDzIds,
+  DZ_PATTERN_TASK_TYPES,
   resolveAgentdbEmbedder,
   cosineSimilarity,
   importVectorsToAgentdb,
@@ -137,6 +138,14 @@ export interface VectorEngine {
   upsert(entries: readonly VectorEntry[]): Promise<{ indexed: number; error?: string | undefined }>;
   search(query: string, limit: number): Promise<{ hits: VectorHit[]; error?: string | undefined }>;
   listIds(): Promise<{ ids: string[]; error?: string | undefined }>;
+  /**
+   * Ids of the PATTERN scope only — the task types `lexicalMirrorable` counts. Optional: an engine
+   * that cannot narrow reports nothing and the caller degrades to `orphaned: undefined`, never to a
+   * fabricated zero. Exists because `listIds()` deliberately enumerates the OWNED SUPERSET, which
+   * also holds `dz-backlog` ideas — and reporting that number beside a pattern-only count once led a
+   * reader to conclude half the index was orphaned when none of it was (ADR-001).
+   */
+  listPatternIds?(): Promise<{ ids: string[]; error?: string | undefined }>;
   /** Portable single-file checkpoint (RVF adapter only — `dz vector export`). */
   exportCheckpoint?(dest: string): Promise<{ error?: string | undefined }>;
   /**
@@ -175,6 +184,39 @@ export interface HybridRecall {
   readonly vectorReason?: string | undefined;
   /** Engine was present but the search failed/timed out — lexical results returned instead. */
   readonly vectorError?: string | undefined;
+  /**
+   * How many candidates the engine RETURNED for this query. Zero on every lexical-only path.
+   *
+   * This and `semanticRanked` exist so the OUTPUT can state what the run did instead of what the
+   * config allows. Every false claim this feature removed came from deriving a statement about the
+   * run from `vectorEngine !== 'none'`, which is engine RESOLVABILITY (ADR-001).
+   */
+  readonly semanticCandidates: number;
+  /**
+   * How many of those candidates survived the orphan drop and entered the merge — the honest test of
+   * "did a vector rank anything". It can be 0 while `semanticCandidates` is positive: an engine that
+   * returns only ids the lexical store no longer has has participated in nothing.
+   */
+  readonly semanticRanked: number;
+}
+
+
+/**
+ * How many of `mirrorable` are absent from the mirror, given the set of ids the mirror and the
+ * pending queue account for.
+ *
+ * A record counts as mirrored under EITHER key, because there are two write seams and they do NOT
+ * agree: `patternVectorEntry` (teach) writes the CONTENT-addressed `patternRecordId`, while
+ * `memoryRecordVectorEntry` (backfill/dream) writes the record's own `r.id`. MEASURED on the real
+ * store (2026-08-22): 270 of 272 records have the same value for both, and the 2 that differ — a
+ * `dream:` record and one re-keyed teach record — are mirrored under their OWN id. Joining on the
+ * derived key alone reported them as an unpaid debt and advised a reindex with nothing to do.
+ *
+ * Pure and exported so the divergent case is testable without a store: a `dream:`-prefixed id can
+ * never equal `patternRecordId`, which always yields `teach:<hash>`.
+ */
+export function countUnmirrored(mirrorable: readonly MemoryRecord[], accounted: ReadonlySet<string>): number {
+  return mirrorable.filter((r) => !accounted.has(r.id) && !accounted.has(patternRecordId(recordToPattern(r)))).length;
 }
 
 /** Field observability for `dz vector status` (I-2/I-5 in the field). */
@@ -186,8 +228,44 @@ export interface VectorTierStatus {
   readonly embeddingModel?: string | undefined;
   readonly lexicalTotal: number;
   readonly lexicalMirrorable: number;
+  /**
+   * How many MIRRORABLE RECORDS are in the vector store, counted in the same scope
+   * `lexicalMirrorable` counts — so the two may be read as a pair.
+   *
+   * `undefined` when the engine cannot report per-scope counts: under the narrower meaning this
+   * field now carries, the unnarrowed store total is NOT an answer to the question it asks.
+   */
   readonly mirrored?: number | undefined;
   readonly pending: number;
+  /**
+   * Whether the mirror WRITER is enabled — `vectorMirrorEnabled()`'s own answer, reported separately
+   * from `available`, which is about the ENGINE. Conflating the two is what let an unconfigured
+   * project print a fully-healthy status line over a dead writer (ADR-001).
+   */
+  readonly mirrorWriterEnabled: boolean;
+  /**
+   * Vectors of OTHER dz-owned task types (today: `dz-backlog`). Reported separately so a store of N
+   * vectors can be fully accounted for, and never folded into `mirrored`, which must stay comparable
+   * to `lexicalMirrorable`. `undefined` when the engine cannot narrow its listing.
+   */
+  readonly mirroredOther?: number | undefined;
+  /**
+   * Pattern-scope vectors matching NO record under either key — the real meaning of "orphan", and
+   * until this feature nothing computed it. `undefined` when the engine cannot narrow its listing.
+   */
+  readonly orphaned?: number | undefined;
+  /**
+   * Mirrorable records present in NEITHER the mirror NOR the pending queue — an exact set difference
+   * over ids, so an orphan vector can never make it negative.
+   *
+   * The name is deliberately `unmirrored`, not `unqueued`: the difference proves only that a record
+   * is not in the mirror NOW. A vector that was written and later deleted, or a pending entry that
+   * was cleared, is indistinguishable from one never offered — so claiming "never queued" would
+   * overstate what was measured (raised by cross-family review, 2026-08-22).
+   *
+   * `undefined` means UNKNOWN — no engine, or an engine whose `listIds()` failed. It never means zero.
+   */
+  readonly unmirrored?: number | undefined;
 }
 
 /** Wall-time bound applied to EVERY engine call, read and write legs alike (ADR R1 + NC1). */
@@ -735,6 +813,11 @@ const RRF_K = 60;
 /**
  * Reciprocal Rank Fusion merge: `score(p) = Σ 1/(60 + rank)` over the lists containing `p`
  * (semantic ranks weighted by `semanticWeight`). Dedup by id; `backend: 'both'` when a pattern
+ *
+ * Ordering: fused score, then EVIDENCE (`both` before lexical-only before semantic-only), then id.
+ * When `semanticWeight > 1` the lexical top-1 is guaranteed a place in the result, taken from the
+ * last seat unless that seat holds a `both` hit. See `features/semantic-keeps-exact-hits`.
+ *
  * appears in both lists. DETERMINISTIC (AC-6): ties break on id, so fixed inputs always yield
  * the same ordering. Pure — no I/O.
  */
@@ -758,14 +841,59 @@ export function mergeHybridHits(
     cur.score += weight / (RRF_K + rank + 1);
     acc.set(h.id, cur);
   });
-  return [...acc.entries()]
-    .sort((a, b) => b[1].score - a[1].score || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0))
-    .slice(0, opts.limit)
-    .map(([, v]) => ({
-      pattern: v.pattern,
-      backend: v.lex !== undefined && v.sem ? ('both' as const) : v.lex ?? ('vector' as const),
-      score: v.score,
-    }));
+  // Ties break by EVIDENCE, not by the id alphabet: a hit both legs found outranks one only a single
+  // leg found. Before this, an exact-term match lost a tie to an arbitrary semantic hit purely
+  // because its id sorted later (ADR-001 AM-4).
+  const evidence = (v: Acc): number => (v.lex !== undefined && v.sem ? 0 : v.lex !== undefined ? 1 : 2);
+  const ordered = [...acc.entries()]
+    .sort((a, b) => b[1].score - a[1].score
+      || evidence(a[1]) - evidence(b[1])
+      || (a[0] < b[0] ? -1 : a[0] > b[0] ? 1 : 0));
+  const toHit = ([, v]: [string, Acc]): HybridHit => ({
+    pattern: v.pattern,
+    backend: v.lex !== undefined && v.sem ? ('both' as const) : v.lex ?? ('vector' as const),
+    score: v.score,
+  });
+  // `slice(0, -1)` drops the LAST element instead of returning nothing, so a negative limit used to
+  // return almost the whole list (MEASURED: limit -1 over 5 candidates returned 4). `dz recall`
+  // clamps its own --limit, but this function is exported and the failure is silent, so it defends
+  // itself. `Infinity` means NO limit and must keep meaning that — the first version of this clamp
+  // rejected it along with NaN and returned nothing (found by independent review).
+  const limit = opts.limit === Number.POSITIVE_INFINITY
+    ? acc.size
+    : Number.isFinite(opts.limit) ? Math.max(0, Math.trunc(opts.limit)) : 0;
+  const cut = ordered.slice(0, limit);
+
+  // The lexical TOP-1 gets a RESERVED SEAT — but ONLY when the caller asked to emphasise the
+  // semantic leg. `--semantic` is meant to emphasise it, and instead REPLACES the lexical one: with
+  // RRF_K=60 a lexical hit at rank r is beaten by every semantic hit at rank s <= 61+2r, and the
+  // semantic list is capped at limit*2, so at any sane limit EVERY semantic hit outranks EVERY
+  // lexical one and an exact match on a rare identifier vanishes (MEASURED: one exact hit + three
+  // unrelated semantic hits at limit 3 → absent under weight 2, present under weight 1). Tuning the
+  // weight cannot fix it: any weight above ~1.02 has the same total effect.
+  //
+  // Gated on `weight > 1` because in HYBRID mode there is nothing to repair and the seat does real
+  // harm: at limit 1 it evicted a `both` hit scoring 0.0325 to seat a lexical-only hit scoring
+  // 0.0164 — half the score, and a hit BOTH legs had found (MEASURED, found by independent review;
+  // the first version of this ADR wrongly claimed the seat could not alter hybrid results).
+  // A `both` hit is never evicted: it is the strongest evidence the merge has.
+  const top = lexical[0];
+  if (weight > 1 && top !== undefined && limit > 0 && !cut.some(([id]) => id === top.id)) {
+    const seat = ordered.find(([id]) => id === top.id);
+    // Evict the WEAKEST hit that is not `both` — not simply the last one. Checking only the last
+    // seat meant a `both` hit sitting there blocked the reservation entirely, even when an evictable
+    // semantic-only hit stood right beside it: lexical [TOP, SHARED], semantic [S0…S59, SHARED],
+    // limit 2 → the cut is [S0, SHARED(both)] and TOP was dropped anyway (MEASURED, found by
+    // cross-family review). If every seat holds a `both` hit there is nothing to take, and the
+    // reservation is skipped: a hit both legs found is the strongest evidence the merge has.
+    let victim = -1;
+    for (let i = cut.length - 1; i >= 0; i -= 1) {
+      const entry = cut[i];
+      if (entry !== undefined && !(entry[1].lex !== undefined && entry[1].sem)) { victim = i; break; }
+    }
+    if (seat !== undefined && victim >= 0) cut[victim] = seat;
+  }
+  return cut.map(toHit);
 }
 
 function markRecallHits(projectRoot: string, backend: LearningSignalBackend, hits: readonly HybridHit[], idOf: (p: PatternRecord) => string): void {
@@ -840,6 +968,8 @@ export async function recallHybrid(
     hits: enhance(lexical.map((h, rank) => ({ pattern: h.pattern, backend: h.backend, score: 1 / (RRF_K + rank + 1) }))),
     lexicalBackend,
     vectorEngine: 'none',
+    semanticCandidates: 0,
+    semanticRanked: 0,
     ...extra,
   });
 
@@ -874,6 +1004,7 @@ export async function recallHybrid(
     () => ({ hits: [] as VectorHit[], error: `vector search timed out after ${timeoutMs}ms` }),
   );
   if (sr.error !== undefined) {
+    // The engine answered with an error, so nothing was returned and nothing ranked — both zero.
     const out = { ...lexicalOnly({}), vectorEngine: engine.kind, vectorError: sr.error };
     markRecallHits(projectRoot, learning, out.hits, idOf);
     return out;
@@ -901,7 +1032,13 @@ export async function recallHybrid(
   }));
   const hits = enhance(mergeHybridHits(lex, semantic, { limit, semanticWeight: mode === 'semantic' ? 2 : 1 }));
   markRecallHits(projectRoot, learning, hits, idOf);
-  return { hits, lexicalBackend, vectorEngine: engine.kind };
+  return {
+    hits,
+    lexicalBackend,
+    vectorEngine: engine.kind,
+    semanticCandidates: sr.hits.length,
+    semanticRanked: semantic.length,
+  };
 }
 
 export async function teachGuard(
@@ -944,10 +1081,15 @@ export async function vectorTierStatus(
   } catch {
     records = [];
   }
-  const lexicalMirrorable = records.filter((r) => !isVectorNoise(r.text)).length;
-  const pending = readVectorPending(projectRoot).length;
+  const mirrorableRecords = records.filter((r) => !isVectorNoise(r.text));
+  const lexicalMirrorable = mirrorableRecords.length;
+  const pendingEntries = readVectorPending(projectRoot);
+  const pending = pendingEntries.length;
+  const mirrorWriterEnabled = vectorMirrorEnabled(projectRoot);
   const resolved = pickEngine(projectRoot, opts);
   if (resolved.engine === undefined) {
+    // No engine to ask ⇒ the debt is UNKNOWN. Reporting 0 here is precisely the defect this feature
+    // closes: a number nobody computed is not a number anyone may print (ADR-001, AM-5).
     return {
       mode,
       available: false,
@@ -956,6 +1098,7 @@ export async function vectorTierStatus(
       lexicalTotal: records.length,
       lexicalMirrorable,
       pending,
+      mirrorWriterEnabled,
     };
   }
   const engine = resolved.engine;
@@ -964,6 +1107,45 @@ export async function vectorTierStatus(
     opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS,
     () => ({ ids: [] as string[], error: 'vector listIds timed out' }),
   );
+  // The debt is a SET DIFFERENCE over the content-addressed ids the mirror actually writes under,
+  // not `mirrorable - mirrored - pending`: a vector can outlive its record (an orphan left by a
+  // harmonize), which makes the subtraction negative and the flooring a fresh lie (ADR-001 alt-b).
+  const accounted = new Set<string>([...listed.ids, ...pendingEntries.map((e) => e.dzId)]);
+  const unmirrored = listed.error === undefined
+    ? countUnmirrored(mirrorableRecords, accounted)
+    : undefined;
+
+  // `listIds()` enumerates the OWNED SUPERSET — it also holds `dz-backlog` ideas — so its count may
+  // not be printed beside a pattern-only lexical count. MEASURED on this repository: 547 ids against
+  // 274 records, of which 273 were backlog idea ids and ZERO were true orphans; a reader took the
+  // pair at face value and filed a task to prune a healthy index (ADR-001). Narrow it where the
+  // engine can, and report `undefined` where it cannot — never a fabricated zero.
+  const narrowed = listed.error === undefined && engine.listPatternIds !== undefined
+    ? await withVectorTimeout(
+        safeEngineCall(() => engine.listPatternIds!(), (m) => ({ ids: [] as string[], error: `vector listPatternIds failed: ${m}` })),
+        opts.timeoutMs ?? DEFAULT_VECTOR_TIMEOUT_MS,
+        () => ({ ids: [] as string[], error: 'vector listPatternIds timed out' }),
+      )
+    : undefined;
+  // `known` is built from ALL records, not just the mirrorable ones: a vector written for a record
+  // the noise gate later excluded still HAS a record, and calling it an orphan would send the user
+  // pruning something that is merely un-mirrorable (found by cross-family review).
+  const keysOf = (r: MemoryRecord): [string, string] => [r.id, patternRecordId(recordToPattern(r))];
+  const known = new Set<string>();
+  for (const r of records) for (const k of keysOf(r)) known.add(k);
+
+  const scoped = narrowed !== undefined && narrowed.error === undefined ? narrowed.ids : undefined;
+  const scopedSet = scoped !== undefined ? new Set(scoped) : undefined;
+  // Count RECORDS, not ids. A record has two possible keys and the mirror may hold BOTH, which
+  // counting ids turns into two mirrored records where there is one (found by cross-family review).
+  const mirrored = scopedSet !== undefined
+    ? mirrorableRecords.filter((r) => keysOf(r).some((k) => scopedSet.has(k))).length
+    : undefined;
+  const orphaned = scoped !== undefined ? scoped.filter((id) => !known.has(id)).length : undefined;
+  const mirroredOther = scoped !== undefined && listed.error === undefined
+    ? listed.ids.length - scoped.length
+    : undefined;
+
   return {
     mode,
     kind: engine.kind,
@@ -972,8 +1154,12 @@ export async function vectorTierStatus(
     ...(!('error' in model) ? { embeddingModel: model.model } : {}),
     lexicalTotal: records.length,
     lexicalMirrorable,
-    mirrored: listed.error === undefined ? listed.ids.length : undefined,
+    mirrored,
     pending,
+    mirrorWriterEnabled,
+    ...(unmirrored !== undefined ? { unmirrored } : {}),
+    ...(mirroredOther !== undefined ? { mirroredOther } : {}),
+    ...(orphaned !== undefined ? { orphaned } : {}),
   };
 }
 
@@ -1453,6 +1639,10 @@ function agentdbVectorEngine(projectRoot: string): VectorEngine {
     async listIds() {
       return listAgentdbDzIds(projectRoot);
     },
+    async listPatternIds() {
+      // the NARROWED scope — what `lexicalMirrorable` counts, so the two numbers are comparable
+      return listAgentdbDzIds(projectRoot, { taskTypes: DZ_PATTERN_TASK_TYPES });
+    },
     async importVectors(rows) {
       return importVectorsToAgentdb(
         projectRoot,