@dzhechkov/harness-core 0.7.0 → 0.7.3

package/src/sign.ts

@@ -14,7 +14,7 @@
 
 import { createHash, sign as cryptoSign, verify as cryptoVerify, createPrivateKey, createPublicKey, generateKeyPairSync } from 'node:crypto';
 import { readFileSync, existsSync, readdirSync, openSync, fstatSync, closeSync, constants as fsConstants } from 'node:fs';
-import { isAbsolute, join, relative, resolve, sep } from 'node:path';
+import { basename, isAbsolute, join, relative, resolve, sep } from 'node:path';
 
 export const MANIFEST_NAME = '.dz-manifest.json';
 export const SBOM_NAME = 'sbom.json';
@@ -47,9 +47,117 @@ export interface VerifyResult {
   readonly failures: readonly VerifyFailure[];
 }
 
+
+/**
+ * Files the PACKER REWRITES, so their bytes are not stable across two packs of the same unchanged
+ * tree — hashing them raw makes a signature that can never verify.
+ *
+ * MEASURED 2026-08-22: three consecutive `pnpm pack` runs of an unchanged `@dzhechkov/harness-core`
+ * (14 dependencies) produced THREE different `package.json` digests; the same three runs of
+ * `@dzhechkov/memory` (zero dependencies) produced one. The difference is the ORDER of dependency
+ * keys. So a package with dependencies could never satisfy its own signature: the signer packs once,
+ * the publisher packs again, and the consumer receives the second one.
+ *
+ * These files are therefore hashed in a CANONICAL form — parsed, keys sorted, stable separators.
+ * HONEST LIMIT: a pure reordering of keys becomes invisible to the signature. It changes nothing a
+ * consumer can observe, and the alternative — excusing `package.json` from signing altogether — would
+ * leave the dependency pins and the bin map unsigned, which is the opposite of a fix. The two
+ * differences that a consumer COULD observe — duplicate keys and precision-losing integers — are
+ * refused canonicalisation outright; see `canonicalisationIsFaithful`.
+ */
+const CANONICALISED_PACK_FILES: ReadonlySet<string> = new Set(['package.json']);
+
+/** Recursive, key-sorted JSON — the same bytes for any key order the packer chooses. */
+function stableJson(value: unknown): string {
+  if (Array.isArray(value)) return '[' + value.map(stableJson).join(',') + ']';
+  if (value !== null && typeof value === 'object') {
+    const keys = Object.keys(value as Record<string, unknown>).sort();
+    return '{' + keys.map((k) => JSON.stringify(k) + ':' + stableJson((value as Record<string, unknown>)[k])).join(',') + '}';
+  }
+  return JSON.stringify(value) ?? 'null';
+}
+
+/**
+ * Is this JSON text one whose PARSED value is a faithful stand-in for its BYTES?
+ *
+ * Canonical hashing deliberately ignores key order and whitespace — differences no consumer can
+ * observe. Two differences are NOT of that kind, and a cross-family review (codex `gpt-5.6-sol`,
+ * 2026-08-22) named both:
+ *
+ *  - DUPLICATE KEYS. `{"bin":"evil","bin":"signed"}` parses to the signed value in every JavaScript
+ *    parser (last wins), so it would hash as clean — while a first-wins parser in another language
+ *    reads `evil` from the same signed bytes.
+ *  - LOSSY INTEGERS. `9007199254740993` parses to `...992`, so two different byte strings share one
+ *    canonical form.
+ *
+ * Neither is reachable through `pnpm pack` — this is a guard against a hand-crafted artifact, not
+ * against the packer. On either, the caller hashes the RAW bytes instead, so an injected duplicate
+ * mismatches a canonical signature and the pack reads TAMPERED.
+ */
+function canonicalisationIsFaithful(text: string): boolean {
+  const scopes: Array<Set<string> | null> = []; // Set = inside an object, null = inside an array
+  let i = 0;
+  while (i < text.length) {
+    const c = text.charAt(i);
+    if (c === '"') {
+      let j = i + 1;
+      while (j < text.length && text.charAt(j) !== '"') j += text.charAt(j) === '\\' ? 2 : 1;
+      if (j >= text.length) return false; // unterminated — JSON.parse will reject it anyway
+      let key: string;
+      try {
+        key = JSON.parse(text.slice(i, j + 1)) as string;
+      } catch {
+        return false;
+      }
+      i = j + 1;
+      let k = i;
+      while (k < text.length && /\s/.test(text.charAt(k))) k++;
+      const scope = scopes[scopes.length - 1];
+      if (text.charAt(k) === ':' && scope instanceof Set) {
+        if (scope.has(key)) return false;
+        scope.add(key);
+      }
+      continue;
+    }
+    if (c === '{') { scopes.push(new Set()); i++; continue; }
+    if (c === '[') { scopes.push(null); i++; continue; }
+    if (c === '}' || c === ']') { scopes.pop(); i++; continue; }
+    if (c === '-' || (c >= '0' && c <= '9')) {
+      const start = i;
+      while (i < text.length && /[-+0-9eE.]/.test(text.charAt(i))) i++;
+      const lit = text.slice(start, i);
+      // Only plain integers can lose precision silently; `1.0` and `1e0` spell the same value in
+      // every parser, so they are legitimate canonical equivalences, not a differential.
+      if (/^-?\d+$/.test(lit) && String(Number(lit)) !== lit) return false;
+      continue;
+    }
+    i++;
+  }
+  return true;
+}
+
+/**
+ * Hash the bytes of one pack file. A packer-rewritten file is hashed canonically; everything else
+ * byte-for-byte. Unparseable JSON falls back to the RAW bytes rather than passing — a corrupt
+ * `package.json` must be a mismatch, never a free pass.
+ */
+export function hashPackBytes(relPath: string, bytes: Buffer): string {
+  const rel = relPath.split(sep).join('/');
+  if (CANONICALISED_PACK_FILES.has(rel)) {
+    try {
+      const text = bytes.toString('utf-8');
+      if (!canonicalisationIsFaithful(text)) throw new Error('not canonicalisable');
+      return createHash('sha256').update(Buffer.from(stableJson(JSON.parse(text)), 'utf-8')).digest('hex');
+    } catch {
+      /* fall through to the raw bytes */
+    }
+  }
+  return createHash('sha256').update(bytes).digest('hex');
+}
+
 /** sha256 of a file's bytes, hex. Follows symlinks — used only when building a manifest we control. */
-export function hashFile(absPath: string): string {
-  return createHash('sha256').update(readFileSync(absPath)).digest('hex');
+export function hashFile(absPath: string, relPath?: string): string {
+  return hashPackBytes(relPath ?? basename(absPath), readFileSync(absPath));
 }
 
 /**
@@ -59,12 +167,14 @@ export function hashFile(absPath: string): string {
  *
  * Returns `null` when the path is not a regular file, or is a symlink.
  */
-export function hashRegularFileNoFollow(absPath: string): string | null {
+export function hashRegularFileNoFollow(absPath: string, relPath?: string): string | null {
   let fd: number | undefined;
   try {
     fd = openSync(absPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
     if (!fstatSync(fd).isFile()) return null;
-    return createHash('sha256').update(readFileSync(fd)).digest('hex');
+    // Canonicalise the bytes ALREADY READ from this descriptor — never re-open, or the TOCTOU
+    // window this function exists to close would be reopened by the fix.
+    return hashPackBytes(relPath ?? basename(absPath), readFileSync(fd));
   } catch {
     return null; // ELOOP on a symlink, ENOENT, EACCES — all fail closed
   } finally {
@@ -192,7 +302,7 @@ export function canonicalizeSigned(manifest: Manifest): Buffer {
 export function buildManifest(root: string, pack: string, files: readonly string[]): Manifest {
   const entries = files.map((rel) => ({
     path: rel.split(sep).join('/'),
-    sha256: hashFile(join(root, rel)),
+    sha256: hashFile(join(root, rel), rel),
   }));
   return { version: MANIFEST_VERSION, pack, files: entries };
 }
@@ -302,7 +412,7 @@ export function verifyManifest(
       continue;
     }
     // One open, O_NOFOLLOW, fstat the descriptor, hash from it: no symlink follow, no TOCTOU window.
-    const digest = hashRegularFileNoFollow(abs);
+    const digest = hashRegularFileNoFollow(abs, entry.path);
     if (digest === null) {
       failures.push({ path: entry.path, reason: 'is a symlink or not a regular file — refusing to hash it' });
       continue;
@@ -401,6 +511,12 @@ export interface PublishGateInput {
   readonly manifestPresent: boolean;
   readonly verifyOk: boolean;
   readonly requireSigning: boolean;
+  /**
+   * The artifact could not be produced (the packer failed), so nothing was compared against the
+   * signature. Distinct from `verifyOk: false`, which means a comparison RAN and disagreed — the
+   * gate blocks either way, but only an honest reason tells the operator which one to fix.
+   */
+  readonly artifactUnavailable?: boolean;
 }
 
 export interface PublishGateDecision {
@@ -421,6 +537,12 @@ export function decidePublishGate(input: PublishGateInput): PublishGateDecision
   if (!input.manifestPresent) {
     return { action: 'block', reason: 'trust root is present but the pack carries no signature manifest' };
   }
+  if (input.artifactUnavailable === true) {
+    return {
+      action: 'block',
+      reason: 'the artifact could not be packed, so its signature was never checked against what ships',
+    };
+  }
   if (!input.verifyOk) {
     return { action: 'block', reason: 'the pack does not match its signed manifest' };
   }

package/src/vector-tier.ts

@@ -133,8 +133,21 @@ export interface ImportVectorRow {
 }
 
 /** The engine PORT — both adapters implement exactly this surface (04 §4.1). */
+/**
+ * Is this engine's `similarity` a real cosine, or only good enough to RANK by?
+ *
+ * `agentdb` computes a magnitude-normalised cosine over the stored vectors, so its number is
+ * comparable and lives in the space the recall floors were calibrated in. The `rvf` adapter returns
+ * `-distance` — unbounded, and the metric is not established — so its number may only order rows.
+ * Printing it beside a 0.38 cosine floor would hand the reader a figure that looks calibrated and is
+ * not; that is the one lie this feature must not tell (ADR, features/recall-true-closeness).
+ */
+export type SimilarityKind = 'cosine' | 'rank-only';
+
 export interface VectorEngine {
   readonly kind: VectorEngineKind;
+  /** Absent means `rank-only`: an engine must SAY its number is a cosine to have it shown as one. */
+  readonly similarityKind?: SimilarityKind;
   upsert(entries: readonly VectorEntry[]): Promise<{ indexed: number; error?: string | undefined }>;
   search(query: string, limit: number): Promise<{ hits: VectorHit[]; error?: string | undefined }>;
   listIds(): Promise<{ ids: string[]; error?: string | undefined }>;
@@ -173,6 +186,14 @@ export interface HybridHit {
   readonly score: number;
   /** lesson-quarantine: set only for a quarantined hit — display marks ⚠q, ranking was damped. */
   readonly quarantined?: boolean;
+  /**
+   * Raw closeness from the semantic leg, when the engine reports a genuine cosine. ABSENT for a
+   * lexical-only hit and for an engine whose score is not a cosine — the honest display there is a
+   * dash, never a substitute number. On a `both` hit this is the SEMANTIC leg's cosine, which
+   * explains less of the ordering than it may appear to: the list is ranked by RRF plus learning
+   * signals, not by this.
+   */
+  readonly similarity?: number;
 }
 
 /** Outcome of {@link recallHybrid}. With no engine this is content-identical to `recallPatterns`. */
@@ -243,6 +264,8 @@ export interface VectorTierStatus {
    * project print a fully-healthy status line over a dead writer (ADR-001).
    */
   readonly mirrorWriterEnabled: boolean;
+  /** WHY it is on or off — so the surface printing it cannot invent a cause (see mirrorWriterReason). */
+  readonly mirrorWriterState: MirrorWriterState;
   /**
    * Vectors of OTHER dz-owned task types (today: `dz-backlog`). Reported separately so a store of N
    * vectors can be fully accounted for, and never folded into `mirrored`, which must stay comparable
@@ -521,15 +544,63 @@ export function readHarmonizeThreshold(projectRoot: string): number {
  * `false`, so its `dz teach` output stays byte-identical to the pre-feature baseline (AC-1).
  */
 export function vectorMirrorEnabled(projectRoot: string): boolean {
+  return mirrorWriterReason(projectRoot).enabled;
+}
+
+/** Why the mirror writer is on or off — a CLOSED set, so a caller cannot invent a cause. */
+export type MirrorWriterState =
+  | 'on'
+  /** No `.dz/config.json` at all — an unconfigured project, which is the normal quiet case. */
+  | 'no-config'
+  /** The config exists but could not be read or parsed. */
+  | 'config-unreadable'
+  /** The config is readable and simply does not enable a mirror. */
+  | 'not-enabled'
+  /** The config explicitly turns the vector tier off. */
+  | 'engine-off';
+
+/**
+ * The mirror writer's state AND its real cause.
+ *
+ * Every failure used to collapse into `false`, and the one message printed above it named ONE
+ * specific cause: "no memory.backend=agentdb". MEASURED 2026-08-24 with a config that sets
+ * `memory.vector.engine: "off"` — the status line correctly said OFF and then blamed a setting the
+ * config did not mention. A diagnosis that names the wrong cause sends the reader to fix something
+ * that is not broken, which is worse than saying nothing.
+ */
+export function mirrorWriterReason(projectRoot: string): { enabled: boolean; state: MirrorWriterState } {
+  const path = join(projectRoot, '.dz', 'config.json');
+  if (!existsSync(path)) return { enabled: false, state: 'no-config' };
+  let cfg: { memory?: { backend?: string; vector?: { engine?: string } } };
   try {
-    const cfg = JSON.parse(readFileSync(join(projectRoot, '.dz', 'config.json'), 'utf-8')) as {
-      memory?: { backend?: string; vector?: { engine?: string } };
-    };
-    if (cfg.memory?.backend === 'agentdb') return true;
-    const engine = cfg.memory?.vector?.engine;
-    return engine === 'agentdb' || engine === 'rvf';
+    cfg = JSON.parse(readFileSync(path, 'utf-8')) as typeof cfg;
   } catch {
-    return false;
+    return { enabled: false, state: 'config-unreadable' };
+  }
+  const engine = cfg.memory?.vector?.engine;
+  // An explicit `off` WINS over `memory.backend`, because that is what the engine resolution itself
+  // does. With `{"memory":{"backend":"agentdb","vector":{"engine":"off"}}}` the same status output
+  // printed `Engine: none — vector tier disabled` and `Mirror writer: ON` one line apart
+  // (cross-family review, codex `gpt-5.6-sol`, 2026-08-24). A report that contradicts itself inside
+  // one screen is worse than either half alone, and the half that was wrong is this one: nothing can
+  // queue to a tier that is off.
+  if (engine === 'off') return { enabled: false, state: 'engine-off' };
+  if (cfg.memory?.backend === 'agentdb') return { enabled: true, state: 'on' };
+  if (engine === 'agentdb' || engine === 'rvf') return { enabled: true, state: 'on' };
+  return { enabled: false, state: 'not-enabled' };
+}
+
+/** The sentence a reader can act on, for each state. */
+export function mirrorWriterExplanation(state: MirrorWriterState): string {
+  switch (state) {
+    case 'on': return 'teach is queueing to the mirror';
+    // "has never configured" is a claim about HISTORY from an observation about the PRESENT: the
+    // file may be tracked in git and merely deleted from the working tree (cross-family review,
+    // codex gpt-5.6-sol, 2026-08-24). Absence proves only absence.
+    case 'no-config': return 'no .dz/config.json here — memory is not configured in this working tree, and nothing is queueing';
+    case 'config-unreadable': return '.dz/config.json exists but could not be read or parsed — fix the file, not the settings';
+    case 'engine-off': return '.dz/config.json sets memory.vector.engine = "off" — the tier is deliberately disabled';
+    case 'not-enabled': return '.dz/config.json enables no mirror (needs memory.backend=agentdb, or memory.vector.engine=agentdb|rvf) — teach is NOT queueing';
   }
 }
 
@@ -806,6 +877,13 @@ export interface RankedPattern {
   readonly id: string;
   readonly pattern: PatternRecord;
   readonly backend: RecallHit['backend'];
+  /**
+   * The semantic leg's raw closeness, when the engine reports a real cosine. Rides ALONGSIDE the RRF
+   * score and never enters the ranking maths — four things depend on RRF magnitude (the reinforce
+   * cap, quarantine damping, the learning uplift, and an ADR-level note in backlog.ts), so this is a
+   * sibling field, never a repurposing.
+   */
+  readonly similarity?: number;
 }
 
 const RRF_K = 60;
@@ -827,7 +905,7 @@ export function mergeHybridHits(
   opts: { readonly limit: number; readonly semanticWeight?: number | undefined },
 ): HybridHit[] {
   const weight = opts.semanticWeight ?? 1;
-  interface Acc { pattern: PatternRecord; lex?: RecallHit['backend']; sem: boolean; score: number }
+  interface Acc { pattern: PatternRecord; lex?: RecallHit['backend']; sem: boolean; score: number; similarity?: number }
   const acc = new Map<string, Acc>();
   lexical.forEach((h, rank) => {
     const cur = acc.get(h.id) ?? { pattern: h.pattern, sem: false, score: 0 };
@@ -839,6 +917,10 @@ export function mergeHybridHits(
     const cur = acc.get(h.id) ?? { pattern: h.pattern, sem: false, score: 0 };
     cur.sem = true;
     cur.score += weight / (RRF_K + rank + 1);
+    // The cosine rides along untouched by the ranking maths. On a hit both legs found, this is the
+    // SEMANTIC leg's number — stated in the field's own doc, because it explains less of the order
+    // than it looks like it does.
+    if (h.similarity !== undefined) cur.similarity = h.similarity;
     acc.set(h.id, cur);
   });
   // Ties break by EVIDENCE, not by the id alphabet: a hit both legs found outranks one only a single
@@ -853,6 +935,7 @@ export function mergeHybridHits(
     pattern: v.pattern,
     backend: v.lex !== undefined && v.sem ? ('both' as const) : v.lex ?? ('vector' as const),
     score: v.score,
+    ...(v.similarity === undefined ? {} : { similarity: v.similarity }),
   });
   // `slice(0, -1)` drops the LAST element instead of returning nothing, so a negative limit used to
   // return almost the whole list (MEASURED: limit -1 over 5 candidates returned 4). `dz recall`
@@ -1016,6 +1099,8 @@ export async function recallHybrid(
     const p = recordToPattern(r);
     idToPattern.set(r.id, p);
   }
+  // Only a declared cosine is allowed to travel as one (ADR: per-engine honesty gate).
+  const cosineEngine = engine.similarityKind === 'cosine';
   const semantic: RankedPattern[] = [];
   const seen = new Set<string>();
   for (const h of sr.hits) {
@@ -1023,7 +1108,13 @@ export async function recallHybrid(
     const p = idToPattern.get(h.dzId);
     if (p === undefined) continue; // vector-only orphan — the store pruned/expired it; NEVER resurrect
     seen.add(h.dzId);
-    semantic.push({ id: h.dzId, pattern: p, backend: 'vector' });
+    // The cosine was computed by the engine and then thrown away here — `h.similarity` was read
+    // nowhere in this function, and `relevance` in `--json` has been the RRF rank surrogate ever
+    // since (MEASURED: a nonsense query and a meaningful one both score 1/61 at the top). It now
+    // travels with the hit. Only a genuine cosine travels: an engine whose score is a negated
+    // distance reports `similarityKind: 'rank-only'` and contributes nothing here.
+    semantic.push({ id: h.dzId, pattern: p, backend: 'vector',
+      ...(cosineEngine && Number.isFinite(h.similarity) ? { similarity: h.similarity } : {}) });
   }
   const lex: RankedPattern[] = lexical.map((h) => ({
     id: identityToId.get(patternIdentityOf(h.pattern)) ?? patternRecordId(h.pattern),
@@ -1085,7 +1176,8 @@ export async function vectorTierStatus(
   const lexicalMirrorable = mirrorableRecords.length;
   const pendingEntries = readVectorPending(projectRoot);
   const pending = pendingEntries.length;
-  const mirrorWriterEnabled = vectorMirrorEnabled(projectRoot);
+  const mirror = mirrorWriterReason(projectRoot);
+  const mirrorWriterEnabled = mirror.enabled;
   const resolved = pickEngine(projectRoot, opts);
   if (resolved.engine === undefined) {
     // No engine to ask ⇒ the debt is UNKNOWN. Reporting 0 here is precisely the defect this feature
@@ -1099,6 +1191,7 @@ export async function vectorTierStatus(
       lexicalMirrorable,
       pending,
       mirrorWriterEnabled,
+      mirrorWriterState: mirror.state,
     };
   }
   const engine = resolved.engine;
@@ -1157,6 +1250,7 @@ export async function vectorTierStatus(
     mirrored,
     pending,
     mirrorWriterEnabled,
+    mirrorWriterState: mirror.state,
     ...(unmirrored !== undefined ? { unmirrored } : {}),
     ...(mirroredOther !== undefined ? { mirroredOther } : {}),
     ...(orphaned !== undefined ? { orphaned } : {}),
@@ -1613,6 +1707,9 @@ export async function importRvfCheckpoint(projectRoot: string, source: string, o
 function agentdbVectorEngine(projectRoot: string): VectorEngine {
   return {
     kind: 'agentdb',
+    // A magnitude-normalising cosine over the stored vectors — comparable, and in the same space the
+    // recall floors were calibrated in.
+    similarityKind: 'cosine',
     async upsert(entries) {
       const r = await indexPatternsToAgentdb(
         projectRoot,
@@ -1775,6 +1872,8 @@ function rvfVectorEngine(projectRoot: string): VectorEngine {
   const noEmbedder = 'rvf engine present but no embedder — install agentdb (dz setup --memory agentdb)';
   return {
     kind: 'rvf',
+    // `-distance`, unbounded, metric not established: good for ORDER, never for a threshold.
+    similarityKind: 'rank-only',
     async upsert(entries) {
       const idmap = readRvfIdmap(projectRoot);
       const guard = guardRvfEmbedSpace(projectRoot, idmap);