@dzhechkov/harness-core 0.3.78 → 0.3.82

package/src/agentdb-index.ts

@@ -12,7 +12,7 @@
  * @packageDocumentation
  */
 
-import { existsSync } from 'node:fs';
+import { existsSync, mkdirSync } from 'node:fs';
 import { join, dirname } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { createRequire } from 'node:module';
@@ -218,7 +218,12 @@ function hasDzTables(db: ReadonlyDb): boolean {
   return t('reasoning_patterns') && t('pattern_embeddings');
 }
 
-function cosineSimilarity(a: Float32Array, b: Float32Array): number {
+/**
+ * Cosine similarity in [-1, 1] over two embeddings. Exported (was file-private) so
+ * `harmonizeVectorStore` scores near-duplicate pairs with the IDENTICAL math the semantic search
+ * path uses — one cosine implementation, no drift between search and harmonize.
+ */
+export function cosineSimilarity(a: Float32Array, b: Float32Array): number {
   const n = Math.min(a.length, b.length);
   let dot = 0;
   let na = 0;
@@ -328,3 +333,96 @@ export async function listAgentdbDzIds(
     db.close();
   }
 }
+
+/* ------------------------------------------------------------------ */
+/*  IMPORT half (dz-vector-harmonize-import M0.4): upsert-by-dzId       */
+/* ------------------------------------------------------------------ */
+
+/** One precomputed vector to upsert by its content-addressed `dzId`. */
+export interface AgentdbImportRow {
+  /** Join key — the canonical `MemoryRecord.id`; the upsert key. */
+  readonly dzId: string;
+  /** The embedding to store VERBATIM (the checkpoint's space, preserved). */
+  readonly vector: Float32Array;
+  /** Pattern text (`approach`) — used only when INSERTing a dzId not yet present. */
+  readonly text: string;
+  readonly taskType: string;
+  /** Stored as `success_rate` on insert; clamped to [0,1]. */
+  readonly score: number;
+  readonly metadata?: Record<string, unknown>;
+}
+
+/** Minimal better-sqlite3 surface the upsert path uses (run + get + transaction). */
+interface UpsertDb {
+  pragma: (s: string) => void;
+  exec: (s: string) => void;
+  prepare: (q: string) => {
+    run: (...a: unknown[]) => { lastInsertRowid: number | bigint };
+    get: (...a: unknown[]) => unknown;
+  };
+  transaction: <T>(fn: () => T) => () => T;
+  close: () => void;
+}
+
+/**
+ * UPSERT precomputed vectors into the shared AgentDB store, keyed on `metadata.dzId` — the write
+ * half of `dz vector import`. For each row: look up the existing `reasoning_patterns` row for the
+ * dzId; if found, REPLACE its `pattern_embeddings` BLOB in place (never a new row); if absent,
+ * INSERT both the pattern row (`approach = text`, `metadata.dzId`) and its embedding. The vector is
+ * stored VERBATIM. NON-DESTRUCTIVE: only the imported dzIds are inserted/replaced — every other
+ * dzId's vector and pattern are left untouched (no blind table overwrite). Idempotent — re-importing
+ * the same dzIds REPLACEs in place, adding 0 rows. Same dynamic `better-sqlite3` resolve + WAL +
+ * `busy_timeout 5000` as {@link indexPatternsToAgentdb}. Best-effort: honest `{ error }`, never a throw.
+ */
+export async function importVectorsToAgentdb(
+  projectRoot: string,
+  rows: readonly AgentdbImportRow[],
+  opts: { dbPath?: string } = {},
+): Promise<{ imported: number; error?: string }> {
+  if (rows.length === 0) return { imported: 0 };
+  let sqliteUrl: string;
+  try {
+    const req = createRequire(join(projectRoot, 'package.json'));
+    sqliteUrl = pathToFileURL(req.resolve('better-sqlite3')).href;
+  } catch {
+    return { imported: 0, error: DEPS_MISSING };
+  }
+  try {
+    const { default: Database } = (await import(sqliteUrl)) as { default: new (p: string) => UpsertDb };
+    const dbFile = resolveAgentdbPath(projectRoot, opts.dbPath);
+    mkdirSync(dirname(dbFile), { recursive: true }); // better-sqlite3 won't create the parent dir
+    const db = new Database(dbFile);
+    try {
+      db.pragma('journal_mode = WAL');
+      db.pragma('busy_timeout = 5000'); // wait out a brief MCP-server write lock instead of failing
+      db.exec(REASONING_BANK_SCHEMA);
+      const findByDzId = db.prepare("SELECT id FROM reasoning_patterns WHERE json_extract(metadata, '$.dzId') = ?");
+      const insPattern = db.prepare('INSERT INTO reasoning_patterns (task_type, approach, success_rate, uses, avg_reward, tags, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)');
+      const upsertEmb = db.prepare('INSERT OR REPLACE INTO pattern_embeddings (pattern_id, embedding) VALUES (?, ?)');
+      const commit = db.transaction(() => {
+        let imported = 0;
+        for (const row of rows) {
+          const buf = Buffer.from(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+          const existing = findByDzId.get(row.dzId) as { id: number } | undefined;
+          if (existing !== undefined) {
+            upsertEmb.run(existing.id, buf); // REPLACE the embedding in place — never a duplicate row
+          } else {
+            const meta = { ...(row.metadata ?? {}), dzId: row.dzId };
+            const r = insPattern.run(
+              row.taskType, row.text, Math.max(0, Math.min(1, Number.isFinite(row.score) ? row.score : 0)), 0, 0.0,
+              null, JSON.stringify(meta),
+            );
+            upsertEmb.run(Number(r.lastInsertRowid), buf);
+          }
+          imported += 1;
+        }
+        return imported;
+      });
+      return { imported: commit() };
+    } finally {
+      db.close();
+    }
+  } catch (err) {
+    return { imported: 0, error: `import failed: ${err instanceof Error ? err.message : String(err)}` };
+  }
+}

package/src/index.ts

@@ -21,20 +21,24 @@ export { createSkill } from './create-skill.js';
 export type { CreateSkillOptions, CreateSkillResult } from './create-skill.js';
 export { checkUpstream, checkAllUpstream, discoverSourcePackages, loadSourcesManifest } from './sync-upstream.js';
 export type { SyncUpstreamReport, UpstreamCheckResult, SourcesManifest, SourcePackageInfo } from './sync-upstream.js';
+export { sweepSkillDrift, syncCanonicalSkill } from './skill-drift.js';
+export type { SweepResult, DriftedSkill, SyncResult, SyncCanonicalOptions } from './skill-drift.js';
 export { benchmarkSkill, benchmarkSkills, compareSkills } from './benchmark.js';
 export { buildRegistry, searchRegistry, filterByCategory, skillPackBaseDirs, discoverSkillPackDirs } from './registry.js';
 export { recommend } from './recommend.js';
 export { pretrain } from './pretrain.js';
-export { loadPatterns, loadSessions, computePatternBoost, readLearningConfig, BOOST_CAP, recordPattern, loadStorePatternsSync, loadStoreRecords, patternToRecord, recordToPattern, patternRecordId, patternIdentityOf, dreamRecordId, isMirrorableLearning, consolidateSessions, recallPatterns, pruneNoisePatterns } from './patterns.js';
-export type { PatternRecord, SessionRecord, LearningConfig, LoadOptions, ConsolidateResult, ConsolidateOptions, SqliteBackendMode, RecallHit, SessionsSource, PruneNoiseResult } from './patterns.js';
+export { loadPatterns, loadSessions, computePatternBoost, readLearningConfig, BOOST_CAP, recordPattern, loadStorePatternsSync, loadStoreRecords, patternToRecord, recordToPattern, patternRecordId, patternIdentityOf, dreamRecordId, isMirrorableLearning, consolidateSessions, recallPatterns, pruneNoisePatterns, removePatternsByIds, snapshotStore } from './patterns.js';
+export type { PatternRecord, SessionRecord, LearningConfig, LoadOptions, ConsolidateResult, ConsolidateOptions, SqliteBackendMode, RecallHit, SessionsSource, PruneNoiseResult, RemovePatternsResult, SnapshotStoreResult } from './patterns.js';
 export {
   DEFAULT_VECTOR_TIMEOUT_MS,
+  DEFAULT_HARMONIZE_THRESHOLD,
   withVectorTimeout,
   isVectorNoise,
   patternVectorEntry,
   dreamVectorEntry,
   memoryRecordVectorEntry,
   readVectorEngineMode,
+  readHarmonizeThreshold,
   vectorMirrorEnabled,
   resolveVectorEngine,
   mirrorEntriesToVector,
@@ -43,6 +47,9 @@ export {
   mergeHybridHits,
   recallHybrid,
   vectorTierStatus,
+  harmonizeVectorStore,
+  selectClusterKeeper,
+  importRvfCheckpoint,
 } from './vector-tier.js';
 export type {
   VectorEngine,
@@ -50,6 +57,7 @@ export type {
   VectorEngineMode,
   VectorEntry,
   VectorHit,
+  ImportVectorRow,
   MirrorReceipt,
   ResolvedVectorEngine,
   HybridRecall,
@@ -58,12 +66,18 @@ export type {
   RankedPattern,
   VectorServiceOptions,
   VectorTierStatus,
+  HarmonizeItem,
+  HarmonizeCluster,
+  HarmonizeReport,
+  HarmonizeOptions,
+  ImportReport,
+  ImportOptions,
 } from './vector-tier.js';
 export { runSetup, generateHooksConfig, generateAgentdbWriter, writerVersionOf, AGENTDB_WRITER_VERSION } from './setup.js';
 export { statuslineData, readFeatureAdrState, writeFeatureAdrState, featureAdrStatePath } from './statusline.js';
 export type { StatuslineData, FeatureAdrState, WriteFeatureAdrStateInput } from './statusline.js';
-export { indexPatternsToAgentdb, resolveAgentdbPath, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder } from './agentdb-index.js';
-export type { AgentdbSearchHit, AgentdbSearchResult } from './agentdb-index.js';
+export { indexPatternsToAgentdb, resolveAgentdbPath, searchAgentdbPatterns, listAgentdbDzIds, resolveAgentdbEmbedder, cosineSimilarity, importVectorsToAgentdb } from './agentdb-index.js';
+export type { AgentdbSearchHit, AgentdbSearchResult, AgentdbImportRow } from './agentdb-index.js';
 export { putBookKnowledge, queryBookKnowledge, bookKbPath } from './book-kb.js';
 export type { BookKU, BookKUHit } from './book-kb.js';
 export {

package/src/skill-drift.ts

@@ -0,0 +1,273 @@
+/**
+ * Intra-monorepo skill-drift guard.
+ *
+ * The same skill is physically duplicated across many monorepo packages
+ * (`packages/@dzhechkov/*​/<skill>/` + `.claude/skills/<skill>/`). A fix applied to ONE copy
+ * silently leaves the others broken — this is exactly how a CRITICAL `goap-research-ed25519`
+ * self-signed-forgery exploit shipped in 10 of 12 copies, and how a `brutal-honesty-review`
+ * `set -e` crash reached the PUBLISHED `@dzhechkov/skills-qe`. Both were found only by accident.
+ *
+ * `dz sync-upstream` only checks against EXTERNAL repos and is structurally blind to this class of
+ * drift. This module is the intra-monorepo complement:
+ *
+ *   • `sweepSkillDrift(root)`        — detector: which shared skills byte-differ between copies.
+ *   • `syncCanonicalSkill(root, s)`  — healer: overwrite every copy from `skills-meta/<skill>`.
+ *
+ * Both are PURE functions that return plain data — no printing, no `process.exit`, no throwing on
+ * the "canonical missing" / "drift found" business cases. The CLI layer owns exit codes and I/O.
+ * Dependency-free: `node:fs` / `node:path` / `node:crypto` only.
+ *
+ * @packageDocumentation
+ */
+
+import { readdirSync, statSync, readFileSync, writeFileSync, mkdirSync, rmSync, existsSync } from 'node:fs';
+import { join, relative, dirname, basename } from 'node:path';
+import { createHash } from 'node:crypto';
+
+/** One shared skill whose copies byte-differ (`driftFiles > 0`). */
+export interface DriftedSkill {
+  /** Skill dir basename (e.g. `goap-research-ed25519`). */
+  readonly name: string;
+  /** How many locations hold this skill. */
+  readonly copies: number;
+  /** # relative paths where the copies disagree (≥1 ⇒ drift). */
+  readonly driftFiles: number;
+  /** Size of the union of relative file paths across all copies. */
+  readonly totalFiles: number;
+  /** # (copy × relative-path) pairs where the file is absent from a copy. */
+  readonly missingFiles: number;
+  /** Absolute paths of every copy — lets a human / `--json` consumer jump to the drifted dirs. */
+  readonly locations: readonly string[];
+}
+
+/** Options for {@link sweepSkillDrift}. */
+export interface SweepOptions {
+  /**
+   * Which copies to compare.
+   * - `'packages'` (the CI-gate default): PUBLISHED package copies only (`packages/`). The dogfood
+   *   `.claude/skills/<skill>` dev copies are excluded — the repo's own `dz sync` test treats them as
+   *   "legitimately lagging" the published version, so counting them makes the gate red-on-arrival.
+   *   The dangerous drift (goap, brutal-honesty) was always between PUBLISHED packages.
+   * - `'all'`: packages + `.claude/skills` (the raw sweep the audit script does).
+   */
+  readonly scope?: 'packages' | 'all';
+  /** Skill basenames whose drift is ACCEPTED (documented intentional forks) — reported separately, never counted as gate drift. */
+  readonly allowlist?: readonly string[];
+}
+
+/** Result of a read-only intra-monorepo drift sweep. */
+export interface SweepResult {
+  /** # skills present in ≥2 locations (within scope). */
+  readonly duplicated: number;
+  /** Skills that byte-differ AND are not allowlisted, sorted by `driftFiles` desc — the gate keys on this. */
+  readonly drifted: readonly DriftedSkill[];
+  /** Skills that byte-differ but are allowlisted (intentional) — surfaced for transparency, not gated. */
+  readonly allowlisted: readonly DriftedSkill[];
+}
+
+/** Options for {@link syncCanonicalSkill}. */
+export interface SyncCanonicalOptions {
+  /** Report drift only, write NOTHING (CI mode). */
+  readonly check?: boolean;
+  /** Override the canonical source dir (default: `skills-meta/<skill>`). */
+  readonly from?: string;
+}
+
+/** Result of a canonical-wins sync (or a `check:true` dry-run). */
+export interface SyncResult {
+  /** Resolved canonical dir (abs). */
+  readonly canonical: string;
+  /** Whether the canonical source dir exists. `false` ⇒ nothing was done (CLI → non-zero exit). */
+  readonly canonicalExists: boolean;
+  /** # non-canonical copies found. */
+  readonly copies: number;
+  /** # copies overwritten (0 when `check:true`). */
+  readonly synced: number;
+  /** # copies that differ from canonical. */
+  readonly drifted: number;
+  /** Abs paths of copies written (empty when `check:true` — proves no writes). */
+  readonly wrote: readonly string[];
+}
+
+const SKILL_MANIFEST = 'SKILL.md';
+const IGNORED_ENTRIES = new Set(['node_modules', '__pycache__', '.DS_Store']);
+
+/** md5 of a file's bytes (identical to both prototype scripts ⇒ identical drift verdicts). */
+function md5(path: string): string {
+  return createHash('md5').update(readFileSync(path)).digest('hex');
+}
+
+/** Recursive file list under `dir`; skips `node_modules` / `__pycache__` / `.DS_Store`. */
+function walk(dir: string): string[] {
+  const out: string[] = [];
+  for (const entry of readdirSync(dir)) {
+    if (IGNORED_ENTRIES.has(entry)) continue;
+    const p = join(dir, entry);
+    let st;
+    try {
+      st = statSync(p);
+    } catch {
+      continue;
+    }
+    if (st.isDirectory()) out.push(...walk(p));
+    else out.push(p);
+  }
+  return out;
+}
+
+/**
+ * Every skill dir (a dir containing `SKILL.md`) under `packages/` + `.claude/skills`, excluding
+ * `node_modules` / `__pycache__`. Ported verbatim from `scripts/drift-sweep-skills.mjs`.
+ */
+function findSkillDirs(root: string, scope: 'packages' | 'all' = 'all'): string[] {
+  const dirs: string[] = [];
+  const roots = scope === 'packages' ? [join(root, 'packages')] : [join(root, 'packages'), join(root, '.claude', 'skills')];
+  const stack = roots.filter((p) => existsSync(p));
+  while (stack.length) {
+    const d = stack.pop() as string;
+    let entries: string[];
+    try {
+      entries = readdirSync(d);
+    } catch {
+      continue;
+    }
+    if (entries.includes(SKILL_MANIFEST)) dirs.push(d);
+    for (const e of entries) {
+      if (IGNORED_ENTRIES.has(e)) continue;
+      const p = join(d, e);
+      try {
+        if (statSync(p).isDirectory()) stack.push(p);
+      } catch {
+        /* skip unreadable entries */
+      }
+    }
+  }
+  return dirs;
+}
+
+/**
+ * Detect intra-monorepo skill drift: find every skill duplicated across ≥2 locations and report
+ * which copies byte-differ. Pure port of `scripts/drift-sweep-skills.mjs`.
+ *
+ * `result.drifted.length === 0` is the exact condition the CI gate keys on.
+ */
+export function sweepSkillDrift(root: string, opts: SweepOptions = {}): SweepResult {
+  const scope = opts.scope ?? 'all';
+  const allow = new Set(opts.allowlist ?? []);
+
+  // Group skill dirs by basename → Map<name, locations[]>.
+  const byName = new Map<string, string[]>();
+  for (const d of findSkillDirs(root, scope)) {
+    const name = basename(d);
+    const list = byName.get(name);
+    if (list) list.push(d);
+    else byName.set(name, [d]);
+  }
+
+  let duplicated = 0;
+  const drifted: DriftedSkill[] = [];
+  const allowlisted: DriftedSkill[] = [];
+
+  for (const [name, unsorted] of [...byName.entries()].sort((a, b) => (a[0] < b[0] ? -1 : 1))) {
+    if (unsorted.length < 2) continue;
+    duplicated++;
+    const copies = [...unsorted].sort();
+
+    // Union of relative file paths across every copy.
+    const relFiles = new Set<string>();
+    for (const c of copies) for (const f of walk(c)) relFiles.add(relative(c, f));
+
+    let driftFiles = 0;
+    let missingFiles = 0;
+    for (const rel of relFiles) {
+      const hashes = new Set<string>();
+      for (const c of copies) {
+        const p = join(c, rel);
+        if (existsSync(p)) hashes.add(md5(p));
+        else {
+          missingFiles++;
+          hashes.add('__MISSING__');
+        }
+      }
+      if (hashes.size > 1) driftFiles++;
+    }
+
+    if (driftFiles > 0) {
+      const entry: DriftedSkill = {
+        name,
+        copies: copies.length,
+        driftFiles,
+        totalFiles: relFiles.size,
+        missingFiles,
+        locations: copies,
+      };
+      (allow.has(name) ? allowlisted : drifted).push(entry);
+    }
+  }
+
+  const byDrift = (a: DriftedSkill, b: DriftedSkill): number => b.driftFiles - a.driftFiles;
+  drifted.sort(byDrift);
+  allowlisted.sort(byDrift);
+  return { duplicated, drifted, allowlisted };
+}
+
+/**
+ * Heal one skill: treat `from ?? skills-meta/<skill>` as canonical and overwrite every other copy in
+ * the monorepo, proving byte-identity. Pure port of `scripts/sync-canonical-skill.mjs`.
+ *
+ * `check:true` writes NOTHING (`wrote` stays empty) and only reports the drift count.
+ * Default overwrites drifting copies; a subsequent {@link sweepSkillDrift} then reports 0 drift.
+ * When the canonical dir does not exist, returns `canonicalExists:false` and does nothing (the CLI
+ * turns that into a non-zero exit) — this function never throws / never `process.exit`s.
+ */
+export function syncCanonicalSkill(root: string, skill: string, opts: SyncCanonicalOptions = {}): SyncResult {
+  const check = opts.check === true;
+  const canonical = opts.from ?? join(root, 'packages/@dzhechkov/skills-meta', skill);
+
+  if (!existsSync(canonical)) {
+    return { canonical, canonicalExists: false, copies: 0, synced: 0, drifted: 0, wrote: [] };
+  }
+
+  const canonFiles = walk(canonical)
+    .map((p) => relative(canonical, p))
+    .sort();
+
+  // The healer heals EXACTLY what the detector sees (same roots via findSkillDirs) — otherwise a copy
+  // could be silently healed but never gated, or vice-versa. Every <skill>/ dir except the canonical.
+  const copies = findSkillDirs(root, 'all')
+    .filter((d) => basename(d) === skill && relative(canonical, d) !== '')
+    .sort();
+
+  let drifted = 0;
+  let synced = 0;
+  const wrote: string[] = [];
+
+  for (const copy of copies) {
+    const copyFiles = new Set(walk(copy).map((p) => relative(copy, p)));
+    let differs = false;
+    // Extra files in the copy not present in canonical ⇒ drift.
+    for (const f of copyFiles) if (!canonFiles.includes(f)) differs = true;
+    for (const f of canonFiles) {
+      const src = join(canonical, f);
+      const dst = join(copy, f);
+      if (!existsSync(dst) || md5(src) !== md5(dst)) differs = true;
+    }
+    if (!differs) continue;
+    drifted++;
+
+    if (check) continue; // report only — write NOTHING
+
+    // Overwrite: remove extra files, then copy every canonical file byte-for-byte.
+    for (const f of copyFiles) if (!canonFiles.includes(f)) rmSync(join(copy, f));
+    for (const f of canonFiles) {
+      const src = join(canonical, f);
+      const dst = join(copy, f);
+      mkdirSync(dirname(dst), { recursive: true });
+      writeFileSync(dst, readFileSync(src));
+    }
+    synced++;
+    wrote.push(copy);
+  }
+
+  return { canonical, canonicalExists: true, copies: copies.length, synced, drifted, wrote };
+}