@dzhechkov/harness-core 0.3.140 → 0.3.142

package/src/agentdb-index.ts

@@ -160,9 +160,21 @@ export async function indexPatternsToAgentdb(
 /*  READ half (dz-rvf-vector-bridge FR-3): semantic search + id scan   */
 /* ------------------------------------------------------------------ */
 
-/** The dz-owned ReasoningBank task_types (teach mirror + consolidate mirror). */
+/**
+ * The RECALL/search default task_types. Deliberately EXCLUDES `dz-backlog`: `dz recall` (and
+ * feature-adr Step-0) must never surface raw backlog ideas as if they were earned lessons (ADR-005).
+ */
 const DZ_TASK_TYPES = ['dz-teach', 'dz-learning'] as const;
 
+/**
+ * The dz-owned task_types for LIFECYCLE scans (id enumeration + reindex ownership) — a SUPERSET of
+ * the recall default that ALSO owns `dz-backlog` (smart-backlog, ADR-001/005). Reindex must re-embed
+ * these on a model bump (else backlog rows rot in a stale embedding space), and id-scans must see them
+ * (mirror idempotency). Kept SEPARATE from {@link DZ_TASK_TYPES} so ownership never leaks ideas into
+ * lesson recall: search defaults to DZ_TASK_TYPES, lifecycle to DZ_OWNED_TASK_TYPES.
+ */
+export const DZ_OWNED_TASK_TYPES = ['dz-teach', 'dz-learning', 'dz-backlog'] as const;
+
 /** Minimal READONLY better-sqlite3 surface used by the read half. */
 interface ReadonlyDb {
   pragma: (s: string) => void;
@@ -362,7 +374,9 @@ export async function listAgentdbDzIds(
   const { db } = opened;
   try {
     if (!hasDzTables(db)) return { ids: [] };
-    const taskTypes = opts.taskTypes ?? DZ_TASK_TYPES;
+    // LIFECYCLE scan ⇒ the OWNED superset (incl. dz-backlog) so idea ids are visible for mirror
+    // idempotency + `dz vector status`. Recall/search still defaults to the narrower DZ_TASK_TYPES.
+    const taskTypes = opts.taskTypes ?? DZ_OWNED_TASK_TYPES;
     const placeholders = taskTypes.map(() => '?').join(', ');
     const rows = db
       .prepare(`SELECT metadata FROM reasoning_patterns WHERE task_type IN (${placeholders})`)
@@ -380,6 +394,65 @@ export async function listAgentdbDzIds(
   }
 }
 
+/**
+ * Read the rows that ACTUALLY EXIST IN THE STORE for a task_type, as re-indexable {@link AgentdbRow}s
+ * (their stored `approach` text + score/uses/reward/tags/metadata, dzId preserved). This is the correct
+ * source for a REINDEX: a reindex re-embeds what is physically in the store to the new model — reading it
+ * back from the store (not reconstructing from a sidecar file like ideas.jsonl) means an empty/unreadable
+ * sidecar can never leave real store rows un-re-embedded and stale under an advanced manifest (HIGH-G).
+ * Readonly, best-effort ({rows:[]} on absent/unavailable), never throws.
+ */
+export function readAgentdbRowsByTaskType(
+  projectRoot: string,
+  taskType: string,
+  opts: { dbPath?: string } = {},
+): { rows: AgentdbRow[]; error?: string } {
+  const opened = openReadonly(projectRoot, opts.dbPath);
+  if ('absent' in opened) return { rows: [] };
+  if ('error' in opened) return { rows: [], error: opened.error };
+  const { db } = opened;
+  try {
+    if (!hasDzTables(db)) return { rows: [] };
+    const raw = db
+      .prepare('SELECT approach, success_rate, uses, avg_reward, tags, metadata FROM reasoning_patterns WHERE task_type = ?')
+      .all(taskType) as Array<{ approach: string; success_rate: number; uses: number; avg_reward: number; tags: string | null; metadata: string | null }>;
+    const rows: AgentdbRow[] = raw.map((r) => {
+      let metadata: Record<string, unknown> | undefined;
+      if (typeof r.metadata === 'string' && r.metadata !== '') {
+        try {
+          const m = JSON.parse(r.metadata) as unknown;
+          if (m !== null && typeof m === 'object') metadata = m as Record<string, unknown>;
+        } catch {
+          /* drop unparseable metadata */
+        }
+      }
+      let tags: string[] | undefined;
+      if (typeof r.tags === 'string' && r.tags !== '') {
+        try {
+          const t = JSON.parse(r.tags) as unknown;
+          if (Array.isArray(t)) tags = t.filter((x): x is string => typeof x === 'string');
+        } catch {
+          /* drop unparseable tags */
+        }
+      }
+      return {
+        taskType,
+        text: r.approach,
+        score: r.success_rate,
+        uses: r.uses,
+        avgReward: r.avg_reward,
+        ...(tags !== undefined ? { tags } : {}),
+        ...(metadata !== undefined ? { metadata } : {}),
+      };
+    });
+    return { rows };
+  } catch (err) {
+    return { rows: [], error: `row scan failed: ${err instanceof Error ? err.message : String(err)}` };
+  } finally {
+    db.close();
+  }
+}
+
 /* ------------------------------------------------------------------ */
 /*  IMPORT half (dz-vector-harmonize-import M0.4): upsert-by-dzId       */
 /* ------------------------------------------------------------------ */
@@ -437,6 +510,20 @@ export async function importVectorsToAgentdb(
     const { default: Database } = (await import(sqliteUrl)) as { default: new (p: string) => UpsertDb };
     const model = resolveEmbedModel(projectRoot);
     if ('error' in model) return { imported: 0, error: model.error };
+    // MED-C: the store manifest guards the MODEL, but a per-vector guard was missing — a malformed or
+    // TOCTOU vector (wrong length / NaN / ±Infinity) would be stamped compatible. Fail CLOSED and LOUD:
+    // reject the whole batch if any vector's dimensionality ≠ the store dim or any component is non-finite,
+    // so a vector can only enter the store bound to the model+dim the manifest names.
+    for (const row of rows) {
+      if (!(row.vector instanceof Float32Array) || row.vector.length !== model.dim) {
+        return { imported: 0, error: `vector validation failed for ${row.dzId}: length ${row.vector?.length} != store dim ${model.dim}` };
+      }
+      for (let i = 0; i < row.vector.length; i += 1) {
+        if (!Number.isFinite(row.vector[i])) {
+          return { imported: 0, error: `vector validation failed for ${row.dzId}: non-finite component at index ${i}` };
+        }
+      }
+    }
     const dbFile = resolveAgentdbPath(projectRoot, opts.dbPath);
     mkdirSync(dirname(dbFile), { recursive: true }); // better-sqlite3 won't create the parent dir
     const db = new Database(dbFile);
@@ -530,6 +617,62 @@ export function clearAgentdbQuarantine(
   }
 }
 
+/**
+ * DELETE mirrored rows by `metadata.dzId` (pattern + its embedding), optionally scoped to a task_type
+ * set. The write-half of a structured-store removal: when `harmonize --apply` drops ideas from
+ * `ideas.jsonl`, their `dz-backlog` vectors must be pruned too, or a later semantic search matches an
+ * ORPHAN dzId that no longer has a structured record (smart-backlog HIGH-A). Best-effort, same custody
+ * model as {@link clearAgentdbQuarantine} (missing db/deps ⇒ no-op). Never throws.
+ */
+export function deleteAgentdbByDzIds(
+  projectRoot: string,
+  dzIds: readonly string[],
+  opts: { dbPath?: string; taskTypes?: readonly string[] } = {},
+): { deleted: number; error?: string } {
+  if (dzIds.length === 0) return { deleted: 0 };
+  const dbFile = resolveAgentdbPath(projectRoot, opts.dbPath);
+  if (!existsSync(dbFile)) return { deleted: 0 };
+  let Database: new (p: string) => UpsertDb;
+  try {
+    const req = createRequire(join(projectRoot, 'package.json'));
+    Database = req('better-sqlite3') as new (p: string) => UpsertDb;
+  } catch {
+    return { deleted: 0, error: DEPS_MISSING };
+  }
+  try {
+    const db = new Database(dbFile);
+    try {
+      db.pragma('journal_mode = WAL');
+      db.pragma('busy_timeout = 5000');
+      db.exec(REASONING_BANK_SCHEMA);
+      const scope = opts.taskTypes !== undefined && opts.taskTypes.length > 0;
+      const scopeSql = scope ? ` AND task_type IN (${opts.taskTypes!.map(() => '?').join(', ')})` : '';
+      const findIds = db.prepare(`SELECT id FROM reasoning_patterns WHERE json_extract(metadata, '$.dzId') = ?${scopeSql}`) as unknown as {
+        all: (...a: unknown[]) => { id: number }[];
+      };
+      const delEmb = db.prepare('DELETE FROM pattern_embeddings WHERE pattern_id = ?');
+      const delPat = db.prepare('DELETE FROM reasoning_patterns WHERE id = ?');
+      const tx = db.transaction(() => {
+        let deleted = 0;
+        for (const dzId of dzIds) {
+          const rows = scope ? findIds.all(dzId, ...opts.taskTypes!) : findIds.all(dzId);
+          for (const { id } of rows) {
+            delEmb.run(id);
+            delPat.run(id);
+            deleted += 1;
+          }
+        }
+        return deleted;
+      });
+      return { deleted: tx() };
+    } finally {
+      db.close();
+    }
+  } catch (err) {
+    return { deleted: 0, error: `delete by dzId failed: ${err instanceof Error ? err.message : String(err)}` };
+  }
+}
+
 export function bumpAgentdbUses(
   projectRoot: string,
   dzIds: readonly string[],