@yemi33/minions 0.1.2424 → 0.1.2426

package/engine/review-learning-backfill.js

@@ -0,0 +1,482 @@
+// engine/review-learning-backfill.js — strict, offline backfill of durable
+// review-learning lessons from legacy artifacts.
+//
+// The backfill scans deterministic, bounded slices of legacy sources and imports
+// a lesson ONLY when the source already carries everything the live capture path
+// requires: project, canonical PR, author agent, at least one discrete generalized
+// rule, successful current-head resolution evidence, source consistency, and clean
+// security provenance. It never asks an LLM to infer historical rules — records
+// that lack any qualification signal are left non-recallable with an explicit skip
+// reason.
+//
+// Imports go through the exact same stable-key + upsert path as live capture
+// (reviewLearning.buildFindingKey + memoryStore.upsertReviewLearningLesson), so a
+// dry-run classifies identically to a real run and a second real run imports zero
+// duplicates.
+
+const fs = require('fs');
+const path = require('path');
+const shared = require('./shared');
+const reviewLearning = require('./review-learning');
+
+const REASON_BUCKETS = Object.freeze([
+  'already-present',
+  'ambiguous-raw-comment',
+  'missing-generalized-rule',
+  'missing-resolution-evidence',
+  'source-mismatch',
+  'ambiguous-project',
+  'missing-pr',
+  'missing-author',
+  'injection-flagged',
+  'not-qualified',
+]);
+
+const RAW_KINDS = Object.freeze([
+  'completion-learning',
+  'completed-dispatch-metadata',
+  'pr-record',
+  'legacy-feedback',
+  'quarantined-candidate',
+]);
+
+const { POSITIVE_DISPOSITIONS } = reviewLearning;
+
+function _skip(reason, extra = {}) {
+  const bucket = REASON_BUCKETS.includes(reason) ? reason : 'not-qualified';
+  return { status: 'skipped', reason: bucket, ...extra };
+}
+
+// Deterministic classification of a normalization error into a skip bucket. The
+// live normalizer throws typed messages; we map them so per-record reason counts
+// are stable across dry-run and real runs.
+function bucketForError(err) {
+  const msg = String(err && err.message || err || '').toLowerCase();
+  if (/prohibited|not a safe field|injection|chain[- ]?of[- ]?thought|reasoning|transcript/.test(msg)) {
+    return 'injection-flagged';
+  }
+  if (/project does not match|project scope|finding key project/.test(msg)) return 'source-mismatch';
+  if (/project/.test(msg)) return 'ambiguous-project';
+  if (/source does not match|not an allowed source|source key|conflicting disposition|conflicting audience|author agent does not match/.test(msg)) {
+    return 'source-mismatch';
+  }
+  if (/author/.test(msg)) return 'missing-author';
+  if (/canonical pr|pr id|source pr|\bpr\b/.test(msg)) return 'missing-pr';
+  if (/rule/.test(msg)) return 'missing-generalized-rule';
+  if (/evidence/.test(msg)) return 'missing-resolution-evidence';
+  return 'not-qualified';
+}
+
+function _normalizeContext(context = {}) {
+  const project = reviewLearning.normalizeProjectKey(context.project);
+  const prId = reviewLearning.normalizeCanonicalPrId(context.prId || context.pr);
+  const rawAllowed = context.allowedSourceKeys instanceof Set
+    ? [...context.allowedSourceKeys]
+    : (Array.isArray(context.allowedSourceKeys) ? context.allowedSourceKeys : []);
+  const allowedSourceKeys = new Set(rawAllowed
+    .map(value => {
+      try { return reviewLearning.normalizeFindingSourceKey(value); } catch { return null; }
+    })
+    .filter(Boolean));
+  const authorAgent = context.authorAgent == null ? '' : String(context.authorAgent).trim();
+  return { project, prId, authorAgent, allowedSourceKeys };
+}
+
+// Mirrors engine/lifecycle.js#_groupReviewLearningEntries: collapse normalized
+// completion entries into one lesson group per finding key, flagging conflicting
+// dispositions/audiences so they are rejected rather than silently merged.
+function _groupEntries(entries) {
+  const groups = new Map();
+  for (const entry of entries) {
+    const findingKey = reviewLearning.buildFindingKey({
+      project: entry.project,
+      prId: entry.source.pr,
+      sourceKey: entry.source.sourceKey,
+    });
+    let group = groups.get(findingKey);
+    if (!group) {
+      group = {
+        findingKey,
+        project: entry.project,
+        prId: entry.source.pr,
+        sourceKey: entry.source.sourceKey,
+        authorAgent: entry.authorAgent,
+        disposition: entry.disposition,
+        audience: entry.applicability,
+        rules: [],
+        files: [],
+        symbols: [],
+        tags: [],
+        evidence: [],
+        confidences: [],
+      };
+      groups.set(findingKey, group);
+    }
+    if (group.disposition !== entry.disposition) {
+      group.error = `review learning source ${entry.source.sourceKey} has conflicting dispositions`;
+    }
+    if (group.audience !== entry.applicability) {
+      group.error = `review learning source ${entry.source.sourceKey} has conflicting audiences`;
+    }
+    group.rules.push(entry.rule);
+    group.files.push(...entry.files);
+    group.symbols.push(...entry.symbols);
+    group.tags.push(...entry.tags);
+    group.evidence.push(...entry.evidence);
+    group.confidences.push(entry.confidence);
+  }
+  return [...groups.values()].map((group) => {
+    const facets = reviewLearning.normalizeReviewFacets(group);
+    return {
+      ...group,
+      rules: [...new Set(group.rules)].sort((a, b) => a.localeCompare(b)),
+      files: facets.files,
+      symbols: facets.symbols,
+      tags: facets.tags,
+      area: facets.area,
+      evidence: [...new Set(group.evidence)].sort((a, b) => a.localeCompare(b)),
+      confidence: group.confidences.length ? Math.min(...group.confidences) : 0.8,
+    };
+  }).sort((a, b) => a.findingKey.localeCompare(b.findingKey));
+}
+
+// Build the deterministic lesson input consumed by the live upsert path.
+function _lessonInputForGroup(group, rec) {
+  return {
+    findingKey: group.findingKey,
+    project: group.project,
+    prId: group.prId,
+    authorAgent: group.authorAgent,
+    audience: group.audience,
+    disposition: group.disposition,
+    rules: group.rules,
+    files: group.files,
+    symbols: group.symbols,
+    tags: group.tags,
+    evidence: group.evidence,
+    confidence: group.confidence,
+    metadata: {
+      backfill: true,
+      backfillSource: rec.kind,
+      backfillRef: rec.orderKey || null,
+      sourceKey: group.sourceKey,
+      verifiedBy: `exact-${reviewLearning.classifyExactEvidence(group.evidence).join('+') || 'evidence'}`,
+    },
+  };
+}
+
+// Import (or, in dry-run, classify) a single qualified lesson group. Idempotency
+// uses the same stable finding key + body content that live capture writes, so a
+// second real run reports the record as already-present instead of duplicating it.
+function _importGroup(group, rec, deps) {
+  const { memoryStore, dryRun } = deps;
+  if (group.error) return _skip('source-mismatch', { findingKey: group.findingKey });
+  if (!POSITIVE_DISPOSITIONS.has(group.disposition)) {
+    return _skip('missing-resolution-evidence', { findingKey: group.findingKey });
+  }
+  if (reviewLearning.classifyExactEvidence(group.evidence).length === 0) {
+    return _skip('missing-resolution-evidence', { findingKey: group.findingKey });
+  }
+  if (!Array.isArray(group.rules) || group.rules.length === 0) {
+    return _skip('missing-generalized-rule', { findingKey: group.findingKey });
+  }
+
+  const lessonInput = _lessonInputForGroup(group, rec);
+  let record;
+  try {
+    record = reviewLearning.buildLessonMemoryRecord(lessonInput);
+  } catch (err) {
+    return _skip(bucketForError(err), { findingKey: group.findingKey });
+  }
+
+  let existing = [];
+  try {
+    existing = memoryStore.listActiveReviewLearnings({
+      project: group.project,
+      findingKey: group.findingKey,
+      limit: 50,
+    });
+  } catch {
+    existing = [];
+  }
+  if (existing.some(row => row.body === record.body)) {
+    return { status: 'already-present', findingKey: group.findingKey, memoryId: existing.find(row => row.body === record.body)?.id || null };
+  }
+
+  if (dryRun) {
+    return { status: 'imported', findingKey: group.findingKey, dryRun: true };
+  }
+  const lesson = memoryStore.upsertReviewLearningLesson(lessonInput);
+  return { status: 'imported', findingKey: group.findingKey, memoryId: lesson?.id || null };
+}
+
+// Process one raw source record into an array of outcomes. A completion-learning
+// record can yield several findings; every other legacy kind currently lacks a
+// discrete generalized rule and/or verified resolution evidence, so it resolves
+// to a single explicit skip.
+function processRecord(rec, deps) {
+  if (!rec || typeof rec !== 'object') return [{ status: 'error' }];
+
+  if (rec.injectionFlagged === true) {
+    return [_skip('injection-flagged', { ref: rec.orderKey || null })];
+  }
+
+  switch (rec.kind) {
+    case 'completion-learning':
+    case 'completed-dispatch-metadata': {
+      const ctxInput = rec.context;
+      if (!ctxInput || typeof ctxInput !== 'object' || Object.keys(ctxInput).length === 0) {
+        // No trusted dispatch context — the source binding cannot be verified.
+        return [_skip('source-mismatch', { ref: rec.orderKey || null })];
+      }
+      let context;
+      try {
+        context = _normalizeContext(ctxInput);
+      } catch (err) {
+        return [_skip(bucketForError(err), { ref: rec.orderKey || null })];
+      }
+      if (!context.authorAgent) return [_skip('missing-author', { ref: rec.orderKey || null })];
+      const learnings = Array.isArray(rec.learnings) ? rec.learnings : [];
+      if (learnings.length === 0) {
+        return [_skip('missing-generalized-rule', { ref: rec.orderKey || null })];
+      }
+      let normalized;
+      try {
+        normalized = reviewLearning.normalizeCompletionReviewLearnings(learnings, {
+          project: context.project,
+          prId: context.prId,
+          authorAgent: context.authorAgent,
+          allowedSourceKeys: context.allowedSourceKeys,
+        });
+      } catch {
+        // Re-normalize per entry so each malformed entry is bucketed individually.
+        return learnings.map((entry) => {
+          try {
+            reviewLearning.normalizeCompletionReviewLearnings([entry], {
+              project: context.project,
+              prId: context.prId,
+              authorAgent: context.authorAgent,
+              allowedSourceKeys: context.allowedSourceKeys,
+            });
+            return _skip('not-qualified', { ref: rec.orderKey || null });
+          } catch (err) {
+            return _skip(bucketForError(err), { ref: rec.orderKey || null });
+          }
+        });
+      }
+      const groups = _groupEntries(normalized);
+      return groups.map(group => _importGroup(group, rec, deps));
+    }
+    case 'pr-record':
+      // Tracked PR records carry project/PR/author provenance but no discrete
+      // generalized rule; importing would require inferring the lesson.
+      return [_skip('missing-generalized-rule', { ref: rec.orderKey || null })];
+    case 'legacy-feedback':
+      // Raw feedback inbox/archive prose — ambiguous without a generalized rule.
+      return [_skip('ambiguous-raw-comment', { ref: rec.orderKey || null })];
+    case 'quarantined-candidate':
+      // Migration-027 quarantined candidates stay non-recallable: they hold raw
+      // generated feedback, not a discrete rule, so they are reconciled (counted,
+      // left as candidates) rather than promoted.
+      return [_skip('missing-generalized-rule', { ref: rec.orderKey || null, reconciled: true })];
+    default:
+      return [_skip('not-qualified', { ref: rec.orderKey || null })];
+  }
+}
+
+// ─── Default source scanners (offline; read FS + SQL stores) ─────────────────
+
+function _readdirSafe(dir) {
+  try { return fs.readdirSync(dir); } catch { return []; }
+}
+
+function _readJsonSafe(file) {
+  try {
+    const parsed = JSON.parse(fs.readFileSync(file, 'utf8'));
+    return parsed && typeof parsed === 'object' ? parsed : null;
+  } catch { return null; }
+}
+
+function scanCompletionArtifacts(dir, { project } = {}) {
+  const completionsDir = path.join(dir, 'engine', 'completions');
+  const files = _readdirSafe(completionsDir).filter(f => f.endsWith('.json')).sort();
+  if (files.length === 0) return [];
+
+  let dispatch = { pending: [], active: [], completed: [] };
+  try { dispatch = require('./dispatch-store').readDispatchSectioned() || dispatch; } catch {}
+  const entriesById = new Map();
+  for (const list of ['completed', 'active', 'pending']) {
+    for (const entry of (dispatch[list] || [])) {
+      if (entry && entry.id && !entriesById.has(entry.id)) entriesById.set(entry.id, entry);
+    }
+  }
+
+  let buildContext = null;
+  try { buildContext = require('./lifecycle').buildReviewLearningCompletionContext; } catch {}
+
+  const records = [];
+  for (const file of files) {
+    const dispatchId = file.replace(/\.json$/, '');
+    const report = _readJsonSafe(path.join(completionsDir, file));
+    if (!report) continue;
+    const learnings = report.reviewLearnings || report.meta?.reviewLearnings;
+    if (!Array.isArray(learnings) || learnings.length === 0) continue;
+
+    const injectionFlagged = report.securityFlags?.injectionAttempt === true
+      || report.meta?.securityFlags?.injectionAttempt === true;
+    const entry = entriesById.get(dispatchId);
+    let context = null;
+    if (entry && typeof buildContext === 'function') {
+      try {
+        const ctx = buildContext(entry, entry.agent);
+        context = {
+          project: ctx.project,
+          prId: ctx.prId,
+          authorAgent: ctx.authorAgent,
+          allowedSourceKeys: ctx.allowedSourceKeys,
+        };
+      } catch { context = null; }
+    }
+    if (!context) {
+      // No trusted dispatch context — cannot bind sources; leave non-recallable.
+      records.push({ kind: 'completion-learning', orderKey: dispatchId, learnings, context: {}, injectionFlagged });
+      continue;
+    }
+    if (project && context.project !== reviewLearning.normalizeProjectKey(project)) continue;
+    records.push({ kind: 'completion-learning', orderKey: dispatchId, learnings, context, injectionFlagged });
+  }
+  return records;
+}
+
+function scanQuarantinedCandidates(deps, { project } = {}) {
+  const { memoryStore } = deps;
+  let rows = [];
+  try {
+    rows = memoryStore.listReviewLearningCandidates({ project: project || undefined, limit: 500 }) || [];
+  } catch { rows = []; }
+  return rows
+    .filter(row => row?.metadata?.legacyUnverified === true || row?.metadata?.quarantinedBy)
+    .map(row => ({
+      kind: 'quarantined-candidate',
+      orderKey: `qc:${row.id}`,
+      project: row.scopeKey || null,
+      memoryId: row.id,
+    }))
+    .sort((a, b) => a.orderKey.localeCompare(b.orderKey));
+}
+
+function scanLegacyFeedbackFiles(dir) {
+  const roots = [
+    path.join(dir, 'notes', 'inbox'),
+    path.join(dir, 'notes', 'inbox', 'archive'),
+    path.join(dir, 'notes', 'archive'),
+  ];
+  const records = [];
+  for (const root of roots) {
+    for (const name of _readdirSafe(root)) {
+      if (!/^feedback-.+\.md$/i.test(name)) continue;
+      records.push({ kind: 'legacy-feedback', orderKey: `fb:${path.relative(dir, path.join(root, name)).replace(/\\/g, '/')}` });
+    }
+  }
+  return records.sort((a, b) => a.orderKey.localeCompare(b.orderKey));
+}
+
+function scanTrackedPrRecords(dir) {
+  let prs = [];
+  try {
+    const prStore = require('./pull-requests-store');
+    prs = (prStore.readAllPullRequests ? prStore.readAllPullRequests() : prStore.readPullRequests?.()) || [];
+  } catch { prs = []; }
+  return prs
+    .filter(pr => pr && (pr.reviewStatus || pr.minionsReview))
+    .map(pr => ({ kind: 'pr-record', orderKey: `pr:${pr.project || ''}:${pr.id || pr.prNumber || ''}` }))
+    .sort((a, b) => a.orderKey.localeCompare(b.orderKey));
+}
+
+function gatherDefaultSources(dir, deps, { project } = {}) {
+  return [
+    ...scanCompletionArtifacts(dir, { project }),
+    ...scanTrackedPrRecords(dir),
+    ...scanLegacyFeedbackFiles(dir),
+    ...scanQuarantinedCandidates(deps, { project }),
+  ];
+}
+
+// ─── Orchestrator ────────────────────────────────────────────────────────────
+
+function _emptyReasonCounts() {
+  const reasons = {};
+  for (const bucket of REASON_BUCKETS) reasons[bucket] = 0;
+  return reasons;
+}
+
+function backfillReviewLearnings(options = {}) {
+  const {
+    dryRun = false,
+    project = null,
+    dir = shared.MINIONS_DIR,
+    sources = null,
+  } = options;
+  const limit = Number.isFinite(Number(options.limit)) && Number(options.limit) > 0
+    ? Math.floor(Number(options.limit))
+    : 100;
+  const memoryStore = options.memoryStore || require('./memory-store');
+  const deps = { memoryStore, dryRun, project };
+
+  let raw = Array.isArray(sources) ? sources.slice() : gatherDefaultSources(dir, deps, { project });
+  // Deterministic ordering + bounded slice honor the requested limit before any work.
+  raw.sort((a, b) => String((a && a.orderKey) || '').localeCompare(String((b && b.orderKey) || '')));
+  const bounded = raw.slice(0, limit);
+
+  const counts = {
+    dryRun,
+    limit,
+    scannedSources: bounded.length,
+    processed: 0,
+    imported: 0,
+    alreadyPresent: 0,
+    skipped: 0,
+    errors: 0,
+    reasons: _emptyReasonCounts(),
+  };
+
+  for (const rec of bounded) {
+    let outcomes;
+    try {
+      outcomes = processRecord(rec, deps);
+    } catch {
+      // Non-integrity error for this record — count and continue.
+      counts.errors += 1;
+      counts.processed += 1;
+      continue;
+    }
+    for (const outcome of outcomes) {
+      counts.processed += 1;
+      if (outcome.status === 'imported') counts.imported += 1;
+      else if (outcome.status === 'already-present') counts.alreadyPresent += 1;
+      else if (outcome.status === 'error') counts.errors += 1;
+      else {
+        counts.skipped += 1;
+        const reason = REASON_BUCKETS.includes(outcome.reason) ? outcome.reason : 'not-qualified';
+        counts.reasons[reason] += 1;
+      }
+    }
+  }
+  return counts;
+}
+
+module.exports = {
+  REASON_BUCKETS,
+  RAW_KINDS,
+  backfillReviewLearnings,
+  processRecord,
+  bucketForError,
+  gatherDefaultSources,
+  scanCompletionArtifacts,
+  scanQuarantinedCandidates,
+  scanLegacyFeedbackFiles,
+  scanTrackedPrRecords,
+  // exported for testing
+  _groupEntries,
+  _normalizeContext,
+};