@jaimevalasek/aioson 1.7.2 → 1.9.0

package/src/learning-loop-doctor.js

@@ -0,0 +1,217 @@
+'use strict';
+
+/**
+ * Active Learning Loop — Phase 4 doctor curation checks.
+ *
+ * Pure-where-possible helpers consumed by `src/doctor.js#runDoctor`. The shape
+ * mirrors `assessScoutPruning`: each helper returns `{ ok, items?, params?,
+ * skipped?, reason? }` and `doctor.js` is responsible for translating to
+ * `checks[].push(...)` rows with i18n keys.
+ *
+ * BR-ALL-11: MICRO projects opt out of these checks (caller short-circuits).
+ * The helpers themselves do not read classification — they are pure data
+ * functions over (db, fs).
+ */
+
+const fs = require('node:fs/promises');
+const path = require('node:path');
+
+const MIN_STALENESS_FEATURES = 5;
+const MIN_DISTILLATION_LAG = 5;
+
+// Pure: derive N = max(5, ceil(avg_days_between_last_5_features / 7)).
+// Allows low-velocity projects (1 feature every 6 weeks) to stretch the
+// window beyond the floor without penalising high-velocity teams.
+function computeStalenessThreshold(featureCloseDates) {
+  const sorted = (Array.isArray(featureCloseDates) ? featureCloseDates : [])
+    .map((d) => +new Date(d))
+    .filter((n) => Number.isFinite(n))
+    .sort((a, b) => a - b);
+  if (sorted.length < 2) return MIN_STALENESS_FEATURES;
+  const last = sorted.slice(-5);
+  const span = last[last.length - 1] - last[0];
+  const avgDaysBetween = span / (24 * 60 * 60 * 1000) / (last.length - 1);
+  return Math.max(MIN_STALENESS_FEATURES, Math.ceil(avgDaysBetween / 7));
+}
+
+async function readProjectClassification(targetDir) {
+  try {
+    const raw = await fs.readFile(path.join(targetDir, '.aioson/context/project.context.md'), 'utf8');
+    const match = raw.match(/^classification\s*:\s*["']?([A-Z]+)["']?/m);
+    return match ? match[1].trim().toUpperCase() : null;
+  } catch {
+    return null;
+  }
+}
+
+async function readClosedFeatures(targetDir) {
+  try {
+    const raw = await fs.readFile(path.join(targetDir, '.aioson/context/features.md'), 'utf8');
+    const closed = [];
+    const lines = raw.split('\n');
+    for (const line of lines) {
+      // pipe-table row: | slug | status | started | completed |
+      const m = line.match(/^\|\s*([\w-]+)\s*\|\s*(\w+)\s*\|\s*([\d-]+|—)\s*\|\s*([\d-]+|—)\s*\|/);
+      if (!m) continue;
+      const [, slug, status, , completed] = m;
+      if (status === 'status' || slug === 'slug') continue; // header
+      if (status !== 'done') continue;
+      const date = completed && completed !== '—' ? completed : null;
+      closed.push({ slug, completed: date });
+    }
+    return closed;
+  } catch {
+    return [];
+  }
+}
+
+async function listRuleSlugs(targetDir) {
+  try {
+    const dir = path.join(targetDir, '.aioson/rules');
+    const entries = await fs.readdir(dir, { withFileTypes: true });
+    return entries
+      .filter((e) => e.isFile() && e.name.endsWith('.md') && !e.name.startsWith('_'))
+      .map((e) => e.name.slice(0, -3));
+  } catch {
+    return [];
+  }
+}
+
+// assessRuleStaleness — returns the set of rules with zero `rule_loaded` event
+// over the last `N` closed features. `N` = computeStalenessThreshold().
+// payload_json.target_slug is the canonical identifier; payload_json.feature_slug
+// associates the event with a feature (NULL when loaded outside a feature flow —
+// EC-ALL-02). Both paths count toward "loaded at some point".
+async function assessRuleStaleness({ db, targetDir, threshold, recentFeatureSlugs }) {
+  const ruleSlugs = await listRuleSlugs(targetDir);
+  if (ruleSlugs.length === 0) {
+    return { ok: true, items: [], threshold, ruleCount: 0 };
+  }
+
+  // Build set of slugs that DID get loaded inside the last N features OR with
+  // no feature_slug bound (outside any feature flow — still counts as alive).
+  const rows = db.prepare(`
+    SELECT DISTINCT json_extract(payload_json, '$.target_slug') AS slug,
+                    json_extract(payload_json, '$.feature_slug') AS feature_slug
+    FROM execution_events
+    WHERE event_type = 'rule_loaded'
+      AND payload_json IS NOT NULL
+  `).all();
+
+  const recent = new Set(recentFeatureSlugs);
+  const loaded = new Set();
+  for (const r of rows) {
+    if (!r.slug) continue;
+    if (r.feature_slug === null || r.feature_slug === undefined || recent.has(r.feature_slug)) {
+      loaded.add(r.slug);
+    }
+  }
+
+  const stale = ruleSlugs.filter((s) => !loaded.has(s));
+  // Look up last_loaded_at for each stale rule (informational hint).
+  const items = [];
+  const lastLoadedStmt = db.prepare(`
+    SELECT MAX(created_at) AS last
+    FROM execution_events
+    WHERE event_type = 'rule_loaded'
+      AND json_extract(payload_json, '$.target_slug') = ?
+  `);
+  for (const slug of stale) {
+    const r = lastLoadedStmt.get(slug);
+    items.push({ slug, last_loaded_at: r && r.last ? r.last : null });
+  }
+  return { ok: stale.length === 0, items, threshold, ruleCount: ruleSlugs.length };
+}
+
+// assessLearningOrphans — learnings with status='promoted' whose `promoted_to`
+// rule has zero `rule_loaded` events AFTER the promotion timestamp.
+async function assessLearningOrphans({ db }) {
+  const promoted = db.prepare(`
+    SELECT learning_id, promoted_to, updated_at
+    FROM project_learnings
+    WHERE status = 'promoted'
+      AND promoted_to IS NOT NULL
+      AND promoted_to <> ''
+  `).all();
+
+  if (promoted.length === 0) return { ok: true, items: [] };
+
+  const hasPostPromotionLoad = db.prepare(`
+    SELECT 1
+    FROM execution_events
+    WHERE event_type = 'rule_loaded'
+      AND (json_extract(payload_json, '$.target_path') = ?
+           OR json_extract(payload_json, '$.target_slug') = ?)
+      AND created_at > ?
+    LIMIT 1
+  `);
+
+  const orphans = [];
+  for (const row of promoted) {
+    // target_slug is the basename without extension; target_path is the
+    // full posix path (e.g. ".aioson/rules/foo.md"). Match either form.
+    const slugFromPath = (() => {
+      const p = String(row.promoted_to);
+      const base = p.split('/').pop() || p;
+      return base.endsWith('.md') ? base.slice(0, -3) : base;
+    })();
+    const hit = hasPostPromotionLoad.get(row.promoted_to, slugFromPath, row.updated_at || '1970-01-01T00:00:00.000Z');
+    if (!hit) {
+      orphans.push({
+        learning_id: row.learning_id,
+        promoted_to: row.promoted_to,
+        promoted_at: row.updated_at || null
+      });
+    }
+  }
+  return { ok: orphans.length === 0, items: orphans };
+}
+
+// assessDistillationLag — fires when N≥5 features are status='done' in
+// features.md but `evolution_log` has fewer than that many
+// event_type='auto_distillation' rows tied to those slugs. Escape valve when
+// the Phase 5 hook fails silently (BR-ALL-05).
+async function assessDistillationLag({ db, closedFeatures }) {
+  const closed = Array.isArray(closedFeatures) ? closedFeatures : [];
+  if (closed.length < MIN_DISTILLATION_LAG) {
+    return {
+      ok: true,
+      params: { closed: closed.length, distillations: null, threshold: MIN_DISTILLATION_LAG },
+      items: []
+    };
+  }
+  const slugs = closed.map((f) => f.slug);
+  const placeholders = slugs.map(() => '?').join(',');
+  const row = db.prepare(`
+    SELECT COUNT(DISTINCT feature_slug) AS distillations
+    FROM evolution_log
+    WHERE event_type = 'auto_distillation'
+      AND feature_slug IN (${placeholders})
+  `).get(...slugs);
+  const distillations = Number(row && row.distillations) || 0;
+  const missing = slugs.filter((s) => {
+    const hit = db.prepare(`
+      SELECT 1 FROM evolution_log
+      WHERE event_type = 'auto_distillation' AND feature_slug = ?
+      LIMIT 1
+    `).get(s);
+    return !hit;
+  });
+  return {
+    ok: distillations >= closed.length,
+    params: { closed: closed.length, distillations, threshold: MIN_DISTILLATION_LAG },
+    items: missing.map((slug) => ({ slug }))
+  };
+}
+
+module.exports = {
+  MIN_STALENESS_FEATURES,
+  MIN_DISTILLATION_LAG,
+  computeStalenessThreshold,
+  readProjectClassification,
+  readClosedFeatures,
+  listRuleSlugs,
+  assessRuleStaleness,
+  assessLearningOrphans,
+  assessDistillationLag
+};

package/src/learning-loop-engine.js

@@ -0,0 +1,254 @@
+'use strict';
+
+/**
+ * Active Learning Loop — Phase 5 distillation orchestrator.
+ *
+ * `runDistillation({ targetDir, slug, classification, db, timeoutMs })`:
+ *
+ *  1. Bail out with `skipped_micro` when project/feature classification is MICRO
+ *     (PMD-5 / BR-ALL-11).
+ *  2. Acquire a row-level lock via `BEGIN IMMEDIATE` + `INSERT INTO evolution_log`
+ *     with `event_type='auto_distillation'` and `end_at=NULL` (DD-3 resolution).
+ *     If another active distillation for the same slug already exists, return
+ *     `lock_held` without running anything.
+ *  3. Run `runLearningAutoPromote` against the project (project-wide promotion
+ *     is the existing primitive; per-feature filtering is V2 if telemetry shows
+ *     drift). A 5s Promise.race timeout (DD-2) is enforced — note that
+ *     better-sqlite3 is synchronous so the timeout protects the orchestrator
+ *     from runaway external work, not the SQL itself.
+ *  4. Release the lock row: UPDATE end_at + payload_json with the summary. On
+ *     any failure, write a fresh `distillation_failed` row instead.
+ *
+ * Best-effort semantics (BR-ALL-05): this engine NEVER throws back to the
+ * caller. Every internal failure is captured in the returned result and (for
+ * accounting) also persisted to evolution_log.
+ */
+
+const path = require('node:path');
+const fs = require('node:fs/promises');
+const crypto = require('node:crypto');
+
+// NOTE: required lazily inside runDistillation so test-time monkey-patching of
+// `learning-auto-promote.runLearningAutoPromote` (failure injection) takes
+// effect — direct destructuring would bind the original export.
+const autoPromoteModule = require('./commands/learning-auto-promote');
+
+const DEFAULT_TIMEOUT_MS = 5000;
+const MICRO = 'MICRO';
+
+function nowIso() {
+  return new Date().toISOString();
+}
+
+function buildEntryId(eventType, slug) {
+  const suffix = crypto.randomBytes(3).toString('hex');
+  const tsClean = nowIso().replace(/[:.]/g, '');
+  const safeSlug = String(slug || 'unknown').replace(/[^a-zA-Z0-9_.-]/g, '_');
+  return `evo-${eventType}-feature-${safeSlug}-${tsClean}-${suffix}`;
+}
+
+// PRD frontmatter reader — pull the per-feature classification from the
+// canonical `prd-{slug}.md`. Falls back to null when the file is absent.
+async function readFeatureClassification(targetDir, slug) {
+  const prdPath = path.join(targetDir, '.aioson', 'context', `prd-${slug}.md`);
+  try {
+    const raw = await fs.readFile(prdPath, 'utf8');
+    const match = raw.match(/^classification\s*:\s*["']?([A-Z]+)["']?/m);
+    return match ? match[1].trim().toUpperCase() : null;
+  } catch {
+    return null;
+  }
+}
+
+// Insert a distillation lock row inside a BEGIN IMMEDIATE transaction. Returns
+// `{ acquired: true, lockId }` on success or `{ acquired: false }` when another
+// active row already holds the lock.
+function acquireDistillationLock(db, slug) {
+  // BEGIN IMMEDIATE escalates to a write lock so the SELECT-then-INSERT race
+  // window collapses (better-sqlite3 default is autocommit per statement; the
+  // immediate transaction holds the lock across both statements).
+  db.exec('BEGIN IMMEDIATE');
+  try {
+    const existing = db.prepare(`
+      SELECT id FROM evolution_log
+      WHERE feature_slug = ?
+        AND event_type = 'auto_distillation'
+        AND end_at IS NULL
+      LIMIT 1
+    `).get(slug);
+    if (existing) {
+      db.exec('ROLLBACK');
+      return { acquired: false, lockHolder: existing.id };
+    }
+    const lockId = buildEntryId('auto_distillation', slug);
+    const startAt = nowIso();
+    db.prepare(`
+      INSERT INTO evolution_log (
+        id, applied_at, deltas_count,
+        event_type, target_type, target_id,
+        start_at, end_at, reason, actor, feature_slug, payload_json
+      ) VALUES (
+        @id, @applied_at, 0,
+        'auto_distillation', 'feature', @target_id,
+        @start_at, NULL, NULL, 'auto', @feature_slug, @payload_json
+      )
+    `).run({
+      id: lockId,
+      applied_at: startAt,
+      target_id: slug,
+      start_at: startAt,
+      feature_slug: slug,
+      payload_json: JSON.stringify({ state: 'in_progress' })
+    });
+    db.exec('COMMIT');
+    return { acquired: true, lockId, startAt };
+  } catch (err) {
+    try { db.exec('ROLLBACK'); } catch { /* swallow */ }
+    throw err;
+  }
+}
+
+function releaseLockWithSummary(db, lockId, summary) {
+  db.prepare(`
+    UPDATE evolution_log
+    SET end_at = ?, payload_json = ?
+    WHERE id = ? AND end_at IS NULL
+  `).run(nowIso(), JSON.stringify(summary), lockId);
+}
+
+function recordDistillationFailed(db, slug, errorPhase, errorMessage) {
+  const id = buildEntryId('distillation_failed', slug);
+  const ts = nowIso();
+  try {
+    db.prepare(`
+      INSERT INTO evolution_log (
+        id, applied_at, deltas_count,
+        event_type, target_type, target_id,
+        start_at, end_at, reason, actor, feature_slug, payload_json
+      ) VALUES (
+        @id, @applied_at, 0,
+        'distillation_failed', 'feature', @target_id,
+        @start_at, @end_at, NULL, 'auto', @feature_slug, @payload_json
+      )
+    `).run({
+      id,
+      applied_at: ts,
+      target_id: slug,
+      start_at: ts,
+      end_at: ts,
+      feature_slug: slug,
+      payload_json: JSON.stringify({
+        error_phase: String(errorPhase || 'unknown'),
+        error_message: String(errorMessage || '').slice(0, 1024)
+      })
+    });
+  } catch {
+    // Best-effort: if even the failure row cannot be persisted, we still
+    // refuse to throw upwards (Article: Living Memory never blocks workflows).
+  }
+}
+
+// Promise.race-based timeout. better-sqlite3 is sync so this can't actually
+// abort the in-flight statement; it just returns to the orchestrator after
+// `timeoutMs`. The runaway operation may still complete in the background and
+// touch the DB. The lock row still gets stamped via the `distillation_failed`
+// path so doctor checks see the truth.
+function withTimeout(workPromise, timeoutMs) {
+  let timer;
+  const timeoutPromise = new Promise((_resolve, reject) => {
+    timer = setTimeout(() => reject(Object.assign(new Error('timeout'), { _aioson_timeout: true })), timeoutMs);
+  });
+  return Promise.race([workPromise, timeoutPromise]).finally(() => clearTimeout(timer));
+}
+
+async function runDistillation(options) {
+  const targetDir = options.targetDir;
+  const slug = options.slug;
+  const db = options.db;
+  const timeoutMs = Number(options.timeoutMs) || DEFAULT_TIMEOUT_MS;
+  const classification = options.classification
+    || await readFeatureClassification(targetDir, slug)
+    || null;
+
+  if (!slug) return { ok: false, reason: 'missing_slug' };
+  if (!db) return { ok: false, reason: 'missing_db' };
+
+  // BR-ALL-11 / PMD-5 — MICRO opt-out.
+  if (classification === MICRO) {
+    return { ok: false, reason: 'skipped_micro', classification };
+  }
+
+  let lock;
+  try {
+    lock = acquireDistillationLock(db, slug);
+  } catch (err) {
+    recordDistillationFailed(db, slug, 'lock_acquire', err && err.message);
+    return { ok: false, reason: 'lock_failed', error: String(err && err.message || err) };
+  }
+  if (!lock.acquired) {
+    return { ok: false, reason: 'lock_held', lockHolder: lock.lockHolder };
+  }
+
+  const startedAt = Date.now();
+  let result;
+  try {
+    const work = autoPromoteModule.runLearningAutoPromote({
+      args: [targetDir],
+      options: { json: true },
+      logger: { log: () => {}, error: () => {} }
+    });
+    result = await withTimeout(work, timeoutMs);
+  } catch (err) {
+    const phase = err && err._aioson_timeout ? 'timeout' : 'auto_promote';
+    // Release the lock row as a failure-marker so doctor sees the truth.
+    releaseLockWithSummary(db, lock.lockId, {
+      state: 'failed',
+      error_phase: phase,
+      error_message: String(err && err.message || err).slice(0, 1024),
+      duration_ms: Date.now() - startedAt
+    });
+    recordDistillationFailed(db, slug, phase, err && err.message);
+    return {
+      ok: false,
+      reason: 'auto_promote_failed',
+      error_phase: phase,
+      error: String(err && err.message || err),
+      duration_ms: Date.now() - startedAt
+    };
+  }
+
+  const durationMs = Date.now() - startedAt;
+  const promoted = Array.isArray(result && result.promoted) ? result.promoted.length : Number(result && result.promoted_count) || 0;
+  const review = Array.isArray(result && result.noted) ? result.noted.length : Number(result && result.review_count) || 0;
+  const skipped = Array.isArray(result && result.skipped) ? result.skipped.length : Number(result && result.skipped_count) || 0;
+  const summary = {
+    state: 'complete',
+    promoted_count: promoted,
+    review_count: review,
+    skipped_count: skipped,
+    merge_candidate_count: 0, // DD-5 deferred — always 0 in V1
+    duration_ms: durationMs
+  };
+  releaseLockWithSummary(db, lock.lockId, summary);
+
+  return {
+    ok: true,
+    feature_slug: slug,
+    lock_id: lock.lockId,
+    promoted_count: promoted,
+    review_count: review,
+    skipped_count: skipped,
+    merge_candidate_count: 0,
+    duration_ms: durationMs
+  };
+}
+
+module.exports = {
+  DEFAULT_TIMEOUT_MS,
+  readFeatureClassification,
+  acquireDistillationLock,
+  releaseLockWithSummary,
+  recordDistillationFailed,
+  withTimeout,
+  runDistillation
+};

package/src/learning-loop-fts5.js

@@ -0,0 +1,132 @@
+'use strict';
+
+/**
+ * FTS5 query helpers for project_learnings_fts (Phase 2 — memory-search-fts5).
+ *
+ * DD-4 guardrails (.aioson/plans/active-learning-loop/decision-search-ranking.md):
+ *  - Bind parameters only; no string interpolation.
+ *  - Phrase-query wrap as default sanitization; FTS5 operators are V2 opt-in.
+ *  - Length cap 500 chars (EC-ALL-08); empty queries rejected.
+ *  - ORDER BY rank ASC mandatory.
+ *  - Snippet config: snippet(project_learnings_fts, -1, ...).
+ *  - target_type derived via CASE on promoted_to (column does not live in
+ *    project_learnings; rules-promoted-from-learnings carry promoted_to set).
+ */
+
+const QUERY_MAX_CHARS = 500;
+const SURFACE_VALUES = new Set(['all', 'rules', 'learnings']);
+const STATUS_DEFAULT_FILTER = ['active', 'promoted']; // AC-ALL-202
+
+function sanitizeFtsQuery(raw) {
+  if (raw === undefined || raw === null) return '';
+  const text = String(raw).trim();
+  if (!text) return '';
+  // DD-4 sanitization (refined): tokenize on whitespace, strip embedded quotes and FTS5
+  // operator chars per token, wrap each token as a phrase, AND across tokens. Achieves
+  // "all keywords present" UX while keeping every token literal — FTS5 operators
+  // (*, (, ), :, ^, +, -, ", AND, OR, NOT) cannot leak from user input because each
+  // surviving token is quoted. The literal phrase-only default in the architect's
+  // guardrail was too tight (≤7/10 precision in the canonical fixture); this token-AND
+  // form preserves the same security envelope and unblocks AC-ALL-205.
+  return text
+    .split(/\s+/)
+    .map((tok) => tok.replace(/["*()^:+\-]/g, ''))
+    .filter((tok) => tok.length > 0)
+    .map((tok) => `"${tok}"`)
+    .join(' ');
+}
+
+function validateQuery(raw) {
+  if (raw === undefined || raw === null || String(raw).trim() === '') {
+    return { ok: false, reason: 'query_empty' };
+  }
+  if (String(raw).length > QUERY_MAX_CHARS) {
+    return { ok: false, reason: 'query_too_long', max: QUERY_MAX_CHARS };
+  }
+  return { ok: true };
+}
+
+function normalizeSurface(value) {
+  if (!value) return 'all';
+  const text = String(value).trim().toLowerCase();
+  return SURFACE_VALUES.has(text) ? text : null;
+}
+
+function buildSearchSql(options = {}) {
+  const surface = options.surface || 'all';
+  const includeArchived = Boolean(options.includeArchived);
+
+  let surfaceClause = '';
+  if (surface === 'rules') {
+    surfaceClause = " AND pl.promoted_to IS NOT NULL AND pl.promoted_to != ''";
+  } else if (surface === 'learnings') {
+    surfaceClause = " AND (pl.promoted_to IS NULL OR pl.promoted_to = '')";
+  }
+
+  // Phase 2 default uses pl.status as the active-vs-archived signal. When Phase 3
+  // ships the evolution_log validity-window, swap this to a JOIN on evolution_log
+  // filtered by end_at IS NULL (no API change required — flag stays the same).
+  const statusClause = includeArchived
+    ? ''
+    : ` AND pl.status IN (${STATUS_DEFAULT_FILTER.map(() => '?').join(', ')})`;
+
+  return `
+    SELECT
+      CASE
+        WHEN pl.promoted_to IS NOT NULL AND pl.promoted_to != '' THEN 'rule'
+        ELSE 'learning'
+      END AS target_type,
+      COALESCE(NULLIF(pl.promoted_to, ''), pl.learning_id) AS target_id,
+      pl.feature_slug,
+      pl.status,
+      snippet(project_learnings_fts, -1, '«', '»', '…', 24) AS snippet,
+      project_learnings_fts.rank AS score
+    FROM project_learnings_fts
+    JOIN project_learnings AS pl ON pl.rowid = project_learnings_fts.rowid
+    WHERE project_learnings_fts MATCH ?
+      ${surfaceClause}
+      ${statusClause}
+    ORDER BY project_learnings_fts.rank ASC
+    LIMIT ?
+  `;
+}
+
+function searchProjectLearnings(db, options = {}) {
+  const validation = validateQuery(options.query);
+  if (!validation.ok) {
+    return { ok: false, ...validation };
+  }
+  const surface = normalizeSurface(options.surface);
+  if (surface === null) {
+    return { ok: false, reason: 'invalid_surface', value: options.surface };
+  }
+
+  const limit = Math.max(1, Math.min(Number(options.limit) || 5, 50));
+  const sanitized = sanitizeFtsQuery(options.query);
+  // H-01 (QA Phase 2): inputs reducible to empty after sanitization (operator-only,
+  // quote-only, or any string whose tokens are all stripped) must not reach FTS5 —
+  // SQLite would raise `fts5: syntax error near ""`. Return a structured error so the
+  // CLI/JSON contract is preserved.
+  if (sanitized === '') {
+    return { ok: false, reason: 'query_unparseable', value: String(options.query) };
+  }
+
+  const sql = buildSearchSql({ surface, includeArchived: Boolean(options.includeArchived) });
+
+  const bindings = [sanitized];
+  if (!options.includeArchived) bindings.push(...STATUS_DEFAULT_FILTER);
+  bindings.push(limit);
+
+  const rows = db.prepare(sql).all(...bindings);
+  return { ok: true, query: String(options.query), surface, limit, results: rows };
+}
+
+module.exports = {
+  QUERY_MAX_CHARS,
+  SURFACE_VALUES,
+  sanitizeFtsQuery,
+  validateQuery,
+  normalizeSurface,
+  buildSearchSql,
+  searchProjectLearnings
+};