@ngockhoale/ukit 3.0.7 → 3.0.9

package/src/core/memory/memoryHit.js

@@ -0,0 +1,144 @@
+// MemoryHit provenance API (M05-01/M05-03, SPEC §5 FR-001/FR-002, §8).
+// Pure module: no I/O, deterministic. buildMemoryHit carries provenance
+// (citation, as-of, freshness slot, reason codes) plus backward-compat
+// fields; rankHits applies eligibility-first dual-score ranking.
+// The record's stored certainty hint is NEVER read for ranking —
+// eligibility and the composite score are the only gates.
+
+import { eligible } from './policy.js';
+import { scoreRecordTokens } from './recordIndex.js';
+import { LEGACY_SCOPE_MATCH, RECORD_TYPE_WEIGHTS } from './retrieval.js';
+
+const SNIPPET_MAX = 160;
+const TYPE_WEIGHT_SCALE = 10; // typeWeight bounded to ≤0.7 bonus
+const RECENCY_HALF_LIFE_MS = 30 * 24 * 60 * 60 * 1000; // bounded decay on created_at
+
+function firstCitation(evidence) {
+  if (!Array.isArray(evidence)) return null;
+  for (const entry of evidence) {
+    if (entry != null && typeof entry === 'object'
+        && typeof entry.locator === 'string' && entry.locator.length > 0) {
+      // Locator is an opaque passthrough — never resolved, never rendered
+      // as an instruction.
+      return {
+        kind: entry.kind ?? null,
+        locator: entry.locator,
+        observedAt: entry.observed_at ?? null,
+      };
+    }
+  }
+  return null;
+}
+
+function maxObservedAt(evidence, fallback) {
+  let max = null;
+  if (Array.isArray(evidence)) {
+    for (const entry of evidence) {
+      if (entry != null && typeof entry.observed_at === 'number'
+          && (max == null || entry.observed_at > max)) {
+        max = entry.observed_at;
+      }
+    }
+  }
+  return max ?? fallback;
+}
+
+// buildMemoryHit(record, {freshness?, score?, reasons?}) → MemoryHit
+// MemoryHit = {id, type, scope, projectId, snippet, citation, observedAt,
+//   validFrom, validUntil, revision, freshness, reasons, relevanceScore,
+//   summary, source, timestamp}
+export function buildMemoryHit(record, { freshness = null, score = 0, reasons = [] } = {}) {
+  const text = String(record?.text ?? '');
+  const snippet = text.length > SNIPPET_MAX ? text.slice(0, SNIPPET_MAX) : text;
+  const createdAt = record?.created_at ?? 0;
+  return {
+    id: record?.id ?? null,
+    type: record?.type ?? null,
+    scope: record?.scope ?? null,
+    projectId: record?.project_id ?? null,
+    snippet,
+    citation: firstCitation(record?.evidence),
+    observedAt: maxObservedAt(record?.evidence, createdAt),
+    validFrom: record?.valid_from ?? null,
+    validUntil: record?.valid_until ?? null,
+    revision: record?.revision ?? 1,
+    freshness,
+    reasons: Array.isArray(reasons) ? [...reasons] : [],
+    relevanceScore: score,
+    // Backward-compat fields for existing search-result consumers.
+    summary: snippet,
+    source: record?.type ?? null,
+    timestamp: createdAt,
+  };
+}
+
+// Requested-scope narrowing only — eligibility is owned by policy.js.
+// Delegates to the shared LEGACY_SCOPE_MATCH table from retrieval.js.
+function matchesRequestedScope(record, scope) {
+  if (scope === 'all') return true;
+  const matcher = LEGACY_SCOPE_MATCH[scope];
+  return matcher ? matcher(record) : (record.scope === scope || record.type === scope);
+}
+
+function hasLocatorEvidence(record) {
+  return Array.isArray(record.evidence)
+    && record.evidence.some((entry) => entry != null
+      && typeof entry.locator === 'string' && entry.locator.length > 0);
+}
+
+// rankHits(records, {queryTokens, eligibleCtx, policy, scope?, limit?, now?})
+//   → MemoryHit[]
+// Eligibility first (eligible() + requested-scope narrowing), then composite
+// score = lexical + typeBonus + evidenceBonus + recencyBonus. Dedup by id.
+export function rankHits(records, {
+  queryTokens = [],
+  eligibleCtx = {},
+  policy = {},
+  scope = 'all',
+  limit = 10,
+  now,
+} = {}) {
+  if (!Array.isArray(records) || records.length === 0) return [];
+  if (!Array.isArray(queryTokens) || queryTokens.length === 0) return [];
+  const nowMs = typeof now === 'number' ? now : Date.now();
+  const ctx = { ...eligibleCtx, now: nowMs };
+
+  const seen = new Set();
+  const scored = [];
+  for (const record of records) {
+    if (record == null || seen.has(record.id)) continue;
+    seen.add(record.id);
+    if (!eligible(record, ctx, policy).ok) continue;
+    if (!matchesRequestedScope(record, scope)) continue;
+
+    const lexical = scoreRecordTokens(record, queryTokens);
+    const typeWeight = RECORD_TYPE_WEIGHTS[record.type] ?? 1;
+    const typeBonus = typeWeight / TYPE_WEIGHT_SCALE;
+    const evidenceBonus = hasLocatorEvidence(record) ? 1 : 0;
+    const age = nowMs - (record.created_at ?? 0);
+    const recencyBonus = record.created_at > 0 && age >= 0
+      ? Math.exp(-Math.LN2 * age / RECENCY_HALF_LIFE_MS)
+      : 0;
+    const score = lexical + typeBonus + evidenceBonus + recencyBonus;
+
+    const reasons = [];
+    if (lexical > 0) reasons.push('lexical');
+    reasons.push(`type:${record.type}`);
+    if (evidenceBonus > 0) reasons.push('evidence');
+    if (recencyBonus > 0) reasons.push('recency');
+
+    scored.push({ record, score, reasons });
+  }
+
+  return scored
+    .sort((left, right) => (
+      right.score - left.score
+      || (right.record.created_at ?? 0) - (left.record.created_at ?? 0)
+      || String(left.record.text ?? '').localeCompare(String(right.record.text ?? ''))
+    ))
+    .slice(0, limit)
+    .map((entry) => buildMemoryHit(entry.record, {
+      score: entry.score,
+      reasons: entry.reasons,
+    }));
+}

package/src/core/memory/migrate.js

@@ -1,16 +1,19 @@
-// Memory v1 → v2 migration — SPEC §4.
+// Memory v1 → v2 migration — SPEC §4 / §5 FR-001..003.
 // Idempotent: marker v2/migrated-from-v1.json; re-run is a no-op.
 // Backup-first: legacy memory dir copied to v1-backup-<ts>/ before writing.
 // Legacy files are never mutated.
+// Record mapping lives in migrateMapping.js (pure, deterministic); this file
+// only collects legacy sources, resolves identities/tombstones, and runs the
+// guarded store merge.
 
-import crypto from 'node:crypto';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 import { buildRuntimePaths } from '../runtimePaths.js';
 import { readJsonIfExists, writeJson } from '../fileOps.js';
 import { loadRuntimeConfig } from '../runtimeConfig.js';
-import { createRecord } from './records.js';
-import { mutateRecordStore } from './recordStore.js';
+import { mapLegacySources } from './migrateMapping.js';
+import { readRecordStore } from './recordStore.js';
+import { lookupRegisteredProjectId } from './projectIdentity.js';
 
 const MARKER_NAME = 'migrated-from-v1.json';
 const DAY_MS = 24 * 60 * 60 * 1000;
@@ -62,176 +65,38 @@ async function copyDirRecursive(srcDir, destDir) {
   }
 }
 
-function fingerprint(type, text) {
-  const normalized = String(text).toLowerCase().replace(/\s+/g, ' ').trim();
-  return crypto.createHash('sha1').update(`v1:${type}:${normalized}`).digest('hex');
-}
-
-// Per-run skip counter — threaded explicitly so concurrent runMigration calls
-// can never cross-report each other's skips (a module-level counter did).
-
-function makeRecord({ type, scope, text, confidence, provenance, projectId = null, validUntil = null, createdAt = null, meta = {}, status = null }, skip) {
-  try {
-    const record = createRecord({
-      type,
-      scope,
-      text,
-      provenance,
-      sourceFingerprint: fingerprint(type, text),
-      confidence,
-      createdBy: 'migration',
-      projectId,
-      validUntil,
-      meta,
-    });
-    if (createdAt != null && Number.isFinite(createdAt)) record.created_at = createdAt;
-    if (status) record.status = status;
-    return record;
-  } catch {
-    skip.count += 1;
-    return null;
-  }
-}
-
-function migrateUserMemory(user, records, skip) {
-  if (!user || typeof user !== 'object') return;
-  for (const rule of Array.isArray(user.rules) ? user.rules : []) {
-    if (typeof rule !== 'string' || !rule) continue;
-    const rec = makeRecord({
-      type: 'project_rule', scope: 'user', text: rule,
-      confidence: 1.0, provenance: 'legacy:user-rules',
-      createdAt: user.updatedAt,
-    }, skip);
-    if (rec) records.push(rec);
-  }
-  const prefs = user.preferences && typeof user.preferences === 'object' ? user.preferences : {};
-  for (const [key, value] of Object.entries(prefs)) {
-    const text = `${key} = ${typeof value === 'string' ? value : JSON.stringify(value)}`;
-    const rec = makeRecord({
-      type: 'derived_fact', scope: 'user', text,
-      confidence: 1.0, provenance: 'legacy:user-preferences',
-      createdAt: user.updatedAt,
-    }, skip);
-    if (rec) records.push(rec);
-  }
-}
-
-function migrateProjectMemory(project, projectId, records, skip) {
-  if (!project || typeof project !== 'object') return;
-  const createdAt = project.updatedAt;
-
-  for (const text of Array.isArray(project.conventions) ? project.conventions : []) {
-    if (typeof text !== 'string' || !text) continue;
-    const rec = makeRecord({
-      type: 'project_rule', scope: 'repo', text,
-      confidence: 0.9, provenance: 'legacy:conventions',
-      projectId, createdAt,
-    }, skip);
-    if (rec) records.push(rec);
-  }
-
-  for (const text of Array.isArray(project.activeRules) ? project.activeRules : []) {
-    if (typeof text !== 'string' || !text) continue;
-    const rec = makeRecord({
-      type: 'project_rule', scope: 'repo', text,
-      confidence: 1.0, provenance: 'legacy:activeRules',
-      projectId, createdAt,
-    }, skip);
-    if (rec) records.push(rec);
-  }
-
-  for (const decision of Array.isArray(project.decisions) ? project.decisions : []) {
-    if (!decision || typeof decision !== 'object') continue;
-    const text = decision.why ? `${decision.what} — ${decision.why}` : String(decision.what ?? '');
-    if (!text) continue;
-    const rec = makeRecord({
-      type: 'derived_fact', scope: 'repo', text,
-      confidence: 0.9, provenance: 'legacy:decisions',
-      projectId, createdAt: decision.when ?? createdAt,
-    }, skip);
-    if (rec) records.push(rec);
-  }
-
-  const profileEntries = [
-    ...(typeof project.architecture === 'string' && project.architecture ? [project.architecture] : []),
-    ...(Array.isArray(project.techStack) ? project.techStack.filter((t) => typeof t === 'string' && t) : []),
-  ];
-  for (const text of profileEntries) {
-    const rec = makeRecord({
-      type: 'derived_fact', scope: 'repo', text,
-      confidence: 0.8, provenance: 'legacy:project-profile',
-      projectId, createdAt,
-    }, skip);
-    if (rec) records.push(rec);
-  }
-
-  for (const pc of Array.isArray(project.patternCandidates) ? project.patternCandidates : []) {
-    if (!pc || typeof pc !== 'object' || typeof pc.text !== 'string' || !pc.text) continue;
-    const legacyStatus = pc.status ?? 'pending';
-    const meta = { ...pc, legacyStatus };
-    const base = {
-      text: pc.text,
-      scope: 'task', confidence: 0.4, provenance: 'legacy:pattern-candidate',
-      projectId, createdAt: pc.detectedAt ?? createdAt, meta,
-    };
-    let rec = null;
-    if (legacyStatus === 'approved') {
-      rec = makeRecord({ ...base, type: 'project_rule', scope: 'repo', confidence: 0.9 }, skip);
-    } else if (legacyStatus === 'rejected') {
-      rec = makeRecord({ ...base, type: 'derived_fact', status: 'archived' }, skip);
-    } else {
-      rec = makeRecord({ ...base, type: 'derived_fact' }, skip);
-    }
-    if (rec) records.push(rec);
-  }
-}
-
-function sessionText(session) {
-  return [
-    session.taskDescription,
-    session.outcome ? `outcome: ${session.outcome}` : null,
-    ...(Array.isArray(session.keyActions) ? session.keyActions : []),
-  ].filter(Boolean).join(' — ');
-}
-
-function migrateSession(session, episodeTtlMs, records, skip) {
-  if (!session || typeof session !== 'object') return;
-  const text = sessionText(session);
-  if (!text) return;
-  const createdAt = session.startedAt ?? session.endedAt ?? null;
-  const validUntil = createdAt != null ? createdAt + episodeTtlMs : null;
-  const rec = makeRecord({
-    type: 'episode', scope: 'session', text,
-    confidence: 0.7, provenance: 'legacy:session',
-    projectId: typeof session.projectId === 'string' ? session.projectId : null,
-    createdAt, validUntil,
-    meta: { legacyId: session.id ?? null },
-  }, skip);
-  if (rec) records.push(rec);
-}
-
+// Source shape consumed by mapLegacySources:
+//   { user, userFile, projects:[{id,file,content}],
+//     sessions:[{file,session}], corruptFiles, corrupt:[fileName] }
 async function collectLegacySources(paths) {
-  const sources = { user: null, projects: [], sessions: [], corruptFiles: 0 };
+  const sources = { user: null, userFile: null, projects: [], sessions: [], corruptFiles: 0, corrupt: [] };
 
   const user = await readJsonTolerant(paths.userMemoryPath);
-  if (user.corrupt) sources.corruptFiles += 1;
-  if (user.value) sources.user = user.value;
+  if (user.corrupt) {
+    sources.corruptFiles += 1;
+    sources.corrupt.push(path.basename(paths.userMemoryPath));
+  }
+  if (user.value) {
+    sources.user = user.value;
+    sources.userFile = path.basename(paths.userMemoryPath);
+  }
 
   for (const name of await listJsonFiles(paths.projectsDir)) {
     const filePath = path.join(paths.projectsDir, name);
     const { value: content, corrupt } = await readJsonTolerant(filePath);
     if (corrupt) {
       sources.corruptFiles += 1;
+      sources.corrupt.push(name);
       continue;
     }
     if (!content) continue;
     if (name.endsWith('.archive.json')) {
       for (const s of Array.isArray(content.sessions) ? content.sessions : []) {
-        sources.sessions.push(s);
+        sources.sessions.push({ file: name, session: s });
       }
       continue;
     }
-    sources.projects.push({ id: content.id ?? name.replace(/\.json$/, ''), content });
+    sources.projects.push({ id: content.id ?? name.replace(/\.json$/, ''), file: name, content });
   }
 
   for (const name of await listJsonFiles(paths.sessionsDir)) {
@@ -239,13 +104,14 @@ async function collectLegacySources(paths) {
     const { value: content, corrupt } = await readJsonTolerant(filePath);
     if (corrupt) {
       sources.corruptFiles += 1;
+      sources.corrupt.push(name);
       continue;
     }
     if (!content) continue;
     if (name.endsWith('.archive.json') && Array.isArray(content.sessions)) {
-      sources.sessions.push(...content.sessions);
+      for (const s of content.sessions) sources.sessions.push({ file: name, session: s });
     } else {
-      sources.sessions.push(content);
+      sources.sessions.push({ file: name, session: content });
     }
   }
 
@@ -257,18 +123,12 @@ function hasLegacyContent(sources) {
     || sources.sessions.length > 0 || sources.corruptFiles > 0;
 }
 
-async function buildRecords(projectRoot, paths, sources, skip) {
+async function episodeTtlMsFor(projectRoot) {
   const config = await loadRuntimeConfig(projectRoot);
   const ttlDays = Number.isFinite(config?.memoryV2?.episodeTtlDays) && config.memoryV2.episodeTtlDays > 0
     ? config.memoryV2.episodeTtlDays
     : 90;
-  const episodeTtlMs = ttlDays * DAY_MS;
-
-  const records = [];
-  if (sources.user) migrateUserMemory(sources.user, records, skip);
-  for (const { id, content } of sources.projects) migrateProjectMemory(content, id, records, skip);
-  for (const session of sources.sessions) migrateSession(session, episodeTtlMs, records, skip);
-  return records;
+  return ttlDays * DAY_MS;
 }
 
 /**
@@ -285,25 +145,41 @@ export async function needsMigration(projectRoot) {
 
 /**
  * runMigration(projectRoot, { dryRun? } = {})
- * → { migrated, skipped, markerPath, backupDir }
- * No-op when the marker exists or no legacy files exist. dryRun reports
- * would-be counts without writing records/marker/backup.
+ * → { migrated, skipped, plan, markerPath, backupDir }
+ * No-op when the marker exists or no legacy files exist. dryRun returns the
+ * full per-record plan ({id, source, action, reason}) and writes nothing.
  */
 export async function runMigration(projectRoot, { dryRun = false } = {}) {
   const paths = buildRuntimePaths(projectRoot);
   const markerPath = path.join(paths.memoryV2Dir, MARKER_NAME);
-  const empty = { migrated: 0, skipped: 0, markerPath, backupDir: null };
+  const empty = { migrated: 0, skipped: 0, plan: [], markerPath, backupDir: null };
 
   if (await pathExists(markerPath)) return empty;
 
   const sources = await collectLegacySources(paths);
   if (!hasLegacyContent(sources)) return empty;
 
-  const skip = { count: 0 };
-  const records = await buildRecords(projectRoot, paths, sources, skip);
-  const skipped = skip.count + sources.corruptFiles;
+  // Tombstones + alias resolution feed the pure mapper — tombstone
+  // precedence is planned here and enforced again inside mutateMemory.
+  // Registry lookups are async, so legacy ids are pre-resolved and handed
+  // to the mapper as a synchronous closure.
+  const prior = await readRecordStore(paths.memoryV2RecordsPath);
+  const resolvedIds = new Map();
+  for (const p of sources.projects) {
+    if (p.id && !resolvedIds.has(p.id)) {
+      resolvedIds.set(p.id, await lookupRegisteredProjectId(p.id));
+    }
+  }
+  const { records, plan, skipped } = mapLegacySources(sources, {
+    projectId: null,
+    now: Date.now(),
+    episodeTtlMs: await episodeTtlMsFor(projectRoot),
+    tombstones: prior.tombstones,
+    resolveProjectId: (legacyId) => resolvedIds.get(legacyId) ?? null,
+  });
+
   if (dryRun) {
-    return { migrated: records.length, skipped, markerPath, backupDir: null };
+    return { migrated: records.length, skipped, plan, markerPath, backupDir: null };
   }
 
   // Backup-first: copy legacy memory dir (excluding v2 + prior backups) before writing.
@@ -321,20 +197,24 @@ export async function runMigration(projectRoot, { dryRun = false } = {}) {
     }
   }
 
-  // Merge into the existing store under the shared lock — never clobber.
-  // records.json can already hold records when the marker is absent (a crash
-  // between saveRecords and the marker write, or a manual re-run); overwriting
-  // it would silently destroy post-migration data. source_fingerprint dedupes
-  // re-migrated records so the merge stays idempotent.
-  await mutateRecordStore(paths.memoryV2RecordsPath, (existing) => {
-    const fingerprints = new Set(
-      existing.map((r) => r.source_fingerprint).filter(Boolean),
-    );
-    const additions = records.filter(
-      (r) => !r.source_fingerprint || !fingerprints.has(r.source_fingerprint),
+  // Merge into the existing store through mutateMemory's restore op — the
+  // single guarded write entry. Tombstone precedence lives there, so a
+  // purged record can never resurrect via re-migration; source_fingerprint
+  // + deterministic-id dedupe keeps the merge idempotent. A non-ok status
+  // (corrupt/disabled/rejected) is fail-closed: surface it as an error
+  // rather than writing the marker over a store that never received the
+  // records.
+  const { mutateMemory } = await import('./mutateMemory.js');
+  const restore = await mutateMemory(
+    { kind: 'project', projectRoot },
+    { op: 'restore', payload: { records } },
+  );
+  if (restore.status !== 'ok') {
+    throw new Error(
+      `runMigration: restore rejected — ${restore.status}${restore.reason ? `:${restore.reason}` : ''}`,
     );
-    return { result: additions.length, records: [...existing, ...additions] };
-  });
+  }
+  const migrated = restore.added ?? 0;
 
   const counts = {
     total: records.length,
@@ -347,5 +227,5 @@ export async function runMigration(projectRoot, { dryRun = false } = {}) {
     backupDir,
   });
 
-  return { migrated: records.length, skipped, markerPath, backupDir };
+  return { migrated, skipped, plan, markerPath, backupDir };
 }