@ngockhoale/ukit 2.6.7 → 2.6.8

package/src/core/memory/migrate.js

@@ -0,0 +1,324 @@
+// Memory v1 → v2 migration — SPEC §4.
+// Idempotent: marker v2/migrated-from-v1.json; re-run is a no-op.
+// Backup-first: legacy memory dir copied to v1-backup-<ts>/ before writing.
+// Legacy files are never mutated.
+
+import crypto from 'node:crypto';
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import { buildRuntimePaths } from '../runtimePaths.js';
+import { readJsonIfExists, writeJson } from '../fileOps.js';
+import { loadRuntimeConfig } from '../runtimeConfig.js';
+import { createRecord } from './records.js';
+import { saveRecords } from './storeV2.js';
+
+const MARKER_NAME = 'migrated-from-v1.json';
+const DAY_MS = 24 * 60 * 60 * 1000;
+
+async function pathExists(targetPath) {
+  try {
+    await fs.access(targetPath);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+async function readJsonTolerant(filePath) {
+  try {
+    return await readJsonIfExists(filePath);
+  } catch {
+    return null;
+  }
+}
+
+async function listJsonFiles(dirPath) {
+  let entries = [];
+  try {
+    entries = await fs.readdir(dirPath, { withFileTypes: true });
+  } catch {
+    return [];
+  }
+  return entries
+    .filter((e) => e.isFile() && e.name.endsWith('.json'))
+    .map((e) => e.name)
+    .sort();
+}
+
+async function copyDirRecursive(srcDir, destDir) {
+  await fs.mkdir(destDir, { recursive: true });
+  const entries = await fs.readdir(srcDir, { withFileTypes: true });
+  for (const entry of entries) {
+    const src = path.join(srcDir, entry.name);
+    const dest = path.join(destDir, entry.name);
+    if (entry.isDirectory()) {
+      await copyDirRecursive(src, dest);
+    } else if (entry.isFile()) {
+      await fs.copyFile(src, dest);
+    }
+  }
+}
+
+function fingerprint(type, text) {
+  const normalized = String(text).toLowerCase().replace(/\s+/g, ' ').trim();
+  return crypto.createHash('sha1').update(`v1:${type}:${normalized}`).digest('hex');
+}
+
+const skippedCounter = { count: 0 };
+
+function makeRecord({ type, scope, text, confidence, provenance, projectId = null, validUntil = null, createdAt = null, meta = {}, status = null }) {
+  try {
+    const record = createRecord({
+      type,
+      scope,
+      text,
+      provenance,
+      sourceFingerprint: fingerprint(type, text),
+      confidence,
+      createdBy: 'migration',
+      projectId,
+      validUntil,
+      meta,
+    });
+    if (createdAt != null && Number.isFinite(createdAt)) record.created_at = createdAt;
+    if (status) record.status = status;
+    return record;
+  } catch {
+    skippedCounter.count += 1;
+    return null;
+  }
+}
+
+function migrateUserMemory(user, records) {
+  if (!user || typeof user !== 'object') return;
+  for (const rule of Array.isArray(user.rules) ? user.rules : []) {
+    if (typeof rule !== 'string' || !rule) continue;
+    const rec = makeRecord({
+      type: 'project_rule', scope: 'user', text: rule,
+      confidence: 1.0, provenance: 'legacy:user-rules',
+      createdAt: user.updatedAt,
+    });
+    if (rec) records.push(rec);
+  }
+  const prefs = user.preferences && typeof user.preferences === 'object' ? user.preferences : {};
+  for (const [key, value] of Object.entries(prefs)) {
+    const text = `${key} = ${typeof value === 'string' ? value : JSON.stringify(value)}`;
+    const rec = makeRecord({
+      type: 'derived_fact', scope: 'user', text,
+      confidence: 1.0, provenance: 'legacy:user-preferences',
+      createdAt: user.updatedAt,
+    });
+    if (rec) records.push(rec);
+  }
+}
+
+function migrateProjectMemory(project, projectId, records) {
+  if (!project || typeof project !== 'object') return;
+  const createdAt = project.updatedAt;
+
+  for (const text of Array.isArray(project.conventions) ? project.conventions : []) {
+    if (typeof text !== 'string' || !text) continue;
+    const rec = makeRecord({
+      type: 'project_rule', scope: 'repo', text,
+      confidence: 0.9, provenance: 'legacy:conventions',
+      projectId, createdAt,
+    });
+    if (rec) records.push(rec);
+  }
+
+  for (const text of Array.isArray(project.activeRules) ? project.activeRules : []) {
+    if (typeof text !== 'string' || !text) continue;
+    const rec = makeRecord({
+      type: 'project_rule', scope: 'repo', text,
+      confidence: 1.0, provenance: 'legacy:activeRules',
+      projectId, createdAt,
+    });
+    if (rec) records.push(rec);
+  }
+
+  for (const decision of Array.isArray(project.decisions) ? project.decisions : []) {
+    if (!decision || typeof decision !== 'object') continue;
+    const text = decision.why ? `${decision.what} — ${decision.why}` : String(decision.what ?? '');
+    if (!text) continue;
+    const rec = makeRecord({
+      type: 'derived_fact', scope: 'repo', text,
+      confidence: 0.9, provenance: 'legacy:decisions',
+      projectId, createdAt: decision.when ?? createdAt,
+    });
+    if (rec) records.push(rec);
+  }
+
+  const profileEntries = [
+    ...(typeof project.architecture === 'string' && project.architecture ? [project.architecture] : []),
+    ...(Array.isArray(project.techStack) ? project.techStack.filter((t) => typeof t === 'string' && t) : []),
+  ];
+  for (const text of profileEntries) {
+    const rec = makeRecord({
+      type: 'derived_fact', scope: 'repo', text,
+      confidence: 0.8, provenance: 'legacy:project-profile',
+      projectId, createdAt,
+    });
+    if (rec) records.push(rec);
+  }
+
+  for (const pc of Array.isArray(project.patternCandidates) ? project.patternCandidates : []) {
+    if (!pc || typeof pc !== 'object' || typeof pc.text !== 'string' || !pc.text) continue;
+    const legacyStatus = pc.status ?? 'pending';
+    const meta = { ...pc, legacyStatus };
+    const base = {
+      text: pc.text,
+      scope: 'task', confidence: 0.4, provenance: 'legacy:pattern-candidate',
+      projectId, createdAt: pc.detectedAt ?? createdAt, meta,
+    };
+    let rec = null;
+    if (legacyStatus === 'approved') {
+      rec = makeRecord({ ...base, type: 'project_rule', scope: 'repo', confidence: 0.9 });
+    } else if (legacyStatus === 'rejected') {
+      rec = makeRecord({ ...base, type: 'derived_fact', status: 'archived' });
+    } else {
+      rec = makeRecord({ ...base, type: 'derived_fact' });
+    }
+    if (rec) records.push(rec);
+  }
+}
+
+function sessionText(session) {
+  return [
+    session.taskDescription,
+    session.outcome ? `outcome: ${session.outcome}` : null,
+    ...(Array.isArray(session.keyActions) ? session.keyActions : []),
+  ].filter(Boolean).join(' — ');
+}
+
+function migrateSession(session, episodeTtlMs, records) {
+  if (!session || typeof session !== 'object') return;
+  const text = sessionText(session);
+  if (!text) return;
+  const createdAt = session.startedAt ?? session.endedAt ?? null;
+  const validUntil = createdAt != null ? createdAt + episodeTtlMs : null;
+  const rec = makeRecord({
+    type: 'episode', scope: 'session', text,
+    confidence: 0.7, provenance: 'legacy:session',
+    projectId: typeof session.projectId === 'string' ? session.projectId : null,
+    createdAt, validUntil,
+    meta: { legacyId: session.id ?? null },
+  });
+  if (rec) records.push(rec);
+}
+
+async function collectLegacySources(paths) {
+  const sources = { user: null, projects: [], sessions: [] };
+
+  const user = await readJsonTolerant(paths.userMemoryPath);
+  if (user) sources.user = user;
+
+  for (const name of await listJsonFiles(paths.projectsDir)) {
+    const filePath = path.join(paths.projectsDir, name);
+    const content = await readJsonTolerant(filePath);
+    if (!content) continue;
+    if (name.endsWith('.archive.json')) {
+      for (const s of Array.isArray(content.sessions) ? content.sessions : []) {
+        sources.sessions.push(s);
+      }
+      continue;
+    }
+    sources.projects.push({ id: content.id ?? name.replace(/\.json$/, ''), content });
+  }
+
+  for (const name of await listJsonFiles(paths.sessionsDir)) {
+    const filePath = path.join(paths.sessionsDir, name);
+    const content = await readJsonTolerant(filePath);
+    if (!content) continue;
+    if (name.endsWith('.archive.json') && Array.isArray(content.sessions)) {
+      sources.sessions.push(...content.sessions);
+    } else {
+      sources.sessions.push(content);
+    }
+  }
+
+  return sources;
+}
+
+function hasLegacyContent(sources) {
+  return Boolean(sources.user) || sources.projects.length > 0 || sources.sessions.length > 0;
+}
+
+async function buildRecords(projectRoot, paths, sources) {
+  const config = await loadRuntimeConfig(projectRoot);
+  const ttlDays = Number.isFinite(config?.memoryV2?.episodeTtlDays) && config.memoryV2.episodeTtlDays > 0
+    ? config.memoryV2.episodeTtlDays
+    : 90;
+  const episodeTtlMs = ttlDays * DAY_MS;
+
+  const records = [];
+  if (sources.user) migrateUserMemory(sources.user, records);
+  for (const { id, content } of sources.projects) migrateProjectMemory(content, id, records);
+  for (const session of sources.sessions) migrateSession(session, episodeTtlMs, records);
+  return records;
+}
+
+/**
+ * needsMigration(projectRoot) → boolean
+ * True when the marker is absent AND legacy memory files exist.
+ */
+export async function needsMigration(projectRoot) {
+  const paths = buildRuntimePaths(projectRoot);
+  const markerPath = path.join(paths.memoryV2Dir, MARKER_NAME);
+  if (await pathExists(markerPath)) return false;
+  const sources = await collectLegacySources(paths);
+  return hasLegacyContent(sources);
+}
+
+/**
+ * runMigration(projectRoot, { dryRun? } = {})
+ * → { migrated, skipped, markerPath, backupDir }
+ * No-op when the marker exists or no legacy files exist. dryRun reports
+ * would-be counts without writing records/marker/backup.
+ */
+export async function runMigration(projectRoot, { dryRun = false } = {}) {
+  const paths = buildRuntimePaths(projectRoot);
+  const markerPath = path.join(paths.memoryV2Dir, MARKER_NAME);
+  const empty = { migrated: 0, skipped: 0, markerPath, backupDir: null };
+
+  if (await pathExists(markerPath)) return empty;
+
+  const sources = await collectLegacySources(paths);
+  if (!hasLegacyContent(sources)) return empty;
+
+  skippedCounter.count = 0;
+  const records = await buildRecords(projectRoot, paths, sources);
+  const skipped = skippedCounter.count;
+  if (dryRun) {
+    return { migrated: records.length, skipped, markerPath, backupDir: null };
+  }
+
+  // Backup-first: copy legacy memory dir (excluding v2 + prior backups) before writing.
+  const backupDir = path.join(paths.memoryRoot, `v1-backup-${Date.now()}`);
+  const entries = await fs.readdir(paths.memoryRoot, { withFileTypes: true });
+  for (const entry of entries) {
+    if (entry.name === 'v2' || entry.name.startsWith('v1-backup-')) continue;
+    const src = path.join(paths.memoryRoot, entry.name);
+    const dest = path.join(backupDir, entry.name);
+    if (entry.isDirectory()) {
+      await copyDirRecursive(src, dest);
+    } else if (entry.isFile()) {
+      await fs.mkdir(backupDir, { recursive: true });
+      await fs.copyFile(src, dest);
+    }
+  }
+
+  await saveRecords(projectRoot, records);
+
+  const counts = {
+    total: records.length,
+    skipped,
+    byType: records.reduce((acc, r) => ({ ...acc, [r.type]: (acc[r.type] ?? 0) + 1 }), {}),
+  };
+  await writeJson(markerPath, {
+    migratedAt: Date.now(),
+    counts,
+    backupDir,
+  });
+
+  return { migrated: records.length, skipped, markerPath, backupDir };
+}

package/src/core/memory/records.js

@@ -0,0 +1,172 @@
+// Memory v2 record schema — SPEC §2.
+// Schema constants, validation, normalization, creation, usability check.
+
+import crypto from 'node:crypto';
+
+export const RECORD_TYPES = Object.freeze([
+  'project_rule',
+  'derived_fact',
+  'episode',
+  'procedure',
+]);
+
+export const RECORD_SCOPES = Object.freeze([
+  'repo',
+  'branch',
+  'task',
+  'session',
+  'user',
+]);
+
+export const RECORD_STATUSES = Object.freeze([
+  'active',
+  'stale',
+  'contradicted',
+  'archived',
+]);
+
+function newId() {
+  return `mem_${crypto.randomBytes(6).toString('hex')}`;
+}
+
+/**
+ * validateRecord(record) → { ok: boolean, errors: string[] }
+ */
+export function validateRecord(record) {
+  const errors = [];
+  if (!record || typeof record !== 'object' || Array.isArray(record)) {
+    return { ok: false, errors: ['record is not an object'] };
+  }
+  if (typeof record.id !== 'string' || !/^mem_[0-9a-f]{12}$/.test(record.id)) {
+    errors.push('id must match mem_<12 hex>');
+  }
+  if (!RECORD_TYPES.includes(record.type)) {
+    errors.push(`type must be one of ${RECORD_TYPES.join('|')}`);
+  }
+  if (!RECORD_SCOPES.includes(record.scope)) {
+    errors.push(`scope must be one of ${RECORD_SCOPES.join('|')}`);
+  }
+  if (typeof record.text !== 'string' || record.text.length === 0) {
+    errors.push('text must be a non-empty string');
+  }
+  if (record.provenance != null && typeof record.provenance !== 'string') {
+    errors.push('provenance must be a string');
+  }
+  if (record.source_fingerprint != null && typeof record.source_fingerprint !== 'string') {
+    errors.push('source_fingerprint must be string | null');
+  }
+  if (typeof record.confidence !== 'number' || Number.isNaN(record.confidence)
+      || record.confidence < 0 || record.confidence > 1) {
+    errors.push('confidence must be a number 0..1');
+  }
+  if (typeof record.created_by !== 'string' || record.created_by.length === 0) {
+    errors.push('created_by must be a non-empty string');
+  }
+  if (typeof record.created_at !== 'number' || !Number.isFinite(record.created_at)) {
+    errors.push('created_at must be epoch ms');
+  }
+  if (record.valid_until != null
+      && (typeof record.valid_until !== 'number' || !Number.isFinite(record.valid_until))) {
+    errors.push('valid_until must be epoch ms | null');
+  }
+  if (!RECORD_STATUSES.includes(record.status)) {
+    errors.push(`status must be one of ${RECORD_STATUSES.join('|')}`);
+  }
+  if (record.project_id != null && typeof record.project_id !== 'string') {
+    errors.push('project_id must be string | null');
+  }
+  if (record.meta == null || typeof record.meta !== 'object' || Array.isArray(record.meta)) {
+    errors.push('meta must be an object');
+  }
+  return { ok: errors.length === 0, errors };
+}
+
+/**
+ * createRecord(input) → record — validates + fills defaults. Throws on invalid input.
+ */
+export function createRecord({
+  type,
+  scope,
+  text,
+  provenance = null,
+  sourceFingerprint = null,
+  confidence = 1.0,
+  createdBy,
+  projectId = null,
+  validUntil = null,
+  meta = {},
+} = {}) {
+  const record = {
+    id: newId(),
+    type,
+    scope,
+    text,
+    provenance,
+    source_fingerprint: sourceFingerprint,
+    confidence,
+    created_by: createdBy,
+    created_at: Date.now(),
+    valid_until: validUntil,
+    status: 'active',
+    project_id: projectId,
+    meta,
+  };
+  const { ok, errors } = validateRecord(record);
+  if (!ok) {
+    throw new Error(`createRecord: invalid record — ${errors.join('; ')}`);
+  }
+  return record;
+}
+
+/**
+ * normalizeRecord(raw) → record | null — best-effort coerce; null when unrecoverable.
+ */
+export function normalizeRecord(raw) {
+  if (!raw || typeof raw !== 'object' || Array.isArray(raw)) return null;
+
+  const type = RECORD_TYPES.includes(raw.type) ? raw.type : null;
+  const text = typeof raw.text === 'string' && raw.text.length > 0 ? raw.text : null;
+  if (!type || !text) return null;
+
+  const scope = RECORD_SCOPES.includes(raw.scope) ? raw.scope : 'repo';
+  const status = RECORD_STATUSES.includes(raw.status) ? raw.status : 'active';
+
+  let confidence = typeof raw.confidence === 'number' && !Number.isNaN(raw.confidence)
+    ? raw.confidence : 1.0;
+  confidence = Math.min(1, Math.max(0, confidence));
+
+  const createdAt = typeof raw.created_at === 'number' && Number.isFinite(raw.created_at)
+    ? raw.created_at : Date.now();
+  const validUntil = typeof raw.valid_until === 'number' && Number.isFinite(raw.valid_until)
+    ? raw.valid_until : null;
+
+  const record = {
+    id: typeof raw.id === 'string' && /^mem_[0-9a-f]{12}$/.test(raw.id) ? raw.id : newId(),
+    type,
+    scope,
+    text,
+    provenance: typeof raw.provenance === 'string' ? raw.provenance : null,
+    source_fingerprint: typeof raw.source_fingerprint === 'string' ? raw.source_fingerprint : null,
+    confidence,
+    created_by: typeof raw.created_by === 'string' && raw.created_by.length > 0
+      ? raw.created_by : 'system',
+    created_at: createdAt,
+    valid_until: validUntil,
+    status,
+    project_id: typeof raw.project_id === 'string' ? raw.project_id : null,
+    meta: raw.meta && typeof raw.meta === 'object' && !Array.isArray(raw.meta) ? raw.meta : {},
+  };
+
+  const { ok } = validateRecord(record);
+  return ok ? record : null;
+}
+
+/**
+ * isRecordUsable(record, now = Date.now()) → boolean
+ */
+export function isRecordUsable(record, now = Date.now()) {
+  if (!record || typeof record !== 'object') return false;
+  if (record.status !== 'active') return false;
+  if (record.valid_until != null && record.valid_until <= now) return false;
+  return true;
+}