@ngockhoale/ukit 3.0.3 → 3.0.4

package/src/core/memory/deltaOverlays.js

@@ -0,0 +1,448 @@
+// deltaOverlays.js — M04.2 delta overlays (SPEC §5 FR-017, FR-018; C11).
+//
+// A DeltaOverlay is a narrow, versioned delta over a base contract/workflow
+// version — never a full copy:
+//   { overlayVersion:1, id, scope, baseContract, baseVersion, condition,
+//     operations[], evidenceRefs[], enabled, status }
+// operations ∈ insert|suppress|replace-field on narrow policy fields only.
+//
+// Contracts:
+//   * Authority: only status 'approved' AND enabled overlays apply. Proposed,
+//     rejected, expired, conflict, or disabled overlays are skipped — the
+//     stable policy NEVER reads them (FR-018; promotion stays manual).
+//   * Versioning: overlay.baseVersion must equal the base's current version
+//     (and baseContract must match when the base declares one). A mismatch
+//     yields status 'conflict' + a rebase-review descriptor — never a silent
+//     apply. With { projectRoot } the conflict is persisted on the stored
+//     overlay record so rebase review can list it.
+//   * Delta-only: applyOverlay returns a patched VIEW; the base object is
+//     never mutated. Whole-copy attempts (root targets, the base contract
+//     name, wildcard/empty paths, oversized payloads) fail validation.
+//   * Rollback: rollbackOverlay expires the overlay record (status 'expired',
+//     enabled false) — the stable view is the base again, exactly.
+//   * Stage gating: store-backed ops (registerOverlay, listOverlays) are
+//     inert when learning.overlays.stage is 'off' (default). rollbackOverlay
+//     stays live at any stage — rollback is a safety path, not a feature.
+//   * Persistence: overlays are memoryV2 records (provenance 'delta-overlay'),
+//     full overlay shape in meta. No second store.
+//   * NEVER THROWS on malformed input — validateOverlay reports errors;
+//     applyOverlay returns a typed result.
+
+import crypto from 'node:crypto';
+import { loadRuntimeConfig, resolveConfigStage } from '../runtimeConfig.js';
+import { addRecord, queryRecords, updateRecord } from './storeV2.js';
+
+export const OVERLAY_PROVENANCE = 'delta-overlay';
+export const OVERLAY_OPS = Object.freeze(['insert', 'suppress', 'replace-field']);
+export const OVERLAY_STATUSES = Object.freeze([
+  'proposed', 'approved', 'rejected', 'expired', 'conflict',
+]);
+export const OVERLAY_SCOPES = Object.freeze(['session', 'project', 'user']);
+
+// C11 overlay scope → memoryV2 record scope (same mapping as candidates).
+const OVERLAY_TO_RECORD_SCOPE = {
+  session: 'session',
+  project: 'repo',
+  user: 'user',
+};
+const RECORD_TO_OVERLAY_SCOPE = {
+  session: 'session',
+  repo: 'project',
+  user: 'user',
+};
+
+const MAX_OPERATIONS = 16;
+const MAX_OPERATION_BYTES = 8 * 1024;
+const MAX_EVIDENCE_REFS = 24;
+const MAX_STRING_LENGTH = 512;
+const MAX_FIELD_DEPTH = 8;
+
+// Field paths that would replace the whole document — a full copy wearing an
+// overlay costume. `workflow`/`policy` are rejected as a FIRST segment too:
+// `workflow.blocks` rewrites the workflow document, not a narrow field. The
+// overlay's own baseContract name is rejected separately.
+const WHOLE_COPY_ROOTS = new Set(['workflow', 'policy']);
+const WHOLE_COPY_FIELDS = new Set(['*', '.']);
+const FORBIDDEN_PATH_SEGMENTS = new Set(['__proto__', 'prototype', 'constructor']);
+
+function isPlainObject(value) {
+  return value !== null && typeof value === 'object' && !Array.isArray(value);
+}
+
+function isNonEmptyString(value) {
+  return typeof value === 'string' && value.trim().length > 0;
+}
+
+function fieldSegments(field) {
+  if (!isNonEmptyString(field)) return null;
+  const trimmed = field.trim();
+  const segments = trimmed.split('.');
+  if (segments.some((s) => s.length === 0)) return null;
+  if (segments.length > MAX_FIELD_DEPTH) return null;
+  if (segments.some((s) => FORBIDDEN_PATH_SEGMENTS.has(s))) return null;
+  return segments;
+}
+
+function isWholeCopyField(field, baseContract) {
+  const trimmed = String(field ?? '').trim();
+  if (!trimmed) return true;
+  if (WHOLE_COPY_FIELDS.has(trimmed)) return true;
+  const firstSegment = trimmed.split('.')[0];
+  if (WHOLE_COPY_ROOTS.has(firstSegment)) return true;
+  // Targeting the base contract itself (or its root) is a full replacement.
+  if (isNonEmptyString(baseContract) && trimmed === baseContract.trim()) return true;
+  return false;
+}
+
+function validateOperation(op, index, baseContract, errors) {
+  const label = `operations[${index}]`;
+  if (!isPlainObject(op)) {
+    errors.push(`${label} must be an object.`);
+    return;
+  }
+  if (!OVERLAY_OPS.includes(op.op)) {
+    errors.push(`${label}.op '${String(op.op)}' is not one of ${OVERLAY_OPS.join('|')}.`);
+  }
+  const segments = fieldSegments(op.field);
+  if (segments === null) {
+    errors.push(`${label}.field must be a non-empty dotted path (depth ≤ ${MAX_FIELD_DEPTH}, no prototype keys).`);
+  } else if (isWholeCopyField(op.field, baseContract)) {
+    errors.push(`${label}.field '${op.field}' targets the whole document — overlays are deltas, never full copies.`);
+  }
+  if ((op.op === 'insert' || op.op === 'replace-field')
+      && !Object.prototype.hasOwnProperty.call(op, 'value')) {
+    errors.push(`${label} (${op.op}) requires a value.`);
+  }
+  try {
+    if (JSON.stringify(op).length > MAX_OPERATION_BYTES) {
+      errors.push(`${label} exceeds ${MAX_OPERATION_BYTES} bytes — a delta op must stay narrow.`);
+    }
+  } catch {
+    errors.push(`${label} is not serializable.`);
+  }
+}
+
+/**
+ * validateOverlay(overlay) → { valid: boolean, errors: string[] }
+ * Pure shape check — no store access, never throws.
+ */
+export function validateOverlay(overlay) {
+  const errors = [];
+  if (!isPlainObject(overlay)) {
+    return { valid: false, errors: ['overlay must be an object.'] };
+  }
+
+  if (!Number.isInteger(overlay.overlayVersion) || overlay.overlayVersion < 1) {
+    errors.push('overlayVersion must be an integer ≥ 1.');
+  }
+  if (!isNonEmptyString(overlay.id)) errors.push('id must be a non-empty string.');
+  if (!OVERLAY_SCOPES.includes(overlay.scope)) {
+    errors.push(`scope must be one of ${OVERLAY_SCOPES.join('|')}.`);
+  }
+  if (!isNonEmptyString(overlay.baseContract)) {
+    errors.push('baseContract must be a non-empty string.');
+  }
+  if (!isNonEmptyString(overlay.baseVersion)) {
+    errors.push('baseVersion must be a non-empty string.');
+  }
+  if (overlay.condition !== undefined && overlay.condition !== null
+      && typeof overlay.condition !== 'string') {
+    errors.push('condition must be a string when present.');
+  }
+  if (overlay.enabled !== undefined && typeof overlay.enabled !== 'boolean') {
+    errors.push('enabled must be a boolean when present.');
+  }
+  if (overlay.status !== undefined && !OVERLAY_STATUSES.includes(overlay.status)) {
+    errors.push(`status must be one of ${OVERLAY_STATUSES.join('|')}.`);
+  }
+  if (overlay.evidenceRefs !== undefined) {
+    if (!Array.isArray(overlay.evidenceRefs)
+        || overlay.evidenceRefs.length > MAX_EVIDENCE_REFS
+        || overlay.evidenceRefs.some((r) => !isNonEmptyString(r))) {
+      errors.push(`evidenceRefs must be an array of ≤ ${MAX_EVIDENCE_REFS} non-empty strings.`);
+    }
+  }
+
+  if (!Array.isArray(overlay.operations) || overlay.operations.length === 0) {
+    errors.push('operations must be a non-empty array.');
+  } else if (overlay.operations.length > MAX_OPERATIONS) {
+    errors.push(`operations must contain ≤ ${MAX_OPERATIONS} entries — overlays are deltas, never full copies.`);
+  } else {
+    overlay.operations.forEach((op, index) => {
+      validateOperation(op, index, overlay.baseContract, errors);
+    });
+  }
+
+  return { valid: errors.length === 0, errors };
+}
+
+function cloneBase(base) {
+  try {
+    return structuredClone(base);
+  } catch {
+    return JSON.parse(JSON.stringify(base));
+  }
+}
+
+// Numeric segments index into arrays — a path like `rules.0` must patch the
+// element, not clobber the array into a plain object (the old isPlainObject
+// check destroyed every array it traversed).
+function isIndexSegment(segment) {
+  return /^\d+$/.test(segment);
+}
+
+function setPath(root, segments, value) {
+  let node = root;
+  for (let i = 0; i < segments.length - 1; i += 1) {
+    const key = segments[i];
+    const wantArray = isIndexSegment(segments[i + 1]);
+    const child = node[key];
+    if (wantArray ? !Array.isArray(child) : !isPlainObject(child)) {
+      node[key] = wantArray ? [] : {};
+    }
+    node = node[key];
+  }
+  node[segments[segments.length - 1]] = value;
+}
+
+function deletePath(root, segments) {
+  let node = root;
+  for (let i = 0; i < segments.length - 1; i += 1) {
+    node = node?.[segments[i]];
+    if (!isPlainObject(node) && !Array.isArray(node)) return;
+  }
+  const last = segments[segments.length - 1];
+  if (Array.isArray(node) && isIndexSegment(last)) {
+    node.splice(Number(last), 1);
+  } else {
+    delete node[last];
+  }
+}
+
+function applyOperation(patched, op) {
+  const segments = fieldSegments(op.field);
+  if (!segments) return;
+  if (op.op === 'suppress') {
+    deletePath(patched, segments);
+  } else {
+    setPath(patched, segments, op.value);
+  }
+}
+
+async function overlaysStage(projectRoot) {
+  const config = await loadRuntimeConfig(projectRoot);
+  return resolveConfigStage(config, 'learning.overlays.stage');
+}
+
+function overlayFromRecord(record) {
+  const meta = record?.meta ?? {};
+  return {
+    overlayVersion: meta.overlayVersion ?? 1,
+    id: meta.id ?? record.id,
+    scope: meta.scope ?? RECORD_TO_OVERLAY_SCOPE[record.scope] ?? 'session',
+    baseContract: meta.baseContract ?? null,
+    baseVersion: meta.baseVersion ?? null,
+    condition: meta.condition ?? null,
+    operations: Array.isArray(meta.operations) ? meta.operations : [],
+    evidenceRefs: Array.isArray(meta.evidenceRefs) ? meta.evidenceRefs : [],
+    enabled: meta.enabled !== false,
+    status: meta.status ?? 'proposed',
+    recordId: record.id,
+    projectId: record.project_id ?? null,
+  };
+}
+
+async function findOverlayRecord(projectRoot, overlayId) {
+  const records = await queryRecords(projectRoot, {});
+  return records.find(
+    (r) => r.provenance === OVERLAY_PROVENANCE
+      && r.status !== 'archived'
+      && (r.meta?.id === overlayId || r.id === overlayId),
+  ) ?? null;
+}
+
+async function persistOverlayStatus(projectRoot, overlayId, status, extraMeta = {}) {
+  try {
+    const record = await findOverlayRecord(projectRoot, overlayId);
+    if (!record) return null;
+    return await updateRecord(projectRoot, record.id, {
+      meta: { ...record.meta, status, ...extraMeta },
+    });
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * applyOverlay(base, overlay, { projectRoot } = {}) → result
+ *   { status:'applied', patched }        — delta applied to a cloned view
+ *   { status:'conflict', review }        — base version/contract mismatch;
+ *                                          never silently applied
+ *   { status:'skipped', reason }         — unapproved/disabled/invalid
+ * The base object is never mutated. When projectRoot is given and the overlay
+ * is stored, a conflict is persisted on the record for rebase review.
+ */
+export async function applyOverlay(base, overlay, { projectRoot } = {}) {
+  if (!isPlainObject(base)) {
+    return { status: 'skipped', reason: 'base-not-an-object' };
+  }
+  const { valid, errors } = validateOverlay(overlay);
+  if (!valid) {
+    return { status: 'skipped', reason: 'invalid-overlay', errors };
+  }
+  if (overlay.status !== 'approved') {
+    return { status: 'skipped', reason: `status-${overlay.status ?? 'missing'}` };
+  }
+  if (overlay.enabled === false) {
+    return { status: 'skipped', reason: 'disabled' };
+  }
+
+  const actualContract = base.contract ?? base.baseContract ?? null;
+  if (actualContract !== null && actualContract !== overlay.baseContract) {
+    const review = {
+      reason: 'base-contract-mismatch',
+      expectedBaseContract: overlay.baseContract,
+      actualBaseContract: actualContract,
+      overlayId: overlay.id,
+    };
+    if (projectRoot) {
+      await persistOverlayStatus(projectRoot, overlay.id, 'conflict', { conflict: review });
+    }
+    return { status: 'conflict', review };
+  }
+
+  const actualVersion = base.version ?? base.baseVersion ?? null;
+  if (actualVersion !== overlay.baseVersion) {
+    const review = {
+      reason: 'base-version-mismatch',
+      expectedBaseVersion: overlay.baseVersion,
+      actualBaseVersion: actualVersion,
+      overlayId: overlay.id,
+    };
+    if (projectRoot) {
+      await persistOverlayStatus(projectRoot, overlay.id, 'conflict', { conflict: review });
+    }
+    return { status: 'conflict', review };
+  }
+
+  const patched = cloneBase(base);
+  for (const op of overlay.operations) applyOperation(patched, op);
+  return { status: 'applied', patched };
+}
+
+/**
+ * registerOverlay(projectRoot, overlay) → record | null
+ * Validates and persists an overlay as a memoryV2 record. Idempotent on
+ * overlay.id — re-registering updates the existing record's meta. Returns
+ * null when the stage is 'off' or the overlay fails validation.
+ */
+export async function registerOverlay(projectRoot, overlay) {
+  if (await overlaysStage(projectRoot) === 'off') return null;
+  const { valid } = validateOverlay(overlay);
+  if (!valid) return null;
+
+  const meta = {
+    id: overlay.id,
+    overlayVersion: overlay.overlayVersion,
+    schemaVersion: 1,
+    scope: overlay.scope,
+    baseContract: overlay.baseContract,
+    baseVersion: overlay.baseVersion,
+    condition: overlay.condition ?? null,
+    operations: overlay.operations,
+    evidenceRefs: overlay.evidenceRefs ?? [],
+    enabled: overlay.enabled !== false,
+    status: overlay.status ?? 'proposed',
+    registeredAt: Date.now(),
+  };
+
+  const existing = await findOverlayRecord(projectRoot, overlay.id);
+  if (existing) {
+    return updateRecord(projectRoot, existing.id, {
+      meta: { ...existing.meta, ...meta, registeredAt: existing.meta?.registeredAt ?? meta.registeredAt },
+    });
+  }
+
+  return addRecord(projectRoot, {
+    type: 'derived_fact',
+    scope: OVERLAY_TO_RECORD_SCOPE[overlay.scope] ?? 'session',
+    text: `Delta overlay ${overlay.id} on ${overlay.baseContract}@${overlay.baseVersion} (${overlay.operations.length} op${overlay.operations.length === 1 ? '' : 's'})`,
+    provenance: OVERLAY_PROVENANCE,
+    confidence: 0.8,
+    createdBy: 'delta-overlay',
+    projectId: overlay.projectId ?? null,
+    meta,
+  });
+}
+
+/**
+ * rollbackOverlay(projectRoot, overlayId) → record | null
+ * Expires the overlay (status 'expired', enabled false) so the stable view is
+ * the base again. Live at every stage — rollback is a safety path.
+ */
+export async function rollbackOverlay(projectRoot, overlayId) {
+  if (!isNonEmptyString(overlayId)) return null;
+  const record = await findOverlayRecord(projectRoot, overlayId);
+  if (!record) return null;
+  return updateRecord(projectRoot, record.id, {
+    meta: {
+      ...record.meta,
+      status: 'expired',
+      enabled: false,
+      rolledBackAt: Date.now(),
+    },
+  });
+}
+
+/**
+ * listOverlays(projectRoot, { scope, enabled, status } = {}) → overlay[]
+ * Returns [] when the stage is 'off'. Filters are exact-match on the overlay
+ * fields; omit a filter to match everything.
+ */
+export async function listOverlays(projectRoot, { scope, enabled, status } = {}) {
+  if (await overlaysStage(projectRoot) === 'off') return [];
+  const records = await queryRecords(projectRoot, {});
+  return records
+    .filter((r) => r.provenance === OVERLAY_PROVENANCE && r.status !== 'archived')
+    .map(overlayFromRecord)
+    .filter((o) => (scope === undefined || o.scope === scope)
+      && (enabled === undefined || o.enabled === enabled)
+      && (status === undefined || o.status === status));
+}
+
+/**
+ * overlayFromCandidate(candidate, { baseContract, baseVersion }) → overlay | null
+ * Bridges a TASK-008 LearningCandidate record into an overlay draft. Only
+ * approved candidates produce overlays — promotion stays manual (FR-018).
+ * proposalType 'constraint' maps to a suppress op; everything else maps to
+ * replace-field on proposedDelta.field.
+ */
+export function overlayFromCandidate(candidate, { baseContract, baseVersion } = {}) {
+  const meta = candidate?.meta ?? {};
+  if (candidate?.provenance !== 'learning-candidate' && meta.status === undefined) {
+    return null;
+  }
+  if (meta.status !== 'approved') return null;
+  const delta = meta.proposedDelta;
+  if (!isPlainObject(delta) || !isNonEmptyString(delta.field)) return null;
+  if (!isNonEmptyString(baseContract) || !isNonEmptyString(baseVersion)) return null;
+
+  const isSuppress = meta.proposalType === 'constraint' || !Object.prototype.hasOwnProperty.call(delta, 'value');
+  const operation = isSuppress
+    ? { op: 'suppress', field: delta.field }
+    : { op: 'replace-field', field: delta.field, value: delta.value };
+
+  return {
+    overlayVersion: 1,
+    id: `ov_${crypto.randomBytes(6).toString('hex')}`,
+    scope: OVERLAY_SCOPES.includes(meta.scope) ? meta.scope : 'session',
+    baseContract,
+    baseVersion,
+    condition: meta.condition ?? null,
+    operations: [operation],
+    evidenceRefs: Array.isArray(meta.evidenceRefs) ? meta.evidenceRefs.slice(0, MAX_EVIDENCE_REFS) : [],
+    enabled: true,
+    status: 'approved',
+    candidateId: meta.candidateId ?? meta.id ?? candidate.id ?? null,
+  };
+}