@shadowforge0/aquifer-memory 1.8.0 → 1.9.0

package/core/memory-explain.js

@@ -0,0 +1,624 @@
+'use strict';
+
+const {
+  assertAllowedScopeRequest,
+  assertCuratedBootstrapOpts,
+  assertCuratedRecallOpts,
+  normalizeScopeList,
+  splitScopePath,
+} = require('./memory-serving');
+
+const BOOTSTRAP_TYPE_PRIORITY = {
+  constraint: 0,
+  state: 1,
+  open_loop: 2,
+  decision: 3,
+  preference: 4,
+  fact: 5,
+  conclusion: 6,
+  entity_note: 7,
+};
+
+const BOOTSTRAP_AUTHORITY_PRIORITY = {
+  user_explicit: 0,
+  executable_evidence: 1,
+  manual: 2,
+  system: 3,
+  verified_summary: 4,
+  llm_inference: 5,
+  raw_transcript: 6,
+};
+
+const RECALL_TYPE_RANK = {
+  constraint: 80,
+  preference: 70,
+  state: 60,
+  open_loop: 55,
+  decision: 50,
+  fact: 40,
+  conclusion: 30,
+  entity_note: 20,
+};
+
+const FEEDBACK_WEIGHT = {
+  helpful: 0.15,
+  confirm: 0.10,
+  irrelevant: -0.20,
+  scope_mismatch: -0.25,
+  stale: -0.30,
+  incorrect: -0.50,
+};
+
+const RECALL_TYPE_BOOST = 0.05;
+const RECALL_SIGNAL_PRIORITY = {
+  memory_row: 3,
+};
+
+const QUARANTINED_OR_INCORRECT = new Set(['quarantined', 'incorrect']);
+const DEFAULT_SCAN_LIMIT = 200;
+const MAX_SCAN_LIMIT = 500;
+
+function recordId(record) {
+  const value = record.memoryId ?? record.memory_id ?? record.id ?? record.canonicalKey ?? record.canonical_key ?? null;
+  return value === null || value === undefined || value === '' ? null : String(value);
+}
+
+function canonicalKey(record) {
+  return String(record.canonicalKey ?? record.canonical_key ?? recordId(record) ?? '');
+}
+
+function scopeKey(record) {
+  return String(record.scopeKey ?? record.scope_key ?? '');
+}
+
+function inheritanceMode(record) {
+  return String(record.inheritanceMode ?? record.inheritance_mode ?? record.scope_inheritance_mode ?? 'defaultable');
+}
+
+function parseTime(value) {
+  const parsed = Date.parse(value || '');
+  return Number.isFinite(parsed) ? parsed : null;
+}
+
+function normalizeScopePath(activeScopePath, activeScopeKey) {
+  const fromPath = splitScopePath(activeScopePath);
+  const source = fromPath && fromPath.length > 0
+    ? fromPath
+    : (activeScopeKey ? [String(activeScopeKey).trim()] : ['global']);
+  const seen = new Set();
+  const normalized = [];
+  for (const value of source) {
+    const key = String(value || '').trim();
+    if (!key || seen.has(key)) continue;
+    seen.add(key);
+    normalized.push(key);
+  }
+  return normalized.length > 0 ? normalized : ['global'];
+}
+
+function normalizeAsOf(value) {
+  if (!value) return null;
+  const parsed = Date.parse(value);
+  return Number.isFinite(parsed) ? new Date(parsed).toISOString() : null;
+}
+
+function normalizeRow(row = {}) {
+  return {
+    ...row,
+    memoryId: recordId(row),
+    canonicalKey: canonicalKey(row),
+    memoryType: row.memoryType ?? row.memory_type ?? null,
+    scopeKey: row.scopeKey ?? row.scope_key ?? null,
+    scopeKind: row.scopeKind ?? row.scope_kind ?? null,
+    inheritanceMode: inheritanceMode(row),
+    status: row.status ?? 'candidate',
+    visibleInBootstrap: row.visibleInBootstrap ?? row.visible_in_bootstrap ?? false,
+    visibleInRecall: row.visibleInRecall ?? row.visible_in_recall ?? false,
+    authority: row.authority ?? null,
+    acceptedAt: row.acceptedAt ?? row.accepted_at ?? null,
+    validFrom: row.validFrom ?? row.valid_from ?? null,
+    validTo: row.validTo ?? row.valid_to ?? null,
+    staleAfter: row.staleAfter ?? row.stale_after ?? null,
+    title: row.title ?? null,
+    summary: row.summary ?? null,
+    contextKey: row.contextKey ?? row.context_key ?? null,
+    topicKey: row.topicKey ?? row.topic_key ?? null,
+  };
+}
+
+function createContext(opts = {}) {
+  const activeScopePath = normalizeScopePath(opts.activeScopePath, opts.activeScopeKey);
+  const activeScopeKey = opts.activeScopeKey || activeScopePath[activeScopePath.length - 1] || null;
+  const asOf = normalizeAsOf(opts.asOf);
+  return {
+    activeScopeKey,
+    activeScopePath,
+    asOf,
+    scopePositions: new Map(activeScopePath.map((key, index) => [key, index])),
+  };
+}
+
+function compareAcceptedDesc(left, right) {
+  return (parseTime(right.acceptedAt) ?? 0) - (parseTime(left.acceptedAt) ?? 0);
+}
+
+function compareRecordIdAsc(left, right) {
+  const leftNum = Number(left.memoryId);
+  const rightNum = Number(right.memoryId);
+  if (Number.isFinite(leftNum) && Number.isFinite(rightNum) && leftNum !== rightNum) return leftNum - rightNum;
+  return String(left.memoryId ?? '').localeCompare(String(right.memoryId ?? ''));
+}
+
+function compareBootstrapRows(left, right, context) {
+  const leftScope = context.scopePositions.get(scopeKey(left)) ?? -1;
+  const rightScope = context.scopePositions.get(scopeKey(right)) ?? -1;
+  if (rightScope !== leftScope) return rightScope - leftScope;
+
+  const leftType = BOOTSTRAP_TYPE_PRIORITY[left.memoryType] ?? 99;
+  const rightType = BOOTSTRAP_TYPE_PRIORITY[right.memoryType] ?? 99;
+  if (leftType !== rightType) return leftType - rightType;
+
+  const leftAuthority = BOOTSTRAP_AUTHORITY_PRIORITY[left.authority] ?? 99;
+  const rightAuthority = BOOTSTRAP_AUTHORITY_PRIORITY[right.authority] ?? 99;
+  if (leftAuthority !== rightAuthority) return leftAuthority - rightAuthority;
+
+  const accepted = compareAcceptedDesc(left, right);
+  if (accepted !== 0) return accepted;
+
+  const keyCompare = canonicalKey(left).localeCompare(canonicalKey(right));
+  if (keyCompare !== 0) return keyCompare;
+
+  return compareRecordIdAsc(left, right);
+}
+
+function rankValue(record, key) {
+  const value = record[key];
+  if (typeof value === 'number' && Number.isFinite(value)) return value;
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : 0;
+}
+
+function sortRecallRows(left, right) {
+  const leftSignal = rankValue(left, 'signal_priority');
+  const rightSignal = rankValue(right, 'signal_priority');
+  if (rightSignal !== leftSignal) return rightSignal - leftSignal;
+
+  const leftTitleMatch = left.title_match === true ? 1 : 0;
+  const rightTitleMatch = right.title_match === true ? 1 : 0;
+  if (rightTitleMatch !== leftTitleMatch) return rightTitleMatch - leftTitleMatch;
+
+  const leftScore = rankValue(left, 'recall_score') || rankValue(left, 'score');
+  const rightScore = rankValue(right, 'recall_score') || rankValue(right, 'score');
+  if (rightScore !== leftScore) return rightScore - leftScore;
+
+  const accepted = compareAcceptedDesc(left, right);
+  if (accepted !== 0) return accepted;
+
+  const keyCompare = canonicalKey(left).localeCompare(canonicalKey(right));
+  if (keyCompare !== 0) return keyCompare;
+
+  return compareRecordIdAsc(left, right);
+}
+
+function compareExplainRows(left, right, context) {
+  const bootstrapOrder = compareBootstrapRows(left, right, context);
+  if (bootstrapOrder !== 0) return bootstrapOrder;
+  const scopeCompare = scopeKey(left).localeCompare(scopeKey(right));
+  if (scopeCompare !== 0) return scopeCompare;
+  return canonicalKey(left).localeCompare(canonicalKey(right));
+}
+
+function lineFor(record) {
+  const type = record.memoryType || 'memory';
+  const text = record.summary || record.title || '';
+  return `- ${type}: ${String(text).trim()}`;
+}
+
+function buildBootstrapText(records, meta) {
+  const lines = records.map(lineFor);
+  return [
+    `<memory-bootstrap memories="${records.length}" overflow="${meta.overflow}" degraded="${meta.degraded}">`,
+    ...lines,
+    '</memory-bootstrap>',
+  ].join('\n');
+}
+
+function scopedReason(record, context) {
+  const recordScope = scopeKey(record);
+  const exactScope = context.activeScopeKey && recordScope === context.activeScopeKey;
+  const inPath = context.scopePositions.has(recordScope);
+  if (inheritanceMode(record) === 'non_inheritable' && !exactScope) return 'non_inheritable_scope';
+  if (!inPath) return 'out_of_scope';
+  return null;
+}
+
+function scopeInheritanceDetails(record, context, reason = null) {
+  const recordScopeKey = scopeKey(record);
+  const pathIndex = context.scopePositions.has(recordScopeKey)
+    ? context.scopePositions.get(recordScopeKey)
+    : null;
+  const exactActiveScope = Boolean(context.activeScopeKey && recordScopeKey === context.activeScopeKey);
+  const inActiveScopePath = pathIndex !== null;
+  const mode = inheritanceMode(record);
+  const appliesByScope = mode === 'non_inheritable'
+    ? exactActiveScope
+    : inActiveScopePath;
+  return {
+    mode,
+    recordScopeKey,
+    activeScopeKey: context.activeScopeKey,
+    inActiveScopePath,
+    exactActiveScope,
+    pathIndex,
+    appliesByScope,
+    decision: reason || (appliesByScope ? 'eligible_by_scope' : 'excluded_by_scope'),
+  };
+}
+
+function timeReason(record, context) {
+  if (!context.asOf) return null;
+  const at = Date.parse(context.asOf);
+  const validFrom = parseTime(record.validFrom);
+  const validTo = parseTime(record.validTo);
+  const staleAfter = parseTime(record.staleAfter);
+  if (validFrom !== null && validFrom > at) return 'valid_from_future';
+  if (validTo !== null && validTo <= at) return 'valid_to_expired';
+  if (staleAfter !== null && staleAfter <= at) return 'stale_after_expired';
+  return null;
+}
+
+function baselineReason(record, lane, context) {
+  if (QUARANTINED_OR_INCORRECT.has(record.status)) return 'quarantined_or_incorrect';
+  if (record.status !== 'active') return 'inactive_status';
+  if (lane === 'bootstrap' && record.visibleInBootstrap !== true) return 'not_visible_in_bootstrap';
+  if (lane === 'memory' && record.visibleInRecall !== true) return 'not_visible_in_recall';
+  const timing = timeReason(record, context);
+  if (timing) return timing;
+  return scopedReason(record, context);
+}
+
+function shadowWinnerCandidate(left, right, lane, context) {
+  const leftScope = context.scopePositions.get(scopeKey(left)) ?? -1;
+  const rightScope = context.scopePositions.get(scopeKey(right)) ?? -1;
+  if (leftScope !== rightScope) return leftScope > rightScope ? left : right;
+
+  if (lane === 'memory') {
+    return sortRecallRows(left, right) <= 0 ? left : right;
+  }
+  return compareBootstrapRows(left, right, context) <= 0 ? left : right;
+}
+
+function resolveShadowing(records, lane, context) {
+  const additive = [];
+  const shadowed = [];
+  const winnersByCanonical = new Map();
+  const ordered = records.slice().sort((left, right) => compareExplainRows(left, right, context));
+
+  for (const record of ordered) {
+    if (inheritanceMode(record) === 'additive') {
+      additive.push(record);
+      continue;
+    }
+    const key = canonicalKey(record);
+    const existing = winnersByCanonical.get(key);
+    if (!existing) {
+      winnersByCanonical.set(key, record);
+      continue;
+    }
+    const winner = shadowWinnerCandidate(existing, record, lane, context);
+    const loser = winner === existing ? record : existing;
+    winnersByCanonical.set(key, winner);
+    shadowed.push(loser);
+  }
+
+  return {
+    winners: [...winnersByCanonical.values(), ...additive],
+    shadowed: shadowed.sort((left, right) => compareExplainRows(left, right, context)),
+  };
+}
+
+function serializeRecord(record, reason, extra = {}, context = null) {
+  const selected = reason === 'selected';
+  const redactedIdentity = selected ? null : (record.memoryId ? `memory:${record.memoryId}` : null);
+  return {
+    memoryId: record.memoryId,
+    canonicalKey: selected ? record.canonicalKey : null,
+    canonicalKeyRedacted: !selected && Boolean(record.canonicalKey),
+    redactedIdentity,
+    memoryType: record.memoryType,
+    scopeKey: record.scopeKey,
+    scopeKind: record.scopeKind,
+    inheritanceMode: record.inheritanceMode,
+    status: record.status,
+    authority: record.authority,
+    title: selected ? record.title : null,
+    summary: selected ? record.summary : null,
+    contentRedacted: !selected,
+    acceptedAt: record.acceptedAt,
+    validFrom: record.validFrom,
+    validTo: record.validTo,
+    staleAfter: record.staleAfter,
+    visibleInBootstrap: record.visibleInBootstrap,
+    visibleInRecall: record.visibleInRecall,
+    reason,
+    ...(context ? { scopeInheritance: scopeInheritanceDetails(record, context, reason) } : {}),
+    ...extra,
+  };
+}
+
+function bootstrapLimit(limit) {
+  if (!Number.isFinite(limit)) return null;
+  return Math.max(1, Math.min(100, Math.floor(limit)));
+}
+
+function bootstrapMaxChars(maxChars) {
+  return Math.max(120, Number.isFinite(maxChars) ? Math.floor(maxChars) : 4000);
+}
+
+function feedbackScore(record, feedbackEvents = []) {
+  const id = record.memoryId;
+  if (!id || feedbackEvents.length === 0) return 0;
+  let score = 0;
+  for (const event of feedbackEvents) {
+    const targetId = String(event.targetId ?? event.target_id ?? '');
+    if (targetId !== id) continue;
+    const type = event.feedbackType ?? event.feedback_type ?? event.verdict;
+    score += FEEDBACK_WEIGHT[type] || 0;
+  }
+  return score;
+}
+
+function textOf(record) {
+  return [
+    record.title,
+    record.summary,
+    record.contextKey,
+    record.topicKey,
+  ].filter(Boolean).join(' ');
+}
+
+function lexicalScore(haystack, query) {
+  if (haystack.includes(query)) return 1;
+  const tokens = query.split(/\s+/).filter(Boolean);
+  if (tokens.length === 0) return 0;
+  const hits = tokens.filter(token => haystack.includes(token)).length;
+  return hits / tokens.length;
+}
+
+function scoreRecallRows(records, query, opts = {}) {
+  const q = String(query || '').trim().toLowerCase();
+  if (!q) throw new Error('memory.explainMemory(query): query must be a non-empty string');
+  const feedbackEvents = opts.feedbackEvents || [];
+
+  return records
+    .map(record => {
+      const haystack = textOf(record).toLowerCase();
+      const lexical = lexicalScore(haystack, q);
+      const typeRank = ((RECALL_TYPE_RANK[record.memoryType] || 0) / 100) * RECALL_TYPE_BOOST;
+      const feedback = feedbackScore(record, feedbackEvents);
+      const title = String(record.title || '').toLowerCase();
+      const scored = {
+        ...record,
+        signal_priority: RECALL_SIGNAL_PRIORITY.memory_row,
+        title_match: title.includes(q),
+        score: lexical + typeRank + feedback,
+        recall_score: lexical + typeRank + feedback,
+        _debug: { lexical, typeRank, feedback },
+      };
+      return scored;
+    })
+    .sort(sortRecallRows);
+}
+
+function buildBootstrapExplanation(rows = [], opts = {}) {
+  const context = createContext(opts);
+  assertAllowedScopeRequest({
+    ...opts,
+    activeScopeKey: context.activeScopeKey,
+    activeScopePath: context.activeScopePath,
+  });
+  const normalizedRows = rows.map(normalizeRow).sort((left, right) => compareExplainRows(left, right, context));
+  const excluded = [];
+  const eligible = [];
+
+  for (const row of normalizedRows) {
+    const reason = baselineReason(row, 'bootstrap', context);
+    if (reason) {
+      excluded.push(serializeRecord(row, reason, {}, context));
+      continue;
+    }
+    eligible.push(row);
+  }
+
+  const { winners, shadowed } = resolveShadowing(eligible, 'bootstrap', context);
+  for (const row of shadowed) excluded.push(serializeRecord(row, 'scope_shadowed', {}, context));
+
+  const ordered = winners.sort((left, right) => compareBootstrapRows(left, right, context));
+  const limit = bootstrapLimit(opts.limit);
+  const maxChars = bootstrapMaxChars(opts.maxChars);
+  let selected = limit ? ordered.slice(0, limit) : ordered.slice();
+  const overflowByLimit = limit ? ordered.slice(limit) : [];
+  for (const row of overflowByLimit) excluded.push(serializeRecord(row, 'limit_trimmed', {}, context));
+
+  const meta = {
+    overflow: overflowByLimit.length > 0,
+    degraded: overflowByLimit.length > 0,
+  };
+  let text = buildBootstrapText(selected, meta);
+  const budgetTrimmed = [];
+  while (text.length > maxChars && selected.length > 1) {
+    const removed = selected.pop();
+    budgetTrimmed.push(removed);
+    meta.overflow = true;
+    meta.degraded = true;
+    text = buildBootstrapText(selected, meta);
+  }
+  for (const row of budgetTrimmed) excluded.push(serializeRecord(row, 'budget_trimmed', {}, context));
+  if (text.length > maxChars) {
+    meta.overflow = true;
+    meta.degraded = true;
+  }
+
+  return {
+    readOnly: true,
+    lane: 'bootstrap',
+    scope: {
+      activeScopeKey: context.activeScopeKey,
+      activeScopePath: context.activeScopePath,
+      asOf: context.asOf,
+    },
+    selected: selected.map(row => serializeRecord(row, 'selected', {}, context)),
+    excluded,
+    budget: {
+      limit,
+      maxChars,
+      overflow: meta.overflow,
+      degraded: meta.degraded,
+    },
+  };
+}
+
+function recallLimit(limit) {
+  return Math.max(1, Math.min(50, Number.isFinite(limit) ? Math.floor(limit) : 10));
+}
+
+function buildMemoryExplanation(rows = [], query, opts = {}) {
+  const context = createContext(opts);
+  assertAllowedScopeRequest({
+    ...opts,
+    activeScopeKey: context.activeScopeKey,
+    activeScopePath: context.activeScopePath,
+  });
+  const normalizedRows = rows.map(normalizeRow).sort((left, right) => compareExplainRows(left, right, context));
+  const excluded = [];
+  const eligible = [];
+
+  for (const row of normalizedRows) {
+    const reason = baselineReason(row, 'memory', context);
+    if (reason) {
+      excluded.push(serializeRecord(row, reason, {}, context));
+      continue;
+    }
+    eligible.push(row);
+  }
+
+  const { winners, shadowed } = resolveShadowing(eligible, 'memory', context);
+  for (const row of shadowed) excluded.push(serializeRecord(row, 'scope_shadowed', {}, context));
+
+  const scored = scoreRecallRows(winners, query, opts);
+  const matched = scored.filter(row => row._debug.lexical > 0);
+  const unmatched = scored.filter(row => row._debug.lexical <= 0);
+  const limit = recallLimit(opts.limit);
+  const selected = matched.slice(0, limit);
+  const trimmed = matched.slice(limit);
+
+  for (const row of trimmed) excluded.push(serializeRecord(row, 'limit_trimmed', { score: row.recall_score }, context));
+  for (const row of unmatched) excluded.push(serializeRecord(row, 'query_no_match', { score: row.recall_score }, context));
+
+  return {
+    readOnly: true,
+    lane: 'memory',
+    scope: {
+      activeScopeKey: context.activeScopeKey,
+      activeScopePath: context.activeScopePath,
+      asOf: context.asOf,
+    },
+    query: String(query || ''),
+    selected: selected.map(row => serializeRecord(row, 'selected', { score: row.recall_score }, context)),
+    excluded,
+    budget: {
+      limit,
+      matched: matched.length,
+    },
+  };
+}
+
+function scanLimit(input = {}) {
+  const value = Number.isFinite(input.scanLimit) ? Math.floor(input.scanLimit) : DEFAULT_SCAN_LIMIT;
+  return Math.max(1, Math.min(MAX_SCAN_LIMIT, value));
+}
+
+function recordsQuery(schema, hasAllowedScopeKeys = false) {
+  return `SELECT
+            m.*, s.scope_kind, s.scope_key, s.inheritance_mode AS scope_inheritance_mode
+          FROM ${schema}.memory_records m
+          JOIN ${schema}.scopes s ON s.id = m.scope_id
+          WHERE m.tenant_id = $1
+            ${hasAllowedScopeKeys ? 'AND s.scope_key = ANY($4::text[])' : ''}
+          ORDER BY
+            CASE WHEN s.scope_key = ANY($2::text[]) THEN 0 ELSE 1 END ASC,
+            m.accepted_at DESC NULLS LAST,
+            m.id ASC
+          LIMIT $3`;
+}
+
+function feedbackQuery(schema) {
+  return `SELECT target_id, feedback_type
+          FROM ${schema}.feedback
+          WHERE tenant_id = $1
+            AND target_kind = 'memory_record'
+            AND target_id = ANY($2::text[])`;
+}
+
+function createMemoryExplain({ pool, schema, defaultTenantId }) {
+  function assertPool() {
+    if (!pool || typeof pool.query !== 'function') {
+      throw new Error('memory.explain requires a pool with query(sql, params)');
+    }
+  }
+
+  async function loadRows(opts = {}) {
+    assertPool();
+    const tenantId = opts.tenantId || defaultTenantId;
+    const activeScopePath = normalizeScopePath(opts.activeScopePath, opts.activeScopeKey);
+    const activeScopeKey = opts.activeScopeKey || activeScopePath[activeScopePath.length - 1] || null;
+    assertAllowedScopeRequest({
+      ...opts,
+      activeScopeKey,
+      activeScopePath,
+    });
+    const allowedScopeKeys = normalizeScopeList(opts.allowedScopeKeys);
+    const params = [
+      tenantId,
+      activeScopePath,
+      scanLimit(opts),
+    ];
+    if (allowedScopeKeys.length > 0) params.push(allowedScopeKeys);
+    const result = await pool.query(recordsQuery(schema, allowedScopeKeys.length > 0), params);
+    return result.rows || [];
+  }
+
+  async function loadFeedbackEvents(rows, opts = {}) {
+    const ids = rows.map(recordId).filter(Boolean);
+    if (ids.length === 0) return [];
+    assertPool();
+    const tenantId = opts.tenantId || defaultTenantId;
+    const result = await pool.query(feedbackQuery(schema), [tenantId, ids]);
+    return result.rows || [];
+  }
+
+  async function explainBootstrap(opts = {}) {
+    assertCuratedBootstrapOpts(opts);
+    const rows = await loadRows(opts);
+    return buildBootstrapExplanation(rows, opts);
+  }
+
+  async function explainMemory(query, opts = {}) {
+    assertCuratedRecallOpts(opts);
+    const rows = await loadRows(opts);
+    const feedbackEvents = await loadFeedbackEvents(rows, opts);
+    return buildMemoryExplanation(rows, query, { ...opts, feedbackEvents });
+  }
+
+  return {
+    explainBootstrap,
+    explainCurrent: explainMemory,
+    explainMemory,
+  };
+}
+
+module.exports = {
+  buildBootstrapExplanation,
+  buildMemoryExplanation,
+  createMemoryExplain,
+};