@shadowforge0/aquifer-memory 1.8.1 → 1.9.1

package/core/memory-promotion.js

@@ -1,49 +1,16 @@
 'use strict';
 
 const crypto = require('crypto');
-
-const ACTIVE_V1_TYPES = new Set([
-  'fact',
-  'state',
-  'decision',
-  'preference',
-  'constraint',
-  'entity_note',
-  'open_loop',
-  'conclusion',
-]);
-
-const AUTHORITY_RANK = {
-  raw_transcript: 0,
-  llm_inference: 1,
-  verified_summary: 2,
-  manual: 3,
-  system: 4,
-  executable_evidence: 5,
-  user_explicit: 6,
-};
-
-const DEFAULT_INHERITANCE = {
-  constraint: 'additive',
-  preference: 'defaultable',
-  state: 'defaultable',
-  fact: 'defaultable',
-  conclusion: 'defaultable',
-  entity_note: 'defaultable',
-  decision: 'non_inheritable',
-  open_loop: 'non_inheritable',
-};
-
-const DEFAULT_ASPECT = {
-  fact: 'fact',
-  state: 'state',
-  decision: 'decision',
-  preference: 'preference',
-  constraint: 'constraint',
-  entity_note: 'entity_note',
-  open_loop: 'open_loop',
-  conclusion: 'conclusion',
-};
+const { assessTextForEnrich } = require('./memory-safety-gate');
+const {
+  assistantShapingKindAllowed,
+  assistantShapingServingImpactAllowed,
+  assistantShapingServingImpactKnown,
+  authorityStrength,
+  defaultAspectForMemoryType,
+  defaultInheritanceForMemoryType,
+  isActiveMemoryType,
+} = require('./memory-type-policy');
 
 const FORBIDDEN_TAGS = new Set([
   'commentary',
@@ -59,6 +26,28 @@ const FORBIDDEN_TAGS = new Set([
   'env_dump',
 ]);
 
+const ASSISTANT_BEHAVIOR_LANGUAGE_LEVELS = new Set(['user_behavior', 'assistant_behavior']);
+
+const CANDIDATE_TEXT_FIELDS = [
+  'title',
+  'summary',
+  'evidenceText',
+  'evidence_text',
+  'evidenceExcerpt',
+  'evidence_excerpt',
+  'sourceText',
+  'source_text',
+  'quote',
+];
+
+const CANDIDATE_STRUCTURED_TEXT_FIELDS = [
+  'payload',
+  'evidenceItems',
+  'evidence_items',
+  'evidenceTexts',
+  'evidence_texts',
+];
+
 function normalizeText(value) {
   return String(value || '').trim().replace(/\s+/g, ' ');
 }
@@ -67,6 +56,185 @@ function normalizeType(value) {
   return normalizeText(value).toLowerCase();
 }
 
+function normalizePolicyToken(value) {
+  return normalizeText(value)
+    .toLowerCase()
+    .replace(/[\s-]+/g, '_');
+}
+
+function candidatePayload(candidate = {}) {
+  return candidate.payload && typeof candidate.payload === 'object' && !Array.isArray(candidate.payload)
+    ? candidate.payload
+    : {};
+}
+
+function pickPolicyToken(candidate = {}, aliases = []) {
+  const payload = candidatePayload(candidate);
+  for (const key of aliases) {
+    const value = payload[key] ?? candidate[key];
+    const normalized = normalizePolicyToken(value);
+    if (normalized) return normalized;
+  }
+  return '';
+}
+
+function pickPolicyText(candidate = {}, aliases = []) {
+  const payload = candidatePayload(candidate);
+  for (const key of aliases) {
+    const value = normalizeText(payload[key] ?? candidate[key]);
+    if (value) return value;
+  }
+  return '';
+}
+
+function numericPolicyValue(value) {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : 0;
+}
+
+function arrayPolicyCount(value) {
+  return Array.isArray(value) ? value.filter(Boolean).length : 0;
+}
+
+function temporalSupportForAssistantShaping(candidate = {}) {
+  const payload = candidatePayload(candidate);
+  const support = payload.temporalSupport || payload.temporal_support || candidate.temporalSupport || candidate.temporal_support || {};
+  const refs = normalizeEvidenceRefs(candidate);
+  const distinctRefs = new Set(
+    (Array.isArray(refs) ? refs : [])
+      .map(ref => `${ref.sourceKind || ref.source_kind || 'source'}:${ref.sourceRef || ref.source_ref || ''}`)
+      .filter(value => !value.endsWith(':')),
+  );
+  const countSignals = [
+    support.sessionCount,
+    support.session_count,
+    support.observationCount,
+    support.observation_count,
+    payload.sessionCount,
+    payload.session_count,
+    payload.observationCount,
+    payload.observation_count,
+    candidate.sessionCount,
+    candidate.session_count,
+    candidate.observationCount,
+    candidate.observation_count,
+  ].map(numericPolicyValue);
+  const arraySignals = [
+    support.sessions,
+    support.sessionIds,
+    support.session_ids,
+    support.observations,
+    support.evidence,
+    payload.observations,
+    payload.temporalObservations,
+    payload.temporal_observations,
+    candidate.observations,
+  ].map(arrayPolicyCount);
+  const maxCount = Math.max(0, distinctRefs.size, ...countSignals, ...arraySignals);
+  const consolidationWindow = normalizeText(
+    support.window ||
+    support.cadence ||
+    support.period ||
+    payload.distillationWindow ||
+    payload.distillation_window ||
+    candidate.distillationWindow ||
+    candidate.distillation_window,
+  );
+  const repeated = support.repeated === true
+    || payload.repeated === true
+    || candidate.repeated === true
+    || support.pattern === true
+    || payload.pattern === true
+    || candidate.pattern === true;
+  return {
+    supported: maxCount >= 2 || repeated || Boolean(consolidationWindow),
+    observationCount: maxCount,
+    consolidationWindow: consolidationWindow || null,
+    repeated,
+  };
+}
+
+function assistantBehaviorAbstraction(candidate = {}) {
+  const payload = candidatePayload(candidate);
+  const distillation = payload.distillation && typeof payload.distillation === 'object' ? payload.distillation : {};
+  const promotionTarget = normalizePolicyToken(
+    distillation.promotionTarget
+    || distillation.promotion_target
+    || payload.promotionTarget
+    || payload.promotion_target,
+  );
+  if (promotionTarget !== 'assistant_behavior_memory') {
+    return { required: false, eligible: true };
+  }
+  const abstraction = payload.abstraction && typeof payload.abstraction === 'object'
+    ? payload.abstraction
+    : (payload.generalization && typeof payload.generalization === 'object' ? payload.generalization : {});
+  const boolValue = value => {
+    if (value === true || value === false) return value;
+    if (typeof value === 'string') {
+      const normalized = value.trim().toLowerCase();
+      if (normalized === 'true') return true;
+      if (normalized === 'false') return false;
+    }
+    return null;
+  };
+  const appliesBeyondSource = boolValue(
+    abstraction.appliesBeyondSource
+    ?? abstraction.applies_beyond_source
+    ?? payload.appliesBeyondSource
+    ?? payload.applies_beyond_source,
+  );
+  const sourceBound = boolValue(
+    abstraction.sourceBound
+    ?? abstraction.source_bound
+    ?? payload.sourceBound
+    ?? payload.source_bound,
+  );
+  const principle = normalizeText(
+    abstraction.principle
+    || abstraction.behaviorPrinciple
+    || abstraction.behavior_principle
+    || payload.behaviorPrinciple
+    || payload.behavior_principle,
+  );
+  const languageLevel = normalizePolicyToken(
+    abstraction.languageLevel
+    || abstraction.language_level
+    || payload.languageLevel
+    || payload.language_level,
+  );
+  const generalized = appliesBeyondSource === true && sourceBound === false && Boolean(principle);
+  const behaviorLanguage = ASSISTANT_BEHAVIOR_LANGUAGE_LEVELS.has(languageLevel);
+  return {
+    required: true,
+    eligible: generalized && behaviorLanguage,
+    generalized,
+    behaviorLanguage,
+    appliesBeyondSource,
+    sourceBound,
+    principle,
+    languageLevel,
+  };
+}
+
+function authorityBypassesTemporalGate(authority) {
+  return authorityStrength(authority) >= authorityStrength('manual');
+}
+
+function buildAssistantShapingAspect(candidate = {}, subject = '') {
+  const shapingKind = pickPolicyToken(candidate, ['shapingKind', 'shaping_kind', 'kind', 'category']);
+  const servingImpact = pickPolicyToken(candidate, ['servingImpact', 'serving_impact', 'impact']);
+  const policyKey = pickPolicyToken(candidate, ['policyKey', 'policy_key', 'ruleKey', 'rule_key', 'policyId', 'policy_id']);
+  const explicitAspect = pickPolicyToken(candidate, ['aspect', 'attribute', 'predicate']);
+  const stableSubject = normalizePolicyToken(subject) || 'session';
+  const identityToken = policyKey || explicitAspect || stableSubject;
+  return [
+    shapingKind || defaultAspectForMemoryType('assistant_shaping'),
+    servingImpact || 'assistant_behavior',
+    identityToken,
+  ].filter(Boolean).join(':');
+}
+
 function stableHash(value) {
   return crypto
     .createHash('sha256')
@@ -92,11 +260,11 @@ function assertionHash(value) {
 }
 
 function defaultInheritanceForType(memoryType) {
-  return DEFAULT_INHERITANCE[normalizeType(memoryType)] || 'defaultable';
+  return defaultInheritanceForMemoryType(memoryType);
 }
 
 function authorityRank(authority) {
-  return AUTHORITY_RANK[String(authority || 'llm_inference').toLowerCase()] ?? 0;
+  return authorityStrength(authority || 'llm_inference');
 }
 
 function buildCanonicalKey({ memoryType, scopeKey, contextKey, topicKey, subject, aspect }) {
@@ -106,7 +274,7 @@ function buildCanonicalKey({ memoryType, scopeKey, contextKey, topicKey, subject
   const topic = normalizeType(topicKey || '');
   const scopePart = [scope, context, topic].filter(Boolean).join('|');
   const subj = normalizeType(subject || 'session');
-  const asp = normalizeType(aspect || DEFAULT_ASPECT[type] || type);
+  const asp = normalizeType(aspect || defaultAspectForMemoryType(type));
   return [type, scopePart, subj, asp].join(':');
 }
 
@@ -267,6 +435,87 @@ function sameClaim(a, b) {
     === normalizeText(b && (b.summary || b.title)).toLowerCase();
 }
 
+function rowValue(obj, camel, snake = null) {
+  if (!obj || typeof obj !== 'object') return null;
+  if (obj[camel] !== undefined && obj[camel] !== null && obj[camel] !== '') return obj[camel];
+  if (snake && obj[snake] !== undefined && obj[snake] !== null && obj[snake] !== '') return obj[snake];
+  return null;
+}
+
+function activeIdentityMismatch(candidate = {}, existing = {}) {
+  const candidateType = normalizeType(rowValue(candidate, 'memoryType', 'memory_type'));
+  const existingType = normalizeType(rowValue(existing, 'memoryType', 'memory_type'));
+  if (candidateType && existingType && candidateType !== existingType) return 'memory_type';
+
+  const candidateScopeId = rowValue(candidate, 'scopeId', 'scope_id');
+  const existingScopeId = rowValue(existing, 'scopeId', 'scope_id');
+  if (candidateScopeId && existingScopeId && String(candidateScopeId) !== String(existingScopeId)) return 'scope_id';
+
+  const candidateScopeKey = rowValue(candidate, 'scopeKey', 'scope_key');
+  const existingScopeKey = rowValue(existing, 'scopeKey', 'scope_key');
+  if (candidateScopeKey && existingScopeKey && String(candidateScopeKey) !== String(existingScopeKey)) return 'scope_key';
+
+  return null;
+}
+
+function sanitizeCandidateText(value, tags) {
+  if (typeof value === 'string') {
+    const assessment = assessTextForEnrich(value, 'assistant');
+    for (const tag of assessment.tags || []) tags.add(tag);
+    if (assessment.action === 'drop') return undefined;
+    return assessment.text;
+  }
+
+  if (Array.isArray(value)) {
+    return value
+      .map(item => sanitizeCandidateText(item, tags))
+      .filter(item => item !== undefined);
+  }
+
+  if (value && typeof value === 'object') {
+    const out = {};
+    for (const [key, child] of Object.entries(value)) {
+      const sanitized = sanitizeCandidateText(child, tags);
+      if (sanitized !== undefined) out[key] = sanitized;
+    }
+    return Object.keys(out).length > 0 ? out : undefined;
+  }
+
+  return value;
+}
+
+function sanitizePromotionCandidate(candidate = {}) {
+  const next = { ...candidate };
+  const tags = new Set([
+    ...(candidate.pollutionTags || []),
+    ...(candidate.tags || []),
+  ].map(tag => String(tag || '').trim().toLowerCase()).filter(Boolean));
+
+  for (const field of CANDIDATE_TEXT_FIELDS) {
+    if (next[field] === undefined) continue;
+    const sanitized = sanitizeCandidateText(next[field], tags);
+    if (sanitized === undefined) delete next[field];
+    else next[field] = sanitized;
+  }
+
+  for (const field of CANDIDATE_STRUCTURED_TEXT_FIELDS) {
+    if (next[field] === undefined) continue;
+    const sanitized = sanitizeCandidateText(next[field], tags);
+    if (sanitized === undefined) delete next[field];
+    else next[field] = sanitized;
+  }
+
+  if (tags.size > 0) {
+    next.pollutionTags = [...tags];
+    next._safetyGate = {
+      tags: [...tags],
+      sanitized: true,
+    };
+  }
+
+  return next;
+}
+
 function pushStructuredCandidates(candidates, items, spec) {
   for (const item of items) {
     const text = textFromItem(item, spec.keys);
@@ -285,7 +534,11 @@ function pushStructuredCandidates(candidates, items, spec) {
     const explicitAspect = normalizeText(itemObj && (itemObj.aspect || itemObj.predicate || itemObj.attribute));
     const subject = explicitSubject || normalizeText(spec.subject);
     let aspect = explicitAspect || normalizeText(spec.aspect);
-    if (!explicitAspect) aspect = `${aspect}:${stableHash(text)}`;
+    if (spec.memoryType === 'assistant_shaping') {
+      aspect = buildAssistantShapingAspect(itemObj || {}, subject);
+    } else if (!explicitAspect) {
+      aspect = `${aspect}:${stableHash(text)}`;
+    }
     candidates.push({
       memoryType: spec.memoryType,
       canonicalKey: buildCanonicalKey({
@@ -304,7 +557,7 @@ function pushStructuredCandidates(candidates, items, spec) {
       title: text.slice(0, 120),
       summary: text,
       payload: typeof item === 'string' ? { [spec.payloadKey]: text } : { ...item, [spec.payloadKey]: text },
-      authority: spec.authority,
+      authority: normalizeText(itemObj && (itemObj.authority || itemObj.payload?.authority)) || spec.authority,
       evidenceRefs: spec.evidenceRefs,
       evidenceText: evidenceText || undefined,
       visibleInBootstrap: true,
@@ -333,6 +586,7 @@ function extractCandidatesFromStructuredSummary(input = {}) {
     ['fact', ['important_facts', 'facts'], ['statement', 'fact', 'summary', 'text'], 'statement'],
     ['preference', ['preferences'], ['preference', 'summary', 'text'], 'preference'],
     ['constraint', ['constraints'], ['constraint', 'summary', 'text'], 'constraint'],
+    ['assistant_shaping', ['assistant_shaping', 'assistant_shaping_memories'], ['policy', 'guidance', 'preference', 'boundary', 'rule', 'summary', 'text'], 'guidance'],
     ['conclusion', ['conclusions'], ['conclusion', 'summary', 'text'], 'conclusion'],
     ['state', ['states', 'state'], ['state', 'summary', 'text', 'value'], 'state'],
     ['entity_note', ['entity_notes'], ['note', 'summary', 'text'], 'note'],
@@ -355,7 +609,7 @@ function extractCandidatesFromStructuredSummary(input = {}) {
       contextKey,
       topicKey,
       subject,
-      aspect: DEFAULT_ASPECT[memoryType],
+      aspect: defaultAspectForMemoryType(memoryType),
       inheritanceMode,
       authority,
       evidenceRefs,
@@ -365,8 +619,66 @@ function extractCandidatesFromStructuredSummary(input = {}) {
   return candidates;
 }
 
+function assessAssistantShapingCandidate(candidate = {}) {
+  const shapingKind = pickPolicyToken(candidate, ['shapingKind', 'shaping_kind', 'kind', 'category']);
+  if (!shapingKind) return { action: 'quarantine', reason: 'missing_shaping_kind' };
+  const knownShapingKind = assistantShapingKindAllowed(shapingKind);
+
+  const servingImpact = pickPolicyToken(candidate, ['servingImpact', 'serving_impact', 'impact']);
+  if (!servingImpact) return { action: 'quarantine', reason: 'missing_serving_impact' };
+  const knownServingImpact = assistantShapingServingImpactKnown(servingImpact);
+  if (!assistantShapingServingImpactAllowed(servingImpact)) {
+    return { action: 'quarantine', reason: 'unsupported_serving_impact' };
+  }
+
+  const userRelevance = pickPolicyText(candidate, ['userRelevance', 'user_relevance', 'whyUserFacing', 'why_user_facing']);
+  const payload = candidatePayload(candidate);
+  if (!userRelevance && payload.userFacing !== true && candidate.userFacing !== true) {
+    return { action: 'quarantine', reason: 'missing_user_relevance' };
+  }
+
+  const authority = String(candidate.authority || 'llm_inference').toLowerCase();
+  const temporalSupport = temporalSupportForAssistantShaping(candidate);
+  if (!authorityBypassesTemporalGate(authority) && !temporalSupport.supported) {
+    return {
+      action: 'quarantine',
+      reason: 'assistant_shaping_needs_temporal_evidence',
+      temporalSupport,
+    };
+  }
+
+  const abstraction = assistantBehaviorAbstraction(candidate);
+  if (abstraction.required && !abstraction.generalized) {
+    return {
+      action: 'quarantine',
+      reason: 'assistant_behavior_requires_generalized_abstraction',
+      abstraction,
+    };
+  }
+  if (abstraction.required && !abstraction.behaviorLanguage) {
+    return {
+      action: 'quarantine',
+      reason: 'assistant_behavior_requires_behavior_level_language',
+      abstraction,
+    };
+  }
+
+  return {
+    action: 'promote',
+    reason: 'assistant_shaping_allowed',
+    shapingKind,
+    servingImpact,
+    knownShapingKind,
+    knownServingImpact,
+    temporalSupport,
+    abstraction,
+  };
+}
+
 function assessCandidate(candidate = {}, opts = {}) {
   const memoryType = normalizeType(candidate.memoryType || candidate.memory_type);
+  let allowedReason = 'v1_foundation_allowed';
+  let allowedDetails = {};
   const tags = new Set([
     ...(candidate.pollutionTags || []),
     ...(candidate.tags || []),
@@ -378,7 +690,7 @@ function assessCandidate(candidate = {}, opts = {}) {
     }
   }
 
-  if (!ACTIVE_V1_TYPES.has(memoryType)) {
+  if (!isActiveMemoryType(memoryType)) {
     return { action: 'quarantine', reason: 'unsupported_memory_type' };
   }
 
@@ -399,9 +711,33 @@ function assessCandidate(candidate = {}, opts = {}) {
     return { action: 'quarantine', reason: 'missing_scope' };
   }
 
+  if (memoryType === 'assistant_shaping') {
+    const shapingAssessment = assessAssistantShapingCandidate(candidate);
+    if (shapingAssessment.action !== 'promote') return shapingAssessment;
+    allowedReason = shapingAssessment.reason;
+    const { action: _action, reason: _reason, ...details } = shapingAssessment;
+    void _action;
+    void _reason;
+    allowedDetails = details;
+  }
+
   const activeConflicts = opts.existingActiveRecords || [];
   for (const existing of activeConflicts) {
-    if (sameClaim(candidate, existing)) continue;
+    if (sameClaim(candidate, existing)) {
+      const mismatch = activeIdentityMismatch(candidate, existing);
+      if (mismatch) {
+        return {
+          action: 'quarantine',
+          reason: `active_identity_mismatch_${mismatch}`,
+          conflictWith: existing.id || existing.memory_id,
+        };
+      }
+      return {
+        action: 'skip',
+        reason: 'already_active',
+        activeId: existing.id || existing.memory_id,
+      };
+    }
     const incomingRank = authorityRank(authority);
     const existingRank = authorityRank(existing.authority);
     if (incomingRank < existingRank) {
@@ -414,15 +750,16 @@ function assessCandidate(candidate = {}, opts = {}) {
       action: 'promote',
       reason: 'higher_authority_supersedes',
       supersedeId: existing.id || existing.memory_id,
+      ...allowedDetails,
     };
   }
 
-  return { action: 'promote', reason: 'v1_foundation_allowed' };
+  return { action: 'promote', reason: allowedReason, ...allowedDetails };
 }
 
 function createMemoryPromotion({ records, embedFn = null }) {
   async function prepareCandidates(candidates = []) {
-    const prepared = candidates.map(candidate => ({ ...candidate }));
+    const prepared = candidates.map(candidate => sanitizePromotionCandidate(candidate));
     if (typeof embedFn !== 'function' || prepared.length === 0) return prepared;
 
     const pendingMemoryRows = [];
@@ -663,13 +1000,13 @@ function createMemoryPromotion({ records, embedFn = null }) {
 }
 
 module.exports = {
-  ACTIVE_V1_TYPES,
   FORBIDDEN_TAGS,
-  AUTHORITY_RANK,
   defaultInheritanceForType,
   buildCanonicalKey,
   buildMemoryEmbeddingText,
   extractCandidatesFromStructuredSummary,
   assessCandidate,
+  assessAssistantShapingCandidate,
+  sanitizePromotionCandidate,
   createMemoryPromotion,
 };