@shadowforge0/aquifer-memory 1.5.12 → 1.6.0

package/core/memory-safety-gate.js

@@ -0,0 +1,224 @@
+'use strict';
+
+const REDACTION = '[REDACTED_SECRET]';
+
+const SECRET_PATTERNS = [
+  /\bsk-[A-Za-z0-9_-]{16,}\b/g,
+  /\bgh[pousr]_[A-Za-z0-9_]{20,}\b/g,
+  /\bxox[baprs]-[A-Za-z0-9-]{20,}\b/g,
+  /\b(AWS_SECRET_ACCESS_KEY|OPENAI_API_KEY|ANTHROPIC_API_KEY|GITHUB_TOKEN|DATABASE_URL)\s*=\s*[^\s]+/gi,
+  /\bAuthorization:\s*Bearer\s+[A-Za-z0-9._~+/=-]{12,}/gi,
+  /\b(cookie|set-cookie)\s*:\s*[^\n]+/gi,
+  /\bpostgres(?:ql)?:\/\/[^:\s/]+:[^@\s/]+@[^\s]+/gi,
+  /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?-----END [A-Z ]*PRIVATE KEY-----/g,
+];
+
+const SESSION_INJECTION_RE = [
+  /^\s*\[AQUIFER CONTEXT\]/i,
+  /<session-bootstrap\b/i,
+  /<memory-bootstrap\b/i,
+  /^# AGENTS\.md instructions/i,
+  /<environment_context>/i,
+  /<developer_context>/i,
+];
+
+const TOOL_OUTPUT_RE = [
+  /^\s*(tool|command|shell|exec)_?output\s*:/i,
+  /^\s*Exit code:\s*\d+/im,
+  /^\s*Wall time:\s*[\d.]+/im,
+  /^\s*Output:\s*$/im,
+  /\bstdout\b[\s\S]*\bstderr\b/i,
+];
+
+const STACK_TRACE_RE = [
+  /Traceback \(most recent call last\):/,
+  /^\s+at .+\(.+:\d+:\d+\)$/m,
+  /^\s+at .+:\d+:\d+$/m,
+  /\bSQLSTATE\s+[A-Z0-9]{5}\b/i,
+  /\bduplicate key value violates unique constraint\b/i,
+  /\bsyntax error at or near\b/i,
+];
+
+const COMMENTARY_RE = [
+  /^(I will|I'll|I'm going to|I’m going to|I will now)\b/i,
+  /^(我先|我會|接下來我|現在我會|我現在)\b/,
+];
+
+function extractText(message) {
+  if (!message || typeof message !== 'object') return '';
+  if (typeof message.content === 'string') return message.content;
+  if (Array.isArray(message.content)) {
+    return message.content
+      .filter(part => part && part.type === 'text' && typeof part.text === 'string')
+      .map(part => part.text)
+      .join('\n');
+  }
+  if (typeof message.text === 'string') return message.text;
+  return '';
+}
+
+function replaceText(message, text) {
+  const next = { ...message };
+  if (typeof next.content === 'string') {
+    next.content = text;
+  } else if (Array.isArray(next.content)) {
+    let replaced = false;
+    next.content = next.content.map(part => {
+      if (!part || part.type !== 'text' || typeof part.text !== 'string') return part;
+      if (replaced) return { ...part, text: '' };
+      replaced = true;
+      return { ...part, text };
+    });
+    if (!replaced) next.content = text;
+  } else if (typeof next.text === 'string') {
+    next.text = text;
+  } else {
+    next.content = text;
+  }
+  return next;
+}
+
+function redactSecrets(text) {
+  let redacted = String(text || '');
+  for (const re of SECRET_PATTERNS) {
+    redacted = redacted.replace(re, REDACTION);
+  }
+  return {
+    text: redacted,
+    redacted: redacted !== String(text || ''),
+  };
+}
+
+function isEnvDump(text) {
+  const lines = String(text || '').split(/\r?\n/);
+  let envLines = 0;
+  for (const line of lines) {
+    if (/^[A-Z_][A-Z0-9_]{2,}=/.test(line.trim())) envLines++;
+  }
+  return envLines >= 3;
+}
+
+function hasAny(patterns, text) {
+  return patterns.some(re => re.test(text));
+}
+
+function assessTextForEnrich(text, role) {
+  const raw = String(text || '').trim();
+  const tags = [];
+  if (!raw) return { action: 'drop', reason: 'empty', tags: ['empty'], text: '' };
+
+  if (hasAny(SESSION_INJECTION_RE, raw)) tags.push('session_injected_context');
+  if (role === 'tool' || hasAny(TOOL_OUTPUT_RE, raw)) tags.push('tool_output');
+  if (hasAny(STACK_TRACE_RE, raw)) tags.push('stack_trace');
+  if (isEnvDump(raw)) tags.push('env_dump');
+  if (role === 'assistant' && hasAny(COMMENTARY_RE, raw)) tags.push('commentary');
+
+  const secretResult = redactSecrets(raw);
+  if (secretResult.redacted) tags.push('secret_risk');
+
+  const dropTags = new Set([
+    'session_injected_context',
+    'tool_output',
+    'stack_trace',
+    'env_dump',
+    'commentary',
+  ]);
+  const dropReason = tags.find(tag => dropTags.has(tag));
+  if (dropReason) {
+    return { action: 'drop', reason: dropReason, tags, text: '' };
+  }
+
+  return {
+    action: 'keep',
+    reason: secretResult.redacted ? 'redacted' : 'clean',
+    tags,
+    text: secretResult.text,
+    redacted: secretResult.redacted,
+  };
+}
+
+function applyEnrichSafetyGate(messages = []) {
+  const input = Array.isArray(messages) ? messages : [];
+  const safe = [];
+  const quarantined = [];
+  const stats = {
+    total: input.length,
+    kept: 0,
+    dropped: 0,
+    redacted: 0,
+  };
+
+  for (let i = 0; i < input.length; i++) {
+    const message = input[i];
+    const role = message && message.role ? String(message.role) : 'unknown';
+    const assessment = assessTextForEnrich(extractText(message), role);
+    if (assessment.action === 'drop') {
+      stats.dropped++;
+      quarantined.push({ index: i, role, reason: assessment.reason, tags: assessment.tags });
+      continue;
+    }
+    if (assessment.redacted) stats.redacted++;
+    stats.kept++;
+    safe.push(replaceText(message, assessment.text));
+  }
+
+  return {
+    messages: safe,
+    meta: {
+      stats,
+      quarantined,
+      redacted: stats.redacted,
+      dropped: stats.dropped,
+    },
+  };
+}
+
+function sanitizeStructuredValue(value, meta, role = 'assistant') {
+  if (typeof value === 'string') {
+    const assessment = assessTextForEnrich(value, role);
+    if (assessment.action === 'drop') {
+      meta.dropped++;
+      return undefined;
+    }
+    if (assessment.redacted) meta.redacted++;
+    return assessment.text;
+  }
+  if (Array.isArray(value)) {
+    return value
+      .map(item => sanitizeStructuredValue(item, meta, role))
+      .filter(item => item !== undefined);
+  }
+  if (value && typeof value === 'object') {
+    const out = {};
+    for (const [key, child] of Object.entries(value)) {
+      const sanitized = sanitizeStructuredValue(child, meta, role);
+      if (sanitized !== undefined) out[key] = sanitized;
+    }
+    return Object.keys(out).length > 0 ? out : undefined;
+  }
+  return value;
+}
+
+function sanitizeSummaryResult(summaryResult) {
+  if (!summaryResult) return { summaryResult: null, meta: { redacted: 0, dropped: 0 } };
+  const meta = { redacted: 0, dropped: 0 };
+  const next = { ...summaryResult };
+  if (typeof next.summaryText === 'string') {
+    const assessment = assessTextForEnrich(next.summaryText, 'assistant');
+    next.summaryText = assessment.action === 'drop' ? '' : assessment.text;
+    if (assessment.action === 'drop') meta.dropped++;
+    if (assessment.redacted) meta.redacted++;
+  }
+  if (next.structuredSummary) {
+    next.structuredSummary = sanitizeStructuredValue(next.structuredSummary, meta, 'assistant');
+  }
+  return { summaryResult: next, meta };
+}
+
+module.exports = {
+  REDACTION,
+  redactSecrets,
+  assessTextForEnrich,
+  applyEnrichSafetyGate,
+  sanitizeSummaryResult,
+};

package/core/session-finalization.js

@@ -0,0 +1,350 @@
+'use strict';
+
+const storage = require('./storage');
+const { createMemoryRecords } = require('./memory-records');
+const { createMemoryPromotion } = require('./memory-promotion');
+const { sanitizeSummaryResult } = require('./memory-safety-gate');
+const { buildFinalizationReview, buildSessionStartContext } = require('./finalization-review');
+
+function qi(identifier) { return `"${identifier}"`; }
+
+function requireField(obj, field) {
+  if (!obj || obj[field] === undefined || obj[field] === null || obj[field] === '') {
+    throw new Error(`${field} is required`);
+  }
+}
+
+function hasStructuredContent(value) {
+  return value && typeof value === 'object' && Object.keys(value).length > 0;
+}
+
+const TERMINAL_SUPPRESSION_STATUSES = new Set(['skipped', 'declined', 'deferred']);
+
+function countByReason(results) {
+  const reasons = {};
+  for (const result of results || []) {
+    const reason = result && result.reason ? result.reason : 'unknown';
+    reasons[reason] = (reasons[reason] || 0) + 1;
+  }
+  return reasons;
+}
+
+function summarizeMemoryResults(results = [], extra = {}) {
+  return {
+    candidates: results.length,
+    promoted: results.filter(result => result.action === 'promote').length,
+    quarantined: results.filter(result => result.action === 'quarantine').length,
+    skipped: results.filter(result => result.action && !['promote', 'quarantine'].includes(result.action)).length,
+    reasons: countByReason(results),
+    ...extra,
+  };
+}
+
+function normalizeFinalizationInput(input = {}, defaults = {}) {
+  const tenantId = input.tenantId || defaults.defaultTenantId || 'default';
+  return {
+    tenantId,
+    sessionId: input.sessionId,
+    agentId: input.agentId || 'main',
+    source: input.source || 'codex',
+    host: input.host || 'codex',
+    transcriptHash: input.transcriptHash,
+    phase: input.phase || 'curated_memory_v1',
+    mode: input.mode || 'handoff',
+  };
+}
+
+function createSessionFinalization({
+  pool,
+  schema,
+  recordsSchema,
+  defaultTenantId = 'default',
+}) {
+  const memorySchema = recordsSchema || qi(schema);
+
+  async function createTask(input = {}) {
+    const base = normalizeFinalizationInput(input, { defaultTenantId });
+    requireField(base, 'sessionId');
+    requireField(base, 'agentId');
+    requireField(base, 'source');
+    requireField(base, 'transcriptHash');
+
+    let sessionRowId = input.sessionRowId || null;
+    if (!sessionRowId) {
+      const session = await storage.getSession(
+        pool,
+        base.sessionId,
+        base.agentId,
+        { tenantId: base.tenantId, source: base.source },
+        { schema, tenantId: base.tenantId },
+      );
+      if (!session) {
+        throw new Error(`Session not found: ${base.sessionId} (agentId=${base.agentId}, source=${base.source})`);
+      }
+      sessionRowId = session.id;
+    }
+
+    return storage.upsertSessionFinalization(pool, {
+      ...base,
+      sessionRowId,
+      status: input.status || 'pending',
+      finalizerModel: input.finalizerModel || null,
+      scopeKind: input.scopeKind || null,
+      scopeKey: input.scopeKey || null,
+      contextKey: input.contextKey || null,
+      topicKey: input.topicKey || null,
+      memoryResult: input.memoryResult || {},
+      error: input.error || null,
+      metadata: input.metadata || {},
+      claimedAt: input.claimedAt || null,
+      finalizedAt: input.finalizedAt || null,
+    }, { schema, tenantId: base.tenantId });
+  }
+
+  async function get(input = {}) {
+    const base = normalizeFinalizationInput(input, { defaultTenantId });
+    return storage.getSessionFinalization(pool, base, { schema, tenantId: base.tenantId });
+  }
+
+  async function list(input = {}) {
+    const tenantId = input.tenantId || defaultTenantId || 'default';
+    return storage.listSessionFinalizations(pool, input, { schema, tenantId });
+  }
+
+  async function updateStatus(input = {}) {
+    const tenantId = input.tenantId || defaultTenantId || 'default';
+    return storage.updateSessionFinalizationStatus(pool, input, { schema, tenantId });
+  }
+
+  async function finalizeSession(input = {}) {
+    const base = normalizeFinalizationInput(input, { defaultTenantId });
+    requireField(base, 'sessionId');
+    requireField(base, 'agentId');
+    requireField(base, 'source');
+    requireField(base, 'transcriptHash');
+
+    const summaryText = String(input.summaryText || '').trim();
+    const structuredSummary = input.structuredSummary || {};
+    if (!summaryText && !hasStructuredContent(structuredSummary)) {
+      throw new Error('summaryText or structuredSummary is required');
+    }
+
+    const client = await pool.connect();
+    let failureSession = null;
+    let processingTask = null;
+    try {
+      await client.query('BEGIN');
+
+      const sessionResult = await client.query(
+        `SELECT *
+           FROM ${qi(schema)}.sessions
+          WHERE tenant_id = $1
+            AND agent_id = $2
+            AND session_id = $3
+            AND source = $4
+          FOR UPDATE`,
+        [base.tenantId, base.agentId, base.sessionId, base.source],
+      );
+      const session = sessionResult.rows[0] || null;
+      if (!session) {
+        throw new Error(`Session not found: ${base.sessionId} (agentId=${base.agentId}, source=${base.source})`);
+      }
+      failureSession = session;
+
+      const existing = await storage.getSessionFinalization(client, base, {
+        schema,
+        tenantId: base.tenantId,
+      });
+      if (existing && existing.status === 'finalized') {
+        await client.query('COMMIT');
+        return {
+          status: 'already_finalized',
+          finalization: existing,
+          memoryResult: existing.memory_result || {},
+        };
+      }
+      if (existing && TERMINAL_SUPPRESSION_STATUSES.has(existing.status)) {
+        await client.query('COMMIT');
+        return {
+          status: 'suppressed',
+          finalizationStatus: existing.status,
+          finalization: existing,
+          memoryResult: existing.memory_result || {},
+        };
+      }
+
+      processingTask = await storage.upsertSessionFinalization(client, {
+        ...base,
+        sessionRowId: session.id,
+        status: 'processing',
+        finalizerModel: input.finalizerModel || input.model || null,
+        scopeKind: input.scopeKind || null,
+        scopeKey: input.scopeKey || null,
+        contextKey: input.contextKey || null,
+        topicKey: input.topicKey || null,
+        metadata: input.metadata || {},
+        claimedAt: input.claimedAt || new Date().toISOString(),
+      }, { schema, tenantId: base.tenantId });
+
+      const sanitized = sanitizeSummaryResult({ summaryText, structuredSummary });
+      const safeSummary = sanitized.summaryResult || {};
+      const safeStructuredSummary = safeSummary.structuredSummary || {};
+      const safeSummaryText = safeSummary.summaryText || summaryText;
+      const finalizerModel = input.finalizerModel || input.model || session.model || null;
+
+      const summaryRow = await storage.upsertSummary(client, session.id, {
+        schema,
+        tenantId: base.tenantId,
+        agentId: base.agentId,
+        sessionId: base.sessionId,
+        summaryText: safeSummaryText,
+        structuredSummary: safeStructuredSummary,
+        model: finalizerModel,
+        sourceHash: base.transcriptHash,
+        msgCount: input.msgCount || input.messageCount || session.msg_count || 0,
+        userCount: input.userCount || session.user_count || 0,
+        assistantCount: input.assistantCount || session.assistant_count || 0,
+        startedAt: input.startedAt || session.started_at || null,
+        endedAt: input.endedAt || session.ended_at || session.last_message_at || null,
+        embedding: input.embedding || null,
+      });
+
+      const records = createMemoryRecords({
+        pool: client,
+        schema: memorySchema,
+        defaultTenantId: base.tenantId,
+        inTransaction: true,
+      });
+      const promotion = createMemoryPromotion({ records });
+      const evidenceRefs = [{
+        sourceKind: 'session_summary',
+        sourceRef: base.sessionId,
+        relationKind: 'primary',
+        metadata: {
+          transcriptHash: base.transcriptHash,
+          finalizationId: processingTask ? processingTask.id : null,
+          mode: base.mode,
+          phase: base.phase,
+        },
+      }];
+      const candidates = Array.isArray(input.candidates)
+        ? input.candidates
+        : promotion.extractCandidates({
+            sessionId: base.sessionId,
+            structuredSummary: safeStructuredSummary,
+            scopeKind: input.scopeKind || null,
+            scopeKey: input.scopeKey || null,
+            contextKey: input.contextKey || null,
+            topicKey: input.topicKey || null,
+            authority: input.authority || 'verified_summary',
+            evidenceRefs,
+          });
+
+      const memoryResults = candidates.length > 0
+        ? await promotion.promote(candidates, {
+            tenantId: base.tenantId,
+            acceptedAt: input.acceptedAt || new Date().toISOString(),
+            createdByFinalizationId: processingTask ? processingTask.id : null,
+          })
+        : [];
+      if (processingTask && memoryResults.length > 0) {
+        await storage.upsertFinalizationCandidates(client, memoryResults, {
+          tenantId: base.tenantId,
+          finalizationId: processingTask.id,
+          sessionId: base.sessionId,
+        }, { schema, tenantId: base.tenantId });
+      }
+      const memoryResult = summarizeMemoryResults(memoryResults, {
+        safetyGate: sanitized.meta || {},
+      });
+      const promotedMemories = memoryResults
+        .filter(result => result && result.action === 'promote' && result.memory)
+        .map(result => result.memory);
+      const humanReviewText = buildFinalizationReview({
+        summary: {
+          summaryText: safeSummaryText,
+          structuredSummary: safeStructuredSummary,
+        },
+        memoryResult,
+        memoryResults,
+        sessionId: base.sessionId,
+        transcriptHash: base.transcriptHash,
+        finalization: processingTask,
+      });
+      const sessionStartText = buildSessionStartContext(promotedMemories);
+
+      const finalization = await storage.upsertSessionFinalization(client, {
+        ...base,
+        sessionRowId: session.id,
+        status: 'finalized',
+        finalizerModel,
+        scopeKind: input.scopeKind || null,
+        scopeKey: input.scopeKey || null,
+        contextKey: input.contextKey || null,
+        topicKey: input.topicKey || null,
+        summaryRowId: summaryRow ? summaryRow.session_row_id : session.id,
+        memoryResult,
+        summaryText: safeSummaryText,
+        structuredSummary: safeStructuredSummary,
+        humanReviewText,
+        sessionStartText,
+        metadata: {
+          ...(input.metadata || {}),
+          safetyGate: sanitized.meta || {},
+        },
+      }, { schema, tenantId: base.tenantId });
+
+      await storage.markStatus(client, session.id, 'succeeded', null, { schema });
+      await client.query('COMMIT');
+
+      return {
+        status: 'finalized',
+        finalization,
+        summary: {
+          summaryText: safeSummaryText,
+          structuredSummary: safeStructuredSummary,
+        },
+        memoryResult,
+        memoryResults,
+        humanReviewText,
+        sessionStartText,
+      };
+    } catch (error) {
+      await client.query('ROLLBACK').catch(() => {});
+      if (failureSession) {
+        try {
+          await storage.upsertSessionFinalization(pool, {
+            ...base,
+            sessionRowId: failureSession.id,
+            status: 'failed',
+            finalizerModel: input.finalizerModel || input.model || null,
+            scopeKind: input.scopeKind || null,
+            scopeKey: input.scopeKey || null,
+            contextKey: input.contextKey || null,
+            topicKey: input.topicKey || null,
+            metadata: input.metadata || {},
+            error: error.message,
+          }, { schema, tenantId: base.tenantId });
+        } catch {
+          // The original finalization failure is the useful error. A secondary
+          // ledger failure should not hide it.
+        }
+      }
+      throw error;
+    } finally {
+      client.release();
+    }
+  }
+
+  return {
+    createTask,
+    get,
+    list,
+    updateStatus,
+    finalizeSession,
+  };
+}
+
+module.exports = {
+  createSessionFinalization,
+};