cipher-security 2.0.0

package/lib/memory/index.js

@@ -0,0 +1,58 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * CIPHER Memory Core — Tri-Stage Pipeline + Evolution Engine
+ *
+ * Public API surface for the memory module. All downstream consumers
+ * (gateway, API, MCP) import from this barrel.
+ */
+
+// Core engine
+export {
+  CipherMemory,
+  MemoryEntry,
+  MemoryType,
+  Confidence,
+  SymbolicStore,
+  MemoryConsolidator,
+  reciprocalRankFusion,
+} from './engine.js';
+
+// Stage 1: Compressor
+export {
+  SemanticCompressor,
+  DensityGate,
+  DialogueTurn,
+  CompressedEntry,
+  extractSecurityEntities,
+} from './compressor.js';
+
+// Stage 2: Synthesizer
+export {
+  SemanticSynthesizer,
+  SynthesisResult,
+} from './synthesizer.js';
+
+// Stage 3: Retriever
+export {
+  AdaptiveRetriever,
+  IntentClassifier,
+  QueryDecomposer,
+  ContextBuilder,
+  RetrievalPlan,
+} from './retriever.js';
+
+// Evolution
+export {
+  ResponseScorer,
+  SkillEvolver,
+  ScoredResponse,
+} from './evolution.js';
+
+// Orchestrator
+export {
+  CipherOrchestrator,
+  createOrchestrator,
+} from './orchestrator.js';

package/lib/memory/orchestrator.js

@@ -0,0 +1,506 @@
+// Copyright (c) 2026 defconxt. All rights reserved.
+// Licensed under AGPL-3.0 — see LICENSE file for details.
+// CIPHER is a trademark of defconxt.
+
+/**
+ * CIPHER Memory — Session Orchestrator
+ *
+ * Ties together the 3-stage memory pipeline + evolution engine:
+ *   Stage 1: SemanticCompressor (dialogue → atomic entries)
+ *   Stage 2: SemanticSynthesizer (dedup, merge, link)
+ *   Stage 3: AdaptiveRetriever (intent-aware search + context building)
+ *   Evolution: ResponseScorer + SkillEvolver (learn from failures)
+ *
+ * Manages the full session lifecycle:
+ *   startSession → recordTurn / recordToolUse / scoreResponse → stopSession
+ *
+ * Ported from Python memory/core/orchestrator.py.
+ */
+
+import { randomUUID } from 'node:crypto';
+import { join } from 'node:path';
+import { homedir } from 'node:os';
+
+import { CipherMemory, MemoryEntry, MemoryType, Confidence } from './engine.js';
+import { SemanticCompressor, CompressedEntry, DialogueTurn } from './compressor.js';
+import { SemanticSynthesizer } from './synthesizer.js';
+import { AdaptiveRetriever, ContextBuilder } from './retriever.js';
+import { ResponseScorer, SkillEvolver } from './evolution.js';
+
+// ---------------------------------------------------------------------------
+// Session data structures
+// ---------------------------------------------------------------------------
+
+/**
+ * Create a new SessionState object.
+ * @param {{ sessionId: string, engagementId: string, startedAt: string }} opts
+ * @returns {object}
+ */
+function createSessionState(opts) {
+  return {
+    sessionId: opts.sessionId,
+    engagementId: opts.engagementId,
+    startedAt: opts.startedAt,
+    turns: [],
+    scoredResponses: [],
+    entriesStored: 0,
+    active: true,
+  };
+}
+
+/**
+ * Create a SessionReport from processing results.
+ * @param {object} opts
+ * @returns {object}
+ */
+function createSessionReport(opts) {
+  return {
+    sessionId: opts.sessionId ?? '',
+    engagementId: opts.engagementId ?? '',
+    durationSeconds: opts.durationSeconds ?? 0,
+    turnsRecorded: opts.turnsRecorded ?? 0,
+    entriesStored: opts.entriesStored ?? 0,
+    entriesDeduplicated: opts.entriesDeduplicated ?? 0,
+    entriesMerged: opts.entriesMerged ?? 0,
+    evolutionTriggered: opts.evolutionTriggered ?? false,
+    skillsGenerated: opts.skillsGenerated ?? 0,
+    memoryStats: opts.memoryStats ?? {},
+  };
+}
+
+// ---------------------------------------------------------------------------
+// MemoryType / Confidence value lookup helpers
+// ---------------------------------------------------------------------------
+
+/** Set of valid MemoryType values for fast lookup. */
+const _VALID_MEMORY_TYPES = new Set(Object.values(MemoryType));
+
+/** Set of valid Confidence values for fast lookup. */
+const _VALID_CONFIDENCE = new Set(Object.values(Confidence));
+
+/**
+ * Resolve a string to a MemoryType value, defaulting to NOTE if invalid.
+ * @param {string} value
+ * @returns {string}
+ */
+function _resolveMemoryType(value) {
+  return _VALID_MEMORY_TYPES.has(value) ? value : MemoryType.NOTE;
+}
+
+/**
+ * Resolve a string to a Confidence value, defaulting to CONFIRMED if invalid.
+ * @param {string} value
+ * @returns {string}
+ */
+function _resolveConfidence(value) {
+  return _VALID_CONFIDENCE.has(value) ? value : Confidence.CONFIRMED;
+}
+
+// ---------------------------------------------------------------------------
+// CipherOrchestrator
+// ---------------------------------------------------------------------------
+
+/**
+ * Full session lifecycle orchestrator for CIPHER memory.
+ *
+ * Usage:
+ *   const orch = new CipherOrchestrator({ memoryDir: '/tmp/mem' });
+ *   const ctx = orch.startSession('ENG-001', 'Pentest of ACME Corp');
+ *   orch.recordTurn(ctx.sessionId, 'user', 'Scan 10.10.14.5 for open ports');
+ *   orch.recordTurn(ctx.sessionId, 'assistant', 'Found ports 22, 80, 443...');
+ *   orch.recordToolUse(ctx.sessionId, 'nmap', '-sV 10.10.14.5', '22/tcp ssh...');
+ *   orch.scoreResponse(ctx.sessionId, query, response, 'RED');
+ *   const report = await orch.stopSession(ctx.sessionId);
+ *   orch.close();
+ */
+class CipherOrchestrator {
+  /**
+   * @param {{
+   *   memoryDir?: string,
+   *   skillsDir?: string,
+   *   llmClient?: any,
+   *   enableEvolution?: boolean,
+   *   evolutionThreshold?: number,
+   *   contextTokens?: number,
+   * }} opts
+   */
+  constructor(opts = {}) {
+    const {
+      memoryDir,
+      skillsDir = 'skills',
+      llmClient = null,
+      enableEvolution = true,
+      evolutionThreshold = 0.4,
+      contextTokens = 4000,
+    } = opts;
+
+    if (!memoryDir) {
+      throw new Error('CipherOrchestrator requires memoryDir');
+    }
+
+    // Core memory engine
+    this.memory = new CipherMemory(memoryDir);
+
+    // Stage 1: Compressor
+    this.compressor = new SemanticCompressor({ llmClient });
+
+    // Stage 2: Synthesizer
+    this.synthesizer = new SemanticSynthesizer();
+
+    // Stage 3: Retriever
+    this.retriever = new AdaptiveRetriever(this.memory, {
+      enableReflection: true,
+    });
+
+    // Context builder
+    this.contextBuilder = new ContextBuilder({ maxTokens: contextTokens });
+
+    // Evolution engine
+    this.scorer = new ResponseScorer({ llmClient });
+    this.evolver = enableEvolution
+      ? new SkillEvolver({
+          skillsDir,
+          llmClient,
+          historyPath: join(memoryDir, 'evolution_history.jsonl'),
+        })
+      : null;
+    this.evolutionThreshold = evolutionThreshold;
+
+    // Active sessions
+    this._sessions = new Map();
+  }
+
+  // -------------------------------------------------------------------------
+  // Session lifecycle
+  // -------------------------------------------------------------------------
+
+  /**
+   * Start a new memory session.
+   *
+   * Automatically retrieves and injects context from previous sessions
+   * for the same engagement.
+   *
+   * @param {string} [engagementId='']
+   * @param {string} [userPrompt='']
+   * @returns {{ sessionId: string, engagementId: string, context: string, contextEntries: number }}
+   */
+  startSession(engagementId = '', userPrompt = '') {
+    const sessionId = randomUUID();
+    const now = new Date().toISOString();
+
+    const effectiveEngagement = engagementId || `session-${sessionId.slice(0, 8)}`;
+
+    const state = createSessionState({
+      sessionId,
+      engagementId: effectiveEngagement,
+      startedAt: now,
+    });
+    this._sessions.set(sessionId, state);
+
+    // Retrieve context from previous sessions
+    let context = '';
+    if (userPrompt || engagementId) {
+      const query = userPrompt || `engagement ${engagementId}`;
+      const entries = this.retriever.retrieve(query, engagementId, 20);
+      if (entries.length > 0) {
+        context = this.contextBuilder.build(entries, query);
+      }
+    }
+
+    return {
+      sessionId,
+      engagementId: effectiveEngagement,
+      context,
+      contextEntries: context ? context.split('\n').length : 0,
+    };
+  }
+
+  /**
+   * Record a dialogue turn in the session.
+   * @param {string} sessionId
+   * @param {string} role
+   * @param {string} content
+   * @param {string} [timestamp='']
+   * @returns {boolean}
+   */
+  recordTurn(sessionId, role, content, timestamp = '') {
+    const state = this._sessions.get(sessionId);
+    if (!state || !state.active) {
+      return false;
+    }
+
+    const turn = new DialogueTurn({
+      turnId: state.turns.length + 1,
+      role,
+      content,
+      timestamp: timestamp || new Date().toISOString(),
+    });
+    state.turns.push(turn);
+    return true;
+  }
+
+  /**
+   * Record a tool use event in the session.
+   * @param {string} sessionId
+   * @param {string} toolName
+   * @param {string} toolInput
+   * @param {string} toolOutput
+   * @returns {boolean}
+   */
+  recordToolUse(sessionId, toolName, toolInput, toolOutput) {
+    const state = this._sessions.get(sessionId);
+    if (!state || !state.active) {
+      return false;
+    }
+
+    const turn = new DialogueTurn({
+      turnId: state.turns.length + 1,
+      role: 'tool',
+      content: `Used ${toolName}: ${toolInput}`,
+      toolName,
+      toolOutput,
+      timestamp: new Date().toISOString(),
+    });
+    state.turns.push(turn);
+    return true;
+  }
+
+  /**
+   * Score a response and track for evolution.
+   * @param {string} sessionId
+   * @param {string} query
+   * @param {string} response
+   * @param {string} [mode='']
+   * @param {string} [skillUsed='']
+   * @returns {import('./evolution.js').ScoredResponse}
+   */
+  scoreResponse(sessionId, query, response, mode = '', skillUsed = '') {
+    const scored = this.scorer.score(query, response, mode, skillUsed);
+
+    const state = this._sessions.get(sessionId);
+    if (state) {
+      state.scoredResponses.push(scored);
+    }
+
+    return scored;
+  }
+
+  // -------------------------------------------------------------------------
+  // Session finalization (async — compressor.compress is async)
+  // -------------------------------------------------------------------------
+
+  /**
+   * Finalize session: compress, synthesize, store, and optionally evolve.
+   *
+   * This is the main processing step that converts dialogue into
+   * persistent memory entries.
+   *
+   * @param {string} sessionId
+   * @returns {Promise<object>} SessionReport
+   */
+  async stopSession(sessionId) {
+    const state = this._sessions.get(sessionId);
+    if (!state) {
+      throw new Error(`Session not found: ${sessionId}`);
+    }
+
+    state.active = false;
+    const started = new Date(state.startedAt);
+    const duration = (Date.now() - started.getTime()) / 1000;
+
+    // Stage 1: Compress dialogue into atomic entries
+    const compressed = await this.compressor.compress(
+      state.turns,
+      state.engagementId,
+    );
+
+    // Stage 2: Synthesize with existing memory
+    const existing = this.memory.getEngagementContext(state.engagementId);
+    const existingCompressed = existing.map(
+      (e) =>
+        new CompressedEntry({
+          entryId: e.entryId,
+          losslessRestatement: e.content,
+          memoryType: e.memoryType,
+          targets: e.targets,
+          cveIds: e.cveIds,
+          mitreAttack: e.mitreAttack,
+          keywords: e.keywords,
+        }),
+    );
+    const synthesis = this.synthesizer.synthesize(compressed, existingCompressed);
+
+    // Store new entries
+    let stored = 0;
+    for (const entry of synthesis.newEntries) {
+      const memEntry = new MemoryEntry({
+        entryId: entry.entryId,
+        content: entry.losslessRestatement,
+        memoryType: _resolveMemoryType(entry.memoryType),
+        confidence: _resolveConfidence(entry.confidence),
+        engagementId: entry.engagementId,
+        sourceSkill: entry.sourceSkill,
+        timestamp: entry.timestamp,
+        targets: entry.targets,
+        mitreAttack: entry.mitreAttack,
+        cveIds: entry.cveIds,
+        severity: entry.severity,
+        keywords: entry.keywords,
+      });
+      this.memory.store(memEntry);
+      stored++;
+    }
+
+    // Update merged entries
+    for (const merged of synthesis.mergedEntries) {
+      const memEntry = new MemoryEntry({
+        entryId: merged.entryId,
+        content: merged.losslessRestatement,
+        memoryType: _resolveMemoryType(merged.memoryType),
+        engagementId: merged.engagementId,
+        targets: merged.targets,
+        mitreAttack: merged.mitreAttack,
+        cveIds: merged.cveIds,
+        keywords: merged.keywords,
+      });
+      this.memory.store(memEntry);
+    }
+
+    state.entriesStored = stored;
+
+    // Evolution check
+    let evolutionTriggered = false;
+    let skillsGenerated = 0;
+    if (this.evolver && state.scoredResponses.length > 0) {
+      const failed = state.scoredResponses.filter((r) => r.score <= 0);
+      if (this.evolver.shouldEvolve(state.scoredResponses, this.evolutionThreshold)) {
+        const generated = this.evolver.evolve(failed);
+        skillsGenerated = generated.length;
+        evolutionTriggered = true;
+      }
+    }
+
+    // Memory maintenance
+    this.memory.consolidate();
+
+    const report = createSessionReport({
+      sessionId,
+      engagementId: state.engagementId,
+      durationSeconds: duration,
+      turnsRecorded: state.turns.length,
+      entriesStored: stored,
+      entriesDeduplicated: synthesis.stats.deduplicated ?? 0,
+      entriesMerged: synthesis.stats.merged ?? 0,
+      evolutionTriggered,
+      skillsGenerated,
+      memoryStats: this.memory.stats(),
+    });
+
+    return report;
+  }
+
+  // -------------------------------------------------------------------------
+  // Search & context
+  // -------------------------------------------------------------------------
+
+  /**
+   * Search memory with intent-aware retrieval.
+   * @param {string} query
+   * @param {string} [engagementId='']
+   * @param {number} [limit=10]
+   * @returns {MemoryEntry[]}
+   */
+  search(query, engagementId = '', limit = 10) {
+    return this.retriever.retrieve(query, engagementId, limit);
+  }
+
+  /**
+   * Get formatted context for prompt injection.
+   * @param {string} query
+   * @param {string} [engagementId='']
+   * @param {number} [maxTokens=4000]
+   * @returns {string}
+   */
+  getContext(query, engagementId = '', maxTokens = 4000) {
+    const entries = this.retriever.retrieve(query, engagementId);
+    const builder = new ContextBuilder({ maxTokens });
+    return builder.build(entries, query);
+  }
+
+  // -------------------------------------------------------------------------
+  // Diagnostics
+  // -------------------------------------------------------------------------
+
+  /**
+   * Get orchestrator statistics.
+   * @returns {{ memory: object, evolution: object, active_sessions: number }}
+   */
+  stats() {
+    const memoryStats = this.memory.stats();
+    const evolutionStats = this.evolver ? this.evolver.getSummary() : {};
+    return {
+      memory: memoryStats,
+      evolution: evolutionStats,
+      active_sessions: [...this._sessions.values()].filter((s) => s.active).length,
+    };
+  }
+
+  /**
+   * Clean up session state without closing memory.
+   * @param {string} sessionId
+   */
+  endSession(sessionId) {
+    this._sessions.delete(sessionId);
+  }
+
+  /**
+   * Clean up all resources.
+   */
+  close() {
+    this.memory.close();
+    this._sessions.clear();
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Factory function
+// ---------------------------------------------------------------------------
+
+/**
+ * Factory function to create a configured CipherOrchestrator.
+ *
+ * @param {{
+ *   project?: string,
+ *   memoryDir?: string,
+ *   skillsDir?: string,
+ *   [key: string]: any,
+ * }} opts
+ * @returns {CipherOrchestrator}
+ */
+function createOrchestrator(opts = {}) {
+  const {
+    project = 'cipher',
+    memoryDir,
+    skillsDir = 'skills',
+    ...rest
+  } = opts;
+
+  const effectiveDir = memoryDir || join(homedir(), '.cipher', 'memory', project);
+
+  return new CipherOrchestrator({
+    memoryDir: effectiveDir,
+    skillsDir,
+    ...rest,
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Exports
+// ---------------------------------------------------------------------------
+
+export {
+  CipherOrchestrator,
+  createOrchestrator,
+  createSessionState,
+  createSessionReport,
+};