agentic-flow 1.7.2 → 1.7.4

package/dist/agentdb/controllers/CausalMemoryGraph.js

@@ -0,0 +1,322 @@
+/**
+ * CausalMemoryGraph - Causal Reasoning over Agent Memories
+ *
+ * Implements intervention-based reasoning rather than correlation.
+ * Stores p(y|do(x)) estimates and tracks causal uplift across episodes.
+ *
+ * Based on:
+ * - Pearl's do-calculus and causal inference
+ * - Uplift modeling from A/B testing
+ * - Instrumental variable methods
+ */
+export class CausalMemoryGraph {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    /**
+     * Add a causal edge between memories
+     */
+    addCausalEdge(edge) {
+        const stmt = this.db.prepare(`
+      INSERT INTO causal_edges (
+        from_memory_id, from_memory_type, to_memory_id, to_memory_type,
+        similarity, uplift, confidence, sample_size,
+        evidence_ids, experiment_ids, confounder_score,
+        mechanism, metadata
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+        const result = stmt.run(edge.fromMemoryId, edge.fromMemoryType, edge.toMemoryId, edge.toMemoryType, edge.similarity, edge.uplift || null, edge.confidence, edge.sampleSize || null, edge.evidenceIds ? JSON.stringify(edge.evidenceIds) : null, edge.experimentIds ? JSON.stringify(edge.experimentIds) : null, edge.confounderScore || null, edge.mechanism || null, edge.metadata ? JSON.stringify(edge.metadata) : null);
+        return result.lastInsertRowid;
+    }
+    /**
+     * Create a causal experiment (A/B test)
+     */
+    createExperiment(experiment) {
+        const stmt = this.db.prepare(`
+      INSERT INTO causal_experiments (
+        name, hypothesis, treatment_id, treatment_type, control_id,
+        start_time, sample_size, status, metadata
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+        const result = stmt.run(experiment.name, experiment.hypothesis, experiment.treatmentId, experiment.treatmentType, experiment.controlId || null, experiment.startTime, experiment.sampleSize, experiment.status, experiment.metadata ? JSON.stringify(experiment.metadata) : null);
+        return result.lastInsertRowid;
+    }
+    /**
+     * Record an observation in an experiment
+     */
+    recordObservation(observation) {
+        const stmt = this.db.prepare(`
+      INSERT INTO causal_observations (
+        experiment_id, episode_id, is_treatment, outcome_value, outcome_type, context
+      ) VALUES (?, ?, ?, ?, ?, ?)
+    `);
+        stmt.run(observation.experimentId, observation.episodeId, observation.isTreatment ? 1 : 0, observation.outcomeValue, observation.outcomeType, observation.context ? JSON.stringify(observation.context) : null);
+        // Update sample size
+        this.db.prepare(`
+      UPDATE causal_experiments
+      SET sample_size = sample_size + 1
+      WHERE id = ?
+    `).run(observation.experimentId);
+    }
+    /**
+     * Calculate uplift for an experiment
+     */
+    calculateUplift(experimentId) {
+        // Get treatment and control observations
+        const observations = this.db.prepare(`
+      SELECT is_treatment, outcome_value
+      FROM causal_observations
+      WHERE experiment_id = ?
+    `).all(experimentId);
+        const treatmentValues = observations
+            .filter(o => o.is_treatment === 1)
+            .map(o => o.outcome_value);
+        const controlValues = observations
+            .filter(o => o.is_treatment === 0)
+            .map(o => o.outcome_value);
+        if (treatmentValues.length === 0 || controlValues.length === 0) {
+            return { uplift: 0, pValue: 1.0, confidenceInterval: [0, 0] };
+        }
+        // Calculate means
+        const treatmentMean = this.mean(treatmentValues);
+        const controlMean = this.mean(controlValues);
+        const uplift = treatmentMean - controlMean;
+        // Calculate standard errors
+        const treatmentSE = this.standardError(treatmentValues);
+        const controlSE = this.standardError(controlValues);
+        const pooledSE = Math.sqrt(treatmentSE ** 2 + controlSE ** 2);
+        // t-statistic and p-value (two-tailed)
+        const tStat = uplift / pooledSE;
+        const df = treatmentValues.length + controlValues.length - 2;
+        const pValue = 2 * (1 - this.tCDF(Math.abs(tStat), df));
+        // 95% confidence interval
+        const tCritical = this.tInverse(0.025, df);
+        const marginOfError = tCritical * pooledSE;
+        const confidenceInterval = [
+            uplift - marginOfError,
+            uplift + marginOfError
+        ];
+        // Update experiment with results
+        this.db.prepare(`
+      UPDATE causal_experiments
+      SET treatment_mean = ?,
+          control_mean = ?,
+          uplift = ?,
+          p_value = ?,
+          confidence_interval_low = ?,
+          confidence_interval_high = ?,
+          status = 'completed'
+      WHERE id = ?
+    `).run(treatmentMean, controlMean, uplift, pValue, confidenceInterval[0], confidenceInterval[1], experimentId);
+        return { uplift, pValue, confidenceInterval };
+    }
+    /**
+     * Query causal effects
+     */
+    queryCausalEffects(query) {
+        const { interventionMemoryId, interventionMemoryType, outcomeMemoryId, minConfidence = 0.5, minUplift = 0.0 } = query;
+        let sql = `
+      SELECT * FROM causal_edges
+      WHERE from_memory_id = ?
+        AND from_memory_type = ?
+        AND confidence >= ?
+        AND ABS(uplift) >= ?
+    `;
+        const params = [
+            interventionMemoryId,
+            interventionMemoryType,
+            minConfidence,
+            minUplift
+        ];
+        if (outcomeMemoryId) {
+            sql += ' AND to_memory_id = ?';
+            params.push(outcomeMemoryId);
+        }
+        sql += ' ORDER BY ABS(uplift) * confidence DESC';
+        const rows = this.db.prepare(sql).all(...params);
+        return rows.map(row => this.rowToCausalEdge(row));
+    }
+    /**
+     * Get causal chain (multi-hop reasoning)
+     */
+    getCausalChain(fromMemoryId, toMemoryId, maxDepth = 5) {
+        // Use recursive CTE from view
+        const chains = this.db.prepare(`
+      WITH RECURSIVE chain(from_id, to_id, depth, path, total_uplift, min_confidence) AS (
+        SELECT
+          from_memory_id,
+          to_memory_id,
+          1,
+          from_memory_id || '->' || to_memory_id,
+          uplift,
+          confidence
+        FROM causal_edges
+        WHERE from_memory_id = ? AND confidence >= 0.5
+
+        UNION ALL
+
+        SELECT
+          chain.from_id,
+          ce.to_memory_id,
+          chain.depth + 1,
+          chain.path || '->' || ce.to_memory_id,
+          chain.total_uplift + ce.uplift,
+          MIN(chain.min_confidence, ce.confidence)
+        FROM chain
+        JOIN causal_edges ce ON chain.to_id = ce.from_memory_id
+        WHERE chain.depth < ?
+          AND ce.confidence >= 0.5
+          AND chain.path NOT LIKE '%' || ce.to_memory_id || '%'
+      )
+      SELECT path, total_uplift, min_confidence
+      FROM chain
+      WHERE to_id = ?
+      ORDER BY total_uplift DESC
+      LIMIT 10
+    `).all(fromMemoryId, maxDepth, toMemoryId);
+        return chains.map(row => ({
+            path: row.path.split('->').map(Number),
+            totalUplift: row.total_uplift,
+            confidence: row.min_confidence
+        }));
+    }
+    /**
+     * Calculate causal gain: E[outcome|do(treatment)] - E[outcome]
+     */
+    calculateCausalGain(treatmentId, outcomeType) {
+        // Get episodes where treatment was applied
+        const withTreatment = this.db.prepare(`
+      SELECT AVG(CASE WHEN ? = 'reward' THEN reward
+                     WHEN ? = 'success' THEN success
+                     WHEN ? = 'latency' THEN latency_ms
+                END) as avg_outcome
+      FROM episodes
+      WHERE id IN (
+        SELECT to_memory_id FROM causal_edges
+        WHERE from_memory_id = ? AND confidence >= 0.6
+      )
+    `).get(outcomeType, outcomeType, outcomeType, treatmentId);
+        // Get baseline (no treatment)
+        const baseline = this.db.prepare(`
+      SELECT AVG(CASE WHEN ? = 'reward' THEN reward
+                     WHEN ? = 'success' THEN success
+                     WHEN ? = 'latency' THEN latency_ms
+                END) as avg_outcome
+      FROM episodes
+      WHERE id NOT IN (
+        SELECT to_memory_id FROM causal_edges
+        WHERE from_memory_id = ?
+      )
+    `).get(outcomeType, outcomeType, outcomeType, treatmentId);
+        const causalGain = (withTreatment?.avg_outcome || 0) - (baseline?.avg_outcome || 0);
+        // Get most confident edge for mechanism
+        const edge = this.db.prepare(`
+      SELECT mechanism, confidence
+      FROM causal_edges
+      WHERE from_memory_id = ?
+      ORDER BY confidence DESC
+      LIMIT 1
+    `).get(treatmentId);
+        return {
+            causalGain,
+            confidence: edge?.confidence || 0,
+            mechanism: edge?.mechanism || 'unknown'
+        };
+    }
+    /**
+     * Detect confounders using correlation analysis
+     */
+    detectConfounders(edgeId) {
+        const edge = this.db.prepare('SELECT * FROM causal_edges WHERE id = ?').get(edgeId);
+        if (!edge) {
+            return { confounders: [] };
+        }
+        // Find memories correlated with both treatment and outcome
+        // This is a simplified version - production would use proper statistical tests
+        const potentialConfounders = this.db.prepare(`
+      SELECT DISTINCT e.id, e.task
+      FROM episodes e
+      WHERE e.id != ? AND e.id != ?
+        AND e.session_id IN (
+          SELECT session_id FROM episodes WHERE id = ?
+          UNION
+          SELECT session_id FROM episodes WHERE id = ?
+        )
+    `).all(edge.from_memory_id, edge.to_memory_id, edge.from_memory_id, edge.to_memory_id);
+        const confounders = potentialConfounders.map((conf) => {
+            // Calculate correlation scores (simplified)
+            const treatmentCorr = this.calculateCorrelation(conf.id, edge.from_memory_id);
+            const outcomeCorr = this.calculateCorrelation(conf.id, edge.to_memory_id);
+            const confounderScore = Math.sqrt(treatmentCorr ** 2 * outcomeCorr ** 2);
+            return {
+                memoryId: conf.id,
+                correlationWithTreatment: treatmentCorr,
+                correlationWithOutcome: outcomeCorr,
+                confounderScore
+            };
+        }).filter(c => c.confounderScore > 0.3);
+        // Update edge with confounder score
+        if (confounders.length > 0) {
+            const maxConfounderScore = Math.max(...confounders.map(c => c.confounderScore));
+            this.db.prepare(`
+        UPDATE causal_edges
+        SET confounder_score = ?
+        WHERE id = ?
+      `).run(maxConfounderScore, edgeId);
+        }
+        return { confounders };
+    }
+    // ========================================================================
+    // Private Helper Methods
+    // ========================================================================
+    rowToCausalEdge(row) {
+        return {
+            id: row.id,
+            fromMemoryId: row.from_memory_id,
+            fromMemoryType: row.from_memory_type,
+            toMemoryId: row.to_memory_id,
+            toMemoryType: row.to_memory_type,
+            similarity: row.similarity,
+            uplift: row.uplift,
+            confidence: row.confidence,
+            sampleSize: row.sample_size,
+            evidenceIds: row.evidence_ids ? JSON.parse(row.evidence_ids) : undefined,
+            experimentIds: row.experiment_ids ? JSON.parse(row.experiment_ids) : undefined,
+            confounderScore: row.confounder_score,
+            mechanism: row.mechanism,
+            metadata: row.metadata ? JSON.parse(row.metadata) : undefined
+        };
+    }
+    mean(values) {
+        return values.reduce((a, b) => a + b, 0) / values.length;
+    }
+    variance(values) {
+        const avg = this.mean(values);
+        return values.reduce((sum, val) => sum + (val - avg) ** 2, 0) / values.length;
+    }
+    standardError(values) {
+        return Math.sqrt(this.variance(values) / values.length);
+    }
+    tCDF(t, df) {
+        // Simplified t-distribution CDF (use proper stats library in production)
+        // This is an approximation
+        return 0.5 + 0.5 * Math.sign(t) * (1 - Math.pow(1 + t * t / df, -df / 2));
+    }
+    tInverse(p, df) {
+        // Simplified inverse t-distribution (use proper stats library)
+        // Approximation for 95% CI
+        return 1.96; // Standard normal approximation
+    }
+    calculateCorrelation(id1, id2) {
+        // Simplified correlation calculation
+        // In production, use proper correlation metrics
+        const sharedSessions = this.db.prepare(`
+      SELECT COUNT(DISTINCT e1.session_id) as shared
+      FROM episodes e1
+      JOIN episodes e2 ON e1.session_id = e2.session_id
+      WHERE e1.id = ? AND e2.id = ?
+    `).get(id1, id2);
+        return Math.min(sharedSessions?.shared || 0, 1.0);
+    }
+}

package/dist/agentdb/controllers/CausalRecall.js

@@ -0,0 +1,281 @@
+/**
+ * CausalRecall - Utility-Based Reranking + Certificate Issuer
+ *
+ * Combines:
+ * 1. Vector similarity search
+ * 2. Causal uplift from CausalMemoryGraph
+ * 3. Utility-based reranking: U = α*similarity + β*uplift − γ*latencyCost
+ * 4. Automatic certificate issuance via ExplainableRecall
+ *
+ * This is the main entry point for production retrieval with:
+ * - Causal-aware ranking
+ * - Explainable provenance
+ * - Policy compliance
+ */
+import { CausalMemoryGraph } from './CausalMemoryGraph.js';
+import { ExplainableRecall } from './ExplainableRecall.js';
+export class CausalRecall {
+    config;
+    db;
+    causalGraph;
+    explainableRecall;
+    embedder;
+    constructor(db, embedder, config = {
+        alpha: 0.7,
+        beta: 0.2,
+        gamma: 0.1,
+        minConfidence: 0.6
+    }) {
+        this.config = config;
+        this.db = db;
+        this.embedder = embedder;
+        this.causalGraph = new CausalMemoryGraph(db);
+        this.explainableRecall = new ExplainableRecall(db);
+    }
+    /**
+     * Main recall function with utility-based reranking and certificate issuance
+     *
+     * @param queryId Unique query identifier
+     * @param queryText Natural language query
+     * @param k Number of results to return (default: 12)
+     * @param requirements Optional list of requirements for completeness checking
+     * @param accessLevel Security access level for certificate
+     * @returns Reranked results with certificate
+     */
+    async recall(queryId, queryText, k = 12, requirements, accessLevel = 'internal') {
+        const startTime = Date.now();
+        const metrics = {
+            vectorSearchMs: 0,
+            causalLookupMs: 0,
+            rerankMs: 0,
+            certificateMs: 0
+        };
+        // Step 1: Vector similarity search
+        const vectorStart = Date.now();
+        const queryEmbedding = await this.embedder.embed(queryText);
+        const candidates = await this.vectorSearch(queryEmbedding, k * 2); // Fetch 2k for reranking
+        metrics.vectorSearchMs = Date.now() - vectorStart;
+        // Step 2: Load causal edges for candidates
+        const causalStart = Date.now();
+        const causalEdges = await this.loadCausalEdges(candidates.map(c => c.id));
+        metrics.causalLookupMs = Date.now() - causalStart;
+        // Step 3: Rerank by utility
+        const rerankStart = Date.now();
+        const reranked = this.rerankByUtility(candidates, causalEdges);
+        const topK = reranked.slice(0, k);
+        metrics.rerankMs = Date.now() - rerankStart;
+        // Step 4: Issue certificate
+        const certStart = Date.now();
+        const certificate = this.issueCertificate({
+            queryId,
+            queryText,
+            candidates: topK,
+            requirements: requirements || this.extractRequirements(queryText),
+            accessLevel
+        });
+        metrics.certificateMs = Date.now() - certStart;
+        const totalLatencyMs = Date.now() - startTime;
+        return {
+            candidates: topK,
+            certificate,
+            queryId,
+            totalLatencyMs,
+            metrics
+        };
+    }
+    /**
+     * Vector similarity search using cosine similarity
+     */
+    async vectorSearch(queryEmbedding, k) {
+        const results = [];
+        // Search episode embeddings
+        const episodes = this.db.prepare(`
+      SELECT
+        e.id,
+        'episode' as type,
+        e.task || ' ' || COALESCE(e.output, '') as content,
+        ee.embedding,
+        e.latency_ms
+      FROM episodes e
+      JOIN episode_embeddings ee ON e.id = ee.episode_id
+      ORDER BY e.ts DESC
+      LIMIT ?
+    `).all(k * 2);
+        for (const ep of episodes) {
+            const episodeRow = ep;
+            const embedding = new Float32Array(JSON.parse(episodeRow.embedding));
+            const similarity = this.cosineSimilarity(queryEmbedding, embedding);
+            results.push({
+                id: episodeRow.id.toString(),
+                type: episodeRow.type,
+                content: episodeRow.content,
+                similarity,
+                latencyMs: episodeRow.latency_ms || 0
+            });
+        }
+        // Sort by similarity and return top k
+        return results
+            .sort((a, b) => b.similarity - a.similarity)
+            .slice(0, k);
+    }
+    /**
+     * Load causal edges for candidates
+     */
+    async loadCausalEdges(candidateIds) {
+        const edgeMap = new Map();
+        if (candidateIds.length === 0) {
+            return edgeMap;
+        }
+        const placeholders = candidateIds.map(() => '?').join(',');
+        const edges = this.db.prepare(`
+      SELECT * FROM causal_edges
+      WHERE from_memory_id IN (${placeholders})
+        AND confidence >= ?
+    `).all(...candidateIds.map(id => parseInt(id)), this.config.minConfidence || 0.6);
+        for (const edge of edges) {
+            const fromId = edge.from_memory_id.toString();
+            if (!edgeMap.has(fromId)) {
+                edgeMap.set(fromId, []);
+            }
+            edgeMap.get(fromId).push({
+                id: edge.id,
+                fromMemoryId: edge.from_memory_id,
+                fromMemoryType: edge.from_memory_type,
+                toMemoryId: edge.to_memory_id,
+                toMemoryType: edge.to_memory_type,
+                similarity: edge.similarity,
+                uplift: edge.uplift,
+                confidence: edge.confidence,
+                sampleSize: edge.sample_size,
+                evidenceIds: edge.evidence_ids ? JSON.parse(edge.evidence_ids) : undefined,
+                mechanism: edge.mechanism
+            });
+        }
+        return edgeMap;
+    }
+    /**
+     * Rerank by utility: U = α*similarity + β*uplift − γ*latencyCost
+     */
+    rerankByUtility(candidates, causalEdges) {
+        const { alpha, beta, gamma } = this.config;
+        const reranked = candidates.map(candidate => {
+            // Get causal uplift (average if multiple edges)
+            const edges = causalEdges.get(candidate.id) || [];
+            const avgUplift = edges.length > 0
+                ? edges.reduce((sum, e) => sum + (e.uplift || 0), 0) / edges.length
+                : 0;
+            const avgConfidence = edges.length > 0
+                ? edges.reduce((sum, e) => sum + e.confidence, 0) / edges.length
+                : 0;
+            // Normalize latency (assume max 1000ms)
+            const latencyCost = Math.min(candidate.latencyMs / 1000, 1.0);
+            // Calculate utility
+            const utilityScore = alpha * candidate.similarity + beta * avgUplift - gamma * latencyCost;
+            return {
+                id: candidate.id,
+                type: candidate.type,
+                content: candidate.content,
+                similarity: candidate.similarity,
+                uplift: avgUplift,
+                causalConfidence: avgConfidence,
+                latencyMs: candidate.latencyMs,
+                utilityScore,
+                rank: 0 // Will be set after sorting
+            };
+        });
+        // Sort by utility score descending
+        reranked.sort((a, b) => b.utilityScore - a.utilityScore);
+        // Assign ranks
+        reranked.forEach((candidate, idx) => {
+            candidate.rank = idx + 1;
+        });
+        return reranked;
+    }
+    /**
+     * Issue certificate for the retrieval
+     */
+    issueCertificate(params) {
+        const { queryId, queryText, candidates, requirements, accessLevel } = params;
+        const chunks = candidates.map(c => ({
+            id: c.id,
+            type: c.type,
+            content: c.content,
+            relevance: c.similarity
+        }));
+        return this.explainableRecall.createCertificate({
+            queryId,
+            queryText,
+            chunks,
+            requirements,
+            accessLevel
+        });
+    }
+    /**
+     * Extract requirements from query text (simple keyword extraction)
+     */
+    extractRequirements(queryText) {
+        // Simple extraction: split on common words and filter
+        const stopWords = new Set(['a', 'an', 'the', 'is', 'are', 'was', 'were', 'to', 'from', 'for', 'with', 'how', 'what', 'where', 'when', 'why', 'who']);
+        const words = queryText
+            .toLowerCase()
+            .replace(/[^a-z0-9\s]/g, '')
+            .split(/\s+/)
+            .filter(w => w.length > 3 && !stopWords.has(w));
+        // Return unique words
+        return [...new Set(words)];
+    }
+    /**
+     * Cosine similarity between two vectors
+     */
+    cosineSimilarity(a, b) {
+        if (a.length !== b.length) {
+            throw new Error('Vector dimensions must match');
+        }
+        let dotProduct = 0;
+        let magnitudeA = 0;
+        let magnitudeB = 0;
+        for (let i = 0; i < a.length; i++) {
+            dotProduct += a[i] * b[i];
+            magnitudeA += a[i] * a[i];
+            magnitudeB += b[i] * b[i];
+        }
+        const magnitude = Math.sqrt(magnitudeA) * Math.sqrt(magnitudeB);
+        return magnitude === 0 ? 0 : dotProduct / magnitude;
+    }
+    /**
+     * Batch recall for multiple queries
+     */
+    async batchRecall(queries, requirements, accessLevel = 'internal') {
+        const results = [];
+        for (const query of queries) {
+            const result = await this.recall(query.queryId, query.queryText, query.k || 12, requirements, accessLevel);
+            results.push(result);
+        }
+        return results;
+    }
+    /**
+     * Get recall statistics
+     */
+    getStats() {
+        const causalEdges = this.db.prepare('SELECT COUNT(*) as count FROM causal_edges').get();
+        const certificates = this.db.prepare('SELECT COUNT(*) as count FROM recall_certificates').get();
+        const avgStats = this.db.prepare(`
+      SELECT
+        AVG(redundancy_ratio) as avg_redundancy,
+        AVG(completeness_score) as avg_completeness
+      FROM recall_certificates
+    `).get();
+        return {
+            totalCausalEdges: causalEdges.count,
+            totalCertificates: certificates.count,
+            avgRedundancyRatio: avgStats?.avg_redundancy || 0,
+            avgCompletenessScore: avgStats?.avg_completeness || 0
+        };
+    }
+    /**
+     * Update rerank configuration
+     */
+    updateConfig(config) {
+        this.config = { ...this.config, ...config };
+    }
+}