agentic-flow 1.7.3 → 1.7.4

package/dist/agentdb/controllers/EmbeddingService.js

@@ -0,0 +1,118 @@
+/**
+ * EmbeddingService - Text Embedding Generation
+ *
+ * Handles text-to-vector embedding generation using various models.
+ * Supports both local (transformers.js) and remote (OpenAI, etc.) embeddings.
+ */
+export class EmbeddingService {
+    config;
+    pipeline; // transformers.js pipeline
+    cache;
+    constructor(config) {
+        this.config = config;
+        this.cache = new Map();
+    }
+    /**
+     * Initialize the embedding service
+     */
+    async initialize() {
+        if (this.config.provider === 'transformers') {
+            // Use transformers.js for local embeddings
+            try {
+                const { pipeline } = await import('@xenova/transformers');
+                this.pipeline = await pipeline('feature-extraction', this.config.model);
+            }
+            catch (error) {
+                console.warn('Transformers.js not available, falling back to mock embeddings');
+                this.pipeline = null;
+            }
+        }
+    }
+    /**
+     * Generate embedding for text
+     */
+    async embed(text) {
+        // Check cache
+        const cacheKey = `${this.config.model}:${text}`;
+        if (this.cache.has(cacheKey)) {
+            return this.cache.get(cacheKey);
+        }
+        let embedding;
+        if (this.config.provider === 'transformers' && this.pipeline) {
+            // Use transformers.js
+            const output = await this.pipeline(text, { pooling: 'mean', normalize: true });
+            embedding = new Float32Array(output.data);
+        }
+        else if (this.config.provider === 'openai' && this.config.apiKey) {
+            // Use OpenAI API
+            embedding = await this.embedOpenAI(text);
+        }
+        else {
+            // Mock embedding for testing
+            embedding = this.mockEmbedding(text);
+        }
+        // Cache result
+        if (this.cache.size > 10000) {
+            // Simple LRU: clear half the cache
+            const keysToDelete = Array.from(this.cache.keys()).slice(0, 5000);
+            keysToDelete.forEach(k => this.cache.delete(k));
+        }
+        this.cache.set(cacheKey, embedding);
+        return embedding;
+    }
+    /**
+     * Batch embed multiple texts
+     */
+    async embedBatch(texts) {
+        return Promise.all(texts.map(text => this.embed(text)));
+    }
+    /**
+     * Clear embedding cache
+     */
+    clearCache() {
+        this.cache.clear();
+    }
+    // ========================================================================
+    // Private Methods
+    // ========================================================================
+    async embedOpenAI(text) {
+        const response = await fetch('https://api.openai.com/v1/embeddings', {
+            method: 'POST',
+            headers: {
+                'Authorization': `Bearer ${this.config.apiKey}`,
+                'Content-Type': 'application/json'
+            },
+            body: JSON.stringify({
+                model: this.config.model,
+                input: text
+            })
+        });
+        const data = await response.json();
+        return new Float32Array(data.data[0].embedding);
+    }
+    mockEmbedding(text) {
+        // Simple deterministic mock embedding for testing
+        const embedding = new Float32Array(this.config.dimension);
+        // Use simple hash-based generation
+        let hash = 0;
+        for (let i = 0; i < text.length; i++) {
+            hash = ((hash << 5) - hash) + text.charCodeAt(i);
+            hash = hash & hash; // Convert to 32bit integer
+        }
+        // Fill embedding with pseudo-random values based on hash
+        for (let i = 0; i < this.config.dimension; i++) {
+            const seed = hash + i * 31;
+            embedding[i] = Math.sin(seed) * Math.cos(seed * 0.5);
+        }
+        // Normalize
+        let norm = 0;
+        for (let i = 0; i < embedding.length; i++) {
+            norm += embedding[i] * embedding[i];
+        }
+        norm = Math.sqrt(norm);
+        for (let i = 0; i < embedding.length; i++) {
+            embedding[i] /= norm;
+        }
+        return embedding;
+    }
+}

package/dist/agentdb/controllers/ExplainableRecall.js

@@ -0,0 +1,387 @@
+/**
+ * ExplainableRecall - Provenance and Justification for Memory Retrieval
+ *
+ * Every retrieval returns:
+ * - Minimal hitting set of facts that justify the answer
+ * - Merkle proof chain for provenance
+ * - Policy compliance certificates
+ *
+ * Based on:
+ * - Minimal hitting set algorithms
+ * - Merkle tree provenance
+ * - Explainable AI techniques
+ */
+import * as crypto from 'crypto';
+export class ExplainableRecall {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    /**
+     * Create a recall certificate for a retrieval operation
+     */
+    createCertificate(params) {
+        const { queryId, queryText, chunks, requirements, accessLevel = 'internal' } = params;
+        const startTime = Date.now();
+        // 1. Compute minimal hitting set
+        const minimalWhy = this.computeMinimalHittingSet(chunks, requirements);
+        // 2. Calculate metrics
+        const redundancyRatio = chunks.length / minimalWhy.length;
+        const completenessScore = this.calculateCompleteness(minimalWhy, requirements);
+        // 3. Build provenance chain
+        const sourceHashes = chunks.map(chunk => this.getOrCreateProvenance(chunk.type, parseInt(chunk.id)));
+        const merkleTree = this.buildMerkleTree(sourceHashes);
+        const merkleRoot = merkleTree.root;
+        // 4. Generate chunk metadata first (needed for certificate ID)
+        const chunkIds = chunks.map(c => c.id);
+        const chunkTypes = chunks.map(c => c.type);
+        // 5. Create certificate ID
+        const certificateId = this.generateCertificateId(queryId, chunkIds);
+        // 6. Generate proof chain for each chunk
+        const proofChain = chunks.map((chunk, idx) => this.getMerkleProof(merkleTree, idx)).flat();
+        // 7. Store certificate
+        this.db.prepare(`
+      INSERT INTO recall_certificates (
+        id, query_id, query_text, chunk_ids, chunk_types,
+        minimal_why, redundancy_ratio, completeness_score,
+        merkle_root, source_hashes, proof_chain,
+        access_level, latency_ms
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `).run(certificateId, queryId, queryText, JSON.stringify(chunkIds), JSON.stringify(chunkTypes), JSON.stringify(minimalWhy), redundancyRatio, completenessScore, merkleRoot, JSON.stringify(sourceHashes), JSON.stringify(proofChain), accessLevel, Date.now() - startTime);
+        // 7. Store justification paths
+        this.storeJustificationPaths(certificateId, chunks, minimalWhy, requirements);
+        const certificate = {
+            id: certificateId,
+            queryId,
+            queryText,
+            chunkIds,
+            chunkTypes,
+            minimalWhy,
+            redundancyRatio,
+            completenessScore,
+            merkleRoot,
+            sourceHashes,
+            proofChain,
+            accessLevel: accessLevel,
+            latencyMs: Date.now() - startTime
+        };
+        return certificate;
+    }
+    /**
+     * Verify a recall certificate
+     */
+    verifyCertificate(certificateId) {
+        const cert = this.db.prepare('SELECT * FROM recall_certificates WHERE id = ?').get(certificateId);
+        if (!cert) {
+            return { valid: false, issues: ['Certificate not found'] };
+        }
+        const issues = [];
+        // 1. Verify Merkle root
+        const sourceHashes = JSON.parse(cert.source_hashes);
+        const merkleTree = this.buildMerkleTree(sourceHashes);
+        if (merkleTree.root !== cert.merkle_root) {
+            issues.push('Merkle root mismatch');
+        }
+        // 2. Verify chunk hashes still match
+        const chunkIds = JSON.parse(cert.chunk_ids);
+        const chunkTypes = JSON.parse(cert.chunk_types);
+        for (let i = 0; i < chunkIds.length; i++) {
+            const currentHash = this.getContentHash(chunkTypes[i], parseInt(chunkIds[i]));
+            if (currentHash !== sourceHashes[i]) {
+                issues.push(`Chunk ${chunkIds[i]} hash changed`);
+            }
+        }
+        // 3. Verify completeness
+        const minimalWhy = JSON.parse(cert.minimal_why);
+        if (minimalWhy.length === 0) {
+            issues.push('Empty justification set');
+        }
+        // 4. Verify redundancy ratio
+        if (cert.redundancy_ratio < 1.0) {
+            issues.push('Invalid redundancy ratio');
+        }
+        return {
+            valid: issues.length === 0,
+            issues
+        };
+    }
+    /**
+     * Get justification for why a chunk was included
+     */
+    getJustification(certificateId, chunkId) {
+        const row = this.db.prepare(`
+      SELECT * FROM justification_paths
+      WHERE certificate_id = ? AND chunk_id = ?
+    `).get(certificateId, chunkId);
+        if (!row)
+            return null;
+        return {
+            chunkId: row.chunk_id,
+            chunkType: row.chunk_type,
+            reason: row.reason,
+            necessityScore: row.necessity_score,
+            pathElements: JSON.parse(row.path_elements)
+        };
+    }
+    /**
+     * Get provenance lineage for a source
+     */
+    getProvenanceLineage(contentHash) {
+        const lineage = [];
+        let currentHash = contentHash;
+        while (currentHash) {
+            const source = this.db.prepare(`
+        SELECT * FROM provenance_sources WHERE content_hash = ?
+      `).get(currentHash);
+            if (!source)
+                break;
+            lineage.push({
+                id: source.id,
+                sourceType: source.source_type,
+                sourceId: source.source_id,
+                contentHash: source.content_hash,
+                parentHash: source.parent_hash,
+                derivedFrom: source.derived_from ? JSON.parse(source.derived_from) : undefined,
+                creator: source.creator,
+                metadata: source.metadata ? JSON.parse(source.metadata) : undefined
+            });
+            currentHash = source.parent_hash;
+        }
+        return lineage;
+    }
+    /**
+     * Audit certificate access
+     */
+    auditCertificate(certificateId) {
+        const certRow = this.db.prepare('SELECT * FROM recall_certificates WHERE id = ?').get(certificateId);
+        if (!certRow) {
+            throw new Error(`Certificate ${certificateId} not found`);
+        }
+        const certificate = {
+            id: certRow.id,
+            queryId: certRow.query_id,
+            queryText: certRow.query_text,
+            chunkIds: JSON.parse(certRow.chunk_ids),
+            chunkTypes: JSON.parse(certRow.chunk_types),
+            minimalWhy: JSON.parse(certRow.minimal_why),
+            redundancyRatio: certRow.redundancy_ratio,
+            completenessScore: certRow.completeness_score,
+            merkleRoot: certRow.merkle_root,
+            sourceHashes: JSON.parse(certRow.source_hashes),
+            proofChain: JSON.parse(certRow.proof_chain),
+            policyProof: certRow.policy_proof,
+            policyVersion: certRow.policy_version,
+            accessLevel: certRow.access_level,
+            latencyMs: certRow.latency_ms
+        };
+        // Get justifications
+        const justRows = this.db.prepare(`
+      SELECT * FROM justification_paths WHERE certificate_id = ?
+    `).all(certificateId);
+        const justifications = justRows.map(row => ({
+            chunkId: row.chunk_id,
+            chunkType: row.chunk_type,
+            reason: row.reason,
+            necessityScore: row.necessity_score,
+            pathElements: JSON.parse(row.path_elements)
+        }));
+        // Get provenance for each source
+        const provenance = new Map();
+        for (const hash of certificate.sourceHashes) {
+            provenance.set(hash, this.getProvenanceLineage(hash));
+        }
+        // Calculate quality metrics
+        const avgNecessity = justifications.reduce((sum, j) => sum + j.necessityScore, 0) / justifications.length;
+        return {
+            certificate,
+            justifications,
+            provenance,
+            quality: {
+                completeness: certificate.completenessScore,
+                redundancy: certificate.redundancyRatio,
+                avgNecessity
+            }
+        };
+    }
+    // ========================================================================
+    // Private Helper Methods
+    // ========================================================================
+    /**
+     * Compute minimal hitting set using greedy algorithm
+     * A hitting set contains at least one element from each requirement
+     */
+    computeMinimalHittingSet(chunks, requirements) {
+        if (requirements.length === 0) {
+            return chunks.slice(0, Math.min(3, chunks.length)).map(c => c.id);
+        }
+        const uncovered = new Set(requirements);
+        const selected = [];
+        // Greedy: select chunk that covers most uncovered requirements
+        while (uncovered.size > 0 && chunks.length > 0) {
+            let bestChunk = null;
+            let bestCoverage = 0;
+            for (const chunk of chunks) {
+                const coverage = Array.from(uncovered).filter(req => chunk.content.toLowerCase().includes(req.toLowerCase())).length;
+                if (coverage > bestCoverage) {
+                    bestCoverage = coverage;
+                    bestChunk = chunk;
+                }
+            }
+            if (!bestChunk)
+                break;
+            selected.push(bestChunk.id);
+            // Remove covered requirements
+            for (const req of Array.from(uncovered)) {
+                if (bestChunk.content.toLowerCase().includes(req.toLowerCase())) {
+                    uncovered.delete(req);
+                }
+            }
+            // Remove selected chunk
+            chunks = chunks.filter(c => c.id !== bestChunk.id);
+        }
+        return selected;
+    }
+    /**
+     * Calculate completeness score
+     */
+    calculateCompleteness(minimalWhy, requirements) {
+        if (requirements.length === 0)
+            return 1.0;
+        const chunks = minimalWhy.map(id => {
+            // Get chunk content
+            const episode = this.db.prepare('SELECT output FROM episodes WHERE id = ?').get(parseInt(id));
+            return episode ? episode.output : '';
+        });
+        const satisfied = requirements.filter(req => chunks.some(content => content && content.toLowerCase().includes(req.toLowerCase())));
+        return satisfied.length / requirements.length;
+    }
+    /**
+     * Get or create provenance record
+     */
+    getOrCreateProvenance(sourceType, sourceId) {
+        // Check if provenance exists
+        const existing = this.db.prepare(`
+      SELECT content_hash FROM provenance_sources
+      WHERE source_type = ? AND source_id = ?
+    `).get(sourceType, sourceId);
+        if (existing) {
+            return existing.content_hash;
+        }
+        // Create new provenance
+        const contentHash = this.getContentHash(sourceType, sourceId);
+        this.db.prepare(`
+      INSERT INTO provenance_sources (source_type, source_id, content_hash, creator)
+      VALUES (?, ?, ?, ?)
+    `).run(sourceType, sourceId, contentHash, 'system');
+        return contentHash;
+    }
+    /**
+     * Get content hash for a memory
+     */
+    getContentHash(sourceType, sourceId) {
+        let content = '';
+        switch (sourceType) {
+            case 'episode':
+                const episode = this.db.prepare('SELECT task, output FROM episodes WHERE id = ?').get(sourceId);
+                content = episode ? `${episode.task}:${episode.output}` : '';
+                break;
+            case 'skill':
+                const skill = this.db.prepare('SELECT name, code FROM skills WHERE id = ?').get(sourceId);
+                content = skill ? `${skill.name}:${skill.code}` : '';
+                break;
+            case 'note':
+                const note = this.db.prepare('SELECT text FROM notes WHERE id = ?').get(sourceId);
+                content = note ? note.text : '';
+                break;
+            case 'fact':
+                const fact = this.db.prepare('SELECT subject, predicate, object FROM facts WHERE id = ?').get(sourceId);
+                content = fact ? `${fact.subject}:${fact.predicate}:${fact.object}` : '';
+                break;
+        }
+        return crypto.createHash('sha256').update(content).digest('hex');
+    }
+    /**
+     * Build Merkle tree from hashes
+     */
+    buildMerkleTree(hashes) {
+        if (hashes.length === 0) {
+            return { root: '', tree: [[]] };
+        }
+        const tree = [hashes];
+        while (tree[tree.length - 1].length > 1) {
+            const level = tree[tree.length - 1];
+            const nextLevel = [];
+            for (let i = 0; i < level.length; i += 2) {
+                if (i + 1 < level.length) {
+                    const combined = level[i] + level[i + 1];
+                    nextLevel.push(crypto.createHash('sha256').update(combined).digest('hex'));
+                }
+                else {
+                    nextLevel.push(level[i]);
+                }
+            }
+            tree.push(nextLevel);
+        }
+        return { root: tree[tree.length - 1][0], tree };
+    }
+    /**
+     * Get Merkle proof for a leaf
+     */
+    getMerkleProof(merkleTree, leafIndex) {
+        const proof = [];
+        let index = leafIndex;
+        for (let level = 0; level < merkleTree.tree.length - 1; level++) {
+            const currentLevel = merkleTree.tree[level];
+            const isLeftNode = index % 2 === 0;
+            const siblingIndex = isLeftNode ? index + 1 : index - 1;
+            if (siblingIndex < currentLevel.length) {
+                proof.push({
+                    hash: currentLevel[siblingIndex],
+                    position: isLeftNode ? 'right' : 'left'
+                });
+            }
+            index = Math.floor(index / 2);
+        }
+        return proof;
+    }
+    /**
+     * Generate certificate ID
+     */
+    generateCertificateId(queryId, chunkIds) {
+        const data = `${queryId}:${chunkIds.join(',')}:${Date.now()}`;
+        return crypto.createHash('sha256').update(data).digest('hex');
+    }
+    /**
+     * Store justification paths
+     */
+    storeJustificationPaths(certificateId, chunks, minimalWhy, requirements) {
+        const stmt = this.db.prepare(`
+      INSERT INTO justification_paths (
+        certificate_id, chunk_id, chunk_type, reason, necessity_score, path_elements
+      ) VALUES (?, ?, ?, ?, ?, ?)
+    `);
+        for (const chunk of chunks) {
+            const isNecessary = minimalWhy.includes(chunk.id);
+            const reason = this.determineReason(chunk, requirements);
+            const necessityScore = isNecessary ? chunk.relevance : chunk.relevance * 0.5;
+            const pathElements = [
+                `Retrieved for query`,
+                isNecessary ? `Essential for justification` : `Supporting evidence`,
+                `Relevance: ${(chunk.relevance * 100).toFixed(1)}%`
+            ];
+            stmt.run(certificateId, chunk.id, chunk.type, reason, necessityScore, JSON.stringify(pathElements));
+        }
+    }
+    /**
+     * Determine reason for inclusion
+     */
+    determineReason(chunk, requirements) {
+        if (chunk.relevance > 0.9)
+            return 'semantic_match';
+        if (chunk.relevance > 0.7)
+            return 'causal_link';
+        if (chunk.relevance > 0.5)
+            return 'prerequisite';
+        return 'constraint';
+    }
+}