@aleph-ai/tinyaleph 1.5.7 → 1.6.0

package/examples/semantic/07-search.js

@@ -0,0 +1,304 @@
+/**
+ * @example Semantic Search
+ * @description Build a semantic search engine using TinyAleph
+ * 
+ * TinyAleph enables semantic search that finds results by meaning,
+ * not just keyword matching:
+ * - Index documents as hypercomplex embeddings
+ * - Query using natural language
+ * - Rank results by semantic similarity
+ */
+
+const { SemanticBackend, Hypercomplex } = require('../../modular');
+
+// ===========================================
+// SETUP
+// ===========================================
+
+const backend = new SemanticBackend({ dimension: 16 });
+
+console.log('TinyAleph Semantic Search Example');
+console.log('==================================\n');
+
+// ===========================================
+// SEARCH ENGINE CLASS
+// ===========================================
+
+class SemanticSearchEngine {
+    constructor(backend) {
+        this.backend = backend;
+        this.documents = [];
+        this.index = [];
+    }
+
+    // Add a document to the index
+    addDocument(id, title, content, metadata) {
+        var fullText = title + ' ' + content;
+        var embedding = this.backend.textToOrderedState(fullText);
+        
+        this.documents.push({
+            id: id,
+            title: title,
+            content: content,
+            metadata: metadata || {}
+        });
+        
+        this.index.push({
+            docId: id,
+            embedding: embedding
+        });
+    }
+
+    // Compute similarity between two states
+    similarity(state1, state2) {
+        var dot = 0, mag1 = 0, mag2 = 0;
+        for (var i = 0; i < state1.c.length; i++) {
+            dot += state1.c[i] * state2.c[i];
+            mag1 += state1.c[i] * state1.c[i];
+            mag2 += state2.c[i] * state2.c[i];
+        }
+        return dot / (Math.sqrt(mag1) * Math.sqrt(mag2) || 1);
+    }
+
+    // Search for documents matching query
+    search(query, topK) {
+        topK = topK || 5;
+        var queryEmbedding = this.backend.textToOrderedState(query);
+        
+        var self = this;
+        var results = this.index.map(function(item) {
+            return {
+                docId: item.docId,
+                score: self.similarity(queryEmbedding, item.embedding)
+            };
+        });
+        
+        results.sort(function(a, b) { return b.score - a.score; });
+        
+        var topResults = results.slice(0, topK);
+        
+        return topResults.map(function(result) {
+            var doc = self.documents.find(function(d) { return d.id === result.docId; });
+            return {
+                id: result.docId,
+                title: doc.title,
+                content: doc.content.substring(0, 100) + '...',
+                score: result.score,
+                metadata: doc.metadata
+            };
+        });
+    }
+
+    // Search with filters
+    searchWithFilters(query, filters, topK) {
+        topK = topK || 5;
+        var queryEmbedding = this.backend.textToOrderedState(query);
+        
+        var self = this;
+        
+        // Filter documents first
+        var filteredIndices = this.index.filter(function(item) {
+            var doc = self.documents.find(function(d) { return d.id === item.docId; });
+            
+            for (var key in filters) {
+                if (filters.hasOwnProperty(key)) {
+                    if (doc.metadata[key] !== filters[key]) {
+                        return false;
+                    }
+                }
+            }
+            return true;
+        });
+        
+        // Score filtered documents
+        var results = filteredIndices.map(function(item) {
+            return {
+                docId: item.docId,
+                score: self.similarity(queryEmbedding, item.embedding)
+            };
+        });
+        
+        results.sort(function(a, b) { return b.score - a.score; });
+        
+        return results.slice(0, topK).map(function(result) {
+            var doc = self.documents.find(function(d) { return d.id === result.docId; });
+            return {
+                id: result.docId,
+                title: doc.title,
+                score: result.score,
+                metadata: doc.metadata
+            };
+        });
+    }
+}
+
+// ===========================================
+// BUILD SAMPLE INDEX
+// ===========================================
+
+console.log('Building Document Index:');
+console.log('-'.repeat(50) + '\n');
+
+var engine = new SemanticSearchEngine(backend);
+
+// Add sample documents
+var sampleDocs = [
+    {
+        id: 'doc1',
+        title: 'Introduction to Machine Learning',
+        content: 'Machine learning is a subset of artificial intelligence that enables computers to learn from data without explicit programming. It includes supervised learning, unsupervised learning, and reinforcement learning.',
+        metadata: { category: 'tech', year: 2023 }
+    },
+    {
+        id: 'doc2',
+        title: 'Deep Learning Neural Networks',
+        content: 'Deep learning uses neural networks with multiple layers to process complex patterns. Convolutional neural networks are used for image recognition, while recurrent networks handle sequential data.',
+        metadata: { category: 'tech', year: 2023 }
+    },
+    {
+        id: 'doc3',
+        title: 'Natural Language Processing',
+        content: 'NLP enables computers to understand human language. Techniques include tokenization, parsing, sentiment analysis, and machine translation. Modern NLP uses transformer architectures.',
+        metadata: { category: 'tech', year: 2022 }
+    },
+    {
+        id: 'doc4',
+        title: 'Healthy Mediterranean Diet',
+        content: 'The Mediterranean diet emphasizes fruits, vegetables, whole grains, and olive oil. It has been linked to reduced heart disease, improved cognitive function, and longevity.',
+        metadata: { category: 'health', year: 2023 }
+    },
+    {
+        id: 'doc5',
+        title: 'Exercise and Mental Health',
+        content: 'Regular physical exercise has been shown to reduce symptoms of depression and anxiety. It releases endorphins and promotes neuroplasticity in the brain.',
+        metadata: { category: 'health', year: 2022 }
+    },
+    {
+        id: 'doc6',
+        title: 'Quantum Computing Fundamentals',
+        content: 'Quantum computers use qubits that can exist in superposition. This allows them to solve certain problems exponentially faster than classical computers.',
+        metadata: { category: 'tech', year: 2023 }
+    },
+    {
+        id: 'doc7',
+        title: 'Climate Change Solutions',
+        content: 'Addressing climate change requires reducing carbon emissions, transitioning to renewable energy, and implementing carbon capture technologies. International cooperation is essential.',
+        metadata: { category: 'environment', year: 2023 }
+    },
+    {
+        id: 'doc8',
+        title: 'Renewable Energy Sources',
+        content: 'Solar, wind, and hydroelectric power are major renewable energy sources. They produce no direct carbon emissions and are becoming increasingly cost-competitive with fossil fuels.',
+        metadata: { category: 'environment', year: 2022 }
+    }
+];
+
+for (var i = 0; i < sampleDocs.length; i++) {
+    var doc = sampleDocs[i];
+    engine.addDocument(doc.id, doc.title, doc.content, doc.metadata);
+    console.log('  Indexed: ' + doc.title);
+}
+
+console.log('\n  Total documents: ' + engine.documents.length);
+
+// ===========================================
+// BASIC SEARCH
+// ===========================================
+
+console.log('\n' + '='.repeat(50));
+console.log('Basic Semantic Search:');
+console.log('='.repeat(50) + '\n');
+
+var queries = [
+    'how do computers learn from data',
+    'staying healthy with diet and exercise',
+    'clean energy and sustainability',
+    'understanding AI and neural networks'
+];
+
+for (var q = 0; q < queries.length; q++) {
+    var query = queries[q];
+    console.log('Query: "' + query + '"');
+    var results = engine.search(query, 3);
+    
+    for (var r = 0; r < results.length; r++) {
+        var result = results[r];
+        console.log('  ' + (r + 1) + '. ' + result.title + ' (' + (result.score * 100).toFixed(1) + '%)');
+    }
+    console.log('');
+}
+
+// ===========================================
+// FILTERED SEARCH
+// ===========================================
+
+console.log('='.repeat(50));
+console.log('Filtered Search (by category):');
+console.log('='.repeat(50) + '\n');
+
+var filteredQueries = [
+    { query: 'learning algorithms', filter: { category: 'tech' } },
+    { query: 'wellness and longevity', filter: { category: 'health' } },
+    { query: 'reducing emissions', filter: { category: 'environment' } }
+];
+
+for (var fq = 0; fq < filteredQueries.length; fq++) {
+    var item = filteredQueries[fq];
+    console.log('Query: "' + item.query + '" [category=' + item.filter.category + ']');
+    var results = engine.searchWithFilters(item.query, item.filter, 3);
+    
+    for (var r = 0; r < results.length; r++) {
+        var result = results[r];
+        console.log('  ' + (r + 1) + '. ' + result.title + ' (' + (result.score * 100).toFixed(1) + '%)');
+    }
+    console.log('');
+}
+
+// ===========================================
+// SEMANTIC VS KEYWORD
+// ===========================================
+
+console.log('='.repeat(50));
+console.log('Semantic vs Keyword Search:');
+console.log('='.repeat(50) + '\n');
+
+// Queries that would fail keyword search but succeed semantically
+var semanticQueries = [
+    {
+        query: 'teaching computers without programming them',
+        expected: 'Machine Learning'
+    },
+    {
+        query: 'brain patterns and mood improvement',
+        expected: 'Exercise and Mental Health'
+    },
+    {
+        query: 'alternative power generation',
+        expected: 'Renewable Energy'
+    }
+];
+
+console.log('Queries that work semantically but might fail keyword matching:\n');
+
+for (var sq = 0; sq < semanticQueries.length; sq++) {
+    var item = semanticQueries[sq];
+    var results = engine.search(item.query, 1);
+    var topResult = results[0];
+    
+    console.log('Query: "' + item.query + '"');
+    console.log('  Expected: ' + item.expected);
+    console.log('  Found: ' + topResult.title + ' (' + (topResult.score * 100).toFixed(1) + '%)');
+    console.log('');
+}
+
+// ===========================================
+// KEY TAKEAWAYS
+// ===========================================
+
+console.log('='.repeat(50));
+console.log('KEY TAKEAWAYS:');
+console.log('1. Semantic search finds results by meaning, not keywords');
+console.log('2. Documents are indexed as hypercomplex embeddings');
+console.log('3. Queries are embedded and compared to document embeddings');
+console.log('4. Filters can narrow results before ranking');
+console.log('5. Works even when query words differ from document words');

package/examples/semantic/08-qa-system.js

@@ -0,0 +1,278 @@
+/**
+ * @example Question-Answering System
+ * @description Build a QA system using semantic embeddings
+ * 
+ * TinyAleph can power question-answering systems:
+ * - Embed knowledge base passages
+ * - Match questions to relevant passages
+ * - Extract and rank answers
+ */
+
+const { SemanticBackend, Hypercomplex } = require('../../modular');
+
+// ===========================================
+// SETUP
+// ===========================================
+
+const backend = new SemanticBackend({ dimension: 16 });
+
+console.log('TinyAleph Question-Answering System Example');
+console.log('============================================\n');
+
+// ===========================================
+// KNOWLEDGE BASE
+// ===========================================
+
+var knowledgeBase = [
+    {
+        id: 'kb1',
+        topic: 'Solar System',
+        passage: 'The Solar System consists of the Sun and the objects that orbit it, including eight planets, dwarf planets, moons, asteroids, and comets. The four inner planets are Mercury, Venus, Earth, and Mars. The four outer planets are Jupiter, Saturn, Uranus, and Neptune.'
+    },
+    {
+        id: 'kb2',
+        topic: 'Earth',
+        passage: 'Earth is the third planet from the Sun and the only astronomical object known to harbor life. It has one natural satellite called the Moon. Earth is about 4.5 billion years old and has a diameter of approximately 12,742 kilometers.'
+    },
+    {
+        id: 'kb3',
+        topic: 'Moon',
+        passage: 'The Moon is Earth\'s only natural satellite. It is the fifth-largest satellite in the Solar System. The Moon orbits Earth at an average distance of 384,400 kilometers. It takes about 27.3 days to complete one orbit around Earth.'
+    },
+    {
+        id: 'kb4',
+        topic: 'Photosynthesis',
+        passage: 'Photosynthesis is the process by which plants and other organisms convert sunlight, water, and carbon dioxide into glucose and oxygen. Chlorophyll, the green pigment in plants, absorbs light energy for this process.'
+    },
+    {
+        id: 'kb5',
+        topic: 'Water Cycle',
+        passage: 'The water cycle describes how water evaporates from oceans and land, rises into the atmosphere, condenses into clouds, and falls back as precipitation. This cycle continuously moves water around the Earth.'
+    },
+    {
+        id: 'kb6',
+        topic: 'DNA',
+        passage: 'DNA (deoxyribonucleic acid) is the molecule that carries genetic information in all living organisms. It consists of two strands that form a double helix structure. DNA is made up of four nucleotide bases: adenine, thymine, guanine, and cytosine.'
+    },
+    {
+        id: 'kb7',
+        topic: 'Gravity',
+        passage: 'Gravity is a fundamental force that attracts any two objects with mass. On Earth, gravity gives objects weight and causes them to fall toward the ground. The gravitational force between objects depends on their masses and the distance between them.'
+    },
+    {
+        id: 'kb8',
+        topic: 'Evolution',
+        passage: 'Evolution is the process by which species change over time through natural selection. Organisms with traits better suited to their environment are more likely to survive and reproduce, passing those traits to offspring.'
+    }
+];
+
+// ===========================================
+// QA SYSTEM CLASS
+// ===========================================
+
+function similarity(state1, state2) {
+    var dot = 0, mag1 = 0, mag2 = 0;
+    for (var i = 0; i < state1.c.length; i++) {
+        dot += state1.c[i] * state2.c[i];
+        mag1 += state1.c[i] * state1.c[i];
+        mag2 += state2.c[i] * state2.c[i];
+    }
+    return dot / (Math.sqrt(mag1) * Math.sqrt(mag2) || 1);
+}
+
+function QASystem(backend) {
+    this.backend = backend;
+    this.passages = [];
+    this.embeddings = [];
+}
+
+QASystem.prototype.loadKnowledge = function(passages) {
+    var self = this;
+    this.passages = passages;
+    this.embeddings = passages.map(function(p) {
+        return {
+            id: p.id,
+            embedding: self.backend.textToOrderedState(p.passage)
+        };
+    });
+    console.log('Loaded ' + passages.length + ' knowledge passages');
+};
+
+QASystem.prototype.findRelevantPassages = function(question, topK) {
+    topK = topK || 3;
+    var questionEmb = this.backend.textToOrderedState(question);
+    
+    var self = this;
+    var scored = this.embeddings.map(function(e) {
+        return {
+            id: e.id,
+            score: similarity(questionEmb, e.embedding)
+        };
+    });
+    
+    scored.sort(function(a, b) { return b.score - a.score; });
+    
+    return scored.slice(0, topK).map(function(item) {
+        var passage = self.passages.find(function(p) { return p.id === item.id; });
+        return {
+            passage: passage,
+            score: item.score
+        };
+    });
+};
+
+QASystem.prototype.answer = function(question) {
+    var relevantPassages = this.findRelevantPassages(question, 2);
+    
+    if (relevantPassages.length === 0 || relevantPassages[0].score < 0.2) {
+        return {
+            answer: 'I don\'t have enough information to answer that question.',
+            confidence: 0,
+            sources: []
+        };
+    }
+    
+    // In a real system, you'd use NLP to extract the answer
+    // Here we return the most relevant passage as the answer
+    var topPassage = relevantPassages[0];
+    
+    return {
+        answer: topPassage.passage.passage,
+        confidence: topPassage.score,
+        topic: topPassage.passage.topic,
+        sources: relevantPassages.map(function(r) {
+            return { topic: r.passage.topic, score: r.score };
+        })
+    };
+};
+
+// ===========================================
+// BUILD QA SYSTEM
+// ===========================================
+
+console.log('Building QA System:');
+console.log('-'.repeat(50) + '\n');
+
+var qaSystem = new QASystem(backend);
+qaSystem.loadKnowledge(knowledgeBase);
+
+// ===========================================
+// TEST QUESTIONS
+// ===========================================
+
+console.log('\n' + '='.repeat(50));
+console.log('Testing Questions:');
+console.log('='.repeat(50) + '\n');
+
+var questions = [
+    'How many planets are in our solar system?',
+    'What is the Moon and how far is it from Earth?',
+    'How do plants make their food?',
+    'What is DNA made of?',
+    'Why do things fall down?',
+    'How do species change over time?',
+    'What is the water cycle?'
+];
+
+for (var q = 0; q < questions.length; q++) {
+    var question = questions[q];
+    console.log('Q: ' + question);
+    
+    var result = qaSystem.answer(question);
+    
+    console.log('Topic: ' + result.topic);
+    console.log('Confidence: ' + (result.confidence * 100).toFixed(1) + '%');
+    console.log('Answer: ' + result.answer.substring(0, 120) + '...');
+    console.log('');
+}
+
+// ===========================================
+// MULTI-HOP QUESTIONS
+// ===========================================
+
+console.log('='.repeat(50));
+console.log('Multi-Hop Question Answering:');
+console.log('='.repeat(50) + '\n');
+
+console.log('For complex questions, combine multiple passages:\n');
+
+var complexQuestion = 'What orbits Earth and how does gravity affect it?';
+console.log('Q: ' + complexQuestion + '\n');
+
+var passages = qaSystem.findRelevantPassages(complexQuestion, 3);
+console.log('Relevant passages found:');
+for (var p = 0; p < passages.length; p++) {
+    var passage = passages[p];
+    console.log('  ' + (p + 1) + '. ' + passage.passage.topic + ' (' + (passage.score * 100).toFixed(1) + '%)');
+}
+
+console.log('\nCombined answer would draw from:');
+console.log('  - Moon passage (for what orbits Earth)');
+console.log('  - Gravity passage (for gravitational effects)');
+
+// ===========================================
+// QUESTION TYPES
+// ===========================================
+
+console.log('\n' + '='.repeat(50));
+console.log('Handling Different Question Types:');
+console.log('='.repeat(50) + '\n');
+
+var questionTypes = [
+    { type: 'What', question: 'What is photosynthesis?' },
+    { type: 'How', question: 'How does water cycle work?' },
+    { type: 'Why', question: 'Why do organisms evolve?' },
+    { type: 'When', question: 'When did Earth form?' },
+    { type: 'How many', question: 'How many bases are in DNA?' }
+];
+
+for (var qt = 0; qt < questionTypes.length; qt++) {
+    var item = questionTypes[qt];
+    var result = qaSystem.answer(item.question);
+    
+    console.log('[' + item.type + '] ' + item.question);
+    console.log('  Found: ' + result.topic + ' (' + (result.confidence * 100).toFixed(1) + '%)');
+    console.log('');
+}
+
+// ===========================================
+// CONFIDENCE THRESHOLDING
+// ===========================================
+
+console.log('='.repeat(50));
+console.log('Confidence Thresholding:');
+console.log('='.repeat(50) + '\n');
+
+var outOfDomainQuestions = [
+    'What is the capital of France?',
+    'How do I bake a cake?',
+    'What is the stock price today?'
+];
+
+console.log('Questions outside the knowledge base:\n');
+
+for (var odq = 0; odq < outOfDomainQuestions.length; odq++) {
+    var question = outOfDomainQuestions[odq];
+    var result = qaSystem.answer(question);
+    
+    console.log('Q: ' + question);
+    console.log('  Best match: ' + (result.topic || 'None') + ' (' + (result.confidence * 100).toFixed(1) + '%)');
+    
+    if (result.confidence < 0.3) {
+        console.log('  Status: LOW CONFIDENCE - may not be answerable');
+    }
+    console.log('');
+}
+
+// ===========================================
+// KEY TAKEAWAYS
+// ===========================================
+
+console.log('='.repeat(50));
+console.log('KEY TAKEAWAYS:');
+console.log('1. Embed knowledge passages for semantic retrieval');
+console.log('2. Questions are matched to passages by similarity');
+console.log('3. Confidence scores indicate answer reliability');
+console.log('4. Multi-hop questions need multiple passages');
+console.log('5. Low confidence indicates out-of-domain queries');
+console.log('6. Combine with NLP for answer extraction');

package/examples/semantic/README.md

@@ -0,0 +1,116 @@
+# Semantic Examples
+
+This directory contains examples demonstrating TinyAleph's semantic processing capabilities.
+
+## Examples
+
+### 01-vocabulary.js
+**Word-to-Prime Mapping** - Explore how words map to prime numbers in TinyAleph's vocabulary space.
+
+```bash
+node examples/semantic/01-vocabulary.js
+```
+
+### 02-similarity.js
+**Semantic Similarity** - Compute similarity between texts using hypercomplex embeddings with various metrics.
+
+```bash
+node examples/semantic/02-similarity.js
+```
+
+### 03-word-algebra.js
+**Word Algebra** - Perform algebraic operations on word embeddings (king - man + woman = queen).
+
+```bash
+node examples/semantic/03-word-algebra.js
+```
+
+### 04-clustering.js
+**Text Clustering** - Group similar texts using K-means and hierarchical clustering algorithms.
+
+```bash
+node examples/semantic/04-clustering.js
+```
+
+### 05-classification.js
+**Text Classification** - Classify texts into categories using prototype-based and KNN classifiers.
+
+```bash
+node examples/semantic/05-classification.js
+```
+
+### 06-dna-encoding.js
+**DNA-Inspired Encoding** - Explore biological-inspired text processing with bidirectional encoding, codons, and reading frames.
+
+```bash
+node examples/semantic/06-dna-encoding.js
+```
+
+### 07-search.js
+**Semantic Search** - Build a semantic search engine that finds results by meaning, not just keywords.
+
+```bash
+node examples/semantic/07-search.js
+```
+
+### 08-qa-system.js
+**Question-Answering System** - Build a QA system using semantic embeddings to match questions to knowledge.
+
+```bash
+node examples/semantic/08-qa-system.js
+```
+
+## Key Concepts
+
+### Prime Encoding
+Every word token maps to a unique prime number, creating a mathematical vocabulary space where:
+- Each word has a unique prime identity
+- Word order affects encoding through prime powers
+- Semantic relationships emerge from mathematical structure
+
+### Hypercomplex Embeddings
+Text is embedded into hypercomplex (high-dimensional) vectors where:
+- Similar texts have similar embeddings
+- Vector operations (add, subtract, multiply) are meaningful
+- Cosine similarity measures semantic relatedness
+
+### DNA-Inspired Processing
+Biological concepts enhance text processing:
+- **Bidirectional**: Alternating reading direction
+- **Codons**: Triplet grouping of tokens
+- **Reading Frames**: 6 different starting positions
+- **Sense/Antisense**: Forward and reversed representations
+
+## Usage Pattern
+
+```javascript
+const { SemanticBackend } = require('../../modular');
+
+// Create backend
+const backend = new SemanticBackend({ dimension: 16 });
+
+// Get text embedding
+const state = backend.textToOrderedState('your text here');
+
+// Access components
+console.log(state.c[0]);    // First component
+console.log(state.norm());  // Magnitude
+
+// Compute similarity
+function similarity(s1, s2) {
+    let dot = 0, m1 = 0, m2 = 0;
+    for (let i = 0; i < s1.c.length; i++) {
+        dot += s1.c[i] * s2.c[i];
+        m1 += s1.c[i] * s1.c[i];
+        m2 += s2.c[i] * s2.c[i];
+    }
+    return dot / (Math.sqrt(m1) * Math.sqrt(m2) || 1);
+}
+```
+
+## Next Steps
+
+After understanding semantic processing, explore:
+- `../crypto/` - Cryptographic hashing and key derivation
+- `../scientific/` - Quantum computing simulation
+- `../ai/` - AI and machine learning applications