@unrdf/knowledge-engine 5.0.1 → 26.4.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@unrdf/knowledge-engine",
-  "version": "5.0.1",
+  "version": "26.4.2",
   "description": "UNRDF Knowledge Engine - Rule Engine, Inference, and Pattern Matching (Optional Extension)",
   "type": "module",
   "main": "src/index.mjs",
@@ -8,7 +8,8 @@
     ".": "./src/index.mjs",
     "./query": "./src/query.mjs",
     "./canonicalize": "./src/canonicalize.mjs",
-    "./parse": "./src/parse.mjs"
+    "./parse": "./src/parse.mjs",
+    "./ai-search": "./src/ai-enhanced-search.mjs"
   },
   "sideEffects": false,
   "files": [
@@ -17,6 +18,18 @@
     "README.md",
     "LICENSE"
   ],
+  "scripts": {
+    "test": "vitest run --coverage",
+    "test:fast": "vitest run --coverage",
+    "test:watch": "vitest --coverage",
+    "build": "unbuild || true",
+    "lint": "eslint src/ test/ --max-warnings=0",
+    "lint:fix": "eslint src/ test/ --fix",
+    "format": "prettier --write src/",
+    "format:check": "prettier --check src/",
+    "clean": "rm -rf dist/ .nyc_output/ coverage/",
+    "dev": "echo 'Development mode for @unrdf/knowledge-engine'"
+  },
   "keywords": [
     "rdf",
     "knowledge-engine",
@@ -25,9 +38,14 @@
     "rules"
   ],
   "dependencies": {
+    "@iarna/toml": "^2.2.5",
+    "@noble/hashes": "^1.5.0",
+    "@unrdf/core": "workspace:*",
+    "@unrdf/oxigraph": "workspace:*",
+    "@unrdf/streaming": "workspace:*",
+    "@xenova/transformers": "^2.17.2",
     "eyereasoner": "^18.23.0",
-    "@unrdf/core": "5.0.1",
-    "@unrdf/streaming": "5.0.1"
+    "zod": "^3.25.76"
   },
   "devDependencies": {
     "@types/node": "^24.10.1",
@@ -48,17 +66,5 @@
   "license": "MIT",
   "publishConfig": {
     "access": "public"
-  },
-  "scripts": {
-    "test": "vitest run --coverage",
-    "test:fast": "vitest run --coverage",
-    "test:watch": "vitest --coverage",
-    "build": "node build.config.mjs",
-    "lint": "eslint src/ test/ --max-warnings=0",
-    "lint:fix": "eslint src/ test/ --fix",
-    "format": "prettier --write src/ test/",
-    "format:check": "prettier --check src/ test/",
-    "clean": "rm -rf dist/ .nyc_output/ coverage/",
-    "dev": "echo 'Development mode for @unrdf/knowledge-engine'"
   }
-}
+}

package/src/ai-enhanced-search.mjs

@@ -0,0 +1,371 @@
+/**
+ * AI-Enhanced Knowledge Graph Search
+ * Semantic similarity search on RDF triples using transformer models
+ *
+ * @module @unrdf/knowledge-engine/ai-enhanced-search
+ * @description
+ * Integrates Xenova Transformers (WASM-based) for semantic embeddings
+ * and similarity search over RDF knowledge graphs. Enables natural
+ * language queries against structured RDF data with vector similarity.
+ */
+
+import { z } from 'zod';
+import { pipeline, env } from '@xenova/transformers';
+import { query as sparqlQuery } from './query.mjs';
+import { KnowledgeSubstrateCore } from './knowledge-substrate-core.mjs';
+
+// Configure Transformers.js to use local models (no external downloads in production)
+env.allowRemoteModels = false;
+env.allowLocalModels = true;
+
+// =============================================================================
+// Configuration Schemas
+// =============================================================================
+
+/**
+ * AI search configuration schema
+ */
+export const AISearchConfigSchema = z.object({
+  /** Embedding model name (default: Xenova/all-MiniLM-L6-v2) */
+  model: z.string().default('Xenova/all-MiniLM-L6-v2'),
+  /** Maximum number of results to return */
+  topK: z.number().int().positive().default(10),
+  /** Minimum similarity threshold (0-1) */
+  threshold: z.number().min(0).max(1).default(0.5),
+  /** Enable caching of embeddings */
+  cache: z.boolean().default(true),
+  /** Batch size for embedding generation */
+  batchSize: z.number().int().positive().default(32),
+});
+
+/**
+ * Search result schema
+ */
+export const SearchResultSchema = z.object({
+  /** RDF triple (subject, predicate, object) */
+  triple: z.object({
+    subject: z.string(),
+    predicate: z.string(),
+    object: z.string(),
+  }),
+  /** Similarity score (0-1) */
+  score: z.number().min(0).max(1),
+  /** Embedding vector */
+  embedding: z.array(z.number()).optional(),
+  /** Additional metadata */
+  metadata: z.record(z.string(), z.any()).optional(),
+});
+
+// =============================================================================
+// AI-Enhanced Search Engine
+// =============================================================================
+
+/**
+ * Creates an AI-enhanced semantic search engine for RDF knowledge graphs
+ *
+ * @param {Object} store - UNRDF store instance
+ * @param {Object} config - Search configuration
+ * @returns {Promise<Object>} Search engine instance
+ *
+ * @example
+ * const core = await createKnowledgeSubstrateCore();
+ * const searchEngine = await createAISearchEngine(core.store, {
+ *   model: 'Xenova/all-MiniLM-L6-v2',
+ *   topK: 5,
+ *   threshold: 0.7
+ * });
+ *
+ * const results = await searchEngine.search('machine learning algorithms');
+ */
+export async function createAISearchEngine(store, config = {}) {
+  const validated = AISearchConfigSchema.parse(config);
+
+  // Initialize embedding pipeline
+  let embeddingPipeline = null;
+  const embeddingCache = new Map();
+
+  /**
+   * Initialize the transformer model
+   * @private
+   */
+  async function initModel() {
+    if (!embeddingPipeline) {
+      try {
+        embeddingPipeline = await pipeline('feature-extraction', validated.model);
+      } catch (error) {
+        throw new Error(`Failed to load embedding model: ${error.message}`);
+      }
+    }
+    return embeddingPipeline;
+  }
+
+  /**
+   * Generate embedding for text
+   *
+   * @param {string} text - Text to embed
+   * @returns {Promise<Array<number>>} Embedding vector
+   */
+  async function embed(text) {
+    // Check cache
+    if (validated.cache && embeddingCache.has(text)) {
+      return embeddingCache.get(text);
+    }
+
+    const model = await initModel();
+
+    // Generate embedding
+    const output = await model(text, { pooling: 'mean', normalize: true });
+    const embedding = Array.from(output.data);
+
+    // Cache result
+    if (validated.cache) {
+      embeddingCache.set(text, embedding);
+    }
+
+    return embedding;
+  }
+
+  /**
+   * Calculate cosine similarity between two vectors
+   *
+   * @param {Array<number>} a - First vector
+   * @param {Array<number>} b - Second vector
+   * @returns {number} Similarity score (0-1)
+   */
+  function cosineSimilarity(a, b) {
+    if (a.length !== b.length) {
+      throw new Error('Vectors must have same length');
+    }
+
+    let dotProduct = 0;
+    let normA = 0;
+    let normB = 0;
+
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i] * b[i];
+      normA += a[i] * a[i];
+      normB += b[i] * b[i];
+    }
+
+    return dotProduct / (Math.sqrt(normA) * Math.sqrt(normB));
+  }
+
+  /**
+   * Search knowledge graph using natural language query
+   *
+   * @param {string} queryText - Natural language query
+   * @param {Object} options - Search options
+   * @returns {Promise<Array<Object>>} Search results ranked by similarity
+   */
+  async function search(queryText, options = {}) {
+    const opts = { ...validated, ...options };
+
+    // Generate query embedding
+    const queryEmbedding = await embed(queryText);
+
+    // Fetch all triples from knowledge graph
+    const sparqlQueryText = `
+      SELECT ?s ?p ?o
+      WHERE {
+        ?s ?p ?o .
+        FILTER(isLiteral(?o))
+      }
+      LIMIT 1000
+    `;
+
+    const triples = await sparqlQuery(store, sparqlQueryText);
+
+    // Generate embeddings for each triple and calculate similarity
+    const results = [];
+
+    for (const binding of triples.bindings || []) {
+      const tripleText = `${binding.s?.value || ''} ${binding.p?.value || ''} ${binding.o?.value || ''}`;
+      const tripleEmbedding = await embed(tripleText);
+      const score = cosineSimilarity(queryEmbedding, tripleEmbedding);
+
+      if (score >= opts.threshold) {
+        results.push({
+          triple: {
+            subject: binding.s?.value || '',
+            predicate: binding.p?.value || '',
+            object: binding.o?.value || '',
+          },
+          score,
+          embedding: opts.includeEmbeddings ? tripleEmbedding : undefined,
+        });
+      }
+    }
+
+    // Sort by score descending and take top K
+    results.sort((a, b) => b.score - a.score);
+    const topResults = results.slice(0, opts.topK);
+
+    return topResults.map((r) => SearchResultSchema.parse(r));
+  }
+
+  /**
+   * Find similar triples to a given triple
+   *
+   * @param {Object} triple - Reference triple
+   * @param {Object} options - Search options
+   * @returns {Promise<Array<Object>>} Similar triples
+   */
+  async function findSimilar(triple, options = {}) {
+    const tripleText = `${triple.subject} ${triple.predicate} ${triple.object}`;
+    return search(tripleText, options);
+  }
+
+  /**
+   * Cluster triples by semantic similarity
+   *
+   * @param {number} numClusters - Number of clusters to create
+   * @returns {Promise<Array<Array<Object>>>} Clustered triples
+   */
+  async function cluster(numClusters = 5) {
+    // Fetch all triples
+    const sparqlQueryText = `
+      SELECT ?s ?p ?o
+      WHERE {
+        ?s ?p ?o .
+        FILTER(isLiteral(?o))
+      }
+      LIMIT 500
+    `;
+
+    const triples = await sparqlQuery(store, sparqlQueryText);
+
+    // Generate embeddings for all triples
+    const embeddings = [];
+    const tripleData = [];
+
+    for (const binding of triples.bindings || []) {
+      const tripleText = `${binding.s?.value || ''} ${binding.p?.value || ''} ${binding.o?.value || ''}`;
+      const embedding = await embed(tripleText);
+
+      embeddings.push(embedding);
+      tripleData.push({
+        subject: binding.s?.value || '',
+        predicate: binding.p?.value || '',
+        object: binding.o?.value || '',
+      });
+    }
+
+    // Simple k-means clustering (simplified implementation)
+    const clusters = Array.from({ length: numClusters }, () => []);
+
+    // Initialize centroids randomly
+    const centroids = [];
+    for (let i = 0; i < numClusters; i++) {
+      const randomIdx = Math.floor(Math.random() * embeddings.length);
+      centroids.push([...embeddings[randomIdx]]);
+    }
+
+    // Assign each triple to nearest centroid
+    for (let i = 0; i < embeddings.length; i++) {
+      let maxSim = -1;
+      let bestCluster = 0;
+
+      for (let j = 0; j < numClusters; j++) {
+        const sim = cosineSimilarity(embeddings[i], centroids[j]);
+        if (sim > maxSim) {
+          maxSim = sim;
+          bestCluster = j;
+        }
+      }
+
+      clusters[bestCluster].push({
+        triple: tripleData[i],
+        embedding: embeddings[i],
+      });
+    }
+
+    return clusters;
+  }
+
+  /**
+   * Get embedding statistics
+   *
+   * @returns {Object} Cache statistics
+   */
+  function getStats() {
+    return {
+      cacheSize: embeddingCache.size,
+      modelLoaded: embeddingPipeline !== null,
+      config: validated,
+    };
+  }
+
+  /**
+   * Clear embedding cache
+   */
+  function clearCache() {
+    embeddingCache.clear();
+  }
+
+  return {
+    search,
+    findSimilar,
+    cluster,
+    embed,
+    getStats,
+    clearCache,
+  };
+}
+
+// =============================================================================
+// Utility Functions
+// =============================================================================
+
+/**
+ * Perform a quick semantic search without creating an engine instance
+ *
+ * @param {Object} store - UNRDF store instance
+ * @param {string} query - Natural language query
+ * @param {Object} options - Search options
+ * @returns {Promise<Array<Object>>} Search results
+ *
+ * @example
+ * const results = await semanticSearch(store, 'artificial intelligence', {
+ *   topK: 5,
+ *   threshold: 0.7
+ * });
+ */
+export async function semanticSearch(store, query, options = {}) {
+  const engine = await createAISearchEngine(store, options);
+  const results = await engine.search(query);
+  engine.clearCache();
+  return results;
+}
+
+/**
+ * Generate embeddings for multiple texts in batch
+ *
+ * @param {Array<string>} texts - Array of texts to embed
+ * @param {Object} config - Configuration options
+ * @returns {Promise<Array<Array<number>>>} Array of embeddings
+ *
+ * @example
+ * const embeddings = await batchEmbed([
+ *   'machine learning',
+ *   'artificial intelligence',
+ *   'neural networks'
+ * ]);
+ */
+export async function batchEmbed(texts, config = {}) {
+  const validated = AISearchConfigSchema.parse(config);
+  const model = await pipeline('feature-extraction', validated.model);
+
+  const embeddings = [];
+
+  // Process in batches
+  for (let i = 0; i < texts.length; i += validated.batchSize) {
+    const batch = texts.slice(i, i + validated.batchSize);
+
+    for (const text of batch) {
+      const output = await model(text, { pooling: 'mean', normalize: true });
+      embeddings.push(Array.from(output.data));
+    }
+  }
+
+  return embeddings;
+}

package/src/anomaly-detector.mjs

@@ -0,0 +1,226 @@
+/**
+ * @file Z-Score Anomaly Detection for UNRDF Observability
+ * @module anomaly-detector
+ *
+ * @description
+ * Implements statistical anomaly detection using z-score analysis
+ * for latency spikes and throughput drops in OTEL metrics.
+ */
+
+import { z } from 'zod';
+
+/**
+ * Configuration schema for anomaly detector
+ */
+export const AnomalyDetectorConfigSchema = z.object({
+  zScoreThreshold: z.number().positive().default(3.0), // 3 sigma = 99.7% confidence
+  windowSize: z.number().int().positive().default(100), // Rolling window size
+  minSamples: z.number().int().positive().default(10), // Minimum samples before detection
+  cooldownMs: z.number().int().positive().default(60000), // 1 minute cooldown between alerts
+});
+
+/**
+ * Anomaly event schema
+ */
+export const AnomalyEventSchema = z.object({
+  timestamp: z.number(),
+  metric: z.string(),
+  value: z.number(),
+  mean: z.number(),
+  stdDev: z.number(),
+  zScore: z.number(),
+  severity: z.enum(['low', 'medium', 'high', 'critical']),
+  message: z.string(),
+});
+
+/**
+ * Z-Score based anomaly detector
+ */
+export class AnomalyDetector {
+  /**
+   * Create anomaly detector
+   * @param {Object} [config] - Configuration
+   */
+  constructor(config = {}) {
+    this.config = AnomalyDetectorConfigSchema.parse(config);
+    this.windows = new Map(); // metric -> rolling window of values
+    this.lastAlerts = new Map(); // metric -> timestamp of last alert
+    this.stats = new Map(); // metric -> {mean, stdDev}
+    this.anomalyListeners = [];
+  }
+
+  /**
+   * Add a metric sample
+   * @param {string} metric - Metric name (e.g., 'transaction_latency', 'throughput')
+   * @param {number} value - Metric value
+   * @returns {Object|null} Anomaly event if detected, null otherwise
+   */
+  addSample(metric, value) {
+    // Initialize window if needed
+    if (!this.windows.has(metric)) {
+      this.windows.set(metric, []);
+    }
+
+    const window = this.windows.get(metric);
+    window.push(value);
+
+    // Keep window size bounded
+    if (window.length > this.config.windowSize) {
+      window.shift();
+    }
+
+    // Need minimum samples for statistical significance
+    if (window.length < this.config.minSamples) {
+      return null;
+    }
+
+    // Calculate statistics
+    const stats = this._calculateStats(window);
+    this.stats.set(metric, stats);
+
+    // Check for anomaly
+    const zScore = (value - stats.mean) / stats.stdDev;
+    const absZScore = Math.abs(zScore);
+
+    if (absZScore > this.config.zScoreThreshold) {
+      // Check cooldown to prevent alert spam
+      const lastAlert = this.lastAlerts.get(metric);
+      const now = Date.now();
+      if (lastAlert && now - lastAlert < this.config.cooldownMs) {
+        return null; // Still in cooldown
+      }
+
+      // Create anomaly event
+      const severity = this._calculateSeverity(absZScore);
+      const anomaly = AnomalyEventSchema.parse({
+        timestamp: now,
+        metric,
+        value,
+        mean: stats.mean,
+        stdDev: stats.stdDev,
+        zScore,
+        severity,
+        message: this._generateMessage(metric, value, stats, zScore, severity),
+      });
+
+      // Update last alert time
+      this.lastAlerts.set(metric, now);
+
+      // Notify listeners
+      this._notifyListeners(anomaly);
+
+      return anomaly;
+    }
+
+    return null;
+  }
+
+  /**
+   * Calculate mean and standard deviation
+   * @param {Array<number>} values - Values
+   * @returns {{mean: number, stdDev: number}}
+   * @private
+   */
+  _calculateStats(values) {
+    const n = values.length;
+    const mean = values.reduce((sum, v) => sum + v, 0) / n;
+    const variance = values.reduce((sum, v) => sum + Math.pow(v - mean, 2), 0) / n;
+    const stdDev = Math.sqrt(variance);
+    return { mean, stdDev };
+  }
+
+  /**
+   * Calculate severity based on z-score
+   * @param {number} absZScore - Absolute z-score
+   * @returns {string} Severity level
+   * @private
+   */
+  _calculateSeverity(absZScore) {
+    if (absZScore >= 5.0) return 'critical'; // >5σ = extremely rare
+    if (absZScore >= 4.0) return 'high'; // 4-5σ
+    if (absZScore >= 3.5) return 'medium'; // 3.5-4σ
+    return 'low'; // 3-3.5σ
+  }
+
+  /**
+   * Generate human-readable message
+   * @param {string} metric - Metric name
+   * @param {number} value - Current value
+   * @param {Object} stats - Statistics
+   * @param {number} zScore - Z-score
+   * @param {string} severity - Severity
+   * @returns {string} Message
+   * @private
+   */
+  _generateMessage(metric, value, stats, zScore, severity) {
+    const direction = zScore > 0 ? 'spike' : 'drop';
+    const deviation = Math.abs(zScore).toFixed(2);
+    return `[${severity.toUpperCase()}] ${metric} ${direction}: ${value.toFixed(2)} (${deviation}σ from mean ${stats.mean.toFixed(2)})`;
+  }
+
+  /**
+   * Register anomaly listener
+   * @param {Function} listener - Callback(anomaly)
+   */
+  onAnomaly(listener) {
+    this.anomalyListeners.push(listener);
+  }
+
+  /**
+   * Notify all listeners
+   * @param {Object} anomaly - Anomaly event
+   * @private
+   */
+  _notifyListeners(anomaly) {
+    for (const listener of this.anomalyListeners) {
+      try {
+        listener(anomaly);
+      } catch (error) {
+        console.error('[AnomalyDetector] Listener error:', error);
+      }
+    }
+  }
+
+  /**
+   * Get statistics for a metric
+   * @param {string} metric - Metric name
+   * @returns {{mean: number, stdDev: number, samples: number}|null}
+   */
+  getStats(metric) {
+    const stats = this.stats.get(metric);
+    const window = this.windows.get(metric);
+    if (!stats || !window) return null;
+    return {
+      ...stats,
+      samples: window.length,
+    };
+  }
+
+  /**
+   * Reset detector state
+   */
+  reset() {
+    this.windows.clear();
+    this.lastAlerts.clear();
+    this.stats.clear();
+  }
+
+  /**
+   * Clear metric history
+   * @param {string} metric - Metric name
+   */
+  clearMetric(metric) {
+    this.windows.delete(metric);
+    this.lastAlerts.delete(metric);
+    this.stats.delete(metric);
+  }
+}
+
+/**
+ * Create anomaly detector instance
+ * @param {Object} [config] - Configuration
+ * @returns {AnomalyDetector}
+ */
+export function createAnomalyDetector(config = {}) {
+  return new AnomalyDetector(config);
+}