@sparkleideas/embeddings 3.0.0-alpha.13

package/src/neural-integration.ts

@@ -0,0 +1,295 @@
+/**
+ * Neural Substrate Integration
+ *
+ * Integrates @sparkleideas/agentic-flow's neural embedding features:
+ * - Semantic drift detection
+ * - Memory physics (hippocampal dynamics)
+ * - Embedding state machine
+ * - Swarm coordination
+ * - Coherence monitoring
+ *
+ * These features treat embeddings as a synthetic nervous system.
+ */
+
+// Types from @sparkleideas/agentic-flow/embeddings
+export interface DriftResult {
+  distance: number;
+  velocity: number;
+  acceleration: number;
+  trend: 'stable' | 'drifting' | 'accelerating' | 'recovering';
+  shouldEscalate: boolean;
+  shouldTriggerReasoning: boolean;
+}
+
+export interface MemoryEntry {
+  id: string;
+  embedding: Float32Array;
+  content: string;
+  strength: number;
+  timestamp: number;
+  accessCount: number;
+  associations: string[];
+}
+
+export interface AgentState {
+  id: string;
+  position: Float32Array;
+  velocity: Float32Array;
+  attention: Float32Array;
+  energy: number;
+  lastUpdate: number;
+}
+
+export interface CoherenceResult {
+  isCoherent: boolean;
+  anomalyScore: number;
+  stabilityScore: number;
+  driftDirection: Float32Array | null;
+  warnings: string[];
+}
+
+export interface SubstrateHealth {
+  memoryCount: number;
+  activeAgents: number;
+  avgDrift: number;
+  avgCoherence: number;
+  lastConsolidation: number;
+  uptime: number;
+}
+
+export interface NeuralSubstrateConfig {
+  dimension?: number;
+  driftThreshold?: number;
+  decayRate?: number;
+}
+
+/**
+ * Lazy-loaded Neural Substrate wrapper
+ *
+ * Wraps @sparkleideas/agentic-flow's NeuralSubstrate with graceful fallback
+ */
+export class NeuralEmbeddingService {
+  private substrate: any = null;
+  private initialized = false;
+  private available = false;
+
+  constructor(private config: NeuralSubstrateConfig = {}) {}
+
+  /**
+   * Initialize neural substrate
+   */
+  async init(): Promise<boolean> {
+    if (this.initialized) return this.available;
+
+    try {
+      const { getNeuralSubstrate } = await import('@sparkleideas/agentic-flow/embeddings');
+      this.substrate = await getNeuralSubstrate(this.config);
+      await this.substrate.init();
+      this.available = true;
+    } catch (error) {
+      console.warn('[neural] Neural substrate not available:', error instanceof Error ? error.message : error);
+      this.available = false;
+    }
+
+    this.initialized = true;
+    return this.available;
+  }
+
+  /**
+   * Check if neural features are available
+   */
+  isAvailable(): boolean {
+    return this.available;
+  }
+
+  /**
+   * Detect semantic drift from baseline
+   */
+  async detectDrift(input: string): Promise<DriftResult | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.drift.detect(input);
+  }
+
+  /**
+   * Set baseline for drift detection
+   */
+  async setDriftBaseline(context: string): Promise<void> {
+    if (!this.available || !this.substrate) return;
+    await this.substrate.drift.setBaseline(context);
+  }
+
+  /**
+   * Store memory with interference detection
+   */
+  async storeMemory(id: string, content: string): Promise<{ stored: boolean; interference: string[] } | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.memory.store(id, content);
+  }
+
+  /**
+   * Recall memories by similarity
+   */
+  async recallMemories(query: string, topK = 5): Promise<Array<MemoryEntry & { relevance: number }> | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.memory.recall(query, topK);
+  }
+
+  /**
+   * Consolidate memories (merge similar, forget weak)
+   */
+  consolidateMemories(): { merged: number; forgotten: number; remaining: number } | null {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.memory.consolidate();
+  }
+
+  /**
+   * Register agent for state tracking
+   */
+  async registerAgent(id: string, role: string): Promise<AgentState | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.states.registerAgent(id, role);
+  }
+
+  /**
+   * Update agent state based on observation
+   */
+  async updateAgentState(agentId: string, observation: string): Promise<{
+    newState: AgentState;
+    nearestRegion: string;
+    regionProximity: number;
+  } | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.states.updateState(agentId, observation);
+  }
+
+  /**
+   * Get agent state
+   */
+  getAgentState(agentId: string): AgentState | null {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.states.getAgent(agentId);
+  }
+
+  /**
+   * Coordinate swarm for task
+   */
+  async coordinateSwarm(task: string): Promise<Array<{
+    agentId: string;
+    taskAlignment: number;
+    bestCollaborator: string | null;
+    collaborationScore: number;
+  }> | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.swarm.coordinate(task);
+  }
+
+  /**
+   * Add agent to swarm
+   */
+  async addSwarmAgent(id: string, role: string): Promise<AgentState | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.swarm.addAgent(id, role);
+  }
+
+  /**
+   * Calibrate coherence monitor
+   */
+  async calibrateCoherence(goodOutputs: string[]): Promise<{ calibrated: boolean; sampleCount: number } | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.coherence.calibrate(goodOutputs);
+  }
+
+  /**
+   * Check output coherence
+   */
+  async checkCoherence(output: string): Promise<CoherenceResult | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.coherence.check(output);
+  }
+
+  /**
+   * Process input through full neural substrate
+   */
+  async process(input: string, context?: {
+    agentId?: string;
+    memoryId?: string;
+    checkCoherence?: boolean;
+  }): Promise<{
+    drift: DriftResult;
+    state?: { nearestRegion: string; regionProximity: number };
+    coherence?: CoherenceResult;
+    stored?: boolean;
+  } | null> {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.process(input, context);
+  }
+
+  /**
+   * Get substrate health
+   */
+  health(): SubstrateHealth | null {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.health();
+  }
+
+  /**
+   * Full consolidation pass
+   */
+  consolidate(): { memory: { merged: number; forgotten: number; remaining: number } } | null {
+    if (!this.available || !this.substrate) return null;
+    return this.substrate.consolidate();
+  }
+}
+
+/**
+ * Create neural embedding service
+ */
+export function createNeuralService(config: NeuralSubstrateConfig = {}): NeuralEmbeddingService {
+  return new NeuralEmbeddingService(config);
+}
+
+/**
+ * Check if neural features are available
+ */
+export async function isNeuralAvailable(): Promise<boolean> {
+  try {
+    await import('@sparkleideas/agentic-flow/embeddings');
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * List available ONNX embedding models
+ */
+export async function listEmbeddingModels(): Promise<Array<{
+  id: string;
+  dimension: number;
+  size: string;
+  quantized: boolean;
+  downloaded: boolean;
+}>> {
+  try {
+    const { listAvailableModels } = await import('@sparkleideas/agentic-flow/embeddings');
+    return listAvailableModels();
+  } catch {
+    // Return default models if @sparkleideas/agentic-flow not available
+    return [
+      { id: 'all-MiniLM-L6-v2', dimension: 384, size: '23MB', quantized: false, downloaded: false },
+      { id: 'all-mpnet-base-v2', dimension: 768, size: '110MB', quantized: false, downloaded: false },
+    ];
+  }
+}
+
+/**
+ * Download embedding model
+ */
+export async function downloadEmbeddingModel(
+  modelId: string,
+  targetDir?: string,
+  onProgress?: (progress: { percent: number; bytesDownloaded: number; totalBytes: number }) => void
+): Promise<string> {
+  const { downloadModel } = await import('@sparkleideas/agentic-flow/embeddings');
+  return downloadModel(modelId, targetDir ?? '.models', onProgress);
+}

package/src/normalization.ts

@@ -0,0 +1,267 @@
+/**
+ * Embedding Normalization Utilities
+ *
+ * Features:
+ * - L2 (Euclidean) normalization
+ * - L1 (Manhattan) normalization
+ * - Min-max normalization
+ * - Z-score standardization
+ * - Batch normalization
+ */
+
+/**
+ * Normalization type
+ */
+export type NormalizationType = 'l2' | 'l1' | 'minmax' | 'zscore' | 'none';
+
+/**
+ * Normalization options
+ */
+export interface NormalizationOptions {
+  /** Normalization type (default: 'l2') */
+  type?: NormalizationType;
+  /** Epsilon for numerical stability (default: 1e-12) */
+  epsilon?: number;
+  /** In-place modification (default: false) */
+  inPlace?: boolean;
+}
+
+/**
+ * L2 (Euclidean) normalize embedding to unit length
+ * Most common for cosine similarity
+ *
+ * @param embedding - Input embedding vector
+ * @param epsilon - Small value to prevent division by zero
+ * @returns Normalized embedding with ||v|| = 1
+ */
+export function l2Normalize(
+  embedding: Float32Array | number[],
+  epsilon = 1e-12
+): Float32Array {
+  const result = embedding instanceof Float32Array
+    ? new Float32Array(embedding.length)
+    : new Float32Array(embedding.length);
+
+  // Calculate L2 norm (Euclidean length)
+  let sumSquares = 0;
+  for (let i = 0; i < embedding.length; i++) {
+    sumSquares += embedding[i] * embedding[i];
+  }
+
+  const norm = Math.sqrt(sumSquares);
+  const scale = norm > epsilon ? 1 / norm : 0;
+
+  // Normalize
+  for (let i = 0; i < embedding.length; i++) {
+    result[i] = embedding[i] * scale;
+  }
+
+  return result;
+}
+
+/**
+ * L2 normalize embedding in-place (modifies original array)
+ */
+export function l2NormalizeInPlace(
+  embedding: Float32Array,
+  epsilon = 1e-12
+): Float32Array {
+  let sumSquares = 0;
+  for (let i = 0; i < embedding.length; i++) {
+    sumSquares += embedding[i] * embedding[i];
+  }
+
+  const norm = Math.sqrt(sumSquares);
+  const scale = norm > epsilon ? 1 / norm : 0;
+
+  for (let i = 0; i < embedding.length; i++) {
+    embedding[i] *= scale;
+  }
+
+  return embedding;
+}
+
+/**
+ * L1 (Manhattan) normalize embedding
+ * Sum of absolute values = 1
+ */
+export function l1Normalize(
+  embedding: Float32Array | number[],
+  epsilon = 1e-12
+): Float32Array {
+  const result = new Float32Array(embedding.length);
+
+  // Calculate L1 norm (sum of absolute values)
+  let sumAbs = 0;
+  for (let i = 0; i < embedding.length; i++) {
+    sumAbs += Math.abs(embedding[i]);
+  }
+
+  const scale = sumAbs > epsilon ? 1 / sumAbs : 0;
+
+  for (let i = 0; i < embedding.length; i++) {
+    result[i] = embedding[i] * scale;
+  }
+
+  return result;
+}
+
+/**
+ * Min-max normalize embedding to [0, 1] range
+ */
+export function minMaxNormalize(
+  embedding: Float32Array | number[],
+  epsilon = 1e-12
+): Float32Array {
+  const result = new Float32Array(embedding.length);
+
+  // Find min and max
+  let min = Infinity;
+  let max = -Infinity;
+
+  for (let i = 0; i < embedding.length; i++) {
+    if (embedding[i] < min) min = embedding[i];
+    if (embedding[i] > max) max = embedding[i];
+  }
+
+  const range = max - min;
+  const scale = range > epsilon ? 1 / range : 0;
+
+  for (let i = 0; i < embedding.length; i++) {
+    result[i] = (embedding[i] - min) * scale;
+  }
+
+  return result;
+}
+
+/**
+ * Z-score standardize embedding (mean=0, std=1)
+ */
+export function zScoreNormalize(
+  embedding: Float32Array | number[],
+  epsilon = 1e-12
+): Float32Array {
+  const result = new Float32Array(embedding.length);
+  const n = embedding.length;
+
+  // Calculate mean
+  let sum = 0;
+  for (let i = 0; i < n; i++) {
+    sum += embedding[i];
+  }
+  const mean = sum / n;
+
+  // Calculate standard deviation
+  let sumSquaredDiff = 0;
+  for (let i = 0; i < n; i++) {
+    const diff = embedding[i] - mean;
+    sumSquaredDiff += diff * diff;
+  }
+  const std = Math.sqrt(sumSquaredDiff / n);
+  const scale = std > epsilon ? 1 / std : 0;
+
+  // Standardize
+  for (let i = 0; i < n; i++) {
+    result[i] = (embedding[i] - mean) * scale;
+  }
+
+  return result;
+}
+
+/**
+ * Normalize embedding using specified method
+ */
+export function normalize(
+  embedding: Float32Array | number[],
+  options: NormalizationOptions = {}
+): Float32Array {
+  const { type = 'l2', epsilon = 1e-12, inPlace = false } = options;
+
+  if (type === 'none') {
+    return embedding instanceof Float32Array
+      ? embedding
+      : new Float32Array(embedding);
+  }
+
+  if (inPlace && embedding instanceof Float32Array && type === 'l2') {
+    return l2NormalizeInPlace(embedding, epsilon);
+  }
+
+  switch (type) {
+    case 'l2':
+      return l2Normalize(embedding, epsilon);
+    case 'l1':
+      return l1Normalize(embedding, epsilon);
+    case 'minmax':
+      return minMaxNormalize(embedding, epsilon);
+    case 'zscore':
+      return zScoreNormalize(embedding, epsilon);
+    default:
+      return l2Normalize(embedding, epsilon);
+  }
+}
+
+/**
+ * Batch normalize multiple embeddings
+ */
+export function normalizeBatch(
+  embeddings: Array<Float32Array | number[]>,
+  options: NormalizationOptions = {}
+): Float32Array[] {
+  return embeddings.map(emb => normalize(emb, options));
+}
+
+/**
+ * Calculate L2 norm of embedding
+ */
+export function l2Norm(embedding: Float32Array | number[]): number {
+  let sumSquares = 0;
+  for (let i = 0; i < embedding.length; i++) {
+    sumSquares += embedding[i] * embedding[i];
+  }
+  return Math.sqrt(sumSquares);
+}
+
+/**
+ * Check if embedding is already normalized (L2 norm ≈ 1)
+ */
+export function isNormalized(
+  embedding: Float32Array | number[],
+  tolerance = 1e-6
+): boolean {
+  const norm = l2Norm(embedding);
+  return Math.abs(norm - 1) < tolerance;
+}
+
+/**
+ * Center embeddings by subtracting mean across batch
+ * Useful for improving similarity metrics
+ */
+export function centerEmbeddings(
+  embeddings: Array<Float32Array | number[]>
+): Float32Array[] {
+  if (embeddings.length === 0) return [];
+
+  const dim = embeddings[0].length;
+  const n = embeddings.length;
+
+  // Calculate mean for each dimension
+  const mean = new Float32Array(dim);
+  for (const emb of embeddings) {
+    for (let i = 0; i < dim; i++) {
+      mean[i] += emb[i];
+    }
+  }
+  for (let i = 0; i < dim; i++) {
+    mean[i] /= n;
+  }
+
+  // Subtract mean from each embedding
+  return embeddings.map(emb => {
+    const centered = new Float32Array(dim);
+    for (let i = 0; i < dim; i++) {
+      centered[i] = emb[i] - mean[i];
+    }
+    return centered;
+  });
+}