@defai.digital/semantic-context 13.4.0

package/src/index.ts

@@ -0,0 +1,56 @@
+/**
+ * Semantic Context Domain
+ *
+ * Provides semantic search and vector-indexed storage.
+ *
+ * @packageDocumentation
+ */
+
+// Types and interfaces
+export type {
+  EmbeddingPort,
+  EmbeddingRequest,
+  EmbeddingResult,
+  SemanticStorePort,
+  SemanticStoreStats,
+  SemanticManager,
+  SemanticManagerOptions,
+  SimilarityMethod,
+  SimilarityOptions,
+} from './types.js';
+
+// Stub implementations for testing
+export { StubEmbeddingPort, InMemorySemanticStore } from './types.js';
+
+// Similarity utilities
+export {
+  cosineSimilarity,
+  dotProductSimilarity,
+  euclideanDistance,
+  manhattanDistance,
+  computeSimilarity,
+  normalizeVector,
+  vectorNorm,
+  addVectors,
+  subtractVectors,
+  scaleVector,
+  computeCentroid,
+  findKNearest,
+  filterByThreshold,
+  DEFAULT_SIMILARITY_OPTIONS,
+} from './similarity.js';
+
+// Embedding service
+export {
+  LocalEmbeddingProvider,
+  CachedEmbeddingProvider,
+  createEmbeddingProvider,
+  createTFIDFEmbedding,
+  createTFIDFEmbeddingBatch,
+} from './embedding-service.js';
+
+// Semantic manager
+export {
+  createSemanticManager,
+  SemanticManagerError,
+} from './semantic-manager.js';

package/src/semantic-manager.ts

@@ -0,0 +1,246 @@
+/**
+ * Semantic Manager
+ *
+ * High-level manager for semantic context storage and search.
+ * Combines embedding computation with storage.
+ *
+ * Invariants:
+ * - INV-SEM-001: Embeddings computed on store, cached until content changes
+ * - INV-SEM-002: Search results sorted by similarity descending
+ * - INV-SEM-003: Similarity scores normalized to [0, 1]
+ * - INV-SEM-004: Namespace isolation
+ */
+
+import type {
+  SemanticItem,
+  SemanticSearchRequest,
+  SemanticSearchResponse,
+  SemanticStoreRequest,
+  SemanticStoreResponse,
+  SemanticListRequest,
+  SemanticListResponse,
+  SemanticDeleteResponse,
+  EmbeddingConfig,
+} from '@defai.digital/contracts';
+import { SemanticContextErrorCodes, computeContentHash } from '@defai.digital/contracts';
+import type {
+  SemanticManager,
+  SemanticManagerOptions,
+  SemanticStoreStats,
+} from './types.js';
+
+/**
+ * Error thrown by semantic manager
+ */
+export class SemanticManagerError extends Error {
+  constructor(
+    public readonly code: string,
+    message: string,
+    public readonly details?: Record<string, unknown>
+  ) {
+    super(message);
+    this.name = 'SemanticManagerError';
+  }
+
+  static notFound(key: string, namespace: string): SemanticManagerError {
+    return new SemanticManagerError(
+      SemanticContextErrorCodes.NOT_FOUND,
+      `Item not found: ${namespace}:${key}`,
+      { key, namespace }
+    );
+  }
+
+  static embeddingFailed(message: string): SemanticManagerError {
+    return new SemanticManagerError(
+      SemanticContextErrorCodes.EMBEDDING_FAILED,
+      `Embedding computation failed: ${message}`
+    );
+  }
+
+  static searchFailed(message: string): SemanticManagerError {
+    return new SemanticManagerError(
+      SemanticContextErrorCodes.SEARCH_FAILED,
+      `Search failed: ${message}`
+    );
+  }
+
+  static dimensionMismatch(expected: number, actual: number): SemanticManagerError {
+    return new SemanticManagerError(
+      SemanticContextErrorCodes.DIMENSION_MISMATCH,
+      `Embedding dimension mismatch: expected ${expected}, got ${actual}`,
+      { expected, actual }
+    );
+  }
+}
+
+/**
+ * Creates a semantic manager
+ */
+export function createSemanticManager(options: SemanticManagerOptions): SemanticManager {
+  const {
+    embeddingPort,
+    storePort,
+    defaultNamespace = 'default',
+    autoEmbed = true,
+  } = options;
+
+  // Track namespace embedding dimensions for consistency (INV-SEM-200)
+  const namespaceDimensions = new Map<string, number>();
+
+  /**
+   * Validate embedding dimension for namespace
+   */
+  function validateDimension(namespace: string, dimension: number): void {
+    const expected = namespaceDimensions.get(namespace);
+    if (expected !== undefined && expected !== dimension) {
+      throw SemanticManagerError.dimensionMismatch(expected, dimension);
+    }
+    if (expected === undefined) {
+      namespaceDimensions.set(namespace, dimension);
+    }
+  }
+
+  return {
+    /**
+     * Store content with automatic embedding
+     * INV-SEM-001: Embeddings computed and cached
+     */
+    async store(request: SemanticStoreRequest): Promise<SemanticStoreResponse> {
+      const namespace = request.namespace ?? defaultNamespace;
+
+      try {
+        // Check if content changed (for caching)
+        const contentHash = await computeContentHash(request.content);
+        const existing = await storePort.get(request.key, namespace);
+
+        // Determine if embedding needs computation
+        let embedding = request.embedding;
+        let embeddingComputed = false;
+
+        if (autoEmbed && !embedding) {
+          const needsEmbedding =
+            !existing ||
+            existing.contentHash !== contentHash ||
+            request.forceRecompute;
+
+          if (needsEmbedding) {
+            const result = await embeddingPort.embed({ text: request.content });
+            embedding = result.embedding;
+            embeddingComputed = true;
+
+            // Validate dimension consistency (INV-SEM-200)
+            validateDimension(namespace, result.dimension);
+          } else if (existing?.embedding) {
+            // Reuse existing embedding
+            embedding = existing.embedding;
+          }
+        }
+
+        // Validate provided embedding dimension
+        if (embedding) {
+          const config = embeddingPort.getConfig();
+          if (embedding.length !== config.dimension) {
+            throw SemanticManagerError.dimensionMismatch(config.dimension, embedding.length);
+          }
+          validateDimension(namespace, embedding.length);
+        }
+
+        // Store with embedding
+        const result = await storePort.store({
+          ...request,
+          namespace,
+          embedding,
+        });
+
+        return {
+          ...result,
+          embeddingComputed,
+        };
+      } catch (error) {
+        if (error instanceof SemanticManagerError) throw error;
+
+        const message = error instanceof Error ? error.message : 'Unknown error';
+        throw SemanticManagerError.embeddingFailed(message);
+      }
+    },
+
+    /**
+     * Search by semantic similarity
+     * INV-SEM-002: Results sorted by similarity descending
+     * INV-SEM-003: Scores normalized to [0, 1]
+     * INV-SEM-004: Namespace isolation
+     */
+    async search(request: SemanticSearchRequest): Promise<SemanticSearchResponse> {
+      const namespace = request.namespace;
+
+      try {
+        // Compute query embedding
+        const queryResult = await embeddingPort.embed({ text: request.query });
+
+        // Validate dimension if namespace has items
+        if (namespace) {
+          const stats = await storePort.getStats(namespace);
+          if (stats.embeddingDimension !== null) {
+            validateDimension(namespace, queryResult.dimension);
+          }
+        }
+
+        // Delegate search to store
+        return await storePort.search(request);
+      } catch (error) {
+        if (error instanceof SemanticManagerError) throw error;
+
+        const message = error instanceof Error ? error.message : 'Unknown error';
+        throw SemanticManagerError.searchFailed(message);
+      }
+    },
+
+    /**
+     * Get item by key
+     */
+    async get(key: string, namespace?: string): Promise<SemanticItem | null> {
+      return storePort.get(key, namespace ?? defaultNamespace);
+    },
+
+    /**
+     * List items
+     */
+    async list(request: SemanticListRequest): Promise<SemanticListResponse> {
+      return storePort.list({
+        ...request,
+        namespace: request.namespace ?? defaultNamespace,
+      });
+    },
+
+    /**
+     * Delete item
+     */
+    async delete(key: string, namespace?: string): Promise<SemanticDeleteResponse> {
+      return storePort.delete(key, namespace ?? defaultNamespace);
+    },
+
+    /**
+     * Get statistics
+     */
+    async getStats(namespace?: string): Promise<SemanticStoreStats> {
+      return storePort.getStats(namespace);
+    },
+
+    /**
+     * Clear namespace
+     */
+    async clear(namespace?: string): Promise<number> {
+      const ns = namespace ?? defaultNamespace;
+      // Reset dimension tracking for cleared namespace
+      namespaceDimensions.delete(ns);
+      return storePort.clear(ns);
+    },
+
+    /**
+     * Get embedding configuration
+     */
+    getEmbeddingConfig(): EmbeddingConfig {
+      return embeddingPort.getConfig();
+    },
+  };
+}

package/src/similarity.ts

@@ -0,0 +1,265 @@
+/**
+ * Similarity Computation Utilities
+ *
+ * Provides various methods for computing vector similarity.
+ *
+ * Invariants:
+ * - INV-SEM-003: All scores normalized to [0, 1] range
+ */
+
+import type { SimilarityMethod, SimilarityOptions } from './types.js';
+
+/**
+ * Default similarity options
+ */
+export const DEFAULT_SIMILARITY_OPTIONS: SimilarityOptions = {
+  method: 'cosine',
+  normalize: true,
+};
+
+/**
+ * Compute cosine similarity between two vectors
+ * Returns value in [-1, 1] (or [0, 1] if normalized)
+ *
+ * INV-SEM-003: Normalized to [0, 1] when normalize=true
+ */
+export function cosineSimilarity(a: number[], b: number[], normalize = true): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  if (a.length === 0) return 0;
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i]! * b[i]!;
+    normA += a[i]! * a[i]!;
+    normB += b[i]! * b[i]!;
+  }
+
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+
+  if (denominator === 0) return 0;
+
+  const similarity = dotProduct / denominator;
+
+  // Normalize from [-1, 1] to [0, 1]
+  return normalize ? (similarity + 1) / 2 : similarity;
+}
+
+/**
+ * Compute dot product similarity between two vectors
+ * Returns raw dot product (or normalized if requested)
+ *
+ * INV-SEM-003: When normalize=true, normalizes vectors first and maps to [0, 1]
+ */
+export function dotProductSimilarity(a: number[], b: number[], normalize = true): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  if (a.length === 0) return 0;
+
+  if (!normalize) {
+    // Raw dot product
+    let dotProduct = 0;
+    for (let i = 0; i < a.length; i++) {
+      dotProduct += a[i]! * b[i]!;
+    }
+    return dotProduct;
+  }
+
+  // Normalized: compute cosine similarity (dot product of unit vectors)
+  // This ensures result is in [-1, 1] range, then map to [0, 1]
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i]! * b[i]!;
+    normA += a[i]! * a[i]!;
+    normB += b[i]! * b[i]!;
+  }
+
+  const denominator = Math.sqrt(normA) * Math.sqrt(normB);
+
+  if (denominator === 0) return 0;
+
+  const similarity = dotProduct / denominator;
+
+  // Map from [-1, 1] to [0, 1]
+  return (similarity + 1) / 2;
+}
+
+/**
+ * Compute Euclidean distance between two vectors
+ * Returns distance (or similarity if normalize=true)
+ *
+ * INV-SEM-003: Converted to similarity via 1/(1+distance) when normalize=true
+ */
+export function euclideanDistance(a: number[], b: number[], normalize = true): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  if (a.length === 0) return 0;
+
+  let sumSquares = 0;
+  for (let i = 0; i < a.length; i++) {
+    const diff = a[i]! - b[i]!;
+    sumSquares += diff * diff;
+  }
+
+  const distance = Math.sqrt(sumSquares);
+
+  // Convert distance to similarity: smaller distance = higher similarity
+  return normalize ? 1 / (1 + distance) : distance;
+}
+
+/**
+ * Compute Manhattan distance between two vectors
+ */
+export function manhattanDistance(a: number[], b: number[], normalize = true): number {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+
+  if (a.length === 0) return 0;
+
+  let distance = 0;
+  for (let i = 0; i < a.length; i++) {
+    distance += Math.abs(a[i]! - b[i]!);
+  }
+
+  // Convert to similarity
+  return normalize ? 1 / (1 + distance) : distance;
+}
+
+/**
+ * Compute similarity using specified method
+ */
+export function computeSimilarity(
+  a: number[],
+  b: number[],
+  options: Partial<SimilarityOptions> = {}
+): number {
+  const { method, normalize } = { ...DEFAULT_SIMILARITY_OPTIONS, ...options };
+
+  switch (method) {
+    case 'cosine':
+      return cosineSimilarity(a, b, normalize);
+    case 'dot':
+      return dotProductSimilarity(a, b, normalize);
+    case 'euclidean':
+      return euclideanDistance(a, b, normalize);
+    default:
+      throw new Error(`Unknown similarity method: ${method}`);
+  }
+}
+
+/**
+ * Normalize a vector to unit length
+ */
+export function normalizeVector(v: number[]): number[] {
+  const norm = Math.sqrt(v.reduce((sum, x) => sum + x * x, 0));
+  if (norm === 0) return v;
+  return v.map((x) => x / norm);
+}
+
+/**
+ * Compute vector norm (magnitude)
+ */
+export function vectorNorm(v: number[]): number {
+  return Math.sqrt(v.reduce((sum, x) => sum + x * x, 0));
+}
+
+/**
+ * Add two vectors
+ */
+export function addVectors(a: number[], b: number[]): number[] {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+  return a.map((x, i) => x + b[i]!);
+}
+
+/**
+ * Subtract vectors: a - b
+ */
+export function subtractVectors(a: number[], b: number[]): number[] {
+  if (a.length !== b.length) {
+    throw new Error(`Vector dimension mismatch: ${a.length} vs ${b.length}`);
+  }
+  return a.map((x, i) => x - b[i]!);
+}
+
+/**
+ * Scale a vector by a scalar
+ */
+export function scaleVector(v: number[], scalar: number): number[] {
+  return v.map((x) => x * scalar);
+}
+
+/**
+ * Compute centroid (average) of multiple vectors
+ */
+export function computeCentroid(vectors: number[][]): number[] {
+  if (vectors.length === 0) return [];
+
+  const dim = vectors[0]!.length;
+  const centroid = new Array(dim).fill(0);
+
+  for (const v of vectors) {
+    if (v.length !== dim) {
+      throw new Error(`Inconsistent vector dimensions`);
+    }
+    for (let i = 0; i < dim; i++) {
+      centroid[i] += v[i]!;
+    }
+  }
+
+  return centroid.map((x) => x / vectors.length);
+}
+
+/**
+ * Find k nearest neighbors from candidates
+ * INV-SEM-002: Results sorted by similarity descending
+ */
+export function findKNearest(
+  query: number[],
+  candidates: Array<{ id: string; embedding: number[] }>,
+  k: number,
+  method: SimilarityMethod = 'cosine'
+): Array<{ id: string; similarity: number }> {
+  const scored = candidates.map((c) => ({
+    id: c.id,
+    similarity: computeSimilarity(query, c.embedding, { method, normalize: true }),
+  }));
+
+  // Sort by similarity descending
+  scored.sort((a, b) => b.similarity - a.similarity);
+
+  return scored.slice(0, k);
+}
+
+/**
+ * Filter vectors by minimum similarity threshold
+ * INV-SEM-003: Threshold applied after normalization
+ */
+export function filterByThreshold(
+  query: number[],
+  candidates: Array<{ id: string; embedding: number[] }>,
+  minSimilarity: number,
+  method: SimilarityMethod = 'cosine'
+): Array<{ id: string; similarity: number }> {
+  return candidates
+    .map((c) => ({
+      id: c.id,
+      similarity: computeSimilarity(query, c.embedding, { method, normalize: true }),
+    }))
+    .filter((s) => s.similarity >= minSimilarity)
+    .sort((a, b) => b.similarity - a.similarity);
+}