cognitive-core 0.2.4 → 0.3.1

package/src/bin/commands/skills.ts

@@ -1,146 +0,0 @@
-/**
- * Skills CLI commands — list, stats, refresh.
- */
-
-import { getPlaybookSuccessRate } from '../../types/playbook.js';
-import {
-  type CLIOptions,
-  withAtlas,
-  requireSkillLibrary,
-} from '../cli-utils.js';
-
-export async function skillsCommand(
-  command: string,
-  _positional: string[],
-  options: CLIOptions,
-): Promise<void> {
-  switch (command) {
-    case 'list':
-      return skillsList(options);
-    case 'stats':
-      return skillsStats(options);
-    case 'refresh':
-      return skillsRefresh(options);
-    default:
-      if (command) {
-        console.error(`Unknown skills command: ${command}`);
-      }
-      console.log('Usage: cogcore skills <command>\n');
-      console.log('Commands:');
-      console.log('  list     List skills (--tier core|domain)');
-      console.log('  stats    Show skill library statistics');
-      console.log('  refresh  Refresh skill tiers');
-      process.exit(command ? 1 : 0);
-  }
-}
-
-async function skillsList(options: CLIOptions): Promise<void> {
-  await withAtlas(options, async (atlas) => {
-    const lib = requireSkillLibrary(atlas);
-
-    let playbooks;
-    if (options.tier === 'core') {
-      playbooks = lib.getCoreSkills();
-    } else if (options.tier === 'domain') {
-      playbooks = [];
-      for (const domain of lib.getDomains()) {
-        playbooks.push(...lib.getDomainSkills(domain));
-      }
-    } else {
-      playbooks = [...lib.getCoreSkills()];
-      for (const domain of lib.getDomains()) {
-        for (const pb of lib.getDomainSkills(domain)) {
-          if (!playbooks.some((p) => p.id === pb.id)) {
-            playbooks.push(pb);
-          }
-        }
-      }
-    }
-
-    const coreIds = new Set(lib.getCoreSkills().map((c) => c.id));
-
-    if (options.format === 'json') {
-      const results = playbooks.map((pb) => ({
-        id: pb.id,
-        name: pb.name,
-        confidence: pb.confidence,
-        domains: pb.applicability.domains,
-        tier: coreIds.has(pb.id) ? 'core' : 'domain',
-        successRate: getPlaybookSuccessRate(pb),
-      }));
-      console.log(JSON.stringify(results, null, 2));
-    } else {
-      if (playbooks.length === 0) {
-        console.log('No skills found.');
-        return;
-      }
-      console.log(`Found ${playbooks.length} skill(s):\n`);
-      for (const pb of playbooks) {
-        const tier = coreIds.has(pb.id) ? 'core' : 'domain';
-        const confidence = Math.round(pb.confidence * 100);
-        const successRate = Math.round(getPlaybookSuccessRate(pb) * 100);
-        const domains = pb.applicability.domains.join(', ');
-        console.log(`  ${pb.id}  ${pb.name}`);
-        console.log(`    Tier: ${tier}  Confidence: ${confidence}%  Success: ${successRate}%  Domains: ${domains}`);
-      }
-    }
-  });
-}
-
-async function skillsStats(options: CLIOptions): Promise<void> {
-  await withAtlas(options, async (atlas) => {
-    const lib = requireSkillLibrary(atlas);
-    const stats = lib.getStats();
-
-    if (options.format === 'json') {
-      console.log(JSON.stringify(stats, null, 2));
-    } else {
-      console.log('Skill Library Stats:\n');
-      console.log(`  Core skills: ${stats.coreSkillCount}`);
-      const domainEntries = Object.entries(stats.domainSkillCounts);
-      if (domainEntries.length > 0) {
-        console.log('  Domain skills:');
-        for (const [domain, count] of domainEntries) {
-          console.log(`    ${domain}: ${count}`);
-        }
-      } else {
-        console.log('  Domain skills: (none)');
-      }
-      console.log(`  Last refresh: ${stats.lastRefresh?.toISOString() ?? 'never'}`);
-      const failureEntries = Object.entries(stats.consecutiveFailures);
-      if (failureEntries.length > 0) {
-        console.log('  Consecutive failures:');
-        for (const [id, count] of failureEntries) {
-          console.log(`    ${id}: ${count}`);
-        }
-      }
-    }
-  });
-}
-
-async function skillsRefresh(options: CLIOptions): Promise<void> {
-  await withAtlas(options, async (atlas) => {
-    const lib = requireSkillLibrary(atlas);
-    const result = await lib.refreshTiers();
-
-    if (options.format === 'json') {
-      console.log(JSON.stringify(result, null, 2));
-    } else {
-      console.log('Tier refresh complete.\n');
-      if (result.promoted.length > 0) {
-        console.log(`  Promoted: ${result.promoted.join(', ')}`);
-      }
-      if (result.demoted.length > 0) {
-        console.log(`  Demoted: ${result.demoted.join(', ')}`);
-      }
-      console.log(`  Core count: ${result.coreCount}`);
-      const domainEntries = Object.entries(result.domainCounts);
-      if (domainEntries.length > 0) {
-        console.log('  Domain counts:');
-        for (const [domain, count] of domainEntries) {
-          console.log(`    ${domain}: ${count}`);
-        }
-      }
-    }
-  });
-}

package/src/embeddings/bm25.ts

@@ -1,337 +0,0 @@
-/**
- * BM25 Index
- *
- * A text-based similarity search index using BM25 (Best Match 25) algorithm.
- * Used as a fallback when no embedding provider is configured.
- *
- * BM25 is a ranking function used in information retrieval that considers
- * term frequency, document length, and inverse document frequency.
- */
-
-export type MemoryItemType = 'experience' | 'strategy' | 'concept' | 'skill';
-
-interface Document {
-  id: string;
-  tokens: string[];
-  type: MemoryItemType;
-  content: string;
-}
-
-/**
- * BM25 text similarity search index
- */
-export class BM25Index {
-  private documents: Map<string, Document>;
-  private idf: Map<string, number>;
-  private avgDocLength: number;
-  private k1: number;
-  private b: number;
-  private dirty: boolean;
-
-  constructor(options?: { k1?: number; b?: number }) {
-    this.documents = new Map();
-    this.idf = new Map();
-    this.avgDocLength = 0;
-    // k1 controls term frequency saturation (typical: 1.2-2.0)
-    this.k1 = options?.k1 ?? 1.5;
-    // b controls document length normalization (typical: 0.75)
-    this.b = options?.b ?? 0.75;
-    this.dirty = false;
-  }
-
-  /**
-   * Add a document to the index
-   */
-  add(id: string, content: string, type: MemoryItemType): void {
-    const tokens = this.tokenize(content);
-    this.documents.set(id, { id, tokens, type, content });
-    this.dirty = true;
-  }
-
-  /**
-   * Remove a document from the index
-   */
-  remove(id: string): boolean {
-    const removed = this.documents.delete(id);
-    if (removed) {
-      this.dirty = true;
-    }
-    return removed;
-  }
-
-  /**
-   * Check if a document exists
-   */
-  has(id: string): boolean {
-    return this.documents.has(id);
-  }
-
-  /**
-   * Get document count
-   */
-  get size(): number {
-    return this.documents.size;
-  }
-
-  /**
-   * Search for similar documents
-   */
-  search(
-    query: string,
-    options: {
-      k?: number;
-      type?: MemoryItemType;
-      threshold?: number;
-    } = {}
-  ): Array<{ id: string; score: number; content: string }> {
-    // Rebuild IDF if needed
-    if (this.dirty) {
-      this.rebuildIdf();
-    }
-
-    const queryTokens = this.tokenize(query);
-
-    // Return empty for empty query
-    if (queryTokens.length === 0) {
-      return [];
-    }
-
-    const results: Array<{ id: string; score: number; content: string }> = [];
-
-    for (const doc of this.documents.values()) {
-      // Skip if type filter doesn't match
-      if (options.type && doc.type !== options.type) continue;
-
-      const score = this.computeScore(queryTokens, doc.tokens);
-
-      // Normalize score to 0-1 range (approximate)
-      const normalizedScore = this.normalizeScore(score, queryTokens.length);
-
-      if (normalizedScore >= (options.threshold ?? 0)) {
-        results.push({
-          id: doc.id,
-          score: normalizedScore,
-          content: doc.content,
-        });
-      }
-    }
-
-    return results.sort((a, b) => b.score - a.score).slice(0, options.k ?? 10);
-  }
-
-  /**
-   * Tokenize text for indexing and search
-   */
-  private tokenize(text: string): string[] {
-    return text
-      .toLowerCase()
-      .replace(/[^\w\s]/g, ' ')
-      .split(/\s+/)
-      .filter((t) => t.length > 1)
-      .filter((t) => !STOP_WORDS.has(t));
-  }
-
-  /**
-   * Rebuild IDF values for all terms
-   */
-  private rebuildIdf(): void {
-    this.idf.clear();
-    const N = this.documents.size;
-
-    if (N === 0) {
-      this.avgDocLength = 0;
-      this.dirty = false;
-      return;
-    }
-
-    const termDocCounts = new Map<string, number>();
-    let totalLength = 0;
-
-    for (const doc of this.documents.values()) {
-      totalLength += doc.tokens.length;
-      const uniqueTerms = new Set(doc.tokens);
-      for (const term of uniqueTerms) {
-        termDocCounts.set(term, (termDocCounts.get(term) ?? 0) + 1);
-      }
-    }
-
-    this.avgDocLength = totalLength / N;
-
-    // Calculate IDF for each term
-    for (const [term, df] of termDocCounts) {
-      // IDF formula: log((N - df + 0.5) / (df + 0.5) + 1)
-      this.idf.set(term, Math.log((N - df + 0.5) / (df + 0.5) + 1));
-    }
-
-    this.dirty = false;
-  }
-
-  /**
-   * Compute BM25 score for a document
-   */
-  private computeScore(queryTokens: string[], docTokens: string[]): number {
-    const docLength = docTokens.length;
-
-    // Build term frequency map for document
-    const termFreq = new Map<string, number>();
-    for (const token of docTokens) {
-      termFreq.set(token, (termFreq.get(token) ?? 0) + 1);
-    }
-
-    let score = 0;
-
-    for (const term of queryTokens) {
-      const tf = termFreq.get(term) ?? 0;
-      const idf = this.idf.get(term) ?? 0;
-
-      if (tf === 0 || idf === 0) continue;
-
-      // BM25 formula
-      const numerator = tf * (this.k1 + 1);
-      const denominator =
-        tf +
-        this.k1 * (1 - this.b + this.b * (docLength / this.avgDocLength));
-
-      score += idf * (numerator / denominator);
-    }
-
-    return score;
-  }
-
-  /**
-   * Normalize BM25 score to approximately 0-1 range
-   * This is an approximation since BM25 scores are unbounded
-   */
-  private normalizeScore(score: number, queryLength: number): number {
-    if (score <= 0 || queryLength === 0) return 0;
-
-    // Approximate maximum possible score per term
-    // Assuming perfect TF and high IDF
-    const maxScorePerTerm = 2.5; // Typical upper bound
-    const maxPossibleScore = queryLength * maxScorePerTerm;
-
-    // Sigmoid-like normalization
-    const normalized = score / (score + maxPossibleScore * 0.5);
-
-    return Math.min(1, Math.max(0, normalized));
-  }
-
-  /**
-   * Serialize index to JSON (for persistence)
-   */
-  toJSON(): {
-    documents: Array<{
-      id: string;
-      tokens: string[];
-      type: MemoryItemType;
-      content: string;
-    }>;
-    k1: number;
-    b: number;
-  } {
-    return {
-      documents: Array.from(this.documents.values()),
-      k1: this.k1,
-      b: this.b,
-    };
-  }
-
-  /**
-   * Restore index from JSON
-   */
-  static fromJSON(data: {
-    documents: Array<{
-      id: string;
-      tokens: string[];
-      type: MemoryItemType;
-      content: string;
-    }>;
-    k1: number;
-    b: number;
-  }): BM25Index {
-    const index = new BM25Index({ k1: data.k1, b: data.b });
-    for (const doc of data.documents) {
-      index.documents.set(doc.id, doc);
-    }
-    index.dirty = true;
-    return index;
-  }
-}
-
-/**
- * Common English stop words to filter out
- */
-const STOP_WORDS = new Set([
-  'a',
-  'an',
-  'and',
-  'are',
-  'as',
-  'at',
-  'be',
-  'by',
-  'for',
-  'from',
-  'has',
-  'he',
-  'in',
-  'is',
-  'it',
-  'its',
-  'of',
-  'on',
-  'or',
-  'that',
-  'the',
-  'to',
-  'was',
-  'were',
-  'will',
-  'with',
-  'this',
-  'but',
-  'they',
-  'have',
-  'had',
-  'what',
-  'when',
-  'where',
-  'who',
-  'which',
-  'why',
-  'how',
-  'all',
-  'each',
-  'every',
-  'both',
-  'few',
-  'more',
-  'most',
-  'other',
-  'some',
-  'such',
-  'no',
-  'nor',
-  'not',
-  'only',
-  'own',
-  'same',
-  'so',
-  'than',
-  'too',
-  'very',
-  'can',
-  'just',
-  'should',
-  'now',
-]);
-
-/**
- * Create a new BM25 index
- */
-export function createBM25Index(options?: {
-  k1?: number;
-  b?: number;
-}): BM25Index {
-  return new BM25Index(options);
-}

package/src/embeddings/index.ts

@@ -1,42 +0,0 @@
-/**
- * Embeddings Module
- *
- * Provides embedding generation, storage, and similarity search capabilities.
- * Supports multiple embedding providers (OpenAI, Voyage, local) with automatic
- * fallback to BM25 text search when no provider is configured.
- */
-
-// Provider interface and implementations
-export {
-  type EmbeddingProvider,
-  type EmbeddingProviderConfig,
-  OpenAIEmbeddingProvider,
-  VoyageEmbeddingProvider,
-  LocalEmbeddingProvider,
-  createEmbeddingProvider,
-} from './provider.js';
-
-// BM25 text search (fallback)
-export {
-  BM25Index,
-  createBM25Index,
-  type MemoryItemType,
-} from './bm25.js';
-
-// Vector store using sqlite-vec
-export {
-  SqliteVectorStore,
-  createVectorStore,
-  type VectorSearchResult,
-} from './vector-store.js';
-
-// Inverted index (candidate narrowing)
-export { InvertedIndex } from './inverted-index.js';
-
-// Embedding manager (main interface)
-export {
-  EmbeddingManager,
-  createEmbeddingManager,
-  type EmbeddingManagerOptions,
-  type SearchResult,
-} from './manager.js';

package/src/embeddings/inverted-index.ts

@@ -1,134 +0,0 @@
-/**
- * Inverted Index
- *
- * Lightweight inverted index mapping tokens → document IDs for fast candidate
- * narrowing. Used as an acceleration structure before expensive similarity
- * scoring — not a replacement for scoring.
- *
- * Reuses the same tokenization approach as BM25Index (lowercase, split on
- * whitespace/punctuation, filter stopwords).
- */
-
-/**
- * Inverted index for fast candidate set narrowing
- */
-export class InvertedIndex {
-  private postings: Map<string, Set<string>> = new Map();
-  private docTokens: Map<string, string[]> = new Map();
-
-  /**
-   * Add a document to the index
-   */
-  add(id: string, content: string): void {
-    // Remove existing entry if updating
-    if (this.docTokens.has(id)) {
-      this.remove(id);
-    }
-
-    const tokens = this.tokenize(content);
-    this.docTokens.set(id, tokens);
-
-    for (const token of tokens) {
-      let posting = this.postings.get(token);
-      if (!posting) {
-        posting = new Set();
-        this.postings.set(token, posting);
-      }
-      posting.add(id);
-    }
-  }
-
-  /**
-   * Remove a document from the index
-   */
-  remove(id: string): boolean {
-    const tokens = this.docTokens.get(id);
-    if (!tokens) return false;
-
-    for (const token of tokens) {
-      const posting = this.postings.get(token);
-      if (posting) {
-        posting.delete(id);
-        if (posting.size === 0) {
-          this.postings.delete(token);
-        }
-      }
-    }
-
-    this.docTokens.delete(id);
-    return true;
-  }
-
-  /**
-   * Check if a document exists in the index
-   */
-  has(id: string): boolean {
-    return this.docTokens.has(id);
-  }
-
-  /**
-   * Get candidate document IDs for a query, ranked by number of matching tokens.
-   * Returns at most maxCandidates results.
-   */
-  getCandidates(query: string, maxCandidates = 50): string[] {
-    const queryTokens = this.tokenize(query);
-    if (queryTokens.length === 0) return [];
-
-    // Count matching tokens per document
-    const scores = new Map<string, number>();
-
-    for (const token of queryTokens) {
-      const posting = this.postings.get(token);
-      if (!posting) continue;
-      for (const id of posting) {
-        scores.set(id, (scores.get(id) ?? 0) + 1);
-      }
-    }
-
-    // Sort by match count descending, take top maxCandidates
-    return Array.from(scores.entries())
-      .sort((a, b) => b[1] - a[1])
-      .slice(0, maxCandidates)
-      .map(([id]) => id);
-  }
-
-  /**
-   * Get the number of documents in the index
-   */
-  size(): number {
-    return this.docTokens.size;
-  }
-
-  /**
-   * Clear the entire index
-   */
-  clear(): void {
-    this.postings.clear();
-    this.docTokens.clear();
-  }
-
-  /**
-   * Tokenize text — lowercase, split on non-word chars, filter stopwords and short tokens
-   */
-  private tokenize(text: string): string[] {
-    return text
-      .toLowerCase()
-      .replace(/[^\w\s]/g, ' ')
-      .split(/\s+/)
-      .filter((t) => t.length > 1)
-      .filter((t) => !STOP_WORDS.has(t));
-  }
-}
-
-/**
- * Common English stop words (same set as BM25Index)
- */
-const STOP_WORDS = new Set([
-  'a', 'an', 'and', 'are', 'as', 'at', 'be', 'by', 'for', 'from',
-  'has', 'he', 'in', 'is', 'it', 'its', 'of', 'on', 'or', 'that',
-  'the', 'to', 'was', 'were', 'will', 'with', 'this', 'but', 'they',
-  'have', 'had', 'what', 'when', 'where', 'who', 'which', 'why', 'how',
-  'all', 'each', 'every', 'both', 'few', 'more', 'most', 'other', 'some',
-  'such', 'no', 'nor', 'not', 'only', 'own', 'same', 'so', 'than', 'too',
-  'very', 'can', 'just', 'should', 'now',
-]);