cognitive-core 0.2.5 → 0.3.1

package/src/utils/partitioned-store.ts

@@ -1,299 +0,0 @@
-/**
- * Partitioned Store
- *
- * Domain-partitioned wrapper around JsonStore. Items are routed to partition
- * subdirectories based on a caller-supplied key function. Queries can target
- * a single partition (fast) or scan all partitions (full).
- *
- * On first init with existing flat data (no partition subdirs), items are
- * auto-migrated into partition directories. This is a one-time operation.
- */
-
-import { JsonStore, ensureDir, loadJson, listFiles } from './storage.js';
-import { join } from 'node:path';
-import { existsSync } from 'node:fs';
-import { rename } from 'node:fs/promises';
-
-const GLOBAL_PARTITION = '_global';
-
-export interface PartitionedStoreOptions {
-  autoSaveInterval?: number;
-  pretty?: boolean;
-}
-
-/**
- * Domain-partitioned key-value store backed by JSON files
- */
-export class PartitionedStore<T> {
-  private partitions: Map<string, JsonStore<T>> = new Map();
-  private keyIndex: Map<string, string> = new Map(); // key → partition name
-  private baseDir: string;
-  private collection: string;
-  private partitionKeyFn: (item: T) => string | undefined;
-  private options: PartitionedStoreOptions;
-  private initialized = false;
-
-  constructor(
-    baseDir: string,
-    collection: string,
-    partitionKeyFn: (item: T) => string | undefined,
-    options?: PartitionedStoreOptions,
-  ) {
-    this.baseDir = baseDir;
-    this.collection = collection;
-    this.partitionKeyFn = partitionKeyFn;
-    this.options = options ?? {};
-  }
-
-  /**
-   * Initialize — discover existing partitions, auto-migrate flat data if needed
-   */
-  async init(): Promise<void> {
-    if (this.initialized) return;
-
-    const collectionDir = join(this.baseDir, this.collection);
-    await ensureDir(collectionDir);
-
-    // Check if we need to migrate from flat layout
-    await this.maybeMigrate(collectionDir);
-
-    // Discover partition subdirectories
-    const entries = await listFiles(collectionDir);
-    for (const entry of entries) {
-      const entryPath = join(collectionDir, entry);
-      if (existsSync(entryPath) && !entry.endsWith('.json')) {
-        // It's a directory — treat as partition
-        await this.getOrCreatePartition(entry);
-      }
-    }
-
-    // Ensure global partition exists
-    await this.getOrCreatePartition(GLOBAL_PARTITION);
-
-    this.initialized = true;
-  }
-
-  /**
-   * Get an item by key. If partition is known, searches only that partition.
-   */
-  get(key: string, partition?: string): T | undefined {
-    if (partition) {
-      return this.partitions.get(partition)?.get(key);
-    }
-
-    // Check index first
-    const knownPartition = this.keyIndex.get(key);
-    if (knownPartition) {
-      return this.partitions.get(knownPartition)?.get(key);
-    }
-
-    // Fall back to scanning all partitions
-    for (const store of this.partitions.values()) {
-      const item = store.get(key);
-      if (item !== undefined) return item;
-    }
-    return undefined;
-  }
-
-  /**
-   * Set an item — auto-routes to the correct partition based on partitionKeyFn
-   */
-  set(key: string, value: T): void {
-    const partitionName = this.partitionKeyFn(value) || GLOBAL_PARTITION;
-
-    // If the item was in a different partition, remove it from the old one
-    const oldPartition = this.keyIndex.get(key);
-    if (oldPartition && oldPartition !== partitionName) {
-      this.partitions.get(oldPartition)?.delete(key);
-    }
-
-    const store = this.partitions.get(partitionName);
-    if (store) {
-      store.set(key, value);
-      this.keyIndex.set(key, partitionName);
-    } else {
-      // Partition doesn't exist yet — create it synchronously by using the map
-      // (the directory will be created on flush/save)
-      const newStore = new JsonStore<T>(
-        join(this.baseDir, this.collection),
-        partitionName,
-        this.options,
-      );
-      this.partitions.set(partitionName, newStore);
-      newStore.set(key, value);
-      this.keyIndex.set(key, partitionName);
-    }
-  }
-
-  /**
-   * Delete an item by key
-   */
-  delete(key: string): boolean {
-    const partition = this.keyIndex.get(key);
-    if (partition) {
-      this.keyIndex.delete(key);
-      return this.partitions.get(partition)?.delete(key) ?? false;
-    }
-
-    // Fall back to scanning
-    for (const store of this.partitions.values()) {
-      if (store.has(key)) {
-        store.delete(key);
-        return true;
-      }
-    }
-    return false;
-  }
-
-  /**
-   * Check if a key exists
-   */
-  has(key: string): boolean {
-    if (this.keyIndex.has(key)) return true;
-    for (const store of this.partitions.values()) {
-      if (store.has(key)) return true;
-    }
-    return false;
-  }
-
-  /**
-   * Get all values within a single partition
-   */
-  valuesInPartition(partition: string): T[] {
-    return this.partitions.get(partition)?.values() ?? [];
-  }
-
-  /**
-   * Get all values across all partitions
-   */
-  values(): T[] {
-    const result: T[] = [];
-    for (const store of this.partitions.values()) {
-      result.push(...store.values());
-    }
-    return result;
-  }
-
-  /**
-   * Get all keys across all partitions
-   */
-  keys(): string[] {
-    const result: string[] = [];
-    for (const store of this.partitions.values()) {
-      result.push(...store.keys());
-    }
-    return result;
-  }
-
-  /**
-   * Get total item count across all partitions
-   */
-  size(): number {
-    let total = 0;
-    for (const store of this.partitions.values()) {
-      total += store.size();
-    }
-    return total;
-  }
-
-  /**
-   * Get names of all partitions
-   */
-  getPartitionNames(): string[] {
-    return Array.from(this.partitions.keys());
-  }
-
-  /**
-   * Get all entries across all partitions
-   */
-  entries(): Array<[string, T]> {
-    const result: Array<[string, T]> = [];
-    for (const store of this.partitions.values()) {
-      result.push(...store.entries());
-    }
-    return result;
-  }
-
-  /**
-   * Flush all dirty partitions to disk
-   */
-  async flush(): Promise<void> {
-    for (const store of this.partitions.values()) {
-      await store.flush();
-    }
-  }
-
-  /**
-   * Save a single item immediately
-   */
-  async save(key: string): Promise<void> {
-    const partition = this.keyIndex.get(key);
-    if (partition) {
-      await this.partitions.get(partition)?.save(key);
-    }
-  }
-
-  /**
-   * Close all partitions
-   */
-  async close(): Promise<void> {
-    for (const store of this.partitions.values()) {
-      await store.close();
-    }
-  }
-
-  /**
-   * Get or create a partition's JsonStore, loading its data
-   */
-  private async getOrCreatePartition(name: string): Promise<JsonStore<T>> {
-    let store = this.partitions.get(name);
-    if (store) return store;
-
-    store = new JsonStore<T>(
-      join(this.baseDir, this.collection),
-      name,
-      this.options,
-    );
-    await store.init();
-    this.partitions.set(name, store);
-
-    // Build key index for this partition
-    for (const key of store.keys()) {
-      this.keyIndex.set(key, name);
-    }
-
-    return store;
-  }
-
-  /**
-   * Migrate flat layout (files directly in collection dir) to partitioned layout.
-   * Only runs if flat JSON files exist alongside no partition directories.
-   */
-  private async maybeMigrate(collectionDir: string): Promise<void> {
-    const entries = await listFiles(collectionDir);
-    const jsonFiles = entries.filter(
-      (f) => f.endsWith('.json') && !f.startsWith('_'),
-    );
-    const subdirs = entries.filter((f) => {
-      const p = join(collectionDir, f);
-      return existsSync(p) && !f.endsWith('.json');
-    });
-
-    // Only migrate if there are JSON files and no subdirectories yet
-    if (jsonFiles.length === 0 || subdirs.length > 0) return;
-
-    // Load each file, determine its partition, move it
-    for (const file of jsonFiles) {
-      const filePath = join(collectionDir, file);
-      const data = await loadJson<T>(filePath);
-      if (!data) continue;
-
-      const partitionName = this.partitionKeyFn(data) || GLOBAL_PARTITION;
-      const partitionDir = join(collectionDir, partitionName);
-      await ensureDir(partitionDir);
-
-      const destPath = join(partitionDir, file);
-      await rename(filePath, destPath);
-    }
-  }
-}

package/src/utils/similarity.ts

@@ -1,139 +0,0 @@
-/**
- * Similarity utilities for comparing embeddings and text
- */
-
-/**
- * Compute cosine similarity between two vectors
- */
-export function cosineSimilarity(a: number[], b: number[]): number {
-  if (a.length !== b.length) {
-    throw new Error('Vectors must have the same length');
-  }
-
-  let dotProduct = 0;
-  let normA = 0;
-  let normB = 0;
-
-  for (let i = 0; i < a.length; i++) {
-    dotProduct += a[i] * b[i];
-    normA += a[i] * a[i];
-    normB += b[i] * b[i];
-  }
-
-  normA = Math.sqrt(normA);
-  normB = Math.sqrt(normB);
-
-  if (normA === 0 || normB === 0) {
-    return 0;
-  }
-
-  return dotProduct / (normA * normB);
-}
-
-/**
- * Compute Jaccard similarity between two sets of words
- */
-export function jaccardSimilarity(a: string, b: string): number {
-  const setA = new Set(tokenize(a));
-  const setB = new Set(tokenize(b));
-
-  const intersection = new Set([...setA].filter((x) => setB.has(x)));
-  const union = new Set([...setA, ...setB]);
-
-  if (union.size === 0) {
-    return 0;
-  }
-
-  return intersection.size / union.size;
-}
-
-/**
- * Simple tokenization for text similarity
- */
-export function tokenize(text: string): string[] {
-  return text
-    .toLowerCase()
-    .replace(/[^\w\s]/g, ' ')
-    .split(/\s+/)
-    .filter((token) => token.length > 0);
-}
-
-/**
- * Compute n-gram overlap similarity
- */
-export function ngramSimilarity(
-  a: string,
-  b: string,
-  n = 3
-): number {
-  const ngramsA = getNgrams(a, n);
-  const ngramsB = getNgrams(b, n);
-
-  const intersection = new Set(
-    [...ngramsA].filter((x) => ngramsB.has(x))
-  );
-  const union = new Set([...ngramsA, ...ngramsB]);
-
-  if (union.size === 0) {
-    return 0;
-  }
-
-  return intersection.size / union.size;
-}
-
-/**
- * Get character n-grams from text
- */
-function getNgrams(text: string, n: number): Set<string> {
-  const ngrams = new Set<string>();
-  const normalized = text.toLowerCase();
-
-  for (let i = 0; i <= normalized.length - n; i++) {
-    ngrams.add(normalized.slice(i, i + n));
-  }
-
-  return ngrams;
-}
-
-/**
- * Combined text similarity using multiple methods
- */
-export function textSimilarity(a: string, b: string): number {
-  const jaccard = jaccardSimilarity(a, b);
-  const ngram = ngramSimilarity(a, b);
-
-  // Weighted average
-  return jaccard * 0.6 + ngram * 0.4;
-}
-
-/**
- * Find top-k most similar items
- */
-export function findTopK<T>(
-  query: number[] | string,
-  items: T[],
-  getVector: (item: T) => number[] | string | undefined,
-  k: number
-): Array<{ item: T; score: number }> {
-  const scored = items
-    .map((item) => {
-      const vector = getVector(item);
-      if (!vector) {
-        return { item, score: 0 };
-      }
-
-      let score: number;
-      if (typeof query === 'string' && typeof vector === 'string') {
-        score = textSimilarity(query, vector);
-      } else if (Array.isArray(query) && Array.isArray(vector)) {
-        score = cosineSimilarity(query, vector);
-      } else {
-        score = 0;
-      }
-
-      return { item, score };
-    })
-    .sort((a, b) => b.score - a.score);
-
-  return scored.slice(0, k);
-}

package/src/utils/storage.ts

@@ -1,249 +0,0 @@
-import { readFile, writeFile, mkdir, readdir, stat } from 'node:fs/promises';
-import { join, dirname } from 'node:path';
-import { existsSync } from 'node:fs';
-
-/**
- * Lightweight file-based storage (Claudeception-inspired)
- * Uses JSON files for simplicity and human-readability
- */
-
-export interface StorageOptions {
-  baseDir: string;
-  prettyPrint?: boolean;
-}
-
-/**
- * Ensure directory exists
- */
-export async function ensureDir(path: string): Promise<void> {
-  if (!existsSync(path)) {
-    await mkdir(path, { recursive: true });
-  }
-}
-
-/**
- * Save data to a JSON file
- */
-export async function saveJson<T>(
-  filePath: string,
-  data: T,
-  options?: { pretty?: boolean }
-): Promise<void> {
-  await ensureDir(dirname(filePath));
-  const content = options?.pretty
-    ? JSON.stringify(data, null, 2)
-    : JSON.stringify(data);
-  await writeFile(filePath, content, 'utf-8');
-}
-
-/**
- * Load data from a JSON file
- */
-export async function loadJson<T>(filePath: string): Promise<T | null> {
-  try {
-    const content = await readFile(filePath, 'utf-8');
-    return JSON.parse(content) as T;
-  } catch {
-    return null;
-  }
-}
-
-/**
- * List files in a directory
- */
-export async function listFiles(
-  dir: string,
-  extension?: string
-): Promise<string[]> {
-  try {
-    const files = await readdir(dir);
-    if (extension) {
-      return files.filter((f) => f.endsWith(extension));
-    }
-    return files;
-  } catch {
-    return [];
-  }
-}
-
-/**
- * Check if a path exists
- */
-export async function exists(path: string): Promise<boolean> {
-  try {
-    await stat(path);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-/**
- * Simple key-value store backed by JSON files
- */
-export class JsonStore<T> {
-  private cache: Map<string, T> = new Map();
-  private dirty: Set<string> = new Set();
-  private autoSaveTimer: ReturnType<typeof setInterval> | null = null;
-
-  constructor(
-    private baseDir: string,
-    private collection: string,
-    private options: { autoSaveInterval?: number; pretty?: boolean } = {}
-  ) {}
-
-  /**
-   * Get the file path for a key
-   */
-  private getPath(key: string): string {
-    return join(this.baseDir, this.collection, `${key}.json`);
-  }
-
-  /**
-   * Initialize the store and load existing data
-   */
-  async init(): Promise<void> {
-    const dir = join(this.baseDir, this.collection);
-    await ensureDir(dir);
-
-    // Load existing items
-    const files = await listFiles(dir, '.json');
-    for (const file of files) {
-      const key = file.replace('.json', '');
-      const data = await loadJson<T>(join(dir, file));
-      if (data) {
-        this.cache.set(key, data);
-      }
-    }
-
-    // Start auto-save if configured
-    if (this.options.autoSaveInterval) {
-      this.autoSaveTimer = setInterval(
-        () => this.flush(),
-        this.options.autoSaveInterval
-      );
-    }
-  }
-
-  /**
-   * Get an item by key
-   */
-  get(key: string): T | undefined {
-    return this.cache.get(key);
-  }
-
-  /**
-   * Set an item
-   */
-  set(key: string, value: T): void {
-    this.cache.set(key, value);
-    this.dirty.add(key);
-  }
-
-  /**
-   * Delete an item
-   */
-  delete(key: string): boolean {
-    this.dirty.add(key);
-    return this.cache.delete(key);
-  }
-
-  /**
-   * Check if key exists
-   */
-  has(key: string): boolean {
-    return this.cache.has(key);
-  }
-
-  /**
-   * Get all items
-   */
-  values(): T[] {
-    return Array.from(this.cache.values());
-  }
-
-  /**
-   * Get all keys
-   */
-  keys(): string[] {
-    return Array.from(this.cache.keys());
-  }
-
-  /**
-   * Get all entries
-   */
-  entries(): Array<[string, T]> {
-    return Array.from(this.cache.entries());
-  }
-
-  /**
-   * Get item count
-   */
-  size(): number {
-    return this.cache.size;
-  }
-
-  /**
-   * Flush dirty items to disk
-   */
-  async flush(): Promise<void> {
-    for (const key of this.dirty) {
-      const value = this.cache.get(key);
-      const path = this.getPath(key);
-
-      if (value === undefined) {
-        // Item was deleted - we could delete the file too
-        // For now, we'll leave it (soft delete)
-        continue;
-      }
-
-      await saveJson(path, value, { pretty: this.options.pretty });
-    }
-    this.dirty.clear();
-  }
-
-  /**
-   * Save a single item immediately
-   */
-  async save(key: string): Promise<void> {
-    const value = this.cache.get(key);
-    if (value) {
-      await saveJson(this.getPath(key), value, { pretty: this.options.pretty });
-      this.dirty.delete(key);
-    }
-  }
-
-  /**
-   * Close the store (flush and stop auto-save)
-   */
-  async close(): Promise<void> {
-    if (this.autoSaveTimer) {
-      clearInterval(this.autoSaveTimer);
-      this.autoSaveTimer = null;
-    }
-    await this.flush();
-  }
-}
-
-/**
- * Create a store index file (for quick lookups without loading all items)
- */
-export async function saveIndex(
-  baseDir: string,
-  collection: string,
-  index: Record<string, unknown>
-): Promise<void> {
-  const path = join(baseDir, collection, '_index.json');
-  await saveJson(path, index, { pretty: true });
-}
-
-/**
- * Load a store index
- */
-export async function loadIndex(
-  baseDir: string,
-  collection: string
-): Promise<Record<string, unknown> | null> {
-  const path = join(baseDir, collection, '_index.json');
-  return loadJson(path);
-}