cognitive-core 0.0.2 → 0.1.0

package/src/utils/storage.ts

@@ -0,0 +1,249 @@
+import { readFile, writeFile, mkdir, readdir, stat } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { existsSync } from 'node:fs';
+
+/**
+ * Lightweight file-based storage (Claudeception-inspired)
+ * Uses JSON files for simplicity and human-readability
+ */
+
+export interface StorageOptions {
+  baseDir: string;
+  prettyPrint?: boolean;
+}
+
+/**
+ * Ensure directory exists
+ */
+export async function ensureDir(path: string): Promise<void> {
+  if (!existsSync(path)) {
+    await mkdir(path, { recursive: true });
+  }
+}
+
+/**
+ * Save data to a JSON file
+ */
+export async function saveJson<T>(
+  filePath: string,
+  data: T,
+  options?: { pretty?: boolean }
+): Promise<void> {
+  await ensureDir(dirname(filePath));
+  const content = options?.pretty
+    ? JSON.stringify(data, null, 2)
+    : JSON.stringify(data);
+  await writeFile(filePath, content, 'utf-8');
+}
+
+/**
+ * Load data from a JSON file
+ */
+export async function loadJson<T>(filePath: string): Promise<T | null> {
+  try {
+    const content = await readFile(filePath, 'utf-8');
+    return JSON.parse(content) as T;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * List files in a directory
+ */
+export async function listFiles(
+  dir: string,
+  extension?: string
+): Promise<string[]> {
+  try {
+    const files = await readdir(dir);
+    if (extension) {
+      return files.filter((f) => f.endsWith(extension));
+    }
+    return files;
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Check if a path exists
+ */
+export async function exists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Simple key-value store backed by JSON files
+ */
+export class JsonStore<T> {
+  private cache: Map<string, T> = new Map();
+  private dirty: Set<string> = new Set();
+  private autoSaveTimer: ReturnType<typeof setInterval> | null = null;
+
+  constructor(
+    private baseDir: string,
+    private collection: string,
+    private options: { autoSaveInterval?: number; pretty?: boolean } = {}
+  ) {}
+
+  /**
+   * Get the file path for a key
+   */
+  private getPath(key: string): string {
+    return join(this.baseDir, this.collection, `${key}.json`);
+  }
+
+  /**
+   * Initialize the store and load existing data
+   */
+  async init(): Promise<void> {
+    const dir = join(this.baseDir, this.collection);
+    await ensureDir(dir);
+
+    // Load existing items
+    const files = await listFiles(dir, '.json');
+    for (const file of files) {
+      const key = file.replace('.json', '');
+      const data = await loadJson<T>(join(dir, file));
+      if (data) {
+        this.cache.set(key, data);
+      }
+    }
+
+    // Start auto-save if configured
+    if (this.options.autoSaveInterval) {
+      this.autoSaveTimer = setInterval(
+        () => this.flush(),
+        this.options.autoSaveInterval
+      );
+    }
+  }
+
+  /**
+   * Get an item by key
+   */
+  get(key: string): T | undefined {
+    return this.cache.get(key);
+  }
+
+  /**
+   * Set an item
+   */
+  set(key: string, value: T): void {
+    this.cache.set(key, value);
+    this.dirty.add(key);
+  }
+
+  /**
+   * Delete an item
+   */
+  delete(key: string): boolean {
+    this.dirty.add(key);
+    return this.cache.delete(key);
+  }
+
+  /**
+   * Check if key exists
+   */
+  has(key: string): boolean {
+    return this.cache.has(key);
+  }
+
+  /**
+   * Get all items
+   */
+  values(): T[] {
+    return Array.from(this.cache.values());
+  }
+
+  /**
+   * Get all keys
+   */
+  keys(): string[] {
+    return Array.from(this.cache.keys());
+  }
+
+  /**
+   * Get all entries
+   */
+  entries(): Array<[string, T]> {
+    return Array.from(this.cache.entries());
+  }
+
+  /**
+   * Get item count
+   */
+  size(): number {
+    return this.cache.size;
+  }
+
+  /**
+   * Flush dirty items to disk
+   */
+  async flush(): Promise<void> {
+    for (const key of this.dirty) {
+      const value = this.cache.get(key);
+      const path = this.getPath(key);
+
+      if (value === undefined) {
+        // Item was deleted - we could delete the file too
+        // For now, we'll leave it (soft delete)
+        continue;
+      }
+
+      await saveJson(path, value, { pretty: this.options.pretty });
+    }
+    this.dirty.clear();
+  }
+
+  /**
+   * Save a single item immediately
+   */
+  async save(key: string): Promise<void> {
+    const value = this.cache.get(key);
+    if (value) {
+      await saveJson(this.getPath(key), value, { pretty: this.options.pretty });
+      this.dirty.delete(key);
+    }
+  }
+
+  /**
+   * Close the store (flush and stop auto-save)
+   */
+  async close(): Promise<void> {
+    if (this.autoSaveTimer) {
+      clearInterval(this.autoSaveTimer);
+      this.autoSaveTimer = null;
+    }
+    await this.flush();
+  }
+}
+
+/**
+ * Create a store index file (for quick lookups without loading all items)
+ */
+export async function saveIndex(
+  baseDir: string,
+  collection: string,
+  index: Record<string, unknown>
+): Promise<void> {
+  const path = join(baseDir, collection, '_index.json');
+  await saveJson(path, index, { pretty: true });
+}
+
+/**
+ * Load a store index
+ */
+export async function loadIndex(
+  baseDir: string,
+  collection: string
+): Promise<Record<string, unknown> | null> {
+  const path = join(baseDir, collection, '_index.json');
+  return loadJson(path);
+}

package/src/utils/validation.ts

@@ -0,0 +1,286 @@
+/**
+ * Input validation utilities
+ * Provides validation helpers for API boundaries
+ */
+
+import { z } from 'zod';
+
+/**
+ * Validation error with details
+ */
+export class ValidationError extends Error {
+  constructor(
+    message: string,
+    public readonly field: string,
+    public readonly value: unknown
+  ) {
+    super(`Validation error for '${field}': ${message}`);
+    this.name = 'ValidationError';
+  }
+}
+
+/**
+ * Validate a string field
+ */
+export function validateString(
+  value: unknown,
+  field: string,
+  options?: {
+    minLength?: number;
+    maxLength?: number;
+    pattern?: RegExp;
+    allowEmpty?: boolean;
+  }
+): string {
+  if (typeof value !== 'string') {
+    throw new ValidationError(`Expected string, got ${typeof value}`, field, value);
+  }
+
+  const str = value;
+  const opts = options ?? {};
+
+  if (!opts.allowEmpty && str.trim().length === 0) {
+    throw new ValidationError('Cannot be empty', field, value);
+  }
+
+  if (opts.minLength !== undefined && str.length < opts.minLength) {
+    throw new ValidationError(`Must be at least ${opts.minLength} characters`, field, value);
+  }
+
+  if (opts.maxLength !== undefined && str.length > opts.maxLength) {
+    throw new ValidationError(`Must be at most ${opts.maxLength} characters`, field, value);
+  }
+
+  if (opts.pattern && !opts.pattern.test(str)) {
+    throw new ValidationError(`Does not match required pattern`, field, value);
+  }
+
+  return str;
+}
+
+/**
+ * Validate an optional string field
+ */
+export function validateOptionalString(
+  value: unknown,
+  field: string,
+  options?: {
+    minLength?: number;
+    maxLength?: number;
+    pattern?: RegExp;
+  }
+): string | undefined {
+  if (value === undefined || value === null) {
+    return undefined;
+  }
+  return validateString(value, field, options);
+}
+
+/**
+ * Validate a number field
+ */
+export function validateNumber(
+  value: unknown,
+  field: string,
+  options?: {
+    min?: number;
+    max?: number;
+    integer?: boolean;
+  }
+): number {
+  if (typeof value !== 'number' || isNaN(value)) {
+    throw new ValidationError(`Expected number, got ${typeof value}`, field, value);
+  }
+
+  const opts = options ?? {};
+
+  if (opts.integer && !Number.isInteger(value)) {
+    throw new ValidationError('Must be an integer', field, value);
+  }
+
+  if (opts.min !== undefined && value < opts.min) {
+    throw new ValidationError(`Must be at least ${opts.min}`, field, value);
+  }
+
+  if (opts.max !== undefined && value > opts.max) {
+    throw new ValidationError(`Must be at most ${opts.max}`, field, value);
+  }
+
+  return value;
+}
+
+/**
+ * Validate an optional number field
+ */
+export function validateOptionalNumber(
+  value: unknown,
+  field: string,
+  options?: {
+    min?: number;
+    max?: number;
+    integer?: boolean;
+  }
+): number | undefined {
+  if (value === undefined || value === null) {
+    return undefined;
+  }
+  return validateNumber(value, field, options);
+}
+
+/**
+ * Validate an array field
+ */
+export function validateArray<T>(
+  value: unknown,
+  field: string,
+  itemValidator: (item: unknown, index: number) => T,
+  options?: {
+    minLength?: number;
+    maxLength?: number;
+  }
+): T[] {
+  if (!Array.isArray(value)) {
+    throw new ValidationError(`Expected array, got ${typeof value}`, field, value);
+  }
+
+  const opts = options ?? {};
+
+  if (opts.minLength !== undefined && value.length < opts.minLength) {
+    throw new ValidationError(`Must have at least ${opts.minLength} items`, field, value);
+  }
+
+  if (opts.maxLength !== undefined && value.length > opts.maxLength) {
+    throw new ValidationError(`Must have at most ${opts.maxLength} items`, field, value);
+  }
+
+  return value.map((item, index) => itemValidator(item, index));
+}
+
+/**
+ * Validate a vector (array of numbers) for embeddings
+ */
+export function validateVector(
+  value: unknown,
+  field: string,
+  expectedDimension?: number
+): number[] {
+  if (!Array.isArray(value)) {
+    throw new ValidationError(`Expected array, got ${typeof value}`, field, value);
+  }
+
+  for (let i = 0; i < value.length; i++) {
+    if (typeof value[i] !== 'number' || isNaN(value[i])) {
+      throw new ValidationError(
+        `Element at index ${i} is not a valid number`,
+        field,
+        value[i]
+      );
+    }
+  }
+
+  if (expectedDimension !== undefined && value.length !== expectedDimension) {
+    throw new ValidationError(
+      `Expected dimension ${expectedDimension}, got ${value.length}`,
+      field,
+      value
+    );
+  }
+
+  return value as number[];
+}
+
+/**
+ * Validate task input parameters
+ */
+export const TaskInputSchema = z.object({
+  description: z.string().min(1, 'Task description cannot be empty'),
+  domain: z.string().default('general'),
+  context: z.record(z.unknown()).default({}),
+  constraints: z.array(z.string()).default([]),
+  metadata: z.record(z.unknown()).default({}),
+});
+
+export type TaskInput = z.infer<typeof TaskInputSchema>;
+
+/**
+ * Validate task input
+ */
+export function validateTaskInput(input: unknown): TaskInput {
+  const result = TaskInputSchema.safeParse(input);
+  if (!result.success) {
+    const firstError = result.error.errors[0];
+    throw new ValidationError(
+      firstError.message,
+      firstError.path.join('.'),
+      input
+    );
+  }
+  return result.data;
+}
+
+/**
+ * Validate trajectory step count
+ */
+export function validateTrajectoryStepCount(
+  stepCount: number,
+  options?: {
+    minSteps?: number;
+    maxSteps?: number;
+  }
+): void {
+  const opts = options ?? { minSteps: 1, maxSteps: 10000 };
+
+  if (stepCount < (opts.minSteps ?? 1)) {
+    throw new ValidationError(
+      `Must have at least ${opts.minSteps ?? 1} steps`,
+      'steps',
+      stepCount
+    );
+  }
+
+  if (opts.maxSteps !== undefined && stepCount > opts.maxSteps) {
+    throw new ValidationError(
+      `Must have at most ${opts.maxSteps} steps`,
+      'steps',
+      stepCount
+    );
+  }
+}
+
+/**
+ * MCP tool argument validation schemas
+ */
+export const MCPSearchPlaybooksSchema = z.object({
+  query: z.string().min(1, 'Query cannot be empty'),
+  domain: z.string().optional(),
+  maxResults: z.number().int().min(1).max(50).optional(),
+});
+
+export const MCPGetPlaybookDetailsSchema = z.object({
+  playbookId: z.string().min(1, 'Playbook ID cannot be empty'),
+});
+
+export const MCPGetPlaybooksByDomainSchema = z.object({
+  domain: z.string().min(1, 'Domain cannot be empty'),
+  maxResults: z.number().int().min(1).max(100).optional(),
+});
+
+/**
+ * Validate MCP tool arguments
+ */
+export function validateMCPArgs<T>(
+  schema: z.ZodSchema<T>,
+  args: unknown,
+  toolName: string
+): T {
+  const result = schema.safeParse(args);
+  if (!result.success) {
+    const firstError = result.error.errors[0];
+    throw new ValidationError(
+      `Invalid arguments for ${toolName}: ${firstError.message}`,
+      firstError.path.join('.') || 'arguments',
+      args
+    );
+  }
+  return result.data;
+}

package/tests/embeddings/bm25.test.ts

@@ -0,0 +1,130 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import { BM25Index, createBM25Index } from '../../src/embeddings/bm25.js';
+
+describe('BM25Index', () => {
+  let index: BM25Index;
+
+  beforeEach(() => {
+    index = createBM25Index();
+  });
+
+  describe('basic operations', () => {
+    it('should add and retrieve documents', () => {
+      index.add('doc1', 'hello world', 'experience');
+      expect(index.has('doc1')).toBe(true);
+      expect(index.size).toBe(1);
+    });
+
+    it('should remove documents', () => {
+      index.add('doc1', 'hello world', 'experience');
+      expect(index.remove('doc1')).toBe(true);
+      expect(index.has('doc1')).toBe(false);
+      expect(index.size).toBe(0);
+    });
+
+    it('should handle non-existent document removal', () => {
+      expect(index.remove('non-existent')).toBe(false);
+    });
+  });
+
+  describe('search', () => {
+    beforeEach(() => {
+      index.add('doc1', 'machine learning algorithms for classification', 'experience');
+      index.add('doc2', 'deep learning neural networks', 'strategy');
+      index.add('doc3', 'database query optimization techniques', 'concept');
+      index.add('doc4', 'machine learning for natural language processing', 'skill');
+    });
+
+    it('should find relevant documents', () => {
+      const results = index.search('machine learning');
+      expect(results.length).toBeGreaterThan(0);
+
+      // Machine learning docs should be ranked higher
+      const ids = results.map(r => r.id);
+      expect(ids).toContain('doc1');
+      expect(ids).toContain('doc4');
+    });
+
+    it('should filter by type', () => {
+      const results = index.search('learning', { type: 'experience' });
+      expect(results.every(r => r.content.includes('machine'))).toBe(true);
+    });
+
+    it('should respect k limit', () => {
+      const results = index.search('learning', { k: 2 });
+      expect(results.length).toBeLessThanOrEqual(2);
+    });
+
+    it('should return scores in descending order', () => {
+      const results = index.search('machine learning');
+      for (let i = 1; i < results.length; i++) {
+        expect(results[i - 1].score).toBeGreaterThanOrEqual(results[i].score);
+      }
+    });
+
+    it('should return normalized scores between 0 and 1', () => {
+      const results = index.search('machine learning');
+      for (const result of results) {
+        expect(result.score).toBeGreaterThanOrEqual(0);
+        expect(result.score).toBeLessThanOrEqual(1);
+      }
+    });
+  });
+
+  describe('serialization', () => {
+    it('should serialize and deserialize correctly', () => {
+      index.add('doc1', 'hello world', 'experience');
+      index.add('doc2', 'goodbye world', 'strategy');
+
+      const json = index.toJSON();
+      const restored = BM25Index.fromJSON(json);
+
+      expect(restored.size).toBe(2);
+      expect(restored.has('doc1')).toBe(true);
+      expect(restored.has('doc2')).toBe(true);
+
+      // Search should work after restore
+      const results = restored.search('world');
+      expect(results.length).toBe(2);
+    });
+  });
+
+  describe('edge cases', () => {
+    it('should handle empty index', () => {
+      const results = index.search('anything');
+      expect(results).toEqual([]);
+    });
+
+    it('should handle empty query', () => {
+      index.add('doc1', 'hello world', 'experience');
+      const results = index.search('');
+      expect(results.length).toBe(0);
+    });
+
+    it('should handle special characters', () => {
+      index.add('doc1', 'hello! @world# $test%', 'experience');
+      const results = index.search('hello world test');
+      expect(results.length).toBe(1);
+    });
+
+    it('should filter stop words', () => {
+      index.add('doc1', 'the quick brown fox', 'experience');
+      index.add('doc2', 'a lazy brown dog', 'experience');
+
+      // "the" and "a" are stop words, so searching for them shouldn't find anything
+      // but "brown" should match both
+      const results = index.search('brown');
+      expect(results.length).toBe(2);
+    });
+  });
+
+  describe('BM25 parameters', () => {
+    it('should accept custom k1 and b parameters', () => {
+      const customIndex = createBM25Index({ k1: 2.0, b: 0.5 });
+      customIndex.add('doc1', 'test document', 'experience');
+
+      const results = customIndex.search('test');
+      expect(results.length).toBe(1);
+    });
+  });
+});