cognitive-core 0.1.0 → 0.2.0

package/src/types/trajectory-source.ts

@@ -0,0 +1,47 @@
+import type { Trajectory } from './trajectory.js';
+import type { AnalysisResult } from '../learning/analyzer.js';
+
+// ============================================================================
+// Trajectory Source — formal interface for loading trajectories
+// ============================================================================
+
+/**
+ * A trajectory paired with its optional pre-computed analysis.
+ * If analysis is missing, the consumer should compute it.
+ */
+export interface TrajectoryWithAnalysis {
+  trajectory: Trajectory;
+  analysis?: AnalysisResult;
+}
+
+/**
+ * Options for filtering trajectories when loading.
+ */
+export interface TrajectoryLoadOptions {
+  /** Only load trajectories created after this date */
+  since?: Date;
+  /** Filter to a specific domain */
+  domain?: string;
+  /** Maximum number of trajectories to return */
+  limit?: number;
+  /** Only include successful trajectories */
+  successOnly?: boolean;
+}
+
+/**
+ * TrajectorySource — abstract interface for loading trajectories from
+ * different backends (in-memory arrays, filesystem, pipeline accumulators, etc.).
+ *
+ * Consumers of this interface (e.g. KnowledgeBank.extractFrom) don't care
+ * where trajectories come from — they just load and process them.
+ */
+export interface TrajectorySource {
+  /** Human-readable name for this source */
+  readonly name: string;
+
+  /** Load trajectories, optionally filtered */
+  load(options?: TrajectoryLoadOptions): Promise<TrajectoryWithAnalysis[]>;
+
+  /** Count available trajectories without loading them */
+  count(options?: TrajectoryLoadOptions): Promise<number>;
+}

package/src/utils/error-classifier.ts

@@ -0,0 +1,113 @@
+/**
+ * Unified error classification utility.
+ *
+ * Provides a single canonical error type mapping used by TrajectoryAnalyzer,
+ * ReflexionGenerator, KnowledgeExtractor, and other learning components.
+ */
+
+/**
+ * Canonical error types recognized by the classifier.
+ */
+export type ErrorType =
+  | 'type-error'
+  | 'import-error'
+  | 'syntax-error'
+  | 'timeout'
+  | 'permission-error'
+  | 'not-found'
+  | 'network-error'
+  | 'test-failure'
+  | 'assertion-error'
+  | 'general-error';
+
+/**
+ * Result of classifying an error string.
+ */
+export interface ErrorClassification {
+  /** Canonical error type */
+  type: ErrorType;
+  /** Human-readable description */
+  description: string;
+}
+
+/**
+ * Ordered list of error classification rules.
+ * More specific patterns come first; 'general-error' is the fallback.
+ */
+const ERROR_RULES: Array<{ test: (s: string) => boolean; type: ErrorType; description: string }> = [
+  {
+    test: (s) => /type\s*error/i.test(s) || (s.includes('type') && s.includes('error')),
+    type: 'type-error',
+    description: 'Type mismatch error',
+  },
+  {
+    test: (s) => /import|module not found|cannot find module/i.test(s),
+    type: 'import-error',
+    description: 'Import or module resolution error',
+  },
+  {
+    test: (s) => /syntax\s*error/i.test(s),
+    type: 'syntax-error',
+    description: 'Syntax error in code',
+  },
+  {
+    test: (s) => /timeout|timed?\s*out/i.test(s),
+    type: 'timeout',
+    description: 'Operation timed out',
+  },
+  {
+    test: (s) => /permission\s*denied|forbidden|unauthorized/i.test(s),
+    type: 'permission-error',
+    description: 'Permission denied',
+  },
+  {
+    test: (s) => /not\s*found|missing|no\s*such\s*file/i.test(s),
+    type: 'not-found',
+    description: 'Resource not found',
+  },
+  {
+    test: (s) => /network|connection|ECONNREFUSED|ENOTFOUND/i.test(s),
+    type: 'network-error',
+    description: 'Network or connection error',
+  },
+  {
+    test: (s) => /test.*fail|fail.*test|assertion|assert/i.test(s),
+    type: 'test-failure',
+    description: 'Test or assertion failure',
+  },
+  {
+    test: (s) => /assertion|assert/i.test(s),
+    type: 'assertion-error',
+    description: 'Assertion failed',
+  },
+  {
+    test: (s) => /exception|error/i.test(s),
+    type: 'general-error',
+    description: 'General error',
+  },
+];
+
+/**
+ * Classify an error message into a canonical error type.
+ * Returns `null` if no pattern matches.
+ */
+export function classifyError(errorInfo: string): ErrorClassification | null {
+  if (!errorInfo) return null;
+
+  for (const rule of ERROR_RULES) {
+    if (rule.test(errorInfo)) {
+      return { type: rule.type, description: rule.description };
+    }
+  }
+
+  return null;
+}
+
+/**
+ * Get just the error type string (convenience wrapper).
+ * Returns `null` if no pattern matches.
+ */
+export function classifyErrorType(errorInfo: string): ErrorType | null {
+  const result = classifyError(errorInfo);
+  return result?.type ?? null;
+}

package/src/utils/frontmatter.ts

@@ -0,0 +1,118 @@
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
+import {
+  KnowledgeNoteFrontmatterSchema,
+  type KnowledgeNoteFrontmatter,
+  type KnowledgeNote,
+} from '../types/knowledge.js';
+
+const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/;
+
+/**
+ * Parse markdown content with YAML frontmatter into a KnowledgeNoteFrontmatter + body.
+ *
+ * Uses the `yaml` npm package for full-fidelity parsing (nested objects,
+ * arrays of objects, etc.). Validates against the Zod schema.
+ *
+ * Returns null frontmatter if the content has no frontmatter block or
+ * if the frontmatter fails validation (logged, not thrown).
+ */
+export function parseKnowledgeFrontmatter(content: string): {
+  frontmatter: KnowledgeNoteFrontmatter | null;
+  body: string;
+} {
+  const match = content.match(FRONTMATTER_REGEX);
+  if (!match) {
+    return { frontmatter: null, body: content };
+  }
+
+  const [, yamlBlock, body] = match;
+
+  try {
+    const raw = parseYaml(yamlBlock);
+    if (!raw || typeof raw !== 'object') {
+      return { frontmatter: null, body };
+    }
+
+    const parsed = KnowledgeNoteFrontmatterSchema.safeParse(raw);
+    if (!parsed.success) {
+      console.warn(
+        `[frontmatter] Validation failed for note: ${JSON.stringify(parsed.error.issues)}`
+      );
+      return { frontmatter: null, body };
+    }
+
+    return { frontmatter: parsed.data, body: body.trim() };
+  } catch (error) {
+    console.warn(
+      `[frontmatter] YAML parse error: ${error instanceof Error ? error.message : String(error)}`
+    );
+    return { frontmatter: null, body: content };
+  }
+}
+
+/**
+ * Serialize a KnowledgeNoteFrontmatter into a YAML frontmatter block.
+ *
+ * Keeps simple top-level fields (created, updated, tags) in a format
+ * that minimem's simpler parser can partially read.
+ */
+export function serializeKnowledgeFrontmatter(
+  frontmatter: KnowledgeNoteFrontmatter
+): string {
+  // Build a clean object for serialization, omitting null/undefined/empty values
+  const obj: Record<string, unknown> = {
+    id: frontmatter.id,
+    type: frontmatter.type,
+  };
+
+  if (frontmatter.domain.length > 0) obj.domain = frontmatter.domain;
+  if (frontmatter.entities.length > 0) obj.entities = frontmatter.entities;
+  if (frontmatter.tags.length > 0) obj.tags = frontmatter.tags;
+
+  obj.confidence = frontmatter.confidence;
+  obj.source = frontmatter.source;
+
+  if (frontmatter.links.length > 0) obj.links = frontmatter.links;
+
+  obj.created = frontmatter.created;
+  obj.updated = frontmatter.updated;
+
+  if (frontmatter.supersedes) obj.supersedes = frontmatter.supersedes;
+
+  const yaml = stringifyYaml(obj, {
+    lineWidth: 120,
+    defaultKeyType: 'PLAIN',
+    defaultStringType: 'PLAIN',
+  });
+
+  return `---\n${yaml}---\n`;
+}
+
+/**
+ * Serialize a full KnowledgeNote (frontmatter + body) into markdown content.
+ */
+export function serializeKnowledgeNote(note: KnowledgeNote): string {
+  const header = serializeKnowledgeFrontmatter(note.frontmatter);
+  return `${header}\n${note.body}\n`;
+}
+
+/**
+ * Parse a full markdown file into a KnowledgeNote.
+ * Returns null if frontmatter is missing or invalid.
+ *
+ * @param content - The raw markdown file content
+ * @param filePath - The file path (stored on the note for reference)
+ */
+export function parseKnowledgeNote(
+  content: string,
+  filePath: string
+): KnowledgeNote | null {
+  const { frontmatter, body } = parseKnowledgeFrontmatter(content);
+  if (!frontmatter) return null;
+
+  return {
+    frontmatter,
+    body,
+    filePath,
+  };
+}

package/src/utils/index.ts

@@ -19,6 +19,11 @@ export {
   type StorageOptions,
 } from './storage.js';
 
+export {
+  PartitionedStore,
+  type PartitionedStoreOptions,
+} from './partitioned-store.js';
+
 export {
   ValidationError,
   validateString,
@@ -36,3 +41,23 @@ export {
   MCPGetPlaybooksByDomainSchema,
   type TaskInput,
 } from './validation.js';
+
+export {
+  parseKnowledgeFrontmatter,
+  serializeKnowledgeFrontmatter,
+  serializeKnowledgeNote,
+  parseKnowledgeNote,
+} from './frontmatter.js';
+
+export {
+  classifyError,
+  classifyErrorType,
+  type ErrorType,
+  type ErrorClassification,
+} from './error-classifier.js';
+
+export {
+  detectRepeatedActions,
+  extractToolNames,
+  type RepeatedAction,
+} from './trajectory-helpers.js';

package/src/utils/partitioned-store.ts

@@ -0,0 +1,299 @@
+/**
+ * Partitioned Store
+ *
+ * Domain-partitioned wrapper around JsonStore. Items are routed to partition
+ * subdirectories based on a caller-supplied key function. Queries can target
+ * a single partition (fast) or scan all partitions (full).
+ *
+ * On first init with existing flat data (no partition subdirs), items are
+ * auto-migrated into partition directories. This is a one-time operation.
+ */
+
+import { JsonStore, ensureDir, loadJson, listFiles } from './storage.js';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { rename } from 'node:fs/promises';
+
+const GLOBAL_PARTITION = '_global';
+
+export interface PartitionedStoreOptions {
+  autoSaveInterval?: number;
+  pretty?: boolean;
+}
+
+/**
+ * Domain-partitioned key-value store backed by JSON files
+ */
+export class PartitionedStore<T> {
+  private partitions: Map<string, JsonStore<T>> = new Map();
+  private keyIndex: Map<string, string> = new Map(); // key → partition name
+  private baseDir: string;
+  private collection: string;
+  private partitionKeyFn: (item: T) => string | undefined;
+  private options: PartitionedStoreOptions;
+  private initialized = false;
+
+  constructor(
+    baseDir: string,
+    collection: string,
+    partitionKeyFn: (item: T) => string | undefined,
+    options?: PartitionedStoreOptions,
+  ) {
+    this.baseDir = baseDir;
+    this.collection = collection;
+    this.partitionKeyFn = partitionKeyFn;
+    this.options = options ?? {};
+  }
+
+  /**
+   * Initialize — discover existing partitions, auto-migrate flat data if needed
+   */
+  async init(): Promise<void> {
+    if (this.initialized) return;
+
+    const collectionDir = join(this.baseDir, this.collection);
+    await ensureDir(collectionDir);
+
+    // Check if we need to migrate from flat layout
+    await this.maybeMigrate(collectionDir);
+
+    // Discover partition subdirectories
+    const entries = await listFiles(collectionDir);
+    for (const entry of entries) {
+      const entryPath = join(collectionDir, entry);
+      if (existsSync(entryPath) && !entry.endsWith('.json')) {
+        // It's a directory — treat as partition
+        await this.getOrCreatePartition(entry);
+      }
+    }
+
+    // Ensure global partition exists
+    await this.getOrCreatePartition(GLOBAL_PARTITION);
+
+    this.initialized = true;
+  }
+
+  /**
+   * Get an item by key. If partition is known, searches only that partition.
+   */
+  get(key: string, partition?: string): T | undefined {
+    if (partition) {
+      return this.partitions.get(partition)?.get(key);
+    }
+
+    // Check index first
+    const knownPartition = this.keyIndex.get(key);
+    if (knownPartition) {
+      return this.partitions.get(knownPartition)?.get(key);
+    }
+
+    // Fall back to scanning all partitions
+    for (const store of this.partitions.values()) {
+      const item = store.get(key);
+      if (item !== undefined) return item;
+    }
+    return undefined;
+  }
+
+  /**
+   * Set an item — auto-routes to the correct partition based on partitionKeyFn
+   */
+  set(key: string, value: T): void {
+    const partitionName = this.partitionKeyFn(value) || GLOBAL_PARTITION;
+
+    // If the item was in a different partition, remove it from the old one
+    const oldPartition = this.keyIndex.get(key);
+    if (oldPartition && oldPartition !== partitionName) {
+      this.partitions.get(oldPartition)?.delete(key);
+    }
+
+    const store = this.partitions.get(partitionName);
+    if (store) {
+      store.set(key, value);
+      this.keyIndex.set(key, partitionName);
+    } else {
+      // Partition doesn't exist yet — create it synchronously by using the map
+      // (the directory will be created on flush/save)
+      const newStore = new JsonStore<T>(
+        join(this.baseDir, this.collection),
+        partitionName,
+        this.options,
+      );
+      this.partitions.set(partitionName, newStore);
+      newStore.set(key, value);
+      this.keyIndex.set(key, partitionName);
+    }
+  }
+
+  /**
+   * Delete an item by key
+   */
+  delete(key: string): boolean {
+    const partition = this.keyIndex.get(key);
+    if (partition) {
+      this.keyIndex.delete(key);
+      return this.partitions.get(partition)?.delete(key) ?? false;
+    }
+
+    // Fall back to scanning
+    for (const store of this.partitions.values()) {
+      if (store.has(key)) {
+        store.delete(key);
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Check if a key exists
+   */
+  has(key: string): boolean {
+    if (this.keyIndex.has(key)) return true;
+    for (const store of this.partitions.values()) {
+      if (store.has(key)) return true;
+    }
+    return false;
+  }
+
+  /**
+   * Get all values within a single partition
+   */
+  valuesInPartition(partition: string): T[] {
+    return this.partitions.get(partition)?.values() ?? [];
+  }
+
+  /**
+   * Get all values across all partitions
+   */
+  values(): T[] {
+    const result: T[] = [];
+    for (const store of this.partitions.values()) {
+      result.push(...store.values());
+    }
+    return result;
+  }
+
+  /**
+   * Get all keys across all partitions
+   */
+  keys(): string[] {
+    const result: string[] = [];
+    for (const store of this.partitions.values()) {
+      result.push(...store.keys());
+    }
+    return result;
+  }
+
+  /**
+   * Get total item count across all partitions
+   */
+  size(): number {
+    let total = 0;
+    for (const store of this.partitions.values()) {
+      total += store.size();
+    }
+    return total;
+  }
+
+  /**
+   * Get names of all partitions
+   */
+  getPartitionNames(): string[] {
+    return Array.from(this.partitions.keys());
+  }
+
+  /**
+   * Get all entries across all partitions
+   */
+  entries(): Array<[string, T]> {
+    const result: Array<[string, T]> = [];
+    for (const store of this.partitions.values()) {
+      result.push(...store.entries());
+    }
+    return result;
+  }
+
+  /**
+   * Flush all dirty partitions to disk
+   */
+  async flush(): Promise<void> {
+    for (const store of this.partitions.values()) {
+      await store.flush();
+    }
+  }
+
+  /**
+   * Save a single item immediately
+   */
+  async save(key: string): Promise<void> {
+    const partition = this.keyIndex.get(key);
+    if (partition) {
+      await this.partitions.get(partition)?.save(key);
+    }
+  }
+
+  /**
+   * Close all partitions
+   */
+  async close(): Promise<void> {
+    for (const store of this.partitions.values()) {
+      await store.close();
+    }
+  }
+
+  /**
+   * Get or create a partition's JsonStore, loading its data
+   */
+  private async getOrCreatePartition(name: string): Promise<JsonStore<T>> {
+    let store = this.partitions.get(name);
+    if (store) return store;
+
+    store = new JsonStore<T>(
+      join(this.baseDir, this.collection),
+      name,
+      this.options,
+    );
+    await store.init();
+    this.partitions.set(name, store);
+
+    // Build key index for this partition
+    for (const key of store.keys()) {
+      this.keyIndex.set(key, name);
+    }
+
+    return store;
+  }
+
+  /**
+   * Migrate flat layout (files directly in collection dir) to partitioned layout.
+   * Only runs if flat JSON files exist alongside no partition directories.
+   */
+  private async maybeMigrate(collectionDir: string): Promise<void> {
+    const entries = await listFiles(collectionDir);
+    const jsonFiles = entries.filter(
+      (f) => f.endsWith('.json') && !f.startsWith('_'),
+    );
+    const subdirs = entries.filter((f) => {
+      const p = join(collectionDir, f);
+      return existsSync(p) && !f.endsWith('.json');
+    });
+
+    // Only migrate if there are JSON files and no subdirectories yet
+    if (jsonFiles.length === 0 || subdirs.length > 0) return;
+
+    // Load each file, determine its partition, move it
+    for (const file of jsonFiles) {
+      const filePath = join(collectionDir, file);
+      const data = await loadJson<T>(filePath);
+      if (!data) continue;
+
+      const partitionName = this.partitionKeyFn(data) || GLOBAL_PARTITION;
+      const partitionDir = join(collectionDir, partitionName);
+      await ensureDir(partitionDir);
+
+      const destPath = join(partitionDir, file);
+      await rename(filePath, destPath);
+    }
+  }
+}

package/src/utils/trajectory-helpers.ts

@@ -0,0 +1,79 @@
+/**
+ * Shared trajectory analysis helpers.
+ *
+ * Consolidates action repetition detection and tool name extraction
+ * previously duplicated across analyzer.ts, reflexion-generator.ts,
+ * and meta-learner.ts.
+ */
+
+import type { Step } from '../types/index.js';
+
+/**
+ * An action that was repeated beyond the threshold.
+ */
+export interface RepeatedAction {
+  /** The action string (or normalized prefix) */
+  action: string;
+  /** How many times it occurred */
+  count: number;
+}
+
+/**
+ * Detect repeated actions in a list of steps.
+ *
+ * @param steps - The trajectory steps to analyze
+ * @param threshold - Minimum count to flag as repeated (default: 3)
+ * @param normalize - Optional normalization function for action strings.
+ *   Defaults to lowercasing + truncating to 50 chars.
+ * @returns Array of repeated actions exceeding the threshold.
+ */
+export function detectRepeatedActions(
+  steps: ReadonlyArray<Pick<Step, 'action'>>,
+  threshold = 3,
+  normalize?: (action: string) => string,
+): RepeatedAction[] {
+  const norm = normalize ?? ((a: string) => a.toLowerCase().slice(0, 50));
+  const counts = new Map<string, number>();
+
+  for (const step of steps) {
+    if (!step.action) continue;
+    const key = norm(step.action);
+    counts.set(key, (counts.get(key) ?? 0) + 1);
+  }
+
+  const repeated: RepeatedAction[] = [];
+  for (const [action, count] of counts) {
+    if (count >= threshold) {
+      repeated.push({ action, count });
+    }
+  }
+
+  return repeated;
+}
+
+/**
+ * Extract tool names from trajectory steps.
+ *
+ * Handles both formats:
+ *   - "ToolName args..."
+ *   - "tool_use: ToolName args..."
+ *
+ * @param steps - The trajectory steps to analyze
+ * @returns Set of unique tool names found.
+ */
+export function extractToolNames(
+  steps: ReadonlyArray<Pick<Step, 'action'>>,
+): Set<string> {
+  const tools = new Set<string>();
+
+  for (const step of steps) {
+    if (!step.action) continue;
+    // Matches "tool_use: ToolName ..." or "ToolName ..."
+    const match = step.action.match(/^(?:tool_use:\s*)?(\w+(?:[A-Z]\w*)*)/);
+    if (match) {
+      tools.add(match[1]);
+    }
+  }
+
+  return tools;
+}