cognitive-core 0.1.2 → 0.2.1

package/src/utils/partitioned-store.ts

@@ -0,0 +1,299 @@
+/**
+ * Partitioned Store
+ *
+ * Domain-partitioned wrapper around JsonStore. Items are routed to partition
+ * subdirectories based on a caller-supplied key function. Queries can target
+ * a single partition (fast) or scan all partitions (full).
+ *
+ * On first init with existing flat data (no partition subdirs), items are
+ * auto-migrated into partition directories. This is a one-time operation.
+ */
+
+import { JsonStore, ensureDir, loadJson, listFiles } from './storage.js';
+import { join } from 'node:path';
+import { existsSync } from 'node:fs';
+import { rename } from 'node:fs/promises';
+
+const GLOBAL_PARTITION = '_global';
+
+export interface PartitionedStoreOptions {
+  autoSaveInterval?: number;
+  pretty?: boolean;
+}
+
+/**
+ * Domain-partitioned key-value store backed by JSON files
+ */
+export class PartitionedStore<T> {
+  private partitions: Map<string, JsonStore<T>> = new Map();
+  private keyIndex: Map<string, string> = new Map(); // key → partition name
+  private baseDir: string;
+  private collection: string;
+  private partitionKeyFn: (item: T) => string | undefined;
+  private options: PartitionedStoreOptions;
+  private initialized = false;
+
+  constructor(
+    baseDir: string,
+    collection: string,
+    partitionKeyFn: (item: T) => string | undefined,
+    options?: PartitionedStoreOptions,
+  ) {
+    this.baseDir = baseDir;
+    this.collection = collection;
+    this.partitionKeyFn = partitionKeyFn;
+    this.options = options ?? {};
+  }
+
+  /**
+   * Initialize — discover existing partitions, auto-migrate flat data if needed
+   */
+  async init(): Promise<void> {
+    if (this.initialized) return;
+
+    const collectionDir = join(this.baseDir, this.collection);
+    await ensureDir(collectionDir);
+
+    // Check if we need to migrate from flat layout
+    await this.maybeMigrate(collectionDir);
+
+    // Discover partition subdirectories
+    const entries = await listFiles(collectionDir);
+    for (const entry of entries) {
+      const entryPath = join(collectionDir, entry);
+      if (existsSync(entryPath) && !entry.endsWith('.json')) {
+        // It's a directory — treat as partition
+        await this.getOrCreatePartition(entry);
+      }
+    }
+
+    // Ensure global partition exists
+    await this.getOrCreatePartition(GLOBAL_PARTITION);
+
+    this.initialized = true;
+  }
+
+  /**
+   * Get an item by key. If partition is known, searches only that partition.
+   */
+  get(key: string, partition?: string): T | undefined {
+    if (partition) {
+      return this.partitions.get(partition)?.get(key);
+    }
+
+    // Check index first
+    const knownPartition = this.keyIndex.get(key);
+    if (knownPartition) {
+      return this.partitions.get(knownPartition)?.get(key);
+    }
+
+    // Fall back to scanning all partitions
+    for (const store of this.partitions.values()) {
+      const item = store.get(key);
+      if (item !== undefined) return item;
+    }
+    return undefined;
+  }
+
+  /**
+   * Set an item — auto-routes to the correct partition based on partitionKeyFn
+   */
+  set(key: string, value: T): void {
+    const partitionName = this.partitionKeyFn(value) || GLOBAL_PARTITION;
+
+    // If the item was in a different partition, remove it from the old one
+    const oldPartition = this.keyIndex.get(key);
+    if (oldPartition && oldPartition !== partitionName) {
+      this.partitions.get(oldPartition)?.delete(key);
+    }
+
+    const store = this.partitions.get(partitionName);
+    if (store) {
+      store.set(key, value);
+      this.keyIndex.set(key, partitionName);
+    } else {
+      // Partition doesn't exist yet — create it synchronously by using the map
+      // (the directory will be created on flush/save)
+      const newStore = new JsonStore<T>(
+        join(this.baseDir, this.collection),
+        partitionName,
+        this.options,
+      );
+      this.partitions.set(partitionName, newStore);
+      newStore.set(key, value);
+      this.keyIndex.set(key, partitionName);
+    }
+  }
+
+  /**
+   * Delete an item by key
+   */
+  delete(key: string): boolean {
+    const partition = this.keyIndex.get(key);
+    if (partition) {
+      this.keyIndex.delete(key);
+      return this.partitions.get(partition)?.delete(key) ?? false;
+    }
+
+    // Fall back to scanning
+    for (const store of this.partitions.values()) {
+      if (store.has(key)) {
+        store.delete(key);
+        return true;
+      }
+    }
+    return false;
+  }
+
+  /**
+   * Check if a key exists
+   */
+  has(key: string): boolean {
+    if (this.keyIndex.has(key)) return true;
+    for (const store of this.partitions.values()) {
+      if (store.has(key)) return true;
+    }
+    return false;
+  }
+
+  /**
+   * Get all values within a single partition
+   */
+  valuesInPartition(partition: string): T[] {
+    return this.partitions.get(partition)?.values() ?? [];
+  }
+
+  /**
+   * Get all values across all partitions
+   */
+  values(): T[] {
+    const result: T[] = [];
+    for (const store of this.partitions.values()) {
+      result.push(...store.values());
+    }
+    return result;
+  }
+
+  /**
+   * Get all keys across all partitions
+   */
+  keys(): string[] {
+    const result: string[] = [];
+    for (const store of this.partitions.values()) {
+      result.push(...store.keys());
+    }
+    return result;
+  }
+
+  /**
+   * Get total item count across all partitions
+   */
+  size(): number {
+    let total = 0;
+    for (const store of this.partitions.values()) {
+      total += store.size();
+    }
+    return total;
+  }
+
+  /**
+   * Get names of all partitions
+   */
+  getPartitionNames(): string[] {
+    return Array.from(this.partitions.keys());
+  }
+
+  /**
+   * Get all entries across all partitions
+   */
+  entries(): Array<[string, T]> {
+    const result: Array<[string, T]> = [];
+    for (const store of this.partitions.values()) {
+      result.push(...store.entries());
+    }
+    return result;
+  }
+
+  /**
+   * Flush all dirty partitions to disk
+   */
+  async flush(): Promise<void> {
+    for (const store of this.partitions.values()) {
+      await store.flush();
+    }
+  }
+
+  /**
+   * Save a single item immediately
+   */
+  async save(key: string): Promise<void> {
+    const partition = this.keyIndex.get(key);
+    if (partition) {
+      await this.partitions.get(partition)?.save(key);
+    }
+  }
+
+  /**
+   * Close all partitions
+   */
+  async close(): Promise<void> {
+    for (const store of this.partitions.values()) {
+      await store.close();
+    }
+  }
+
+  /**
+   * Get or create a partition's JsonStore, loading its data
+   */
+  private async getOrCreatePartition(name: string): Promise<JsonStore<T>> {
+    let store = this.partitions.get(name);
+    if (store) return store;
+
+    store = new JsonStore<T>(
+      join(this.baseDir, this.collection),
+      name,
+      this.options,
+    );
+    await store.init();
+    this.partitions.set(name, store);
+
+    // Build key index for this partition
+    for (const key of store.keys()) {
+      this.keyIndex.set(key, name);
+    }
+
+    return store;
+  }
+
+  /**
+   * Migrate flat layout (files directly in collection dir) to partitioned layout.
+   * Only runs if flat JSON files exist alongside no partition directories.
+   */
+  private async maybeMigrate(collectionDir: string): Promise<void> {
+    const entries = await listFiles(collectionDir);
+    const jsonFiles = entries.filter(
+      (f) => f.endsWith('.json') && !f.startsWith('_'),
+    );
+    const subdirs = entries.filter((f) => {
+      const p = join(collectionDir, f);
+      return existsSync(p) && !f.endsWith('.json');
+    });
+
+    // Only migrate if there are JSON files and no subdirectories yet
+    if (jsonFiles.length === 0 || subdirs.length > 0) return;
+
+    // Load each file, determine its partition, move it
+    for (const file of jsonFiles) {
+      const filePath = join(collectionDir, file);
+      const data = await loadJson<T>(filePath);
+      if (!data) continue;
+
+      const partitionName = this.partitionKeyFn(data) || GLOBAL_PARTITION;
+      const partitionDir = join(collectionDir, partitionName);
+      await ensureDir(partitionDir);
+
+      const destPath = join(partitionDir, file);
+      await rename(filePath, destPath);
+    }
+  }
+}

package/src/utils/trajectory-helpers.ts

@@ -0,0 +1,79 @@
+/**
+ * Shared trajectory analysis helpers.
+ *
+ * Consolidates action repetition detection and tool name extraction
+ * previously duplicated across analyzer.ts, reflexion-generator.ts,
+ * and meta-learner.ts.
+ */
+
+import type { Step } from '../types/index.js';
+
+/**
+ * An action that was repeated beyond the threshold.
+ */
+export interface RepeatedAction {
+  /** The action string (or normalized prefix) */
+  action: string;
+  /** How many times it occurred */
+  count: number;
+}
+
+/**
+ * Detect repeated actions in a list of steps.
+ *
+ * @param steps - The trajectory steps to analyze
+ * @param threshold - Minimum count to flag as repeated (default: 3)
+ * @param normalize - Optional normalization function for action strings.
+ *   Defaults to lowercasing + truncating to 50 chars.
+ * @returns Array of repeated actions exceeding the threshold.
+ */
+export function detectRepeatedActions(
+  steps: ReadonlyArray<Pick<Step, 'action'>>,
+  threshold = 3,
+  normalize?: (action: string) => string,
+): RepeatedAction[] {
+  const norm = normalize ?? ((a: string) => a.toLowerCase().slice(0, 50));
+  const counts = new Map<string, number>();
+
+  for (const step of steps) {
+    if (!step.action) continue;
+    const key = norm(step.action);
+    counts.set(key, (counts.get(key) ?? 0) + 1);
+  }
+
+  const repeated: RepeatedAction[] = [];
+  for (const [action, count] of counts) {
+    if (count >= threshold) {
+      repeated.push({ action, count });
+    }
+  }
+
+  return repeated;
+}
+
+/**
+ * Extract tool names from trajectory steps.
+ *
+ * Handles both formats:
+ *   - "ToolName args..."
+ *   - "tool_use: ToolName args..."
+ *
+ * @param steps - The trajectory steps to analyze
+ * @returns Set of unique tool names found.
+ */
+export function extractToolNames(
+  steps: ReadonlyArray<Pick<Step, 'action'>>,
+): Set<string> {
+  const tools = new Set<string>();
+
+  for (const step of steps) {
+    if (!step.action) continue;
+    // Matches "tool_use: ToolName ..." or "ToolName ..."
+    const match = step.action.match(/^(?:tool_use:\s*)?(\w+(?:[A-Z]\w*)*)/);
+    if (match) {
+      tools.add(match[1]);
+    }
+  }
+
+  return tools;
+}