cognitive-core 0.0.2 → 0.1.2

package/src/types/trajectory-source.ts

@@ -0,0 +1,47 @@
+import type { Trajectory } from './trajectory.js';
+import type { AnalysisResult } from '../learning/analyzer.js';
+
+// ============================================================================
+// Trajectory Source — formal interface for loading trajectories
+// ============================================================================
+
+/**
+ * A trajectory paired with its optional pre-computed analysis.
+ * If analysis is missing, the consumer should compute it.
+ */
+export interface TrajectoryWithAnalysis {
+  trajectory: Trajectory;
+  analysis?: AnalysisResult;
+}
+
+/**
+ * Options for filtering trajectories when loading.
+ */
+export interface TrajectoryLoadOptions {
+  /** Only load trajectories created after this date */
+  since?: Date;
+  /** Filter to a specific domain */
+  domain?: string;
+  /** Maximum number of trajectories to return */
+  limit?: number;
+  /** Only include successful trajectories */
+  successOnly?: boolean;
+}
+
+/**
+ * TrajectorySource — abstract interface for loading trajectories from
+ * different backends (in-memory arrays, filesystem, pipeline accumulators, etc.).
+ *
+ * Consumers of this interface (e.g. KnowledgeBank.extractFrom) don't care
+ * where trajectories come from — they just load and process them.
+ */
+export interface TrajectorySource {
+  /** Human-readable name for this source */
+  readonly name: string;
+
+  /** Load trajectories, optionally filtered */
+  load(options?: TrajectoryLoadOptions): Promise<TrajectoryWithAnalysis[]>;
+
+  /** Count available trajectories without loading them */
+  count(options?: TrajectoryLoadOptions): Promise<number>;
+}

package/src/types/trajectory.ts

@@ -0,0 +1,80 @@
+import { z } from 'zod';
+import { TaskSchema, type Task } from './task.js';
+import { StepSchema, type Step } from './step.js';
+import { OutcomeSchema, type Outcome } from './outcome.js';
+
+/**
+ * Trajectory - A complete record of an agent's task-solving attempt
+ * The atomic unit of learning in Atlas
+ */
+export const TrajectorySchema = z.object({
+  /** Unique trajectory identifier */
+  id: z.string(),
+  /** The task being solved */
+  task: TaskSchema,
+  /** Sequence of steps taken */
+  steps: z.array(StepSchema),
+  /** Final outcome */
+  outcome: OutcomeSchema,
+  /** Identifier of the agent that produced this trajectory */
+  agentId: z.string(),
+  /** When the trajectory was created */
+  timestamp: z.date().default(() => new Date()),
+  /** Number of LLM API calls made */
+  llmCalls: z.number().default(0),
+  /** Total tokens used */
+  totalTokens: z.number().default(0),
+  /** Wall clock time in seconds */
+  wallTimeSeconds: z.number().default(0),
+  /** Additional metadata */
+  metadata: z.record(z.unknown()).default({}),
+});
+
+export type Trajectory = z.infer<typeof TrajectorySchema>;
+
+/**
+ * Create a new trajectory
+ */
+export function createTrajectory(params: {
+  id?: string;
+  task: Task;
+  steps: Step[];
+  outcome: Outcome;
+  agentId: string;
+  llmCalls?: number;
+  totalTokens?: number;
+  wallTimeSeconds?: number;
+  metadata?: Record<string, unknown>;
+}): Trajectory {
+  return TrajectorySchema.parse({
+    id: params.id ?? crypto.randomUUID(),
+    task: params.task,
+    steps: params.steps,
+    outcome: params.outcome,
+    agentId: params.agentId,
+    timestamp: new Date(),
+    llmCalls: params.llmCalls ?? 0,
+    totalTokens: params.totalTokens ?? 0,
+    wallTimeSeconds: params.wallTimeSeconds ?? 0,
+    metadata: params.metadata ?? {},
+  });
+}
+
+/**
+ * Check if trajectory was successful
+ */
+export function isSuccessful(trajectory: Trajectory): boolean {
+  return trajectory.outcome.success;
+}
+
+/**
+ * Get key steps (those with high attribution scores)
+ */
+export function getKeySteps(
+  trajectory: Trajectory,
+  threshold = 0.15
+): Step[] {
+  return trajectory.steps.filter(
+    (step) => (step.attributionScore ?? 0) >= threshold
+  );
+}

package/src/utils/frontmatter.ts

@@ -0,0 +1,118 @@
+import { parse as parseYaml, stringify as stringifyYaml } from 'yaml';
+import {
+  KnowledgeNoteFrontmatterSchema,
+  type KnowledgeNoteFrontmatter,
+  type KnowledgeNote,
+} from '../types/knowledge.js';
+
+const FRONTMATTER_REGEX = /^---\n([\s\S]*?)\n---\n?([\s\S]*)$/;
+
+/**
+ * Parse markdown content with YAML frontmatter into a KnowledgeNoteFrontmatter + body.
+ *
+ * Uses the `yaml` npm package for full-fidelity parsing (nested objects,
+ * arrays of objects, etc.). Validates against the Zod schema.
+ *
+ * Returns null frontmatter if the content has no frontmatter block or
+ * if the frontmatter fails validation (logged, not thrown).
+ */
+export function parseKnowledgeFrontmatter(content: string): {
+  frontmatter: KnowledgeNoteFrontmatter | null;
+  body: string;
+} {
+  const match = content.match(FRONTMATTER_REGEX);
+  if (!match) {
+    return { frontmatter: null, body: content };
+  }
+
+  const [, yamlBlock, body] = match;
+
+  try {
+    const raw = parseYaml(yamlBlock);
+    if (!raw || typeof raw !== 'object') {
+      return { frontmatter: null, body };
+    }
+
+    const parsed = KnowledgeNoteFrontmatterSchema.safeParse(raw);
+    if (!parsed.success) {
+      console.warn(
+        `[frontmatter] Validation failed for note: ${JSON.stringify(parsed.error.issues)}`
+      );
+      return { frontmatter: null, body };
+    }
+
+    return { frontmatter: parsed.data, body: body.trim() };
+  } catch (error) {
+    console.warn(
+      `[frontmatter] YAML parse error: ${error instanceof Error ? error.message : String(error)}`
+    );
+    return { frontmatter: null, body: content };
+  }
+}
+
+/**
+ * Serialize a KnowledgeNoteFrontmatter into a YAML frontmatter block.
+ *
+ * Keeps simple top-level fields (created, updated, tags) in a format
+ * that minimem's simpler parser can partially read.
+ */
+export function serializeKnowledgeFrontmatter(
+  frontmatter: KnowledgeNoteFrontmatter
+): string {
+  // Build a clean object for serialization, omitting null/undefined/empty values
+  const obj: Record<string, unknown> = {
+    id: frontmatter.id,
+    type: frontmatter.type,
+  };
+
+  if (frontmatter.domain.length > 0) obj.domain = frontmatter.domain;
+  if (frontmatter.entities.length > 0) obj.entities = frontmatter.entities;
+  if (frontmatter.tags.length > 0) obj.tags = frontmatter.tags;
+
+  obj.confidence = frontmatter.confidence;
+  obj.source = frontmatter.source;
+
+  if (frontmatter.links.length > 0) obj.links = frontmatter.links;
+
+  obj.created = frontmatter.created;
+  obj.updated = frontmatter.updated;
+
+  if (frontmatter.supersedes) obj.supersedes = frontmatter.supersedes;
+
+  const yaml = stringifyYaml(obj, {
+    lineWidth: 120,
+    defaultKeyType: 'PLAIN',
+    defaultStringType: 'PLAIN',
+  });
+
+  return `---\n${yaml}---\n`;
+}
+
+/**
+ * Serialize a full KnowledgeNote (frontmatter + body) into markdown content.
+ */
+export function serializeKnowledgeNote(note: KnowledgeNote): string {
+  const header = serializeKnowledgeFrontmatter(note.frontmatter);
+  return `${header}\n${note.body}\n`;
+}
+
+/**
+ * Parse a full markdown file into a KnowledgeNote.
+ * Returns null if frontmatter is missing or invalid.
+ *
+ * @param content - The raw markdown file content
+ * @param filePath - The file path (stored on the note for reference)
+ */
+export function parseKnowledgeNote(
+  content: string,
+  filePath: string
+): KnowledgeNote | null {
+  const { frontmatter, body } = parseKnowledgeFrontmatter(content);
+  if (!frontmatter) return null;
+
+  return {
+    frontmatter,
+    body,
+    filePath,
+  };
+}

package/src/utils/index.ts

@@ -0,0 +1,45 @@
+export {
+  cosineSimilarity,
+  jaccardSimilarity,
+  textSimilarity,
+  ngramSimilarity,
+  tokenize,
+  findTopK,
+} from './similarity.js';
+
+export {
+  ensureDir,
+  saveJson,
+  loadJson,
+  listFiles,
+  exists,
+  JsonStore,
+  saveIndex,
+  loadIndex,
+  type StorageOptions,
+} from './storage.js';
+
+export {
+  ValidationError,
+  validateString,
+  validateOptionalString,
+  validateNumber,
+  validateOptionalNumber,
+  validateArray,
+  validateVector,
+  validateTaskInput,
+  validateTrajectoryStepCount,
+  validateMCPArgs,
+  TaskInputSchema,
+  MCPSearchPlaybooksSchema,
+  MCPGetPlaybookDetailsSchema,
+  MCPGetPlaybooksByDomainSchema,
+  type TaskInput,
+} from './validation.js';
+
+export {
+  parseKnowledgeFrontmatter,
+  serializeKnowledgeFrontmatter,
+  serializeKnowledgeNote,
+  parseKnowledgeNote,
+} from './frontmatter.js';

package/src/utils/similarity.ts

@@ -0,0 +1,139 @@
+/**
+ * Similarity utilities for comparing embeddings and text
+ */
+
+/**
+ * Compute cosine similarity between two vectors
+ */
+export function cosineSimilarity(a: number[], b: number[]): number {
+  if (a.length !== b.length) {
+    throw new Error('Vectors must have the same length');
+  }
+
+  let dotProduct = 0;
+  let normA = 0;
+  let normB = 0;
+
+  for (let i = 0; i < a.length; i++) {
+    dotProduct += a[i] * b[i];
+    normA += a[i] * a[i];
+    normB += b[i] * b[i];
+  }
+
+  normA = Math.sqrt(normA);
+  normB = Math.sqrt(normB);
+
+  if (normA === 0 || normB === 0) {
+    return 0;
+  }
+
+  return dotProduct / (normA * normB);
+}
+
+/**
+ * Compute Jaccard similarity between two sets of words
+ */
+export function jaccardSimilarity(a: string, b: string): number {
+  const setA = new Set(tokenize(a));
+  const setB = new Set(tokenize(b));
+
+  const intersection = new Set([...setA].filter((x) => setB.has(x)));
+  const union = new Set([...setA, ...setB]);
+
+  if (union.size === 0) {
+    return 0;
+  }
+
+  return intersection.size / union.size;
+}
+
+/**
+ * Simple tokenization for text similarity
+ */
+export function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^\w\s]/g, ' ')
+    .split(/\s+/)
+    .filter((token) => token.length > 0);
+}
+
+/**
+ * Compute n-gram overlap similarity
+ */
+export function ngramSimilarity(
+  a: string,
+  b: string,
+  n = 3
+): number {
+  const ngramsA = getNgrams(a, n);
+  const ngramsB = getNgrams(b, n);
+
+  const intersection = new Set(
+    [...ngramsA].filter((x) => ngramsB.has(x))
+  );
+  const union = new Set([...ngramsA, ...ngramsB]);
+
+  if (union.size === 0) {
+    return 0;
+  }
+
+  return intersection.size / union.size;
+}
+
+/**
+ * Get character n-grams from text
+ */
+function getNgrams(text: string, n: number): Set<string> {
+  const ngrams = new Set<string>();
+  const normalized = text.toLowerCase();
+
+  for (let i = 0; i <= normalized.length - n; i++) {
+    ngrams.add(normalized.slice(i, i + n));
+  }
+
+  return ngrams;
+}
+
+/**
+ * Combined text similarity using multiple methods
+ */
+export function textSimilarity(a: string, b: string): number {
+  const jaccard = jaccardSimilarity(a, b);
+  const ngram = ngramSimilarity(a, b);
+
+  // Weighted average
+  return jaccard * 0.6 + ngram * 0.4;
+}
+
+/**
+ * Find top-k most similar items
+ */
+export function findTopK<T>(
+  query: number[] | string,
+  items: T[],
+  getVector: (item: T) => number[] | string | undefined,
+  k: number
+): Array<{ item: T; score: number }> {
+  const scored = items
+    .map((item) => {
+      const vector = getVector(item);
+      if (!vector) {
+        return { item, score: 0 };
+      }
+
+      let score: number;
+      if (typeof query === 'string' && typeof vector === 'string') {
+        score = textSimilarity(query, vector);
+      } else if (Array.isArray(query) && Array.isArray(vector)) {
+        score = cosineSimilarity(query, vector);
+      } else {
+        score = 0;
+      }
+
+      return { item, score };
+    })
+    .sort((a, b) => b.score - a.score);
+
+  return scored.slice(0, k);
+}

package/src/utils/storage.ts

@@ -0,0 +1,249 @@
+import { readFile, writeFile, mkdir, readdir, stat } from 'node:fs/promises';
+import { join, dirname } from 'node:path';
+import { existsSync } from 'node:fs';
+
+/**
+ * Lightweight file-based storage (Claudeception-inspired)
+ * Uses JSON files for simplicity and human-readability
+ */
+
+export interface StorageOptions {
+  baseDir: string;
+  prettyPrint?: boolean;
+}
+
+/**
+ * Ensure directory exists
+ */
+export async function ensureDir(path: string): Promise<void> {
+  if (!existsSync(path)) {
+    await mkdir(path, { recursive: true });
+  }
+}
+
+/**
+ * Save data to a JSON file
+ */
+export async function saveJson<T>(
+  filePath: string,
+  data: T,
+  options?: { pretty?: boolean }
+): Promise<void> {
+  await ensureDir(dirname(filePath));
+  const content = options?.pretty
+    ? JSON.stringify(data, null, 2)
+    : JSON.stringify(data);
+  await writeFile(filePath, content, 'utf-8');
+}
+
+/**
+ * Load data from a JSON file
+ */
+export async function loadJson<T>(filePath: string): Promise<T | null> {
+  try {
+    const content = await readFile(filePath, 'utf-8');
+    return JSON.parse(content) as T;
+  } catch {
+    return null;
+  }
+}
+
+/**
+ * List files in a directory
+ */
+export async function listFiles(
+  dir: string,
+  extension?: string
+): Promise<string[]> {
+  try {
+    const files = await readdir(dir);
+    if (extension) {
+      return files.filter((f) => f.endsWith(extension));
+    }
+    return files;
+  } catch {
+    return [];
+  }
+}
+
+/**
+ * Check if a path exists
+ */
+export async function exists(path: string): Promise<boolean> {
+  try {
+    await stat(path);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Simple key-value store backed by JSON files
+ */
+export class JsonStore<T> {
+  private cache: Map<string, T> = new Map();
+  private dirty: Set<string> = new Set();
+  private autoSaveTimer: ReturnType<typeof setInterval> | null = null;
+
+  constructor(
+    private baseDir: string,
+    private collection: string,
+    private options: { autoSaveInterval?: number; pretty?: boolean } = {}
+  ) {}
+
+  /**
+   * Get the file path for a key
+   */
+  private getPath(key: string): string {
+    return join(this.baseDir, this.collection, `${key}.json`);
+  }
+
+  /**
+   * Initialize the store and load existing data
+   */
+  async init(): Promise<void> {
+    const dir = join(this.baseDir, this.collection);
+    await ensureDir(dir);
+
+    // Load existing items
+    const files = await listFiles(dir, '.json');
+    for (const file of files) {
+      const key = file.replace('.json', '');
+      const data = await loadJson<T>(join(dir, file));
+      if (data) {
+        this.cache.set(key, data);
+      }
+    }
+
+    // Start auto-save if configured
+    if (this.options.autoSaveInterval) {
+      this.autoSaveTimer = setInterval(
+        () => this.flush(),
+        this.options.autoSaveInterval
+      );
+    }
+  }
+
+  /**
+   * Get an item by key
+   */
+  get(key: string): T | undefined {
+    return this.cache.get(key);
+  }
+
+  /**
+   * Set an item
+   */
+  set(key: string, value: T): void {
+    this.cache.set(key, value);
+    this.dirty.add(key);
+  }
+
+  /**
+   * Delete an item
+   */
+  delete(key: string): boolean {
+    this.dirty.add(key);
+    return this.cache.delete(key);
+  }
+
+  /**
+   * Check if key exists
+   */
+  has(key: string): boolean {
+    return this.cache.has(key);
+  }
+
+  /**
+   * Get all items
+   */
+  values(): T[] {
+    return Array.from(this.cache.values());
+  }
+
+  /**
+   * Get all keys
+   */
+  keys(): string[] {
+    return Array.from(this.cache.keys());
+  }
+
+  /**
+   * Get all entries
+   */
+  entries(): Array<[string, T]> {
+    return Array.from(this.cache.entries());
+  }
+
+  /**
+   * Get item count
+   */
+  size(): number {
+    return this.cache.size;
+  }
+
+  /**
+   * Flush dirty items to disk
+   */
+  async flush(): Promise<void> {
+    for (const key of this.dirty) {
+      const value = this.cache.get(key);
+      const path = this.getPath(key);
+
+      if (value === undefined) {
+        // Item was deleted - we could delete the file too
+        // For now, we'll leave it (soft delete)
+        continue;
+      }
+
+      await saveJson(path, value, { pretty: this.options.pretty });
+    }
+    this.dirty.clear();
+  }
+
+  /**
+   * Save a single item immediately
+   */
+  async save(key: string): Promise<void> {
+    const value = this.cache.get(key);
+    if (value) {
+      await saveJson(this.getPath(key), value, { pretty: this.options.pretty });
+      this.dirty.delete(key);
+    }
+  }
+
+  /**
+   * Close the store (flush and stop auto-save)
+   */
+  async close(): Promise<void> {
+    if (this.autoSaveTimer) {
+      clearInterval(this.autoSaveTimer);
+      this.autoSaveTimer = null;
+    }
+    await this.flush();
+  }
+}
+
+/**
+ * Create a store index file (for quick lookups without loading all items)
+ */
+export async function saveIndex(
+  baseDir: string,
+  collection: string,
+  index: Record<string, unknown>
+): Promise<void> {
+  const path = join(baseDir, collection, '_index.json');
+  await saveJson(path, index, { pretty: true });
+}
+
+/**
+ * Load a store index
+ */
+export async function loadIndex(
+  baseDir: string,
+  collection: string
+): Promise<Record<string, unknown> | null> {
+  const path = join(baseDir, collection, '_index.json');
+  return loadJson(path);
+}