cognitive-core 0.0.2 → 0.1.2

package/src/learning/trajectory-sources/file.ts

@@ -0,0 +1,136 @@
+import { readdir } from 'node:fs/promises';
+import { join } from 'node:path';
+import { TrajectorySchema } from '../../types/trajectory.js';
+import { loadJson } from '../../utils/storage.js';
+import type {
+  TrajectorySource,
+  TrajectoryWithAnalysis,
+  TrajectoryLoadOptions,
+} from '../../types/trajectory-source.js';
+import type { AnalysisResult } from '../analyzer.js';
+
+/**
+ * FileTrajectorySource — loads trajectory JSON files from a directory.
+ *
+ * Expects files named `*.json` containing serialized Trajectory objects.
+ * Optionally reads companion `*-analysis.json` files for pre-computed analyses.
+ *
+ * @example
+ * ```typescript
+ * const source = new FileTrajectorySource('/path/to/trajectories');
+ * const items = await source.load({ domain: 'code' });
+ * ```
+ */
+export class FileTrajectorySource implements TrajectorySource {
+  readonly name = 'file';
+  private dirPath: string;
+
+  constructor(dirPath: string) {
+    this.dirPath = dirPath;
+  }
+
+  async load(options?: TrajectoryLoadOptions): Promise<TrajectoryWithAnalysis[]> {
+    const items = await this.readAll();
+    return this.filter(items, options);
+  }
+
+  async count(options?: TrajectoryLoadOptions): Promise<number> {
+    const items = await this.readAll();
+    return this.filter(items, options).length;
+  }
+
+  private async readAll(): Promise<TrajectoryWithAnalysis[]> {
+    const items: TrajectoryWithAnalysis[] = [];
+
+    let files: string[];
+    try {
+      files = await readdir(this.dirPath);
+    } catch {
+      return [];
+    }
+
+    const trajectoryFiles = files.filter(
+      (f) => f.endsWith('.json') && !f.endsWith('-analysis.json')
+    );
+
+    for (const file of trajectoryFiles) {
+      try {
+        const filePath = join(this.dirPath, file);
+        const raw = await loadJson<Record<string, unknown>>(filePath);
+        if (!raw) continue;
+
+        // Parse dates from JSON (they come as strings)
+        if (typeof raw.timestamp === 'string') {
+          raw.timestamp = new Date(raw.timestamp);
+        }
+        // Task.createdAt
+        const task = raw.task as Record<string, unknown> | undefined;
+        if (task && typeof task.createdAt === 'string') {
+          task.createdAt = new Date(task.createdAt);
+        }
+        if (Array.isArray(raw.steps)) {
+          for (const step of raw.steps) {
+            if (
+              typeof step === 'object' &&
+              step !== null &&
+              typeof (step as Record<string, unknown>).timestamp === 'string'
+            ) {
+              (step as Record<string, unknown>).timestamp = new Date(
+                (step as Record<string, unknown>).timestamp as string
+              );
+            }
+          }
+        }
+
+        const parsed = TrajectorySchema.safeParse(raw);
+        if (!parsed.success) continue;
+
+        // Try loading companion analysis file
+        const analysisFile = file.replace('.json', '-analysis.json');
+        const analysis = await loadJson<AnalysisResult>(
+          join(this.dirPath, analysisFile)
+        );
+
+        items.push({
+          trajectory: parsed.data,
+          analysis: analysis ?? undefined,
+        });
+      } catch {
+        // Skip invalid files
+      }
+    }
+
+    return items;
+  }
+
+  private filter(
+    items: TrajectoryWithAnalysis[],
+    options?: TrajectoryLoadOptions
+  ): TrajectoryWithAnalysis[] {
+    let result = items;
+
+    if (options?.since) {
+      const since = options.since.getTime();
+      result = result.filter(
+        (item) => item.trajectory.timestamp.getTime() >= since
+      );
+    }
+
+    if (options?.domain) {
+      const domain = options.domain;
+      result = result.filter(
+        (item) => item.trajectory.task.domain === domain
+      );
+    }
+
+    if (options?.successOnly) {
+      result = result.filter((item) => item.trajectory.outcome.success);
+    }
+
+    if (options?.limit && options.limit > 0) {
+      result = result.slice(0, options.limit);
+    }
+
+    return result;
+  }
+}

package/src/learning/trajectory-sources/huggingface.ts

@@ -0,0 +1,248 @@
+import {
+  DataclawSessionSchema,
+  type DataclawSession,
+} from '../../types/dataclaw.js';
+import {
+  DataclawTrajectorySource,
+  type DataclawSourceOptions,
+} from './dataclaw.js';
+import type {
+  TrajectorySource,
+  TrajectoryWithAnalysis,
+  TrajectoryLoadOptions,
+} from '../../types/trajectory-source.js';
+
+// ─── HuggingFaceTrajectorySource ─────────────────────────────────────────
+// Loads dataclaw-format datasets directly from the Hugging Face Hub.
+//
+// Uses the HF datasets-server rows API to fetch sessions as JSON, then
+// delegates synthesis to DataclawTrajectorySource.synthesize().
+//
+// Usage:
+//   const source = new HuggingFaceTrajectorySource(
+//     'peteromallet/dataclaw-peteromallet',
+//   );
+//   const items = await source.load({ limit: 50 });
+// ─────────────────────────────────────────────────────────────────────────
+
+const DEFAULT_BASE_URL = 'https://datasets-server.huggingface.co';
+const DEFAULT_PAGE_SIZE = 100;
+
+export interface HuggingFaceSourceOptions extends DataclawSourceOptions {
+  /** HF API token for gated/private datasets */
+  token?: string;
+  /** Dataset config name (default: 'default') */
+  config?: string;
+  /** Dataset split (default: 'train') */
+  split?: string;
+  /** Max rows to fetch per API call (default: 100, max: 100) */
+  pageSize?: number;
+  /** Override the datasets-server base URL (useful for testing) */
+  baseUrl?: string;
+  /** Override fetch for testing / custom HTTP clients */
+  fetchFn?: typeof globalThis.fetch;
+}
+
+/** Shape of a single row from the datasets-server /rows response */
+interface HfRowsResponseRow {
+  row_idx: number;
+  row: Record<string, unknown>;
+}
+
+/** Shape of the datasets-server /rows response */
+interface HfRowsResponse {
+  features: unknown[];
+  rows: HfRowsResponseRow[];
+  num_rows_total: number;
+  num_rows_per_page: number;
+  partial: boolean;
+}
+
+/** Shape of the datasets-server /info response */
+interface HfInfoResponse {
+  dataset_info: Record<string, {
+    num_rows: number;
+    dataset_name: string;
+  }>;
+}
+
+export class HuggingFaceTrajectorySource implements TrajectorySource {
+  readonly name = 'huggingface';
+
+  private readonly datasetId: string;
+  private readonly config: string;
+  private readonly split: string;
+  private readonly pageSize: number;
+  private readonly baseUrl: string;
+  private readonly fetchFn: typeof globalThis.fetch;
+  private readonly token: string | undefined;
+  private readonly synthesizer: DataclawTrajectorySource;
+  private readonly dataclawOptions: DataclawSourceOptions | undefined;
+
+  constructor(
+    datasetId: string,
+    options?: HuggingFaceSourceOptions,
+  ) {
+    this.datasetId = datasetId;
+    this.config = options?.config ?? 'default';
+    this.split = options?.split ?? 'train';
+    this.pageSize = Math.min(options?.pageSize ?? DEFAULT_PAGE_SIZE, 100);
+    this.baseUrl = options?.baseUrl ?? DEFAULT_BASE_URL;
+    this.token = options?.token;
+    this.fetchFn = options?.fetchFn ?? globalThis.fetch;
+
+    // Build dataclaw options (source/model filtering, domain, outcome strategy)
+    this.dataclawOptions = {
+      domain: options?.domain,
+      outcomeStrategy: options?.outcomeStrategy,
+      sources: options?.sources,
+      models: options?.models,
+    };
+
+    // Use a dummy-path DataclawTrajectorySource solely for its synthesize() method
+    this.synthesizer = new DataclawTrajectorySource(
+      '/dev/null',
+      this.dataclawOptions,
+    );
+  }
+
+  async load(
+    options?: TrajectoryLoadOptions,
+  ): Promise<TrajectoryWithAnalysis[]> {
+    const sessions = await this.fetchAllSessions();
+    const items: TrajectoryWithAnalysis[] = [];
+
+    for (const session of sessions) {
+      try {
+        // Pre-filter by dataclaw-specific options
+        if (
+          this.dataclawOptions?.sources?.length &&
+          !this.dataclawOptions.sources.includes(session.source)
+        ) {
+          continue;
+        }
+        if (
+          this.dataclawOptions?.models?.length &&
+          !this.dataclawOptions.models.includes(session.model)
+        ) {
+          continue;
+        }
+
+        const trajectory = this.synthesizer.synthesize(session);
+
+        // Apply standard TrajectoryLoadOptions filters
+        if (options?.since && trajectory.timestamp.getTime() < options.since.getTime()) {
+          continue;
+        }
+        if (options?.domain && trajectory.task.domain !== options.domain) {
+          continue;
+        }
+        if (options?.successOnly && !trajectory.outcome.success) {
+          continue;
+        }
+
+        items.push({ trajectory, analysis: undefined });
+      } catch {
+        // Skip sessions that fail to synthesize
+      }
+    }
+
+    if (options?.limit && options.limit > 0) {
+      return items.slice(0, options.limit);
+    }
+
+    return items;
+  }
+
+  async count(options?: TrajectoryLoadOptions): Promise<number> {
+    // If no filters, use the info endpoint for a fast count
+    if (!options && !this.dataclawOptions?.sources?.length && !this.dataclawOptions?.models?.length) {
+      try {
+        return await this.fetchRowCount();
+      } catch {
+        // Fall back to loading everything
+      }
+    }
+
+    const items = await this.load(options);
+    return items.length;
+  }
+
+  // ── Private: HF API ────────────────────────────────────────────────
+
+  private buildHeaders(): Record<string, string> {
+    const headers: Record<string, string> = {
+      'Accept': 'application/json',
+    };
+    const token =
+      this.token ??
+      process.env.HF_TOKEN ??
+      process.env.HUGGING_FACE_HUB_TOKEN;
+    if (token) {
+      headers['Authorization'] = `Bearer ${token}`;
+    }
+    return headers;
+  }
+
+  private async fetchRowCount(): Promise<number> {
+    const url = `${this.baseUrl}/info?dataset=${encodeURIComponent(this.datasetId)}`;
+    const response = await this.fetchFn(url, { headers: this.buildHeaders() });
+
+    if (!response.ok) {
+      throw new Error(`HF info request failed: ${response.status} ${response.statusText}`);
+    }
+
+    const data = (await response.json()) as HfInfoResponse;
+    const configInfo = data.dataset_info[this.config];
+    if (!configInfo) {
+      throw new Error(`Config "${this.config}" not found in dataset info`);
+    }
+    return configInfo.num_rows;
+  }
+
+  private async fetchAllSessions(): Promise<DataclawSession[]> {
+    const sessions: DataclawSession[] = [];
+    let offset = 0;
+    let total = Infinity;
+
+    while (offset < total) {
+      const page = await this.fetchPage(offset);
+      total = page.num_rows_total;
+
+      for (const { row } of page.rows) {
+        const parsed = DataclawSessionSchema.safeParse(row);
+        if (parsed.success) {
+          sessions.push(parsed.data);
+        }
+      }
+
+      offset += page.rows.length;
+
+      // Safety: if the page returned 0 rows, stop to avoid infinite loop
+      if (page.rows.length === 0) break;
+    }
+
+    return sessions;
+  }
+
+  private async fetchPage(offset: number): Promise<HfRowsResponse> {
+    const params = new URLSearchParams({
+      dataset: this.datasetId,
+      config: this.config,
+      split: this.split,
+      offset: String(offset),
+      length: String(this.pageSize),
+    });
+
+    const url = `${this.baseUrl}/rows?${params}`;
+    const response = await this.fetchFn(url, { headers: this.buildHeaders() });
+
+    if (!response.ok) {
+      throw new Error(
+        `HF rows request failed: ${response.status} ${response.statusText} (offset=${offset})`,
+      );
+    }
+
+    return (await response.json()) as HfRowsResponse;
+  }
+}

package/src/learning/trajectory-sources/in-memory.ts

@@ -0,0 +1,61 @@
+import type {
+  TrajectorySource,
+  TrajectoryWithAnalysis,
+  TrajectoryLoadOptions,
+} from '../../types/trajectory-source.js';
+
+/**
+ * InMemoryTrajectorySource — wraps a user-provided array of trajectories.
+ *
+ * Simplest source: pass in trajectories you already have in memory.
+ *
+ * @example
+ * ```typescript
+ * const source = new InMemoryTrajectorySource(myTrajectories);
+ * await knowledgeBank.extractFrom(source);
+ * ```
+ */
+export class InMemoryTrajectorySource implements TrajectorySource {
+  readonly name = 'in-memory';
+  private items: TrajectoryWithAnalysis[];
+
+  constructor(items: TrajectoryWithAnalysis[]) {
+    this.items = items;
+  }
+
+  async load(options?: TrajectoryLoadOptions): Promise<TrajectoryWithAnalysis[]> {
+    return this.filter(options);
+  }
+
+  async count(options?: TrajectoryLoadOptions): Promise<number> {
+    return this.filter(options).length;
+  }
+
+  private filter(options?: TrajectoryLoadOptions): TrajectoryWithAnalysis[] {
+    let result = this.items;
+
+    if (options?.since) {
+      const since = options.since.getTime();
+      result = result.filter(
+        (item) => item.trajectory.timestamp.getTime() >= since
+      );
+    }
+
+    if (options?.domain) {
+      const domain = options.domain;
+      result = result.filter(
+        (item) => item.trajectory.task.domain === domain
+      );
+    }
+
+    if (options?.successOnly) {
+      result = result.filter((item) => item.trajectory.outcome.success);
+    }
+
+    if (options?.limit && options.limit > 0) {
+      result = result.slice(0, options.limit);
+    }
+
+    return result;
+  }
+}

package/src/learning/trajectory-sources/index.ts

@@ -0,0 +1,15 @@
+export { InMemoryTrajectorySource } from './in-memory.js';
+export { FileTrajectorySource } from './file.js';
+export { PipelineTrajectorySource } from './pipeline.js';
+export {
+  EntireTrajectorySource,
+  type EntireSynthesisOptions,
+} from './entire.js';
+export {
+  DataclawTrajectorySource,
+  type DataclawSourceOptions,
+} from './dataclaw.js';
+export {
+  HuggingFaceTrajectorySource,
+  type HuggingFaceSourceOptions,
+} from './huggingface.js';

package/src/learning/trajectory-sources/pipeline.ts

@@ -0,0 +1,69 @@
+import type {
+  TrajectorySource,
+  TrajectoryWithAnalysis,
+  TrajectoryLoadOptions,
+} from '../../types/trajectory-source.js';
+import type { LearningPipeline } from '../pipeline.js';
+
+/**
+ * PipelineTrajectorySource — reads from LearningPipeline's accumulated trajectories.
+ *
+ * Read-only view: does not clear the pipeline's accumulator.
+ * The pipeline's `runBatchLearning()` or `clearAccumulated()` controls clearing.
+ *
+ * @example
+ * ```typescript
+ * const source = new PipelineTrajectorySource(pipeline);
+ * await knowledgeBank.extractFrom(source);
+ * // Pipeline accumulator is unchanged — batch learning still works
+ * ```
+ */
+export class PipelineTrajectorySource implements TrajectorySource {
+  readonly name = 'pipeline';
+  private pipeline: LearningPipeline;
+
+  constructor(pipeline: LearningPipeline) {
+    this.pipeline = pipeline;
+  }
+
+  async load(options?: TrajectoryLoadOptions): Promise<TrajectoryWithAnalysis[]> {
+    const items = this.pipeline.getAccumulated();
+    return this.filter(items, options);
+  }
+
+  async count(options?: TrajectoryLoadOptions): Promise<number> {
+    const items = this.pipeline.getAccumulated();
+    return this.filter(items, options).length;
+  }
+
+  private filter(
+    items: TrajectoryWithAnalysis[],
+    options?: TrajectoryLoadOptions
+  ): TrajectoryWithAnalysis[] {
+    let result = items;
+
+    if (options?.since) {
+      const since = options.since.getTime();
+      result = result.filter(
+        (item) => item.trajectory.timestamp.getTime() >= since
+      );
+    }
+
+    if (options?.domain) {
+      const domain = options.domain;
+      result = result.filter(
+        (item) => item.trajectory.task.domain === domain
+      );
+    }
+
+    if (options?.successOnly) {
+      result = result.filter((item) => item.trajectory.outcome.success);
+    }
+
+    if (options?.limit && options.limit > 0) {
+      result = result.slice(0, options.limit);
+    }
+
+    return result;
+  }
+}