claude-flow 3.10.3 → 3.10.5

package/.claude/worktrees/adr-120-impl/v3/@claude-flow/memory/benchmarks/longmemeval/adapters/baseline-adapter.ts

@@ -1,104 +0,0 @@
-/**
- * Baseline Memory Adapter for LongMemEval
- *
- * Plain cosine similarity vector search without HNSW.
- * Used as a control to measure how much HNSW indexing helps.
- */
-
-import type { MemoryAdapter, Session } from '../types.js';
-
-export class BaselineAdapter implements MemoryAdapter {
-  readonly name = 'Baseline (Cosine Similarity)';
-  private entries: Array<{
-    key: string;
-    content: string;
-    embedding: Float32Array | null;
-    session_id: string;
-    metadata: Record<string, unknown>;
-  }> = [];
-  private embedder: any = null;
-
-  async init(): Promise<void> {
-    // Try to load ONNX embedder for fair comparison
-    try {
-      const { OnnxEmbedder } = await import('../../../src/onnx-embedder.js');
-      this.embedder = new OnnxEmbedder();
-      await this.embedder.initialize();
-    } catch {
-      console.warn('[BaselineAdapter] ONNX embedder unavailable, using mock embeddings');
-    }
-  }
-
-  async ingestSession(session: Session): Promise<void> {
-    for (const msg of session.messages) {
-      const key = `${session.session_id}:${msg.role}:${msg.timestamp ?? Date.now()}`;
-      let embedding: Float32Array | null = null;
-
-      if (this.embedder) {
-        embedding = await this.embedder.embed(msg.content);
-      }
-
-      this.entries.push({
-        key,
-        content: msg.content,
-        embedding,
-        session_id: session.session_id,
-        metadata: {
-          session_id: session.session_id,
-          role: msg.role,
-          timestamp: msg.timestamp,
-        },
-      });
-    }
-  }
-
-  async retrieve(
-    question: string,
-    topK: number = 10
-  ): Promise<Array<{ content: string; score: number; session_id: string; metadata?: Record<string, unknown> }>> {
-    if (!this.embedder || this.entries.length === 0) return [];
-
-    const queryEmbedding = await this.embedder.embed(question);
-    if (!queryEmbedding) return [];
-
-    // Brute-force cosine similarity
-    const scored = this.entries
-      .filter(e => e.embedding !== null)
-      .map(e => ({
-        content: e.content,
-        score: cosineSimilarity(queryEmbedding, e.embedding!),
-        session_id: e.session_id,
-        metadata: e.metadata,
-      }))
-      .sort((a, b) => b.score - a.score)
-      .slice(0, topK);
-
-    return scored;
-  }
-
-  async getStats(): Promise<{ entries: number; sizeBytes: number }> {
-    const contentBytes = this.entries.reduce((sum, e) => sum + e.content.length * 2, 0);
-    const embeddingBytes = this.entries.reduce((sum, e) => sum + (e.embedding?.byteLength ?? 0), 0);
-    return {
-      entries: this.entries.length,
-      sizeBytes: contentBytes + embeddingBytes,
-    };
-  }
-
-  async close(): Promise<void> {
-    this.entries = [];
-    this.embedder = null;
-  }
-}
-
-/** Cosine similarity between two vectors */
-function cosineSimilarity(a: Float32Array, b: Float32Array): number {
-  let dot = 0, normA = 0, normB = 0;
-  for (let i = 0; i < a.length; i++) {
-    dot += a[i] * b[i];
-    normA += a[i] * a[i];
-    normB += b[i] * b[i];
-  }
-  const denom = Math.sqrt(normA) * Math.sqrt(normB);
-  return denom === 0 ? 0 : dot / denom;
-}

package/.claude/worktrees/adr-120-impl/v3/@claude-flow/memory/benchmarks/longmemeval/harness.ts

@@ -1,374 +0,0 @@
-/**
- * LongMemEval Benchmark Harness for AgentDB
- *
- * ADR-088: Full LongMemEval benchmark implementation
- *
- * Usage:
- *   npx tsx harness.ts --mode raw --limit 50      # Quick test (50 questions)
- *   npx tsx harness.ts --mode raw                  # Full benchmark (500 questions)
- *   npx tsx harness.ts --mode hybrid               # With Haiku reranking
- *   npx tsx harness.ts --mode baseline             # Plain cosine baseline
- */
-
-import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'node:fs';
-import { join, dirname } from 'node:path';
-import { fileURLToPath } from 'node:url';
-import type {
-  Session,
-  Question,
-  QuestionResult,
-  CategoryResult,
-  BenchmarkReport,
-  BenchmarkMode,
-  MemoryAdapter,
-  QuestionType,
-} from './types.js';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
-// ── CLI Args ──────────────────────────────────────────────────
-
-function parseArgs(): { mode: BenchmarkMode; limit: number; dataDir: string; adapter: string } {
-  const args = process.argv.slice(2);
-  let mode: BenchmarkMode = 'raw';
-  let limit = 0; // 0 = all
-  let dataDir = join(__dirname, 'data');
-  let adapter = 'agentdb';
-
-  for (let i = 0; i < args.length; i++) {
-    switch (args[i]) {
-      case '--mode': mode = (args[++i] as BenchmarkMode) || 'raw'; break;
-      case '--limit': limit = parseInt(args[++i], 10) || 0; break;
-      case '--data': dataDir = args[++i]; break;
-      case '--adapter': adapter = args[++i]; break;
-    }
-  }
-
-  return { mode, limit, dataDir, adapter };
-}
-
-// ── Dataset Loading ───────────────────────────────────────────
-
-function loadDataset(dataDir: string): { sessions: Session[]; questions: Question[] } {
-  const oracleFile = join(dataDir, 'longmemeval_oracle.json');
-  if (!existsSync(oracleFile)) {
-    console.error(`Dataset not found at ${oracleFile}`);
-    console.error('Run: bash scripts/download-dataset.sh');
-    process.exit(1);
-  }
-
-  const raw = JSON.parse(readFileSync(oracleFile, 'utf-8'));
-
-  // LongMemEval format: array of items with sessions + question
-  const sessions: Session[] = [];
-  const questions: Question[] = [];
-  const seenSessions = new Set<string>();
-
-  for (const item of raw) {
-    // Extract sessions
-    if (item.sessions) {
-      for (const sess of item.sessions) {
-        if (!seenSessions.has(sess.session_id)) {
-          seenSessions.add(sess.session_id);
-          sessions.push(sess);
-        }
-      }
-    }
-
-    // Extract question
-    if (item.question) {
-      questions.push({
-        question_id: item.question_id ?? item.id ?? `q-${questions.length}`,
-        question: item.question,
-        answer: item.answer ?? item.answers ?? '',
-        question_type: mapQuestionType(item.question_type ?? item.type ?? ''),
-        evidence_session_ids: item.evidence_session_ids ?? [],
-        num_hops: item.num_hops ?? 1,
-      });
-    }
-  }
-
-  return { sessions, questions };
-}
-
-function mapQuestionType(raw: string): QuestionType {
-  const normalized = raw.toLowerCase().replace(/[^a-z-]/g, '-');
-  const mapping: Record<string, QuestionType> = {
-    'single-session-single-hop': 'single-session-single-hop',
-    'single-session-multi-hop': 'single-session-multi-hop',
-    'multi-session-single-hop': 'multi-session-single-hop',
-    'multi-session-multi-hop': 'multi-session-multi-hop',
-    'knowledge-update': 'knowledge-update',
-    'temporal-reasoning': 'temporal-reasoning',
-  };
-  return mapping[normalized] ?? 'single-session-single-hop';
-}
-
-// ── Answer Evaluation ─────────────────────────────────────────
-
-function evaluateAnswer(predicted: string, gold: string | string[]): boolean {
-  const normalize = (s: string) => s.toLowerCase().trim().replace(/[^\w\s]/g, '');
-  const pred = normalize(predicted);
-
-  const golds = Array.isArray(gold) ? gold : [gold];
-
-  for (const g of golds) {
-    const ng = normalize(g);
-    // Exact match
-    if (pred === ng) return true;
-    // Contains match (lenient — official eval is more nuanced)
-    if (pred.includes(ng) || ng.includes(pred)) return true;
-  }
-  return false;
-}
-
-// ── Answer Generation ─────────────────────────────────────────
-
-async function generateAnswer(
-  question: string,
-  context: Array<{ content: string; score: number }>,
-  mode: BenchmarkMode
-): Promise<string> {
-  // Raw mode: extract answer directly from top retrieved chunk
-  if (mode === 'raw' || mode === 'baseline') {
-    // Simple extractive: return the most relevant chunk
-    // A real system would use an LLM here, but raw mode = zero API
-    return context.length > 0 ? context[0].content : '';
-  }
-
-  // Hybrid / Full mode: use Haiku for answer generation
-  if (mode === 'hybrid' || mode === 'full') {
-    const contextText = context
-      .slice(0, 5)
-      .map((c, i) => `[${i + 1}] ${c.content}`)
-      .join('\n\n');
-
-    try {
-      const { default: Anthropic } = await import('@anthropic-ai/sdk');
-      const client = new Anthropic();
-
-      const response = await client.messages.create({
-        model: 'claude-haiku-4-5-20251001',
-        max_tokens: 256,
-        messages: [
-          {
-            role: 'user',
-            content: `Based on the following conversation excerpts, answer the question concisely.\n\nContext:\n${contextText}\n\nQuestion: ${question}\n\nAnswer (be concise, just the answer):`,
-          },
-        ],
-      });
-
-      const block = response.content[0];
-      return block.type === 'text' ? block.text.trim() : '';
-    } catch (err) {
-      console.warn(`[Haiku] Failed: ${(err as Error).message}`);
-      return context.length > 0 ? context[0].content : '';
-    }
-  }
-
-  return '';
-}
-
-// ── Percentile Calculation ────────────────────────────────────
-
-function percentile(arr: number[], p: number): number {
-  if (arr.length === 0) return 0;
-  const sorted = [...arr].sort((a, b) => a - b);
-  const idx = Math.ceil((p / 100) * sorted.length) - 1;
-  return sorted[Math.max(0, idx)];
-}
-
-// ── Main Benchmark ────────────────────────────────────────────
-
-async function main() {
-  const { mode, limit, dataDir, adapter: adapterName } = parseArgs();
-
-  console.log('=== LongMemEval Benchmark for AgentDB ===');
-  console.log(`Mode:    ${mode}`);
-  console.log(`Adapter: ${adapterName}`);
-  console.log(`Limit:   ${limit || 'all'}`);
-  console.log('');
-
-  // Load dataset
-  console.log('[1/5] Loading dataset...');
-  const { sessions, questions } = loadDataset(dataDir);
-  const evalQuestions = limit > 0 ? questions.slice(0, limit) : questions;
-  console.log(`  Sessions: ${sessions.length}, Questions: ${evalQuestions.length}/${questions.length}`);
-
-  // Initialize adapter
-  console.log(`[2/5] Initializing ${adapterName} adapter...`);
-  let adapter: MemoryAdapter;
-
-  if (adapterName === 'baseline') {
-    const { BaselineAdapter } = await import('./adapters/baseline-adapter.js');
-    adapter = new BaselineAdapter();
-  } else {
-    const { AgentDBAdapter } = await import('./adapters/agentdb-adapter.js');
-    adapter = new AgentDBAdapter();
-  }
-
-  await adapter.init();
-
-  // Ingest sessions
-  console.log('[3/5] Ingesting sessions...');
-  const ingestStart = performance.now();
-  let totalMessages = 0;
-
-  for (let i = 0; i < sessions.length; i++) {
-    await adapter.ingestSession(sessions[i]);
-    totalMessages += sessions[i].messages.length;
-    if ((i + 1) % 50 === 0 || i === sessions.length - 1) {
-      process.stdout.write(`  Ingested ${i + 1}/${sessions.length} sessions (${totalMessages} messages)\r`);
-    }
-  }
-  const ingestMs = performance.now() - ingestStart;
-  console.log(`\n  Ingestion complete in ${(ingestMs / 1000).toFixed(1)}s`);
-
-  // Evaluate questions
-  console.log('[4/5] Evaluating questions...');
-  const results: QuestionResult[] = [];
-  let correct = 0;
-
-  for (let i = 0; i < evalQuestions.length; i++) {
-    const q = evalQuestions[i];
-
-    // Retrieve
-    const retStart = performance.now();
-    const retrieved = await adapter.retrieve(q.question);
-    const retrievalMs = performance.now() - retStart;
-
-    // Generate answer
-    const genStart = performance.now();
-    const predicted = await generateAnswer(q.question, retrieved, mode);
-    const generationMs = performance.now() - genStart;
-
-    // Evaluate
-    const isCorrect = evaluateAnswer(predicted, q.answer);
-    if (isCorrect) correct++;
-
-    results.push({
-      question_id: q.question_id,
-      question_type: q.question_type,
-      predicted_answer: predicted.slice(0, 200),
-      gold_answer: q.answer,
-      correct: isCorrect,
-      retrieval_time_ms: Math.round(retrievalMs * 100) / 100,
-      generation_time_ms: Math.round(generationMs * 100) / 100,
-      retrieved_chunks: retrieved.length,
-    });
-
-    if ((i + 1) % 25 === 0 || i === evalQuestions.length - 1) {
-      const pct = ((correct / (i + 1)) * 100).toFixed(1);
-      process.stdout.write(`  ${i + 1}/${evalQuestions.length} — ${correct}/${i + 1} correct (${pct}%)\r`);
-    }
-  }
-
-  const overallAccuracy = (correct / evalQuestions.length) * 100;
-  console.log(`\n  Overall: ${correct}/${evalQuestions.length} (${overallAccuracy.toFixed(1)}%)`);
-
-  // Per-category breakdown
-  const categories = new Map<QuestionType, { total: number; correct: number; retMs: number[]; genMs: number[] }>();
-  for (const r of results) {
-    const cat = categories.get(r.question_type) ?? { total: 0, correct: 0, retMs: [], genMs: [] };
-    cat.total++;
-    if (r.correct) cat.correct++;
-    cat.retMs.push(r.retrieval_time_ms);
-    cat.genMs.push(r.generation_time_ms);
-    categories.set(r.question_type, cat);
-  }
-
-  const byCategory: CategoryResult[] = [...categories.entries()].map(([type, cat]) => ({
-    question_type: type,
-    total: cat.total,
-    correct: cat.correct,
-    accuracy: Math.round((cat.correct / cat.total) * 10000) / 100,
-    avg_retrieval_ms: Math.round(cat.retMs.reduce((a, b) => a + b, 0) / cat.retMs.length * 100) / 100,
-    avg_generation_ms: Math.round(cat.genMs.reduce((a, b) => a + b, 0) / cat.genMs.length * 100) / 100,
-  }));
-
-  // Latency stats
-  const allRetMs = results.map(r => r.retrieval_time_ms);
-  const allGenMs = results.map(r => r.generation_time_ms);
-
-  // Storage stats
-  const stats = await adapter.getStats();
-
-  // Build report
-  const report: BenchmarkReport = {
-    timestamp: new Date().toISOString(),
-    mode,
-    system: adapter.name,
-    version: '3.5.78',
-    overall: {
-      total: evalQuestions.length,
-      correct,
-      accuracy: Math.round(overallAccuracy * 100) / 100,
-    },
-    by_category: byCategory,
-    latency: {
-      retrieval_p50_ms: percentile(allRetMs, 50),
-      retrieval_p95_ms: percentile(allRetMs, 95),
-      retrieval_p99_ms: percentile(allRetMs, 99),
-      generation_p50_ms: percentile(allGenMs, 50),
-      generation_p95_ms: percentile(allGenMs, 95),
-    },
-    storage: {
-      sessions_ingested: sessions.length,
-      total_messages: totalMessages,
-      db_size_bytes: stats.sizeBytes,
-      index_size_bytes: 0,
-    },
-    config: {
-      embedding_model: 'all-MiniLM-L6-v2',
-      embedding_dims: 384,
-      hnsw_m: 16,
-      hnsw_ef_search: 100,
-      top_k: 10,
-      similarity_threshold: 0.3,
-    },
-  };
-
-  // Save report
-  console.log('[5/5] Saving report...');
-  const resultsDir = join(__dirname, 'results');
-  mkdirSync(resultsDir, { recursive: true });
-
-  const reportFile = join(resultsDir, `report-${mode}-${new Date().toISOString().slice(0, 10)}.json`);
-  writeFileSync(reportFile, JSON.stringify(report, null, 2) + '\n', 'utf-8');
-  console.log(`  Report: ${reportFile}`);
-
-  // Save detailed results
-  const detailFile = join(resultsDir, `details-${mode}-${new Date().toISOString().slice(0, 10)}.json`);
-  writeFileSync(detailFile, JSON.stringify(results, null, 2) + '\n', 'utf-8');
-  console.log(`  Details: ${detailFile}`);
-
-  // Print summary table
-  console.log('');
-  console.log('=== Results ===');
-  console.log(`System:   ${adapter.name}`);
-  console.log(`Mode:     ${mode}`);
-  console.log(`Overall:  ${report.overall.accuracy}% (${correct}/${evalQuestions.length})`);
-  console.log('');
-  console.log('By Category:');
-  console.log('  Type                          | Total | Correct | Accuracy');
-  console.log('  ------------------------------|-------|---------|--------');
-  for (const cat of byCategory) {
-    const name = cat.question_type.padEnd(30);
-    console.log(`  ${name}| ${String(cat.total).padStart(5)} | ${String(cat.correct).padStart(7)} | ${cat.accuracy.toFixed(1)}%`);
-  }
-  console.log('');
-  console.log('Latency:');
-  console.log(`  Retrieval p50: ${report.latency.retrieval_p50_ms.toFixed(1)}ms`);
-  console.log(`  Retrieval p95: ${report.latency.retrieval_p95_ms.toFixed(1)}ms`);
-  console.log(`  Retrieval p99: ${report.latency.retrieval_p99_ms.toFixed(1)}ms`);
-
-  // Cleanup
-  await adapter.close();
-  console.log('');
-  console.log('Benchmark complete.');
-}
-
-main().catch(err => {
-  console.error('Benchmark failed:', err);
-  process.exit(1);
-});

package/.claude/worktrees/adr-120-impl/v3/@claude-flow/memory/benchmarks/longmemeval/scripts/download-dataset.sh

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-# Download LongMemEval dataset from HuggingFace
-# Source: https://github.com/xiaowu0162/LongMemEval
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-DATA_DIR="$SCRIPT_DIR/../data"
-
-mkdir -p "$DATA_DIR"
-
-echo "Downloading LongMemEval dataset..."
-
-BASE_URL="https://huggingface.co/datasets/xiaowu0162/longmemeval-cleaned/resolve/main"
-
-for file in longmemeval_oracle.json longmemeval_s_cleaned.json longmemeval_m_cleaned.json; do
-  if [ -f "$DATA_DIR/$file" ]; then
-    echo "  [skip] $file (already exists)"
-  else
-    echo "  [download] $file"
-    curl -L -o "$DATA_DIR/$file" "$BASE_URL/$file"
-  fi
-done
-
-echo ""
-echo "Dataset downloaded to: $DATA_DIR"
-echo "Files:"
-ls -lh "$DATA_DIR"/*.json 2>/dev/null || echo "  (no files found)"

package/.claude/worktrees/adr-120-impl/v3/@claude-flow/memory/benchmarks/longmemeval/scripts/run-benchmark.sh

@@ -1,46 +0,0 @@
-#!/usr/bin/env bash
-# Run the full LongMemEval benchmark for AgentDB
-# Usage: ./run-benchmark.sh [--mode raw|hybrid|full|baseline] [--limit N]
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
-BENCH_DIR="$SCRIPT_DIR/.."
-MODE="${1:---mode}"
-MODE_VAL="${2:-raw}"
-
-# Parse args
-LIMIT=""
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --mode) MODE_VAL="$2"; shift 2 ;;
-    --limit) LIMIT="--limit $2"; shift 2 ;;
-    *) shift ;;
-  esac
-done
-
-echo "=== LongMemEval Benchmark for AgentDB ==="
-echo "Mode: $MODE_VAL"
-echo ""
-
-# Step 1: Ensure dataset exists
-if [ ! -f "$BENCH_DIR/data/longmemeval_oracle.json" ]; then
-  echo "Dataset not found. Downloading..."
-  bash "$SCRIPT_DIR/download-dataset.sh"
-fi
-
-# Step 2: Ingest conversations into AgentDB
-echo "[1/3] Ingesting conversations into AgentDB..."
-npx tsx "$BENCH_DIR/ingest.ts" --data "$BENCH_DIR/data" $LIMIT
-
-# Step 3: Run evaluation
-echo "[2/3] Running evaluation (mode=$MODE_VAL)..."
-npx tsx "$BENCH_DIR/evaluate.ts" --mode "$MODE_VAL" --data "$BENCH_DIR/data" $LIMIT
-
-# Step 4: Generate report
-echo "[3/3] Generating report..."
-npx tsx "$BENCH_DIR/report.ts" --mode "$MODE_VAL"
-
-echo ""
-echo "=== Benchmark complete ==="
-echo "Results saved to: $BENCH_DIR/results/"

package/.claude/worktrees/adr-120-impl/v3/@claude-flow/memory/benchmarks/longmemeval/types.ts

@@ -1,120 +0,0 @@
-/**
- * LongMemEval Benchmark Types
- * Based on: https://github.com/xiaowu0162/LongMemEval
- */
-
-/** A single conversation session from the dataset */
-export interface Session {
-  session_id: string;
-  messages: Array<{
-    role: 'user' | 'assistant';
-    content: string;
-    timestamp?: string;
-  }>;
-}
-
-/** A benchmark question */
-export interface Question {
-  question_id: string;
-  question: string;
-  answer: string | string[];
-  question_type: QuestionType;
-  evidence_session_ids: string[];
-  /** Number of reasoning hops required */
-  num_hops: number;
-}
-
-/** The 6 question types in LongMemEval */
-export type QuestionType =
-  | 'single-session-single-hop'
-  | 'single-session-multi-hop'
-  | 'multi-session-single-hop'
-  | 'multi-session-multi-hop'
-  | 'knowledge-update'
-  | 'temporal-reasoning';
-
-/** Benchmark mode */
-export type BenchmarkMode = 'raw' | 'hybrid' | 'full' | 'baseline';
-
-/** Result for a single question */
-export interface QuestionResult {
-  question_id: string;
-  question_type: QuestionType;
-  predicted_answer: string;
-  gold_answer: string | string[];
-  correct: boolean;
-  retrieval_time_ms: number;
-  generation_time_ms: number;
-  retrieved_chunks: number;
-}
-
-/** Aggregate results per question type */
-export interface CategoryResult {
-  question_type: QuestionType;
-  total: number;
-  correct: number;
-  accuracy: number;
-  avg_retrieval_ms: number;
-  avg_generation_ms: number;
-}
-
-/** Full benchmark report */
-export interface BenchmarkReport {
-  timestamp: string;
-  mode: BenchmarkMode;
-  system: string;
-  version: string;
-  overall: {
-    total: number;
-    correct: number;
-    accuracy: number;
-  };
-  by_category: CategoryResult[];
-  latency: {
-    retrieval_p50_ms: number;
-    retrieval_p95_ms: number;
-    retrieval_p99_ms: number;
-    generation_p50_ms: number;
-    generation_p95_ms: number;
-  };
-  storage: {
-    sessions_ingested: number;
-    total_messages: number;
-    db_size_bytes: number;
-    index_size_bytes: number;
-  };
-  config: {
-    embedding_model: string;
-    embedding_dims: number;
-    hnsw_m: number;
-    hnsw_ef_search: number;
-    top_k: number;
-    similarity_threshold: number;
-  };
-}
-
-/** Memory adapter interface — implementations provide storage + retrieval */
-export interface MemoryAdapter {
-  /** Human-readable name */
-  readonly name: string;
-
-  /** Initialize the adapter and create indices */
-  init(): Promise<void>;
-
-  /** Ingest a conversation session */
-  ingestSession(session: Session): Promise<void>;
-
-  /** Retrieve relevant context for a question */
-  retrieve(question: string, topK?: number): Promise<Array<{
-    content: string;
-    score: number;
-    session_id: string;
-    metadata?: Record<string, unknown>;
-  }>>;
-
-  /** Get storage stats */
-  getStats(): Promise<{ entries: number; sizeBytes: number }>;
-
-  /** Clean up resources */
-  close(): Promise<void>;
-}