@1mbrain/benchmarks 0.1.1

package/src/adapters/unavailable.ts

@@ -0,0 +1,49 @@
+import type {
+  BenchmarkMemoryRecord,
+  BenchmarkRecallRequest,
+  BenchmarkRecallResult,
+  MemoryProviderAdapter,
+  ProviderAvailability,
+} from '../provider.js';
+
+export class UnavailableAdapter implements MemoryProviderAdapter {
+  readonly capabilities = {
+    associations: false,
+    forget: false,
+    decay: false,
+    portability: false,
+  } as const;
+
+  constructor(
+    readonly name: string,
+    readonly label: string,
+    private readonly reason: string,
+  ) {}
+
+  async availability(): Promise<ProviderAvailability> {
+    return {
+      status: 'unsupported',
+      reason: this.reason,
+    };
+  }
+
+  async reset(_agentId: string): Promise<void> {
+    throw new Error(this.reason);
+  }
+
+  async remember(_memory: BenchmarkMemoryRecord, _agentId: string): Promise<void> {
+    throw new Error(this.reason);
+  }
+
+  async recall(
+    _request: BenchmarkRecallRequest & {
+      agentId: string;
+    },
+  ): Promise<BenchmarkRecallResult[]> {
+    throw new Error(this.reason);
+  }
+
+  async close(): Promise<void> {
+    // No-op.
+  }
+}

package/src/adapters/vector-baseline.ts

@@ -0,0 +1,149 @@
+import { rm, stat } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { SqliteDatabaseProvider, logger } from '@1mbrain/core';
+import type {
+  BenchmarkMemoryRecord,
+  BenchmarkRecallRequest,
+  BenchmarkRecallResult,
+  MemoryProviderAdapter,
+  ProviderAvailability,
+  ProviderStats,
+} from '../provider.js';
+import { KeywordEmbeddingProvider } from './keyword-embedding.js';
+
+export class VectorBaselineAdapter implements MemoryProviderAdapter {
+  readonly name = 'vector_baseline';
+  readonly label = 'Vector Baseline (SQLite)';
+  readonly capabilities = {
+    associations: false,
+    forget: true,
+    decay: true,
+    portability: false,
+  } as const;
+
+  private readonly embedder = new KeywordEmbeddingProvider();
+  private db: SqliteDatabaseProvider | null = null;
+  private dbPath: string | null = null;
+
+  async availability(): Promise<ProviderAvailability> {
+    return { status: 'available' };
+  }
+
+  async reset(_agentId: string): Promise<void> {
+    await this.close();
+    logger.level = 'silent';
+
+    this.dbPath = join(tmpdir(), `${this.name}-${process.pid}-${Date.now()}.sqlite`);
+    this.db = new SqliteDatabaseProvider(this.dbPath);
+    await this.db.initialize();
+  }
+
+  async remember(memory: BenchmarkMemoryRecord, agentId: string): Promise<void> {
+    if (!this.db) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    await this.db.createMemory({
+      id: memory.id,
+      agentId,
+      type: memory.type,
+      content: memory.content,
+      embeddingModel: this.embedder.model,
+      embedding: await this.embedder.embed(memory.content),
+      importance: memory.importance ?? 0.75,
+      decayScore: 1,
+      tags: memory.tags,
+      metadata: {
+        ...(memory.metadata ?? {}),
+        benchId: memory.id,
+        benchTimestamp: memory.timestamp,
+      },
+    });
+  }
+
+  async recall(
+    request: BenchmarkRecallRequest & {
+      agentId: string;
+    },
+  ): Promise<BenchmarkRecallResult[]> {
+    if (!this.db) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    const results = await this.db.searchByVector(
+      request.agentId,
+      await this.embedder.embed(request.query ?? ''),
+      {
+        limit: request.limit ?? 5,
+        threshold: request.minScore ?? 0.08,
+      },
+    );
+
+    return results.map((result) => ({
+      memoryId: String(result.memory.metadata?.['benchId'] ?? result.memory.id),
+      content: result.memory.content,
+      score: result.similarity,
+      type: result.memory.type as BenchmarkMemoryRecord['type'],
+      source: 'vector',
+      metadata: result.memory.metadata,
+    }));
+  }
+
+  async forget(memoryId: string, agentId: string): Promise<void> {
+    if (!this.db) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    await this.db.deleteAssociations(memoryId);
+    await this.db.deleteMemory(memoryId, agentId);
+  }
+
+  async applyDecay(decayRate: number, minScore: number): Promise<number> {
+    if (!this.db) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    return this.db.applyDecay(decayRate, minScore);
+  }
+
+  async getStats(): Promise<ProviderStats> {
+    return {
+      storageSizeBytes: await sqliteFootprint(this.dbPath),
+    };
+  }
+
+  async close(): Promise<void> {
+    if (this.db) {
+      await this.db.close();
+      this.db = null;
+    }
+
+    if (this.dbPath) {
+      await removeSqliteArtifacts(this.dbPath);
+      this.dbPath = null;
+    }
+  }
+}
+
+async function sqliteFootprint(dbPath: string | null): Promise<number | null> {
+  if (!dbPath) return null;
+
+  const candidates = [dbPath, `${dbPath}-wal`, `${dbPath}-shm`];
+  let total = 0;
+  for (const candidate of candidates) {
+    try {
+      total += (await stat(candidate)).size;
+    } catch {
+      // Ignore files that do not exist.
+    }
+  }
+  return total;
+}
+
+async function removeSqliteArtifacts(dbPath: string): Promise<void> {
+  const candidates = [dbPath, `${dbPath}-wal`, `${dbPath}-shm`];
+  for (const candidate of candidates) {
+    await rm(candidate, { force: true }).catch(() => undefined);
+  }
+}

package/src/datasets/focused-mini.ts

@@ -0,0 +1,158 @@
+import { readFileSync } from 'node:fs';
+import { resolve } from 'node:path';
+import type {
+  BenchmarkCase,
+  BenchmarkDataset,
+  BenchmarkMemoryRecord,
+  BenchmarkMemoryType,
+  BenchmarkOperation,
+  BenchmarkScenarioType,
+} from '../provider.js';
+
+type FixtureMemoryRecord = {
+  id: string;
+  type: BenchmarkMemoryType;
+  timestamp: string;
+  content: string;
+  tags: string[];
+  importance?: number;
+  metadata?: Record<string, unknown>;
+  associations?: Array<{
+    target_id?: string;
+    targetId?: string;
+    strength: number;
+  }>;
+};
+
+type FixtureQuestion = {
+  question_id: string;
+  category: string;
+  question: string;
+  expected_answer: string;
+  required_memory_ids: string[];
+  forbidden_memory_ids: string[];
+};
+
+type FixtureConversation = {
+  conversation_id: string;
+  agent_id: string;
+  memory_records: FixtureMemoryRecord[];
+  questions: FixtureQuestion[];
+};
+
+type FocusedMiniFixture = {
+  name: string;
+  generated_at: string;
+  conversations: FixtureConversation[];
+};
+
+const CATEGORY_TO_SCENARIO: Record<string, BenchmarkScenarioType> = {
+  atomic_fact_recall: 'basic_semantic_recall',
+  abstention: 'noise_resistance',
+  context_injection: 'agent_task_context',
+  contradiction_resolution: 'memory_update',
+  current_preference: 'memory_update',
+  entity_disambiguation: 'noise_resistance',
+  graph_traversal: 'multi_hop_recall',
+  multi_hop: 'multi_hop_recall',
+  multi_hop_association: 'multi_hop_recall',
+  noise_resistance: 'noise_resistance',
+  paraphrased_semantic_recall: 'basic_semantic_recall',
+  portability: 'portability',
+  procedural_recall: 'basic_semantic_recall',
+  review_behavior: 'agent_task_context',
+  root_cause_recall: 'multi_hop_recall',
+  temporal_update: 'memory_update',
+};
+
+export function createFocusedMiniDataset(packageRoot: string): BenchmarkDataset {
+  return createFixtureDataset(
+    packageRoot,
+    'fixtures/1mbrain-focused-mini/1mbrain-focused-mini.json',
+  );
+}
+
+export function createFixtureDataset(packageRoot: string, relativePath: string): BenchmarkDataset {
+  const fixturePath = resolve(packageRoot, relativePath);
+  const fixture = JSON.parse(readFileSync(fixturePath, 'utf8')) as FocusedMiniFixture;
+  const cases: BenchmarkCase[] = [];
+
+  for (const conversation of fixture.conversations) {
+    const memories = conversation.memory_records.map(toBenchmarkMemory);
+
+    for (const question of conversation.questions) {
+      const scenarioType = CATEGORY_TO_SCENARIO[question.category] ?? 'basic_semantic_recall';
+      const operations: BenchmarkOperation[] =
+        scenarioType === 'portability'
+          ? [
+              {
+                kind: 'export_import',
+                targetAgentId: `${conversation.agent_id}_${question.question_id}_imported`,
+              },
+            ]
+          : [];
+
+      cases.push({
+        scenarioId: question.question_id,
+        scenarioType,
+        title: question.question_id,
+        description: `${conversation.conversation_id}: ${question.category}`,
+        agentId: `${conversation.agent_id}_${question.question_id}`,
+        memories,
+        operations,
+        question: question.question,
+        expectedAnswer: question.expected_answer,
+        recallOptions: {
+          limit: 10,
+          minScore: 0.08,
+          maxHops: 3,
+          activationThreshold: 0.05,
+          blendWeight: 0.45,
+        },
+        expectations: {
+          requiredMemoryIds: question.required_memory_ids,
+          forbiddenMemoryIds: question.forbidden_memory_ids,
+          shouldAbstain: question.category === 'abstention' ? true : undefined,
+          preferredOver: buildPreferredOver(question),
+        },
+      });
+    }
+  }
+
+  return {
+    name: fixture.name,
+    generatedAt: fixture.generated_at,
+    cases,
+  };
+}
+
+function toBenchmarkMemory(memory: FixtureMemoryRecord): BenchmarkMemoryRecord {
+  return {
+    id: memory.id,
+    type: memory.type,
+    timestamp: memory.timestamp,
+    content: memory.content,
+    tags: memory.tags,
+    importance: memory.importance,
+    metadata: memory.metadata,
+    associations: memory.associations?.flatMap((association) => {
+      const targetId = association.targetId ?? association.target_id;
+      return targetId ? [{ targetId, strength: association.strength }] : [];
+    }),
+  };
+}
+
+function buildPreferredOver(question: FixtureQuestion): BenchmarkCase['expectations']['preferredOver'] {
+  if (question.forbidden_memory_ids.length === 0 || question.required_memory_ids.length === 0) {
+    return undefined;
+  }
+
+  if (!['current_preference', 'contradiction_resolution', 'temporal_update'].includes(question.category)) {
+    return undefined;
+  }
+
+  return question.required_memory_ids.map((preferredId) => ({
+    preferredId,
+    competingIds: question.forbidden_memory_ids,
+  }));
+}