@1mbrain/benchmarks 0.1.1

package/src/adapters/1mbrain.ts

@@ -0,0 +1,317 @@
+import { randomUUID } from 'node:crypto';
+import { rm, stat } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import {
+  InMemoryEventBus,
+  MemoryEngine,
+  SqliteDatabaseProvider,
+  logger,
+} from '@1mbrain/core';
+import type { MemoryPassport } from '@1mbrain/core';
+import type {
+  BenchmarkMemoryRecord,
+  BenchmarkRecallRequest,
+  BenchmarkRecallResult,
+  MemoryProviderAdapter,
+  ProviderAvailability,
+  ProviderStats,
+} from '../provider.js';
+import { KeywordEmbeddingProvider } from './keyword-embedding.js';
+import { OpenAIEmbeddingProvider } from '@1mbrain/core';
+
+type OneMBrainMode =
+  | {
+      name: '1mbrain_vector_only';
+      label: '1MBrain Vector Only';
+      useSpreadingActivation: false;
+      maxHops: 0;
+      activationThreshold: 1;
+      blendWeight: 0;
+    }
+  | {
+      name: '1mbrain_graph_light';
+      label: '1MBrain Graph Light';
+      useSpreadingActivation: true;
+      maxHops: 1;
+      activationThreshold: 0.08;
+      blendWeight: 0.25;
+    }
+  | {
+      name: '1mbrain_graph_full';
+      label: '1MBrain Graph Full';
+      useSpreadingActivation: true;
+      maxHops: 3;
+      activationThreshold: 0.05;
+      blendWeight: 0.45;
+    };
+
+const MODES: Record<OneMBrainMode['name'], OneMBrainMode> = {
+  '1mbrain_vector_only': {
+    name: '1mbrain_vector_only',
+    label: '1MBrain Vector Only',
+    useSpreadingActivation: false,
+    maxHops: 0,
+    activationThreshold: 1,
+    blendWeight: 0,
+  },
+  '1mbrain_graph_light': {
+    name: '1mbrain_graph_light',
+    label: '1MBrain Graph Light',
+    useSpreadingActivation: true,
+    maxHops: 1,
+    activationThreshold: 0.08,
+    blendWeight: 0.25,
+  },
+  '1mbrain_graph_full': {
+    name: '1mbrain_graph_full',
+    label: '1MBrain Graph Full',
+    useSpreadingActivation: true,
+    maxHops: 3,
+    activationThreshold: 0.05,
+    blendWeight: 0.45,
+  },
+};
+
+export class OneMBrainBenchmarkAdapter implements MemoryProviderAdapter {
+  readonly name: OneMBrainMode['name'];
+  readonly label: string;
+  readonly capabilities = {
+    associations: true,
+    forget: true,
+    decay: true,
+    portability: true,
+  } as const;
+
+  private readonly mode: OneMBrainMode;
+  private readonly embedder = process.env['OPENAI_API_KEY']
+    ? new OpenAIEmbeddingProvider(process.env['OPENAI_API_KEY'], 'text-embedding-3-small')
+    : new KeywordEmbeddingProvider();
+  private db: SqliteDatabaseProvider | null = null;
+  private engine: MemoryEngine | null = null;
+  private dbPath: string | null = null;
+  private readonly idMap = new Map<string, string>();
+
+  constructor(modeName: OneMBrainMode['name']) {
+    this.mode = MODES[modeName];
+    this.name = this.mode.name;
+    this.label = this.mode.label;
+  }
+
+  async availability(): Promise<ProviderAvailability> {
+    return { status: 'available' };
+  }
+
+  async reset(_agentId: string): Promise<void> {
+    await this.close();
+    logger.level = 'silent';
+
+    this.dbPath = join(tmpdir(), `${this.name}-${process.pid}-${Date.now()}.sqlite`);
+    this.db = new SqliteDatabaseProvider(this.dbPath);
+    await this.db.initialize();
+    this.engine = new MemoryEngine(this.db, this.embedder, new InMemoryEventBus());
+    this.idMap.clear();
+  }
+
+  async remember(memory: BenchmarkMemoryRecord, agentId: string): Promise<void> {
+    if (!this.engine) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    const stored = await this.engine.remember({
+      agentId,
+      type: memory.type,
+      content: memory.content,
+      importance: memory.importance,
+      tags: memory.tags,
+      metadata: {
+        ...(memory.metadata ?? {}),
+        benchId: memory.id,
+        benchTimestamp: memory.timestamp,
+      },
+    });
+
+    this.idMap.set(memory.id, stored.id);
+  }
+
+  async associate(sourceId: string, targetId: string, strength: number, agentId: string): Promise<void> {
+    if (!this.engine) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    const mappedSource = this.requireMappedId(sourceId);
+    const mappedTarget = this.requireMappedId(targetId);
+
+    await this.engine.associate({
+      sourceId: mappedSource,
+      targetId: mappedTarget,
+      agentId,
+      strength,
+      origin: 'explicit',
+      relationType: 'relates_to',
+    });
+  }
+
+  async recall(
+    request: BenchmarkRecallRequest & {
+      agentId: string;
+    },
+  ): Promise<BenchmarkRecallResult[]> {
+    if (!this.engine) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    const results = await this.engine.recall({
+      agentId: request.agentId,
+      query: request.query ?? '',
+      limit: request.limit ?? 5,
+      threshold: request.minScore ?? 0.08,
+      useSpreadingActivation: this.mode.useSpreadingActivation,
+      maxHops: request.maxHops ?? this.mode.maxHops,
+      activationThreshold: request.activationThreshold ?? this.mode.activationThreshold,
+      blendWeight: request.blendWeight ?? this.mode.blendWeight,
+    });
+
+    return results.map((result) => ({
+      memoryId: String(result.memory.metadata?.['benchId'] ?? result.memory.id),
+      content: result.memory.content,
+      score: result.score,
+      type: result.memory.type as BenchmarkMemoryRecord['type'],
+      source: result.source,
+      rankingTrace: result.rankingTrace,
+      metadata: result.memory.metadata,
+    }));
+  }
+
+  async forget(memoryId: string, agentId: string): Promise<void> {
+    if (!this.engine) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    await this.engine.forget(this.requireMappedId(memoryId), agentId);
+  }
+
+  async applyDecay(decayRate: number, minScore: number): Promise<number> {
+    if (!this.db) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    const affectedMemories = await this.db.applyDecay(decayRate, minScore);
+    await this.db.applyAssociationDecay(decayRate, minScore);
+    return affectedMemories;
+  }
+
+  async exportMemory(agentId: string): Promise<unknown> {
+    if (!this.engine) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    return this.engine.exportPassport(agentId);
+  }
+
+  async importMemory(payload: unknown, agentId: string): Promise<void> {
+    if (!this.db) {
+      throw new Error(`${this.name} is not initialized`);
+    }
+
+    const passport = payload as MemoryPassport;
+    const embeddings = await this.embedder.embedBatch(passport.memories.map((memory) => memory.content));
+    const oldToNewIds = new Map<string, string>();
+
+    for (let index = 0; index < passport.memories.length; index++) {
+      const memory = passport.memories[index];
+      const newId = randomUUID();
+      await this.db.createMemory({
+        id: newId,
+        agentId,
+        type: memory.type,
+        content: memory.content,
+        embeddingModel: this.embedder.model,
+        embedding: embeddings[index],
+        importance: memory.importance,
+        decayScore: memory.decayScore,
+        tags: memory.tags,
+        metadata: memory.metadata,
+      });
+
+      oldToNewIds.set(memory.id, newId);
+      const benchId = String(memory.metadata?.['benchId'] ?? newId);
+      this.idMap.set(benchId, newId);
+    }
+
+    for (const association of passport.associations) {
+      const mappedSource = oldToNewIds.get(association.sourceId);
+      const mappedTarget = oldToNewIds.get(association.targetId);
+      if (!mappedSource || !mappedTarget) continue;
+
+      await this.db.createAssociation({
+        sourceId: mappedSource,
+        targetId: mappedTarget,
+        strength: association.strength,
+        origin: association.origin,
+        relationType: association.relationType ?? 'relates_to',
+      });
+    }
+  }
+
+  async getStats(): Promise<ProviderStats> {
+    return {
+      storageSizeBytes: await sqliteFootprint(this.dbPath),
+    };
+  }
+
+  async close(): Promise<void> {
+    if (this.engine) {
+      await this.engine.shutdown();
+      this.engine = null;
+      this.db = null;
+    }
+
+    if (this.dbPath) {
+      await removeSqliteArtifacts(this.dbPath);
+      this.dbPath = null;
+    }
+
+    this.idMap.clear();
+  }
+
+  private requireMappedId(memoryId: string): string {
+    const mapped = this.idMap.get(memoryId);
+    if (!mapped) {
+      throw new Error(`Unknown benchmark memory id ${memoryId}`);
+    }
+    return mapped;
+  }
+}
+
+export function createOneMBrainAdapters(): MemoryProviderAdapter[] {
+  return [
+    new OneMBrainBenchmarkAdapter('1mbrain_vector_only'),
+    new OneMBrainBenchmarkAdapter('1mbrain_graph_light'),
+    new OneMBrainBenchmarkAdapter('1mbrain_graph_full'),
+  ];
+}
+
+async function sqliteFootprint(dbPath: string | null): Promise<number | null> {
+  if (!dbPath) return null;
+
+  const candidates = [dbPath, `${dbPath}-wal`, `${dbPath}-shm`];
+  let total = 0;
+
+  for (const candidate of candidates) {
+    try {
+      total += (await stat(candidate)).size;
+    } catch {
+      // Ignore files that do not exist.
+    }
+  }
+
+  return total;
+}
+
+async function removeSqliteArtifacts(dbPath: string): Promise<void> {
+  const candidates = [dbPath, `${dbPath}-wal`, `${dbPath}-shm`];
+  for (const candidate of candidates) {
+    await rm(candidate, { force: true }).catch(() => undefined);
+  }
+}

package/src/adapters/keyword-embedding.ts

@@ -0,0 +1,48 @@
+import type { EmbeddingProvider } from '@1mbrain/core';
+
+const DIMENSIONS = 256;
+
+export class KeywordEmbeddingProvider implements EmbeddingProvider {
+  readonly name = 'keyword-benchmark';
+  readonly model = 'hashed-token-benchmark-v2';
+  readonly dimensions = DIMENSIONS;
+
+  async embed(text: string): Promise<number[]> {
+    const vector = new Array<number>(this.dimensions).fill(0);
+    const tokens = tokenize(text);
+
+    for (const token of tokens) {
+      const bucket = hashToken(token) % this.dimensions;
+      vector[bucket] += 1;
+    }
+
+    return normalize(vector);
+  }
+
+  async embedBatch(texts: string[]): Promise<number[][]> {
+    return Promise.all(texts.map((text) => this.embed(text)));
+  }
+}
+
+function tokenize(text: string): string[] {
+  return text
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, ' ')
+    .split(/\s+/)
+    .filter(Boolean);
+}
+
+function hashToken(token: string): number {
+  let hash = 2166136261;
+  for (let index = 0; index < token.length; index++) {
+    hash ^= token.charCodeAt(index);
+    hash = Math.imul(hash, 16777619);
+  }
+  return hash >>> 0;
+}
+
+function normalize(vector: number[]): number[] {
+  const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0));
+  if (norm === 0) return vector;
+  return vector.map((value) => value / norm);
+}

package/src/adapters/mem0.ts

@@ -0,0 +1,124 @@
+import { MemoryClient } from 'mem0ai';
+import type {
+  BenchmarkMemoryRecord,
+  BenchmarkRecallRequest,
+  BenchmarkRecallResult,
+  MemoryProviderAdapter,
+  ProviderAvailability,
+} from '../provider.js';
+
+export class Mem0BenchmarkAdapter implements MemoryProviderAdapter {
+  readonly name = 'mem0';
+  readonly label = 'Mem0 (Cloud)';
+  readonly capabilities = {
+    associations: false,
+    forget: true,
+    decay: false,
+    portability: false,
+  } as const;
+
+  private client: MemoryClient | null = null;
+  private readonly apiKey: string;
+  private readonly idMap = new Map<string, string>(); // benchId -> mem0Id
+  private readonly reverseIdMap = new Map<string, string>(); // mem0Id -> benchId
+
+  constructor() {
+    this.apiKey = process.env.MEM0_API_KEY ?? '';
+  }
+
+  async availability(): Promise<ProviderAvailability> {
+    if (!this.apiKey) {
+      return {
+        status: 'unsupported',
+        reason: 'MEM0_API_KEY is not set.',
+      };
+    }
+
+    try {
+      // Basic initialization check
+      const testClient = new MemoryClient({ apiKey: this.apiKey });
+      if (!testClient) {
+        throw new Error('Failed to instantiate MemoryClient');
+      }
+      return { status: 'available' };
+    } catch (error) {
+      return {
+        status: 'unsupported',
+        reason: `Mem0 is not initialized: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async reset(agentId: string): Promise<void> {
+    this.client = new MemoryClient({ apiKey: this.apiKey });
+    this.idMap.clear();
+    this.reverseIdMap.clear();
+    try {
+      await this.client.deleteUsers({ userId: agentId });
+    } catch {
+      // Ignore if user does not exist or API fails on empty reset
+    }
+  }
+
+  async remember(memory: BenchmarkMemoryRecord, agentId: string): Promise<void> {
+    if (!this.client) {
+      throw new Error(`${this.label} is not initialized`);
+    }
+
+    const response = await (this.client as any).add(
+      [{ role: 'user', content: memory.content }],
+      { user_id: agentId, userId: agentId, infer: false }
+    );
+
+    const mem0Id = response?.results?.[0]?.id;
+    if (mem0Id) {
+      this.idMap.set(memory.id, mem0Id);
+      this.reverseIdMap.set(mem0Id, memory.id);
+    }
+  }
+
+  async recall(
+    request: BenchmarkRecallRequest & {
+      agentId: string;
+    },
+  ): Promise<BenchmarkRecallResult[]> {
+    if (!this.client) {
+      throw new Error(`${this.label} is not initialized`);
+    }
+
+    const results = await (this.client as any).search(request.query ?? '', {
+      filters: {
+        user_id: request.agentId,
+      },
+      limit: request.limit ?? 5,
+    });
+
+    const rawResults = Array.isArray(results) ? results : (results as any)?.results ?? [];
+
+    return rawResults.map((result: any, index: number) => {
+      const mem0Id = result.id;
+      const benchId = mem0Id ? this.reverseIdMap.get(mem0Id) : undefined;
+      return {
+        memoryId: benchId ?? result.id ?? `mem0-${index}`,
+        content: result.memory ?? result.content ?? '',
+        score: result.score ?? 1.0,
+        source: 'vector',
+      };
+    });
+  }
+
+  async forget(memoryId: string, _agentId: string): Promise<void> {
+    if (!this.client) {
+      throw new Error(`${this.label} is not initialized`);
+    }
+
+    const mappedId = this.idMap.get(memoryId) || memoryId;
+    await this.client.delete(mappedId);
+  }
+
+  async close(): Promise<void> {
+    this.client = null;
+    this.idMap.clear();
+    this.reverseIdMap.clear();
+  }
+}

package/src/adapters/qdrant.ts

@@ -0,0 +1,214 @@
+import type {
+  BenchmarkMemoryRecord,
+  BenchmarkRecallRequest,
+  BenchmarkRecallResult,
+  MemoryProviderAdapter,
+  ProviderAvailability,
+} from '../provider.js';
+import { KeywordEmbeddingProvider } from './keyword-embedding.js';
+
+type QdrantPoint = {
+  id: number;
+  vector: number[];
+  payload: {
+    benchId: string;
+    agentId: string;
+    type: string;
+    tags: string[];
+    content: string;
+    importance: number;
+    metadata?: Record<string, unknown>;
+  };
+};
+
+type QdrantSearchResult = {
+  id: number;
+  score: number;
+  payload?: {
+    benchId?: string;
+    content?: string;
+    type?: string;
+    metadata?: Record<string, unknown>;
+  };
+};
+
+export class QdrantBenchmarkAdapter implements MemoryProviderAdapter {
+  readonly name = 'qdrant_vector';
+  readonly label = 'Qdrant Vector';
+  readonly capabilities = {
+    associations: false,
+    forget: true,
+    decay: false,
+    portability: false,
+  } as const;
+
+  private readonly embedder = new KeywordEmbeddingProvider();
+  private readonly baseUrl: string;
+  private readonly apiKey?: string;
+  private readonly collectionName: string;
+  private pointCounter = 0;
+  private readonly idMap = new Map<string, number>();
+
+  constructor(options: { url?: string; apiKey?: string; collectionName?: string } = {}) {
+    this.baseUrl = (options.url ?? process.env.QDRANT_URL ?? '').replace(/\/+$/, '');
+    this.apiKey = options.apiKey ?? process.env.QDRANT_API_KEY;
+    this.collectionName =
+      options.collectionName ?? process.env.QDRANT_COLLECTION ?? 'one_million_brain_bench_v2';
+  }
+
+  async availability(): Promise<ProviderAvailability> {
+    if (!this.baseUrl) {
+      return {
+        status: 'unsupported',
+        reason: 'QDRANT_URL is not set.',
+      };
+    }
+
+    try {
+      await this.request('/collections', { method: 'GET' });
+      return { status: 'available' };
+    } catch (error) {
+      return {
+        status: 'unsupported',
+        reason: `Qdrant is not reachable at ${this.baseUrl}: ${error instanceof Error ? error.message : String(error)}`,
+      };
+    }
+  }
+
+  async reset(_agentId: string): Promise<void> {
+    await this.request(`/collections/${encodeURIComponent(this.collectionName)}`, {
+      method: 'DELETE',
+      allowNotFound: true,
+    });
+    await this.request(`/collections/${encodeURIComponent(this.collectionName)}`, {
+      method: 'PUT',
+      body: {
+        vectors: {
+          size: this.embedder.dimensions,
+          distance: 'Cosine',
+        },
+      },
+    });
+
+    this.pointCounter = 0;
+    this.idMap.clear();
+  }
+
+  async remember(memory: BenchmarkMemoryRecord, agentId: string): Promise<void> {
+    this.pointCounter += 1;
+    this.idMap.set(memory.id, this.pointCounter);
+
+    const point: QdrantPoint = {
+      id: this.pointCounter,
+      vector: await this.embedder.embed(memory.content),
+      payload: {
+        benchId: memory.id,
+        agentId,
+        type: memory.type,
+        tags: memory.tags,
+        content: memory.content,
+        importance: memory.importance ?? 0.75,
+        metadata: {
+          ...(memory.metadata ?? {}),
+          benchId: memory.id,
+          benchTimestamp: memory.timestamp,
+        },
+      },
+    };
+
+    await this.request(`/collections/${encodeURIComponent(this.collectionName)}/points?wait=true`, {
+      method: 'PUT',
+      body: {
+        points: [point],
+      },
+    });
+  }
+
+  async recall(
+    request: BenchmarkRecallRequest & {
+      agentId: string;
+    },
+  ): Promise<BenchmarkRecallResult[]> {
+    const response = (await this.request(
+      `/collections/${encodeURIComponent(this.collectionName)}/points/search`,
+      {
+        method: 'POST',
+        body: {
+          vector: await this.embedder.embed(request.query ?? ''),
+          limit: request.limit ?? 5,
+          score_threshold: request.minScore ?? 0.08,
+          with_payload: true,
+          filter: {
+            must: [
+              {
+                key: 'agentId',
+                match: { value: request.agentId },
+              },
+            ],
+          },
+        },
+      },
+    )) as { result: QdrantSearchResult[] };
+
+    return response.result.map((result) => ({
+      memoryId: result.payload?.benchId ?? String(result.id),
+      content: result.payload?.content ?? '',
+      score: result.score,
+      type: result.payload?.type as BenchmarkMemoryRecord['type'] | undefined,
+      source: 'vector',
+      metadata: result.payload?.metadata,
+    }));
+  }
+
+  async forget(memoryId: string, _agentId: string): Promise<void> {
+    const pointId = this.idMap.get(memoryId);
+    if (!pointId) return;
+
+    await this.request(`/collections/${encodeURIComponent(this.collectionName)}/points/delete?wait=true`, {
+      method: 'POST',
+      body: {
+        points: [pointId],
+      },
+    });
+  }
+
+  async close(): Promise<void> {
+    if (!this.baseUrl) return;
+    await this.request(`/collections/${encodeURIComponent(this.collectionName)}`, {
+      method: 'DELETE',
+      allowNotFound: true,
+    }).catch(() => undefined);
+    this.idMap.clear();
+    this.pointCounter = 0;
+  }
+
+  private async request(
+    path: string,
+    options: {
+      method: 'GET' | 'POST' | 'PUT' | 'DELETE';
+      body?: unknown;
+      allowNotFound?: boolean;
+    },
+  ): Promise<unknown> {
+    const response = await fetch(`${this.baseUrl}${path}`, {
+      method: options.method,
+      headers: {
+        'content-type': 'application/json',
+        ...(this.apiKey ? { 'api-key': this.apiKey } : {}),
+      },
+      body: options.body === undefined ? undefined : JSON.stringify(options.body),
+    });
+
+    if (options.allowNotFound && response.status === 404) {
+      return {};
+    }
+
+    if (!response.ok) {
+      const text = await response.text();
+      throw new Error(`Qdrant ${response.status}: ${text}`);
+    }
+
+    const text = await response.text();
+    return text ? JSON.parse(text) : {};
+  }
+}