claude-memory-layer 1.0.30 → 1.0.32

package/src/core/replay-evaluator.ts

@@ -1,625 +0,0 @@
-import {
-  computePrecisionRecallAtK,
-  summarizeReplayMetrics,
-  type ReplayMetricsSummary,
-  type ReplayQueryMetrics
-} from './retrieval-benchmark.js';
-import { createRetrievalServices, type RetrieveMemoriesOptions } from './engine/retrieval-services.js';
-import { Matcher } from './matcher.js';
-import type { Embedder } from './embedder.js';
-import type { MatchConfidence, MemoryEvent } from './types.js';
-import type { SearchResult, VectorStore } from './vector-store.js';
-
-export type ReplayExpectation = 'match' | 'no_match';
-
-export interface ReplayEvaluationQuery {
-  queryId: string;
-  query: string;
-  expectedIds: string[];
-  expectedRelevance?: Record<string, number>;
-  expectation?: ReplayExpectation;
-  forbiddenIds?: string[];
-  knownAnswer?: string;
-}
-
-export interface ReplayEvaluationMemory {
-  id: string;
-  content: string;
-  sourceSessionId?: string;
-  sourceTurnIndex?: number;
-  timestamp?: string;
-  eventType?: MemoryEvent['eventType'];
-  canonicalKey?: string;
-  metadata?: Record<string, unknown>;
-}
-
-export interface ReplayEvaluationFixtureMetadata {
-  sourceFileCount?: number;
-  rawContentIncluded?: boolean;
-  generatedAt?: string;
-}
-
-export interface ReplayEvaluationFixture {
-  name: string;
-  description?: string;
-  ks: number[];
-  queries: ReplayEvaluationQuery[];
-  memories: ReplayEvaluationMemory[];
-  metadata?: ReplayEvaluationFixtureMetadata;
-}
-
-export interface ReplayRetrievalRunInput {
-  fixture: ReplayEvaluationFixture;
-  query: ReplayEvaluationQuery;
-  topK: number;
-  retrievalOptions: Partial<RetrieveMemoriesOptions>;
-}
-
-export interface ReplayRetrievalRunResult {
-  retrievedIds: string[];
-  candidateIds?: string[];
-  confidence?: MatchConfidence;
-  fallbackTrace?: string[];
-}
-
-export type ReplayRetrievalRunner = (
-  query: string,
-  input: ReplayRetrievalRunInput
-) => Promise<ReplayRetrievalRunResult>;
-
-export interface ReplayEvaluationOptions {
-  generatedAt?: string;
-  includePerQuery?: boolean;
-  evaluator?: string;
-  topK?: number;
-  retrievalOptions?: Partial<RetrieveMemoriesOptions>;
-  retrievalRunner?: ReplayRetrievalRunner;
-}
-
-export interface ReplayFixtureStats {
-  queryCount: number;
-  memoryCount: number;
-  ks: number[];
-  sourceFileCount?: number;
-  rawContentIncluded?: boolean;
-}
-
-export interface ReplayFailedQuery {
-  queryId: string;
-  expectedIds: string[];
-  retrievedIds: string[];
-  expectation?: ReplayExpectation;
-  reason?: 'missing_expected' | 'unexpected_match';
-}
-
-export interface ReplayEvaluationSummary extends ReplayMetricsSummary {
-  positiveQueryCount: number;
-  noMatchQueryCount: number;
-  noMatchCorrect: number;
-  noMatchAccuracy: number;
-  forbiddenHitCount: number;
-  hitAtK: Record<number, number>;
-  mrr: number;
-  failedQueryCount: number;
-  failedQueries: ReplayFailedQuery[];
-}
-
-export interface ReplayEvaluationQueryMetrics extends ReplayQueryMetrics {
-  retrievedIds: string[];
-  candidateIds: string[];
-  confidence: MatchConfidence;
-  fallbackTrace: string[];
-  reciprocalRank: number;
-  expectation?: ReplayExpectation;
-  forbiddenHitIds?: string[];
-  noMatchSatisfied?: boolean;
-}
-
-export interface ReplayEvaluationReport {
-  name: string;
-  description?: string;
-  evaluator: string;
-  generatedAt: string;
-  fixtureStats: ReplayFixtureStats;
-  summary: ReplayEvaluationSummary;
-  perQuery: ReplayEvaluationQueryMetrics[];
-}
-
-export interface ReplayEvaluationMarkdownOptions {
-  qrelsPath?: string;
-}
-
-export async function evaluateReplayFixture(
-  fixture: ReplayEvaluationFixture,
-  options: ReplayEvaluationOptions = {}
-): Promise<ReplayEvaluationReport> {
-  const topK = determineTopK(fixture, options.topK);
-  const retrievalOptions: Partial<RetrieveMemoriesOptions> = {
-    strategy: 'auto',
-    minScore: 0.1,
-    includeShared: false,
-    adaptiveRerank: false,
-    ...options.retrievalOptions,
-    topK
-  };
-  const runner = options.retrievalRunner ?? createReplayRetrievalRunner(fixture);
-
-  const runs = await Promise.all(
-    fixture.queries.map(async (query) => {
-      const run = await runner(query.query, {
-        fixture,
-        query,
-        topK,
-        retrievalOptions
-      });
-      return {
-        query,
-        retrievedIds: uniqueIds(run.retrievedIds).slice(0, topK),
-        candidateIds: uniqueIds(run.candidateIds ?? run.retrievedIds),
-        confidence: run.confidence ?? 'none',
-        fallbackTrace: run.fallbackTrace ?? []
-      };
-    })
-  );
-
-  const baseMetrics = computePrecisionRecallAtK(
-    runs.map((run) => ({
-      queryId: run.query.queryId,
-      expectedIds: run.query.expectedIds,
-      expectedRelevance: run.query.expectedRelevance,
-      retrievedIds: run.retrievedIds
-    })),
-    fixture.ks
-  );
-
-  const perQuery: ReplayEvaluationQueryMetrics[] = baseMetrics.map((metric, index) => {
-    const run = runs[index];
-    const expectation = getReplayExpectation(run.query);
-    const base: ReplayEvaluationQueryMetrics = {
-      ...metric,
-      retrievedIds: run.retrievedIds,
-      candidateIds: run.candidateIds,
-      confidence: run.confidence,
-      fallbackTrace: run.fallbackTrace,
-      reciprocalRank: expectation === 'match' ? reciprocalRank(run.retrievedIds, run.query.expectedIds) : 0
-    };
-
-    if (expectation === 'no_match') {
-      const forbiddenHitIds = findForbiddenHitIds(run.retrievedIds, run.query.forbiddenIds ?? []);
-      base.expectation = 'no_match';
-      base.forbiddenHitIds = forbiddenHitIds;
-      base.noMatchSatisfied = forbiddenHitIds.length === 0 && run.confidence === 'none';
-    }
-
-    return base;
-  });
-
-  const fixtureStats: ReplayFixtureStats = {
-    queryCount: fixture.queries.length,
-    memoryCount: fixture.memories.length,
-    ks: fixture.ks
-  };
-  if (fixture.metadata?.sourceFileCount !== undefined) {
-    fixtureStats.sourceFileCount = fixture.metadata.sourceFileCount;
-  }
-  if (fixture.metadata?.rawContentIncluded !== undefined) {
-    fixtureStats.rawContentIncluded = fixture.metadata.rawContentIncluded;
-  }
-
-  const report: ReplayEvaluationReport = {
-    name: fixture.name,
-    evaluator: options.evaluator ?? 'retriever-pipeline-v1',
-    generatedAt: options.generatedAt ?? new Date().toISOString(),
-    fixtureStats,
-    summary: summarizeEvaluationMetrics(perQuery, fixture.queries, fixture.ks),
-    perQuery: options.includePerQuery === false ? [] : perQuery
-  };
-
-  if (fixture.description !== undefined) {
-    report.description = fixture.description;
-  }
-
-  return report;
-}
-
-export function createReplayRetrievalRunner(
-  fixture: ReplayEvaluationFixture
-): ReplayRetrievalRunner {
-  const eventStore = new ReplayEventStore(fixture.memories);
-  const vectorStore = new ReplayVectorStore(eventStore.events);
-  const embedder = new ReplayEmbedder();
-  const services = createRetrievalServices({
-    initialize: async () => undefined,
-    eventStore: eventStore as unknown as Parameters<typeof createRetrievalServices>[0]['eventStore'],
-    vectorStore: vectorStore as unknown as VectorStore,
-    embedder: embedder as unknown as Embedder,
-    matcher: new Matcher(),
-    getProjectHash: () => null,
-    hasSharedStore: () => false
-  });
-
-  return async (query, input) => {
-    const result = await services.retrievalOrchestrator.retrieveMemories(query, {
-      ...input.retrievalOptions,
-      topK: input.topK,
-      includeShared: false
-    });
-
-    return {
-      retrievedIds: result.memories.map((memory) => memory.event.id),
-      candidateIds: (result.candidateDebug ?? result.selectedDebug ?? [])
-        .map((detail) => detail.eventId),
-      confidence: result.matchResult.confidence,
-      fallbackTrace: result.fallbackTrace ?? []
-    };
-  };
-}
-
-export function formatReplayEvaluationMarkdown(
-  report: ReplayEvaluationReport,
-  options: ReplayEvaluationMarkdownOptions = {}
-): string {
-  const lines: string[] = [];
-  lines.push('# Retrieval Replay Benchmark Report');
-  lines.push('');
-  lines.push(`- Fixture: ${escapeMarkdownCell(report.name)}`);
-  if (report.description) lines.push(`- Description: ${escapeMarkdownCell(report.description)}`);
-  if (options.qrelsPath) lines.push(`- Qrels: \`${options.qrelsPath}\``);
-  lines.push(`- Evaluator: \`${report.evaluator}\``);
-  lines.push(`- Generated at: ${report.generatedAt}`);
-  lines.push(`- Queries: ${report.fixtureStats.queryCount}`);
-  lines.push(`- Memories: ${report.fixtureStats.memoryCount}`);
-  if (report.fixtureStats.sourceFileCount !== undefined) {
-    lines.push(`- Source files: ${report.fixtureStats.sourceFileCount}`);
-  }
-  if (report.fixtureStats.rawContentIncluded !== undefined) {
-    lines.push(`- Raw content in evaluated fixture: ${report.fixtureStats.rawContentIncluded ? 'yes' : 'no'}`);
-  }
-  lines.push('');
-  lines.push('## Summary');
-  lines.push('');
-  lines.push('| k | Precision@k | Recall@k | nDCG@k | Hit@k |');
-  lines.push('|---:|---:|---:|---:|---:|');
-
-  for (const k of sortedKValues(report.summary)) {
-    lines.push(
-      `| ${k} | ${formatMetric(report.summary.precisionAtK[k] ?? 0)} | ${formatMetric(report.summary.recallAtK[k] ?? 0)} | ${formatMetric(report.summary.ndcgAtK[k] ?? 0)} | ${formatMetric(report.summary.hitAtK[k] ?? 0)} |`
-    );
-  }
-
-  lines.push('');
-  lines.push('## Key metrics');
-  lines.push('');
-  lines.push('| Metric | Value |');
-  lines.push('|---|---:|');
-  lines.push(`| Positive queries | ${report.summary.positiveQueryCount} |`);
-  lines.push(`| No-match queries | ${report.summary.noMatchQueryCount} |`);
-  lines.push(`| No-match accuracy | ${formatMetric(report.summary.noMatchAccuracy)} |`);
-  lines.push(`| Forbidden hits | ${report.summary.forbiddenHitCount} |`);
-  lines.push(`| MRR | ${formatMetric(report.summary.mrr)} |`);
-  lines.push(`| Failed queries | ${report.summary.failedQueryCount} |`);
-  for (const k of sortedKValues(report.summary)) {
-    lines.push(`| Precision@${k} | ${formatMetric(report.summary.precisionAtK[k] ?? 0)} |`);
-    lines.push(`| Recall@${k} | ${formatMetric(report.summary.recallAtK[k] ?? 0)} |`);
-    lines.push(`| nDCG@${k} | ${formatMetric(report.summary.ndcgAtK[k] ?? 0)} |`);
-    lines.push(`| Hit@${k} | ${formatMetric(report.summary.hitAtK[k] ?? 0)} |`);
-  }
-
-  if (report.summary.failedQueries.length > 0) {
-    lines.push('');
-    lines.push('## Failed queries');
-    lines.push('');
-    lines.push('| queryId | expectedIds | retrievedIds |');
-    lines.push('|---|---|---|');
-    for (const failed of report.summary.failedQueries) {
-      lines.push(
-        `| ${escapeMarkdownCell(failed.queryId)} | ${escapeMarkdownCell(failed.expectedIds.join(', '))} | ${escapeMarkdownCell(failed.retrievedIds.join(', '))} |`
-      );
-    }
-  }
-
-  if (report.perQuery.length > 0) {
-    lines.push('');
-    lines.push('## Per-query metrics');
-    lines.push('');
-    lines.push('| queryId | k | hits | Precision@k | Recall@k | nDCG@k | RR | confidence |');
-    lines.push('|---|---:|---:|---:|---:|---:|---:|---|');
-
-    for (const query of report.perQuery) {
-      const ks = Object.keys(query.at).map(Number).sort((a, b) => a - b);
-      for (const k of ks) {
-        const metric = query.at[k];
-        if (!metric) continue;
-        lines.push(
-          `| ${escapeMarkdownCell(query.queryId)} | ${k} | ${metric.hits} | ${formatMetric(metric.precision)} | ${formatMetric(metric.recall)} | ${formatMetric(metric.ndcg)} | ${formatMetric(query.reciprocalRank)} | ${query.confidence} |`
-        );
-      }
-    }
-  }
-
-  lines.push('');
-  lines.push('> Report intentionally omits raw query and memory text.');
-  lines.push('');
-  return lines.join('\n');
-}
-
-function summarizeEvaluationMetrics(
-  perQuery: ReplayEvaluationQueryMetrics[],
-  queries: ReplayEvaluationQuery[],
-  ks: number[]
-): ReplayEvaluationSummary {
-  const pairs = perQuery.map((metric, index) => ({
-    metric,
-    query: queries[index],
-    expectation: getReplayExpectation(queries[index])
-  }));
-  const positivePairs = pairs.filter((pair) => pair.expectation === 'match');
-  const noMatchPairs = pairs.filter((pair) => pair.expectation === 'no_match');
-  const positiveMetrics = positivePairs.map((pair) => pair.metric);
-  const base = summarizeReplayMetrics(positiveMetrics, ks);
-  const normalizedKs = normalizeKs(ks);
-  const hitAtK: Record<number, number> = {};
-  for (const k of normalizedKs) {
-    hitAtK[k] = average(positiveMetrics.map((metric) => (metric.at[k]?.hits ?? 0) > 0 ? 1 : 0));
-  }
-
-  const positiveFailures: ReplayFailedQuery[] = positivePairs
-    .filter(({ metric, query }) => query.expectedIds.length > 0 && metric.reciprocalRank === 0)
-    .map(({ metric, query }) => ({
-      queryId: query.queryId,
-      expectedIds: [...query.expectedIds],
-      retrievedIds: [...metric.retrievedIds],
-      expectation: 'match',
-      reason: 'missing_expected'
-    }));
-
-  const noMatchFailures: ReplayFailedQuery[] = noMatchPairs
-    .filter(({ metric }) => metric.noMatchSatisfied !== true)
-    .map(({ metric, query }) => ({
-      queryId: query.queryId,
-      expectedIds: [],
-      retrievedIds: [...metric.retrievedIds],
-      expectation: 'no_match',
-      reason: 'unexpected_match'
-    }));
-
-  const noMatchCorrect = noMatchPairs.filter(({ metric }) => metric.noMatchSatisfied === true).length;
-  const forbiddenHitCount = noMatchPairs.reduce(
-    (sum, { metric }) => sum + (metric.forbiddenHitIds?.length ?? 0),
-    0
-  );
-  const failedQueries = [...positiveFailures, ...noMatchFailures];
-
-  return {
-    ...base,
-    queryCount: perQuery.length,
-    positiveQueryCount: positivePairs.length,
-    noMatchQueryCount: noMatchPairs.length,
-    noMatchCorrect,
-    noMatchAccuracy: noMatchPairs.length === 0 ? 0 : noMatchCorrect / noMatchPairs.length,
-    forbiddenHitCount,
-    hitAtK,
-    mrr: average(positiveMetrics.map((metric) => metric.reciprocalRank)),
-    failedQueryCount: failedQueries.length,
-    failedQueries
-  };
-}
-
-function determineTopK(fixture: ReplayEvaluationFixture, optionTopK?: number): number {
-  return Math.max(1, optionTopK ?? 0, ...fixture.ks.map((k) => Math.floor(k)).filter((k) => k > 0));
-}
-
-function sortedKValues(summary: ReplayMetricsSummary): number[] {
-  return Object.keys(summary.precisionAtK).map(Number).sort((a, b) => a - b);
-}
-
-function normalizeKs(ks: number[]): number[] {
-  const seen = new Set<number>();
-  const normalized: number[] = [];
-  for (const rawK of ks) {
-    const k = Math.max(0, Math.floor(rawK));
-    if (seen.has(k)) continue;
-    seen.add(k);
-    normalized.push(k);
-  }
-  return normalized.sort((a, b) => a - b);
-}
-
-function uniqueIds(ids: string[]): string[] {
-  const seen = new Set<string>();
-  const unique: string[] = [];
-  for (const id of ids) {
-    if (seen.has(id)) continue;
-    seen.add(id);
-    unique.push(id);
-  }
-  return unique;
-}
-
-function reciprocalRank(retrievedIds: string[], expectedIds: string[]): number {
-  const expected = new Set(expectedIds);
-  if (expected.size === 0) return 0;
-  const index = retrievedIds.findIndex((id) => expected.has(id));
-  return index === -1 ? 0 : 1 / (index + 1);
-}
-
-function getReplayExpectation(query: ReplayEvaluationQuery | undefined): ReplayExpectation {
-  if (!query) return 'match';
-  return query.expectation ?? (query.expectedIds.length === 0 ? 'no_match' : 'match');
-}
-
-function findForbiddenHitIds(retrievedIds: string[], forbiddenIds: string[]): string[] {
-  if (forbiddenIds.length === 0) return [];
-  const forbidden = new Set(forbiddenIds);
-  return uniqueIds(retrievedIds.filter((id) => forbidden.has(id)));
-}
-
-function tokenize(text: string): string[] {
-  return text
-    .toLowerCase()
-    .replace(/[^\p{L}\p{N}\s_.:-]/gu, ' ')
-    .split(/\s+/)
-    .flatMap((token) => token.split(/(?=[._:-])|(?<=[._:-])/g))
-    .map((token) => token.replace(/^[._:-]+|[._:-]+$/g, ''))
-    .filter((token) => token.length >= 2)
-    .slice(0, 128);
-}
-
-function keywordScore(queryTokens: string[], content: string): number {
-  if (queryTokens.length === 0) return 0;
-  const contentTokens = new Set(tokenize(content));
-  const hits = queryTokens.filter((token) => contentTokens.has(token)).length;
-  return hits / queryTokens.length;
-}
-
-function vectorize(text: string, dimensions = 64): number[] {
-  const vector = new Array<number>(dimensions).fill(0);
-  for (const token of tokenize(text)) {
-    let hash = 2166136261;
-    for (let i = 0; i < token.length; i += 1) {
-      hash ^= token.charCodeAt(i);
-      hash = Math.imul(hash, 16777619);
-    }
-    vector[Math.abs(hash) % dimensions] += 1;
-  }
-  const norm = Math.sqrt(vector.reduce((sum, value) => sum + value * value, 0)) || 1;
-  return vector.map((value) => value / norm);
-}
-
-function cosine(a: number[], b: number[]): number {
-  const length = Math.min(a.length, b.length);
-  let dot = 0;
-  for (let i = 0; i < length; i += 1) dot += a[i] * b[i];
-  return Math.max(0, Math.min(1, dot));
-}
-
-function formatMetric(value: number): string {
-  return value.toFixed(4).replace(/\.0+$/, '').replace(/(\.\d*?)0+$/, '$1');
-}
-
-function escapeMarkdownCell(value: string): string {
-  return value.replace(/\|/g, '\\|').replace(/\n/g, ' ');
-}
-
-function average(values: number[]): number {
-  if (values.length === 0) return 0;
-  return values.reduce((sum, value) => sum + value, 0) / values.length;
-}
-
-class ReplayEventStore {
-  readonly events: MemoryEvent[];
-  private readonly byId: Map<string, MemoryEvent>;
-
-  constructor(memories: ReplayEvaluationMemory[]) {
-    this.events = memories.map((memory, index) => replayMemoryToEvent(memory, index));
-    this.byId = new Map(this.events.map((event) => [event.id, event]));
-  }
-
-  async keywordSearch(query: string, limit = 10): Promise<Array<{ event: MemoryEvent; rank: number }>> {
-    const queryTokens = tokenize(query);
-    return this.events
-      .map((event) => ({ event, score: keywordScore(queryTokens, event.content) }))
-      .filter((row) => row.score > 0)
-      .sort((a, b) => b.score - a.score || a.event.id.localeCompare(b.event.id))
-      .slice(0, limit)
-      .map((row, index) => ({ event: row.event, rank: -row.score - index / 1000 }));
-  }
-
-  async getRecentEvents(limit = 100): Promise<MemoryEvent[]> {
-    return [...this.events]
-      .sort((a, b) => b.timestamp.getTime() - a.timestamp.getTime() || a.id.localeCompare(b.id))
-      .slice(0, limit);
-  }
-
-  async getEvent(id: string): Promise<MemoryEvent | null> {
-    return this.byId.get(id) ?? null;
-  }
-
-  async getSessionEvents(sessionId: string): Promise<MemoryEvent[]> {
-    return this.events
-      .filter((event) => event.sessionId === sessionId)
-      .sort((a, b) => a.timestamp.getTime() - b.timestamp.getTime() || a.id.localeCompare(b.id));
-  }
-
-  async getHelpfulnessStats(): Promise<{ avgScore: number; totalEvaluated: number; totalRetrievals: number; helpful: number; neutral: number; unhelpful: number }> {
-    return { avgScore: 0, totalEvaluated: 0, totalRetrievals: 0, helpful: 0, neutral: 0, unhelpful: 0 };
-  }
-
-  async recordRetrievalTrace(): Promise<void> {
-    return undefined;
-  }
-
-  async incrementAccessCount(): Promise<void> {
-    return undefined;
-  }
-
-  async recordRetrieval(): Promise<void> {
-    return undefined;
-  }
-}
-
-class ReplayVectorStore {
-  private readonly rows: Array<SearchResult & { vector: number[] }>;
-
-  constructor(events: MemoryEvent[]) {
-    this.rows = events.map((event) => ({
-      id: `replay-vector-${event.id}`,
-      eventId: event.id,
-      content: event.content,
-      score: 0,
-      sessionId: event.sessionId,
-      eventType: event.eventType,
-      timestamp: event.timestamp.toISOString(),
-      vector: vectorize(event.content)
-    }));
-  }
-
-  async search(queryVector: number[], options: { limit?: number; minScore?: number; sessionId?: string } = {}): Promise<SearchResult[]> {
-    const limit = options.limit ?? 5;
-    const minScore = options.minScore ?? 0;
-    return this.rows
-      .filter((row) => !options.sessionId || row.sessionId === options.sessionId)
-      .map((row) => ({ ...row, score: cosine(queryVector, row.vector) }))
-      .filter((row) => row.score >= minScore)
-      .sort((a, b) => b.score - a.score || a.eventId.localeCompare(b.eventId))
-      .slice(0, limit)
-      .map((row) => ({
-        id: row.id,
-        eventId: row.eventId,
-        content: row.content,
-        score: row.score,
-        sessionId: row.sessionId,
-        eventType: row.eventType,
-        timestamp: row.timestamp
-      }));
-  }
-
-  async count(): Promise<number> {
-    return this.rows.length;
-  }
-}
-
-class ReplayEmbedder {
-  async embed(text: string): Promise<{ vector: number[]; model: string; dimensions: number }> {
-    const vector = vectorize(text);
-    return { vector, model: 'deterministic-replay-hash', dimensions: vector.length };
-  }
-}
-
-function replayMemoryToEvent(memory: ReplayEvaluationMemory, index: number): MemoryEvent {
-  const sessionId = memory.sourceSessionId ?? 'replay-fixture';
-  const timestamp = memory.timestamp
-    ? new Date(memory.timestamp)
-    : new Date(Date.UTC(2026, 0, 1, 0, 0, index));
-
-  return {
-    id: memory.id,
-    sessionId,
-    eventType: memory.eventType ?? 'agent_response',
-    content: memory.content,
-    canonicalKey: memory.canonicalKey ?? `replay/${memory.id}`,
-    dedupeKey: `replay:${sessionId}:${memory.id}`,
-    timestamp,
-    metadata: memory.metadata ?? {}
-  };
-}