claude-memory-layer 1.0.0

package/src/core/consolidation-worker.ts

@@ -0,0 +1,384 @@
+/**
+ * Consolidation Worker
+ * Periodically consolidates working set into long-term memory
+ * Biomimetic: Simulates memory consolidation during sleep/idle periods
+ */
+
+import type {
+  EndlessModeConfig,
+  MemoryEvent,
+  EventGroup,
+  WorkingSet
+} from './types.js';
+import { WorkingSetStore } from './working-set-store.js';
+import { ConsolidatedStore } from './consolidated-store.js';
+
+export class ConsolidationWorker {
+  private running = false;
+  private timeout: NodeJS.Timeout | null = null;
+  private lastActivity: Date = new Date();
+
+  constructor(
+    private workingSetStore: WorkingSetStore,
+    private consolidatedStore: ConsolidatedStore,
+    private config: EndlessModeConfig
+  ) {}
+
+  /**
+   * Start the consolidation worker
+   */
+  start(): void {
+    if (this.running) return;
+    this.running = true;
+    this.scheduleNext();
+  }
+
+  /**
+   * Stop the consolidation worker
+   */
+  stop(): void {
+    this.running = false;
+    if (this.timeout) {
+      clearTimeout(this.timeout);
+      this.timeout = null;
+    }
+  }
+
+  /**
+   * Record activity (resets idle timer)
+   */
+  recordActivity(): void {
+    this.lastActivity = new Date();
+  }
+
+  /**
+   * Check if currently running
+   */
+  isRunning(): boolean {
+    return this.running;
+  }
+
+  /**
+   * Force a consolidation run (manual trigger)
+   */
+  async forceRun(): Promise<number> {
+    return await this.consolidate();
+  }
+
+  /**
+   * Schedule the next consolidation check
+   */
+  private scheduleNext(): void {
+    if (!this.running) return;
+
+    this.timeout = setTimeout(
+      () => this.run(),
+      this.config.consolidation.triggerIntervalMs
+    );
+  }
+
+  /**
+   * Run consolidation check
+   */
+  private async run(): Promise<void> {
+    if (!this.running) return;
+
+    try {
+      await this.checkAndConsolidate();
+    } catch (error) {
+      console.error('Consolidation error:', error);
+    }
+
+    this.scheduleNext();
+  }
+
+  /**
+   * Check conditions and consolidate if needed
+   */
+  private async checkAndConsolidate(): Promise<void> {
+    const workingSet = await this.workingSetStore.get();
+
+    if (!this.shouldConsolidate(workingSet)) {
+      return;
+    }
+
+    await this.consolidate();
+  }
+
+  /**
+   * Perform consolidation
+   */
+  private async consolidate(): Promise<number> {
+    const workingSet = await this.workingSetStore.get();
+
+    if (workingSet.recentEvents.length < 3) {
+      return 0; // Not enough events to consolidate
+    }
+
+    // Group events by topic
+    const groups = this.groupByTopic(workingSet.recentEvents);
+    let consolidatedCount = 0;
+
+    for (const group of groups) {
+      // Require minimum 3 events per group
+      if (group.events.length < 3) continue;
+
+      // Check if already consolidated
+      const eventIds = group.events.map(e => e.id);
+      const alreadyConsolidated = await this.consolidatedStore.isAlreadyConsolidated(eventIds);
+      if (alreadyConsolidated) continue;
+
+      // Generate summary
+      const summary = await this.summarize(group);
+
+      // Create consolidated memory
+      await this.consolidatedStore.create({
+        summary,
+        topics: group.topics,
+        sourceEvents: eventIds,
+        confidence: this.calculateConfidence(group)
+      });
+
+      consolidatedCount++;
+    }
+
+    // Prune consolidated events from working set
+    if (consolidatedCount > 0) {
+      const consolidatedEventIds = groups
+        .filter(g => g.events.length >= 3)
+        .flatMap(g => g.events.map(e => e.id));
+
+      // Only prune old events (keep recent for context)
+      const oldEventIds = consolidatedEventIds.filter(id => {
+        const event = workingSet.recentEvents.find(e => e.id === id);
+        if (!event) return false;
+        const ageHours = (Date.now() - event.timestamp.getTime()) / (1000 * 60 * 60);
+        return ageHours > this.config.workingSet.timeWindowHours / 2;
+      });
+
+      if (oldEventIds.length > 0) {
+        await this.workingSetStore.prune(oldEventIds);
+      }
+    }
+
+    return consolidatedCount;
+  }
+
+  /**
+   * Check if consolidation should run
+   */
+  private shouldConsolidate(workingSet: WorkingSet): boolean {
+    // Check event count trigger
+    if (workingSet.recentEvents.length >= this.config.consolidation.triggerEventCount) {
+      return true;
+    }
+
+    // Check idle time trigger
+    const idleTime = Date.now() - this.lastActivity.getTime();
+    if (idleTime >= this.config.consolidation.triggerIdleMs) {
+      return true;
+    }
+
+    return false;
+  }
+
+  /**
+   * Group events by topic using simple keyword extraction
+   */
+  private groupByTopic(events: MemoryEvent[]): EventGroup[] {
+    const groups = new Map<string, EventGroup>();
+
+    for (const event of events) {
+      const topics = this.extractTopics(event.content);
+
+      for (const topic of topics) {
+        if (!groups.has(topic)) {
+          groups.set(topic, { topics: [topic], events: [] });
+        }
+        const group = groups.get(topic)!;
+        if (!group.events.find(e => e.id === event.id)) {
+          group.events.push(event);
+        }
+      }
+    }
+
+    // Merge groups with overlapping events
+    const mergedGroups = this.mergeOverlappingGroups(Array.from(groups.values()));
+
+    return mergedGroups;
+  }
+
+  /**
+   * Extract topics from content using simple keyword extraction
+   */
+  private extractTopics(content: string): string[] {
+    const topics: string[] = [];
+
+    // Extract code-related keywords
+    const codePatterns = [
+      /\b(function|class|interface|type|const|let|var)\s+(\w+)/gi,
+      /\b(import|export)\s+.*?from\s+['"]([^'"]+)['"]/gi,
+      /\bfile[:\s]+([^\s,]+)/gi
+    ];
+
+    for (const pattern of codePatterns) {
+      let match;
+      while ((match = pattern.exec(content)) !== null) {
+        const keyword = match[2] || match[1];
+        if (keyword && keyword.length > 2) {
+          topics.push(keyword.toLowerCase());
+        }
+      }
+    }
+
+    // Extract common programming terms
+    const commonTerms = [
+      'bug', 'fix', 'error', 'issue', 'feature',
+      'test', 'refactor', 'implement', 'add', 'remove',
+      'update', 'change', 'modify', 'create', 'delete'
+    ];
+
+    const contentLower = content.toLowerCase();
+    for (const term of commonTerms) {
+      if (contentLower.includes(term)) {
+        topics.push(term);
+      }
+    }
+
+    return [...new Set(topics)].slice(0, 5); // Limit to 5 topics
+  }
+
+  /**
+   * Merge groups that have significant event overlap
+   */
+  private mergeOverlappingGroups(groups: EventGroup[]): EventGroup[] {
+    const merged: EventGroup[] = [];
+
+    for (const group of groups) {
+      let foundMerge = false;
+
+      for (const existing of merged) {
+        const overlap = group.events.filter(e =>
+          existing.events.some(ex => ex.id === e.id)
+        );
+
+        // If > 50% overlap, merge
+        if (overlap.length > group.events.length / 2) {
+          existing.topics = [...new Set([...existing.topics, ...group.topics])];
+          for (const event of group.events) {
+            if (!existing.events.find(e => e.id === event.id)) {
+              existing.events.push(event);
+            }
+          }
+          foundMerge = true;
+          break;
+        }
+      }
+
+      if (!foundMerge) {
+        merged.push(group);
+      }
+    }
+
+    return merged;
+  }
+
+  /**
+   * Generate summary for a group of events
+   * Rule-based extraction (no LLM by default)
+   */
+  private async summarize(group: EventGroup): Promise<string> {
+    if (this.config.consolidation.useLLMSummarization) {
+      // Future: LLM-based summarization
+      return this.ruleBasedSummary(group);
+    }
+
+    return this.ruleBasedSummary(group);
+  }
+
+  /**
+   * Rule-based summary generation
+   */
+  private ruleBasedSummary(group: EventGroup): string {
+    const keyPoints: string[] = [];
+
+    for (const event of group.events.slice(0, 10)) {
+      const keyPoint = this.extractKeyPoint(event.content);
+      if (keyPoint) {
+        keyPoints.push(keyPoint);
+      }
+    }
+
+    const topicsStr = group.topics.slice(0, 3).join(', ');
+    const summary = [
+      `Topics: ${topicsStr}`,
+      '',
+      'Key points:',
+      ...keyPoints.map(kp => `- ${kp}`)
+    ].join('\n');
+
+    return summary;
+  }
+
+  /**
+   * Extract key point from content
+   */
+  private extractKeyPoint(content: string): string | null {
+    // Get first meaningful sentence
+    const sentences = content.split(/[.!?\n]+/).filter(s => s.trim().length > 10);
+    if (sentences.length === 0) return null;
+
+    const firstSentence = sentences[0].trim();
+
+    // Truncate if too long
+    if (firstSentence.length > 100) {
+      return firstSentence.slice(0, 100) + '...';
+    }
+
+    return firstSentence;
+  }
+
+  /**
+   * Calculate confidence score for a group
+   */
+  private calculateConfidence(group: EventGroup): number {
+    // Factor 1: Event count (more events = higher confidence)
+    const eventScore = Math.min(group.events.length / 10, 1);
+
+    // Factor 2: Time proximity (events closer together = higher confidence)
+    const timeScore = this.calculateTimeProximity(group.events);
+
+    // Factor 3: Topic consistency (fewer topics per event = higher confidence)
+    const topicScore = Math.min(3 / group.topics.length, 1);
+
+    return (eventScore * 0.4 + timeScore * 0.4 + topicScore * 0.2);
+  }
+
+  /**
+   * Calculate time proximity score
+   */
+  private calculateTimeProximity(events: MemoryEvent[]): number {
+    if (events.length < 2) return 1;
+
+    const timestamps = events.map(e => e.timestamp.getTime()).sort((a, b) => a - b);
+    const timeSpan = timestamps[timestamps.length - 1] - timestamps[0];
+
+    // Score based on average time between events
+    const avgGap = timeSpan / (events.length - 1);
+    const hourInMs = 60 * 60 * 1000;
+
+    // Within 1 hour average = score 1, 24 hours = score 0.5, etc.
+    return Math.max(0, 1 - (avgGap / (24 * hourInMs)));
+  }
+}
+
+/**
+ * Create a Consolidation Worker instance
+ */
+export function createConsolidationWorker(
+  workingSetStore: WorkingSetStore,
+  consolidatedStore: ConsolidatedStore,
+  config: EndlessModeConfig
+): ConsolidationWorker {
+  return new ConsolidationWorker(workingSetStore, consolidatedStore, config);
+}

package/src/core/context-formatter.ts

@@ -0,0 +1,276 @@
+/**
+ * Context Formatter
+ * Formats progressive search results for Claude context injection
+ */
+
+import type {
+  ProgressiveSearchResult,
+  SearchIndexItem,
+  TimelineItem,
+  FullDetail,
+  CitedSearchResult
+} from './types.js';
+import { formatCitationId } from './citation-generator.js';
+
+export interface FormatOptions {
+  format?: 'inline' | 'footnote' | 'reference';
+  showTokens?: boolean;
+  maxWidth?: number;
+}
+
+export class ContextFormatter {
+  /**
+   * Format progressive search result for Claude context
+   */
+  formatProgressiveResult(
+    result: ProgressiveSearchResult,
+    options?: FormatOptions
+  ): string {
+    const parts: string[] = [];
+
+    // Layer 1: Always included (index)
+    parts.push(this.formatLayer1(result.index));
+
+    // Layer 2: Timeline (if expanded)
+    if (result.timeline && result.timeline.length > 0) {
+      parts.push(this.formatLayer2(result.timeline));
+    }
+
+    // Layer 3: Details (if expanded)
+    if (result.details && result.details.length > 0) {
+      parts.push(this.formatLayer3(result.details, options));
+    }
+
+    // Meta information
+    if (options?.showTokens !== false) {
+      parts.push(this.formatMeta(result.meta));
+    }
+
+    return parts.join('\n\n');
+  }
+
+  /**
+   * Format Layer 1: Search Index
+   */
+  private formatLayer1(items: SearchIndexItem[]): string {
+    if (items.length === 0) {
+      return '## Related Memories\n\nNo relevant memories found.';
+    }
+
+    const header = `## Related Memories (${items.length} matches)\n`;
+    const rows = items.map((item, i) => {
+      const date = item.timestamp.toISOString().split('T')[0];
+      return `${i + 1}. **[${item.id.slice(0, 8)}]** ${item.summary} _(${date}, score: ${item.score.toFixed(2)})_`;
+    }).join('\n');
+
+    return header + rows;
+  }
+
+  /**
+   * Format Layer 2: Timeline
+   */
+  private formatLayer2(items: TimelineItem[]): string {
+    const header = '## Timeline Context\n';
+    const timeline = items.map(item => {
+      const marker = item.isTarget ? '**→**' : '   ';
+      const time = item.timestamp.toLocaleTimeString('en-US', {
+        hour: '2-digit',
+        minute: '2-digit'
+      });
+      const typeIcon = this.getTypeIcon(item.type);
+      return `${marker} ${time} ${typeIcon} ${item.preview}`;
+    }).join('\n');
+
+    return header + timeline;
+  }
+
+  /**
+   * Format Layer 3: Full Details
+   */
+  private formatLayer3(items: FullDetail[], options?: FormatOptions): string {
+    const format = options?.format ?? 'inline';
+
+    switch (format) {
+      case 'inline':
+        return this.formatDetailsInline(items);
+      case 'footnote':
+        return this.formatDetailsFootnote(items);
+      case 'reference':
+        return this.formatDetailsReference(items);
+    }
+  }
+
+  /**
+   * Inline format for details
+   */
+  private formatDetailsInline(items: FullDetail[]): string {
+    return items.map(item => {
+      const date = item.timestamp.toLocaleDateString();
+      const session = item.sessionId.slice(0, 8);
+      const citation = item.citationId ? formatCitationId(item.citationId) : '';
+
+      const header = `## Detail: ${item.id.slice(0, 8)}`;
+      const meta = `_${item.type} | Session: ${session} | ${date}_`;
+      const content = item.content;
+      const footer = citation ? `\n${citation}` : '';
+
+      return [header, meta, '', content, footer].join('\n');
+    }).join('\n\n---\n\n');
+  }
+
+  /**
+   * Footnote format for details
+   */
+  private formatDetailsFootnote(items: FullDetail[]): string {
+    const content = items.map((item, i) => {
+      return `${item.content} [${i + 1}]`;
+    }).join('\n\n');
+
+    const footnotes = items.map((item, i) => {
+      const citation = item.citationId ? formatCitationId(item.citationId) : `[${item.id.slice(0, 8)}]`;
+      const date = item.timestamp.toLocaleDateString();
+      return `[${i + 1}] ${citation} - ${date}`;
+    }).join('\n');
+
+    return `${content}\n\n---\n**References:**\n${footnotes}`;
+  }
+
+  /**
+   * Reference format for details
+   */
+  private formatDetailsReference(items: FullDetail[]): string {
+    const content = items.map(item => {
+      return `### ${item.type}\n${item.content}`;
+    }).join('\n\n');
+
+    const references = items.map(item => {
+      const citation = item.citationId ? formatCitationId(item.citationId) : `[${item.id.slice(0, 8)}]`;
+      const date = item.timestamp.toLocaleDateString();
+      return `- ${citation} Session ${item.sessionId.slice(0, 8)}, ${date}`;
+    }).join('\n');
+
+    return `## Content\n\n${content}\n\n## References\n${references}`;
+  }
+
+  /**
+   * Format meta information
+   */
+  private formatMeta(meta: ProgressiveSearchResult['meta']): string {
+    const parts: string[] = [];
+
+    if (meta.expansionReason) {
+      const reasonText = this.getExpansionReasonText(meta.expansionReason);
+      parts.push(`_${reasonText}_`);
+    }
+
+    parts.push(`_~${meta.estimatedTokens} tokens | ${meta.expandedCount} expanded_`);
+
+    return parts.join(' | ');
+  }
+
+  /**
+   * Get icon for event type
+   */
+  private getTypeIcon(type: string): string {
+    switch (type) {
+      case 'user_prompt':
+        return '👤';
+      case 'agent_response':
+        return '🤖';
+      case 'session_summary':
+        return '📋';
+      case 'tool_observation':
+        return '🔧';
+      default:
+        return '📄';
+    }
+  }
+
+  /**
+   * Get human-readable expansion reason
+   */
+  private getExpansionReasonText(reason: string): string {
+    switch (reason) {
+      case 'high_confidence_single':
+        return 'High confidence match - auto-expanded';
+      case 'clear_winner':
+        return 'Clear best match found';
+      case 'ambiguous_multiple_high':
+        return 'Multiple relevant results - showing timeline';
+      case 'low_confidence':
+        return 'No high confidence matches';
+      case 'no_results':
+        return 'No matches found';
+      default:
+        return reason;
+    }
+  }
+}
+
+/**
+ * Format context with citations (for cited search results)
+ */
+export function formatContextWithCitations(
+  results: CitedSearchResult[],
+  options?: FormatOptions
+): string {
+  const format = options?.format ?? 'inline';
+
+  switch (format) {
+    case 'inline':
+      return formatCitedInline(results);
+    case 'footnote':
+      return formatCitedFootnote(results);
+    case 'reference':
+      return formatCitedReference(results);
+  }
+}
+
+function formatCitedInline(results: CitedSearchResult[]): string {
+  return results.map(r => {
+    const date = r.event.timestamp.toLocaleDateString();
+    const session = r.event.sessionId.slice(0, 8);
+    const citation = formatCitationId(r.citation.citationId);
+
+    return [
+      `> ${r.event.content}`,
+      `>`,
+      `> ${citation} - ${date}, Session ${session}`
+    ].join('\n');
+  }).join('\n\n---\n\n');
+}
+
+function formatCitedFootnote(results: CitedSearchResult[]): string {
+  const content = results.map((r, i) => {
+    return `${r.event.content} [${i + 1}]`;
+  }).join('\n\n');
+
+  const footnotes = results.map((r, i) => {
+    const citation = formatCitationId(r.citation.citationId);
+    const date = r.event.timestamp.toLocaleDateString();
+    return `[${i + 1}] ${citation} - ${date}`;
+  }).join('\n');
+
+  return `${content}\n\n---\n**References:**\n${footnotes}`;
+}
+
+function formatCitedReference(results: CitedSearchResult[]): string {
+  const content = results.map(r => {
+    return `### ${r.event.eventType}\n${r.event.content}`;
+  }).join('\n\n');
+
+  const references = results.map(r => {
+    const citation = formatCitationId(r.citation.citationId);
+    const date = r.event.timestamp.toLocaleDateString();
+    return `- ${citation} Session ${r.event.sessionId.slice(0, 8)}, ${date}`;
+  }).join('\n');
+
+  return `## Content\n\n${content}\n\n## References\n${references}`;
+}
+
+/**
+ * Create a context formatter instance
+ */
+export function createContextFormatter(): ContextFormatter {
+  return new ContextFormatter();
+}