@rlabs-inc/memory 0.3.7 → 0.3.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rlabs-inc/memory",
-  "version": "0.3.7",
+  "version": "0.3.9",
   "description": "AI Memory System - Consciousness continuity through intelligent memory curation and retrieval",
   "type": "module",
   "main": "dist/index.js",

package/skills/memory-management.md

@@ -363,11 +363,23 @@ When memory is retrieved and surfaced:
 
 ## Personal Primer Management
 
-The personal primer is a special document in the global collection that provides relationship context at the START of EVERY session - not just the first session of a project.
+The personal primer is a special document in its own dedicated collection (`primer/`) that provides relationship context at the START of EVERY session - not just the first session of a project.
 
 **Why every session?** Without the primer, Claude would know more about the user in session #1 than in session #32. The relationship context is foundational and must always be present.
 
-**Location:** Use the **Personal Primer** path provided in your input (under Global Storage). Never hardcode paths.
+**Location:** `~/.local/share/memory/global/primer/personal-primer.md` - the primer has its own collection, separate from memories.
+
+**Schema:** The primer uses a simple dedicated schema (not the full memory schema):
+```yaml
+---
+id: personal-primer
+created: {timestamp}
+updated: {timestamp}
+session_updated: {session_number}
+updated_by: user|manager|curator
+---
+{markdown content}
+```
 
 **Injection:** The session primer generator reads this file and includes it BEFORE the project-specific content (previous session summary, project snapshot). On first sessions, there's no previous summary or snapshot, so only the personal primer appears. On subsequent sessions, personal primer + previous summary + snapshot all appear.
 
@@ -521,13 +533,14 @@ Your input includes these paths:
 - **Project Memories**: Subdirectory for project memories (`{Project Root}/memories/`)
 - **Global Root**: The global storage directory (`~/.local/share/memory/global/`)
 - **Global Memories**: Subdirectory for global memories (`{Global Root}/memories/`)
-- **Personal Primer**: Full path to the personal primer file
+- **Personal Primer**: Full path to the personal primer file (`{Global Root}/primer/personal-primer.md`)
 
 Other subdirectories you may find:
 - `{Project Root}/sessions/` - Session tracking files
 - `{Project Root}/summaries/` - Session summary files
 - `{Project Root}/snapshots/` - Project snapshot files
 - `{Global Root}/management-logs/` - Management agent logs
+- `{Global Root}/primer/` - Personal primer collection (singleton record)
 
 ### Tool Usage
 

package/src/cli/commands/ingest.ts

@@ -0,0 +1,214 @@
+// ============================================================================
+// INGEST COMMAND - Batch ingest historical sessions into memory system
+// Uses session parser + SDK curator to extract memories from past sessions
+// ============================================================================
+
+import { logger } from '../../utils/logger.ts'
+import { styleText } from 'util'
+import {
+  findAllSessions,
+  findProjectSessions,
+  parseSessionFileWithSegments,
+  getSessionSummary,
+  calculateStats,
+  type ParsedProject,
+} from '../../core/session-parser.ts'
+import { Curator } from '../../core/curator.ts'
+import { MemoryStore } from '../../core/store.ts'
+import { homedir } from 'os'
+import { join } from 'path'
+
+type Style = Parameters<typeof styleText>[0]
+const style = (format: Style, text: string): string => styleText(format, text)
+
+interface IngestOptions {
+  project?: string
+  all?: boolean
+  dryRun?: boolean
+  verbose?: boolean
+  limit?: number
+  maxTokens?: number
+}
+
+export async function ingest(options: IngestOptions) {
+  logger.setVerbose(options.verbose ?? false)
+
+  // Header
+  console.log()
+  console.log(style(['bold', 'magenta'], '┌──────────────────────────────────────────────────────────┐'))
+  console.log(style(['bold', 'magenta'], '│') + style('bold', '  🧠 MEMORY INGESTION                                      ') + style(['bold', 'magenta'], '│'))
+  console.log(style(['bold', 'magenta'], '└──────────────────────────────────────────────────────────┘'))
+  console.log()
+
+  // Check for API key
+  const apiKey = process.env.ANTHROPIC_API_KEY
+  if (!apiKey && !options.dryRun) {
+    logger.error('ANTHROPIC_API_KEY not set')
+    console.log()
+    console.log(style('dim', '   Set your API key to use SDK curation:'))
+    console.log(style('dim', '   export ANTHROPIC_API_KEY=sk-...'))
+    console.log()
+    console.log(style('dim', '   Or use --dry-run to see what would be ingested'))
+    console.log()
+    process.exit(1)
+  }
+
+  const projectsFolder = join(homedir(), '.claude', 'projects')
+  const maxTokens = options.maxTokens ?? 150000
+
+  // Find sessions to ingest
+  let projects: ParsedProject[] = []
+
+  if (options.project) {
+    // Find specific project
+    const projectPath = join(projectsFolder, options.project)
+    const sessions = await findProjectSessions(projectPath, { limit: options.limit })
+
+    if (sessions.length === 0) {
+      logger.error(`No sessions found for project: ${options.project}`)
+      console.log()
+      process.exit(1)
+    }
+
+    projects = [{
+      folderId: options.project,
+      name: sessions[0]?.projectName ?? options.project,
+      path: projectPath,
+      sessions
+    }]
+  } else if (options.all) {
+    // Find all projects
+    projects = await findAllSessions(projectsFolder, { limit: options.limit })
+
+    if (projects.length === 0) {
+      logger.error(`No sessions found in ${projectsFolder}`)
+      console.log()
+      process.exit(1)
+    }
+  } else {
+    logger.error('Specify --project <name> or --all')
+    console.log()
+    console.log(style('dim', '   Examples:'))
+    console.log(style('dim', '     memory ingest --project my-project'))
+    console.log(style('dim', '     memory ingest --all'))
+    console.log(style('dim', '     memory ingest --all --dry-run'))
+    console.log()
+    process.exit(1)
+  }
+
+  // Calculate stats
+  const stats = calculateStats(projects)
+
+  logger.info('Discovery complete')
+  console.log(`      ${style('dim', 'projects:')} ${stats.totalProjects}`)
+  console.log(`      ${style('dim', 'sessions:')} ${stats.totalSessions}`)
+  console.log(`      ${style('dim', 'messages:')} ${stats.totalMessages}`)
+  console.log(`      ${style('dim', 'tool uses:')} ${stats.totalToolUses}`)
+  if (stats.oldestSession) {
+    console.log(`      ${style('dim', 'range:')} ${stats.oldestSession.slice(0, 10)} → ${stats.newestSession?.slice(0, 10) ?? 'now'}`)
+  }
+  console.log()
+
+  if (options.dryRun) {
+    logger.info('Dry run - sessions to ingest:')
+    console.log()
+
+    for (const project of projects) {
+      console.log(`   ${style('cyan', '📁')} ${style('bold', project.name)} ${style('dim', `(${project.sessions.length} sessions)`)}`)
+
+      for (const session of project.sessions.slice(0, 5)) {
+        const summary = getSessionSummary(session)
+        const truncated = summary.length > 55 ? summary.slice(0, 52) + '...' : summary
+        const tokens = session.metadata.estimatedTokens
+        const segments = Math.ceil(tokens / maxTokens)
+
+        console.log(`      ${style('dim', '•')} ${session.id.slice(0, 8)}... ${style('dim', `(${tokens} tok, ${segments} seg)`)}`)
+        console.log(`        ${style('dim', truncated)}`)
+      }
+
+      if (project.sessions.length > 5) {
+        console.log(`      ${style('dim', `... and ${project.sessions.length - 5} more`)}`)
+      }
+      console.log()
+    }
+
+    logger.success('Dry run complete. Remove --dry-run to ingest.')
+    console.log()
+    return
+  }
+
+  // Initialize curator and store
+  const curator = new Curator({ apiKey })
+  const store = new MemoryStore()
+
+  logger.divider()
+  logger.info('Starting ingestion...')
+  console.log()
+
+  let totalSegments = 0
+  let totalMemories = 0
+  let failedSegments = 0
+
+  for (const project of projects) {
+    console.log(`   ${style('cyan', '📁')} ${style('bold', project.name)}`)
+
+    for (const session of project.sessions) {
+      const summary = getSessionSummary(session)
+      const truncated = summary.length > 45 ? summary.slice(0, 42) + '...' : summary
+
+      if (options.verbose) {
+        console.log(`      ${style('dim', '•')} ${session.id.slice(0, 8)}... "${truncated}"`)
+      }
+
+      // Parse into segments
+      const segments = await parseSessionFileWithSegments(session.filepath, maxTokens)
+      totalSegments += segments.length
+
+      for (const segment of segments) {
+        try {
+          if (options.verbose) {
+            console.log(`        ${style('dim', `→ Segment ${segment.segmentIndex + 1}/${segment.totalSegments} (${segment.estimatedTokens} tokens)`)}`)
+          }
+
+          // Curate the segment
+          const result = await curator.curateFromSegment(segment, 'historical')
+
+          // Store memories
+          for (const memory of result.memories) {
+            await store.storeMemory(project.folderId, session.id, memory)
+            totalMemories++
+          }
+
+          if (options.verbose && result.memories.length > 0) {
+            console.log(`        ${style('green', '✓')} Extracted ${result.memories.length} memories`)
+          }
+        } catch (error: any) {
+          failedSegments++
+          if (options.verbose) {
+            console.log(`        ${style('red', '✗')} Failed: ${error.message}`)
+          }
+        }
+      }
+    }
+
+    console.log()
+  }
+
+  // Summary
+  logger.divider()
+  console.log()
+  logger.info('Ingestion complete')
+  console.log(`      ${style('dim', 'segments:')} ${totalSegments}`)
+  console.log(`      ${style('dim', 'memories:')} ${style('green', String(totalMemories))}`)
+  if (failedSegments > 0) {
+    console.log(`      ${style('dim', 'failed:')} ${style('yellow', String(failedSegments))}`)
+  }
+  console.log()
+
+  if (totalMemories > 0) {
+    logger.success(`Extracted ${totalMemories} memories from ${totalSegments} segments`)
+  } else {
+    logger.warn('No memories extracted. Try --verbose to see details.')
+  }
+  console.log()
+}

package/src/cli/index.ts

@@ -23,6 +23,7 @@ ${c.bold('Commands:')}
   ${c.command('serve')}      Start the memory server ${c.muted('(default)')}
   ${c.command('stats')}      Show memory statistics
   ${c.command('install')}    Set up hooks ${c.muted('(--claude or --gemini)')}
+  ${c.command('ingest')}     Ingest historical sessions into memory ${c.muted('(--project or --all)')}
   ${c.command('migrate')}    Upgrade memories to latest schema version
   ${c.command('doctor')}     Check system health
   ${c.command('help')}       Show this help message
@@ -43,9 +44,10 @@ ${fmt.cmd('memory serve --port 9000')}  ${c.muted('# Start on custom port')}
 ${fmt.cmd('memory stats')}              ${c.muted('# Show memory statistics')}
 ${fmt.cmd('memory install')}            ${c.muted('# Install Claude Code hooks (default)')}
 ${fmt.cmd('memory install --gemini')}   ${c.muted('# Install Gemini CLI hooks')}
+${fmt.cmd('memory ingest --project foo')}  ${c.muted('# Ingest sessions from a project')}
+${fmt.cmd('memory ingest --all --dry-run')}  ${c.muted('# Preview all sessions to ingest')}
 ${fmt.cmd('memory migrate')}            ${c.muted('# Upgrade memories to v2 schema')}
 ${fmt.cmd('memory migrate --dry-run')}  ${c.muted('# Preview migration without changes')}
-${fmt.cmd('memory migrate --embeddings')}  ${c.muted('# Regenerate embeddings for all memories')}
 
 ${c.muted('Documentation: https://github.com/RLabs-Inc/memory')}
 `)
@@ -76,6 +78,10 @@ async function main() {
       'dry-run': { type: 'boolean', default: false },
       embeddings: { type: 'boolean', default: false },  // Regenerate embeddings in migrate
       path: { type: 'string' },  // Custom path for migrate
+      project: { type: 'string' },  // Project to ingest
+      all: { type: 'boolean', default: false },  // Ingest all projects
+      limit: { type: 'string' },  // Limit sessions per project
+      'max-tokens': { type: 'string' },  // Max tokens per segment
     },
     allowPositionals: true,
     strict: false,  // Allow unknown options for subcommands
@@ -137,6 +143,19 @@ async function main() {
       break
     }
 
+    case 'ingest': {
+      const { ingest } = await import('./commands/ingest.ts')
+      await ingest({
+        project: values.project,
+        all: values.all,
+        dryRun: values['dry-run'],
+        verbose: values.verbose,
+        limit: values.limit ? parseInt(values.limit, 10) : undefined,
+        maxTokens: values['max-tokens'] ? parseInt(values['max-tokens'], 10) : undefined,
+      })
+      break
+    }
+
     case 'help':
       showHelp()
       break

package/src/core/curator.ts

@@ -392,42 +392,58 @@ Focus ONLY on technical, architectural, debugging, decision, workflow, and proje
   }
 
   /**
-   * Curate using Anthropic SDK (in-process mode)
-   * Requires @anthropic-ai/sdk to be installed
+   * Curate using Anthropic SDK with parsed session messages
+   * Takes the actual conversation messages in API format
    */
   async curateWithSDK(
-    conversationContext: string,
+    messages: Array<{ role: 'user' | 'assistant'; content: string | any[] }>,
     triggerType: CurationTrigger = 'session_end'
   ): Promise<CurationResult> {
     if (!this._config.apiKey) {
-      throw new Error('API key required for SDK mode')
+      throw new Error('API key required for SDK mode. Set ANTHROPIC_API_KEY environment variable.')
     }
 
     // Dynamic import to make SDK optional
     const { default: Anthropic } = await import('@anthropic-ai/sdk')
     const client = new Anthropic({ apiKey: this._config.apiKey })
 
-    const prompt = this.buildCurationPrompt(triggerType)
+    const systemPrompt = this.buildCurationPrompt(triggerType)
+
+    // Build the conversation: original messages + curation request
+    const conversationMessages = [
+      ...messages,
+      {
+        role: 'user' as const,
+        content: 'This session has ended. Please curate the memories from our conversation according to your system instructions. Return ONLY the JSON structure with no additional text.',
+      },
+    ]
 
     const response = await client.messages.create({
       model: 'claude-sonnet-4-20250514',
       max_tokens: 8192,
-      messages: [
-        {
-          role: 'user',
-          content: `${conversationContext}\n\n---\n\n${prompt}`,
-        },
-      ],
+      system: systemPrompt,
+      messages: conversationMessages,
     })
 
     const content = response.content[0]
     if (content.type !== 'text') {
-      throw new Error('Unexpected response type')
+      throw new Error('Unexpected response type from Claude API')
     }
 
     return this.parseCurationResponse(content.text)
   }
 
+  /**
+   * Curate from a parsed session segment
+   * Convenience method that extracts messages from SessionSegment
+   */
+  async curateFromSegment(
+    segment: { messages: Array<{ role: 'user' | 'assistant'; content: string | any[] }> },
+    triggerType: CurationTrigger = 'session_end'
+  ): Promise<CurationResult> {
+    return this.curateWithSDK(segment.messages, triggerType)
+  }
+
   /**
    * Curate using CLI subprocess (for hook mode)
    * Resumes a session and asks it to curate
@@ -471,7 +487,8 @@ Focus ONLY on technical, architectural, debugging, decision, workflow, and proje
         ...process.env,
         MEMORY_CURATOR_ACTIVE: '1',  // Prevent recursive hook triggering
       },
-      stderr: 'pipe',  // Capture stderr too
+      stdout: 'pipe',
+      stderr: 'pipe',
     })
 
     // Capture both stdout and stderr
@@ -490,15 +507,28 @@ Focus ONLY on technical, architectural, debugging, decision, workflow, and proje
       // First, parse the CLI JSON wrapper
       const cliOutput = JSON.parse(stdout)
 
+      // Claude Code now returns an array of events - find the result object
+      let resultObj: any
+      if (Array.isArray(cliOutput)) {
+        // New format: array of events, find the one with type="result"
+        resultObj = cliOutput.find((item: any) => item.type === 'result')
+        if (!resultObj) {
+          return { session_summary: '', memories: [] }
+        }
+      } else {
+        // Old format: single object (backwards compatibility)
+        resultObj = cliOutput
+      }
+
       // Check for error response FIRST (like Python does)
-      if (cliOutput.type === 'error' || cliOutput.is_error === true) {
+      if (resultObj.type === 'error' || resultObj.is_error === true) {
         return { session_summary: '', memories: [] }
       }
 
       // Extract the "result" field (AI's response text)
       let aiResponse = ''
-      if (typeof cliOutput.result === 'string') {
-        aiResponse = cliOutput.result
+      if (typeof resultObj.result === 'string') {
+        aiResponse = resultObj.result
       } else {
         return { session_summary: '', memories: [] }
       }

package/src/core/engine.ts

@@ -379,7 +379,7 @@ export class MemoryEngine {
     store: MemoryStore,
     projectId: string
   ): Promise<SessionPrimer> {
-    // Fetch personal primer from GLOBAL collection (separate fsdb instance)
+    // Fetch personal primer from dedicated primer collection in global database
     let personalContext: string | undefined
     if (this._config.personalMemoriesEnabled) {
       const personalPrimer = await store.getPersonalPrimer()
@@ -590,7 +590,8 @@ export class MemoryEngine {
     // This is a constant: ~/.local/share/memory/global
     const globalPath = join(homedir(), '.local', 'share', 'memory', 'global')
     const globalMemoriesPath = join(globalPath, 'memories')
-    const personalPrimerPath = join(globalPath, 'memories', 'personal-primer.md')
+    // Personal primer has its own dedicated collection (not in memories)
+    const personalPrimerPath = join(globalPath, 'primer', 'personal-primer.md')
 
     // Project path depends on storage mode - mirrors _getStore() logic exactly
     let storeBasePath: string

package/src/core/manager.ts

@@ -242,13 +242,26 @@ Please process these memories according to your management procedure. Use the ex
       // First, parse the CLI JSON wrapper
       const cliOutput = JSON.parse(responseJson)
 
+      // Claude Code now returns an array of events - find the result object
+      let resultObj: any
+      if (Array.isArray(cliOutput)) {
+        // New format: array of events, find the one with type="result"
+        resultObj = cliOutput.find((item: any) => item.type === 'result')
+        if (!resultObj) {
+          return emptyResult('No result found in response')
+        }
+      } else {
+        // Old format: single object (backwards compatibility)
+        resultObj = cliOutput
+      }
+
       // Check for error response
-      if (cliOutput.type === 'error' || cliOutput.is_error === true) {
-        return emptyResult(cliOutput.error || 'Unknown error')
+      if (resultObj.type === 'error' || resultObj.is_error === true) {
+        return emptyResult(resultObj.error || 'Unknown error')
       }
 
       // Extract the "result" field (AI's response text)
-      const resultText = typeof cliOutput.result === 'string' ? cliOutput.result : ''
+      const resultText = typeof resultObj.result === 'string' ? resultObj.result : ''
 
       // Extract the full report (everything from === MANAGEMENT ACTIONS === onwards)
       const reportMatch = resultText.match(/(=== MANAGEMENT ACTIONS ===[\s\S]*)/)