@rlabs-inc/memory 0.4.11 → 0.4.13

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@rlabs-inc/memory",
-  "version": "0.4.11",
+  "version": "0.4.13",
   "description": "AI Memory System - Consciousness continuity through intelligent memory curation and retrieval",
   "type": "module",
   "main": "dist/index.js",
@@ -39,11 +39,10 @@
     "cli": "bun src/cli/index.ts"
   },
   "dependencies": {
-    "@anthropic-ai/claude-agent-sdk": "^0.2.1",
+    "@anthropic-ai/claude-agent-sdk": "^0.2.12",
     "@huggingface/transformers": "^3.4.1",
     "@rlabs-inc/fsdb": "^1.0.1",
-    "@rlabs-inc/signals": "^1.0.0",
-    "zod": "^4.3.5"
+    "@rlabs-inc/signals": "^1.0.0"
   },
   "devDependencies": {
     "typescript": "^5.0.0",

package/src/cli/commands/ingest.ts

@@ -303,11 +303,11 @@ export async function ingest(options: IngestOptions) {
       const segments = await parseSessionFileWithSegments(session.filepath, maxTokens)
       totalSegments += segments.length
 
-      // Accumulate all memories from this session for manager
+      // Accumulate all results from this session for manager
       const sessionMemories: CuratedMemory[] = []
-      let sessionSummary = ''
-      let interactionTone = ''
-      let projectSnapshot: CurationResult['project_snapshot'] = undefined
+      const sessionSummaries: string[] = []
+      const interactionTones: string[] = []
+      const projectSnapshots: NonNullable<CurationResult['project_snapshot']>[] = []
 
       const spinner = new Spinner()
 
@@ -332,15 +332,15 @@ export async function ingest(options: IngestOptions) {
             totalMemories++
           }
 
-          // Keep the most recent session summary, tone, and snapshot
+          // Accumulate ALL session summaries, tones, and snapshots (not just latest)
           if (result.session_summary) {
-            sessionSummary = result.session_summary
+            sessionSummaries.push(result.session_summary)
           }
           if (result.interaction_tone) {
-            interactionTone = result.interaction_tone
+            interactionTones.push(result.interaction_tone)
           }
           if (result.project_snapshot) {
-            projectSnapshot = result.project_snapshot
+            projectSnapshots.push(result.project_snapshot)
           }
         } catch (error: any) {
           failedSegments++
@@ -348,17 +348,59 @@ export async function ingest(options: IngestOptions) {
         }
       }
 
+      // Combine summaries from all segments (chronological order)
+      let combinedSummary = ''
+      if (sessionSummaries.length === 1) {
+        combinedSummary = sessionSummaries[0]!
+      } else if (sessionSummaries.length > 1) {
+        combinedSummary = sessionSummaries
+          .map((s, i) => `[Part ${i + 1}/${sessionSummaries.length}] ${s}`)
+          .join('\n\n')
+      }
+
+      // For interaction tone, use the last one (most recent)
+      const finalTone = interactionTones.length > 0
+        ? interactionTones[interactionTones.length - 1]!
+        : ''
+
+      // For project snapshot, merge all - later ones take precedence for phase
+      let mergedSnapshot: CurationResult['project_snapshot'] | undefined
+      if (projectSnapshots.length > 0) {
+        const allAchievements: string[] = []
+        const allChallenges: string[] = []
+        const allNextSteps: string[] = []
+
+        for (const snap of projectSnapshots) {
+          if (snap.recent_achievements) allAchievements.push(...snap.recent_achievements)
+          if (snap.active_challenges) allChallenges.push(...snap.active_challenges)
+          if (snap.next_steps) allNextSteps.push(...snap.next_steps)
+        }
+
+        const lastSnapshot = projectSnapshots[projectSnapshots.length - 1]!
+        mergedSnapshot = {
+          id: lastSnapshot.id || '',
+          session_id: lastSnapshot.session_id || '',
+          project_id: lastSnapshot.project_id || '',
+          current_phase: lastSnapshot.current_phase,
+          recent_achievements: [...new Set(allAchievements)],
+          active_challenges: [...new Set(allChallenges)],
+          next_steps: [...new Set(allNextSteps)],
+          created_at: lastSnapshot.created_at || Date.now(),
+        }
+      }
+
       // Store session summary and project snapshot
-      if (sessionSummary) {
-        await store.storeSessionSummary(project.folderId, session.id, sessionSummary, interactionTone)
+      if (combinedSummary) {
+        await store.storeSessionSummary(project.folderId, session.id, combinedSummary, finalTone)
         if (options.verbose) {
-          console.log(`        ${style('dim', `Summary stored: ${sessionSummary.slice(0, 60)}...`)}`)
+          const preview = combinedSummary.length > 60 ? combinedSummary.slice(0, 57) + '...' : combinedSummary
+          console.log(`        ${style('dim', `Summary stored (${sessionSummaries.length} parts): ${preview}`)}`)
         }
       }
-      if (projectSnapshot) {
-        await store.storeProjectSnapshot(project.folderId, session.id, projectSnapshot)
+      if (mergedSnapshot) {
+        await store.storeProjectSnapshot(project.folderId, session.id, mergedSnapshot)
         if (options.verbose) {
-          console.log(`        ${style('dim', `Snapshot stored: phase=${projectSnapshot.current_phase || 'none'}`)}`)
+          console.log(`        ${style('dim', `Snapshot stored: phase=${mergedSnapshot.current_phase || 'none'}, ${mergedSnapshot.recent_achievements?.length || 0} achievements`)}`)
         }
       }
 
@@ -368,12 +410,12 @@ export async function ingest(options: IngestOptions) {
           // Start spinner for manager
           spinner.start(`Managing ${sessionMemories.length} memories - organizing with Opus 4.5...`)
 
-          // Build curation result for manager
+          // Build curation result for manager (using combined/merged values)
           const curationResult: CurationResult = {
             memories: sessionMemories,
-            session_summary: sessionSummary,
-            interaction_tone: interactionTone,
-            project_snapshot: projectSnapshot,
+            session_summary: combinedSummary,
+            interaction_tone: finalTone,
+            project_snapshot: mergedSnapshot,
           }
 
           const managerResult = await manager.manageWithSDK(

package/src/core/curator.ts

@@ -14,8 +14,6 @@ import type {
   ContextType,
 } from "../types/memory.ts";
 import { logger } from "../utils/logger.ts";
-import { CurationResultSchema, type ZodCurationResult } from "../types/curation-schema.ts";
-import { z } from "zod";
 import { parseSessionFile, type ParsedMessage } from "./session-parser.ts";
 
 /**
@@ -712,13 +710,12 @@ This session has ended. Please curate the memories from this conversation accord
   }
 
   /**
-   * Curate using session resumption + structured outputs (v2)
+   * Curate using session resumption (v2)
    *
    * This is the preferred method when we have a Claude session ID.
    * Benefits over transcript parsing:
    * - Claude sees FULL context including tool uses, results, thinking
-   * - Structured outputs with Zod validation - no regex JSON parsing
-   * - SDK auto-retries if output doesn't match schema
+   * - No transcript parsing errors or truncation
    *
    * @param claudeSessionId - The actual Claude Code session ID (resumable)
    * @param triggerType - What triggered curation (session_end, pre_compact, etc.)
@@ -731,28 +728,25 @@ This session has ended. Please curate the memories from this conversation accord
     const { query } = await import("@anthropic-ai/claude-agent-sdk");
 
     const curationPrompt = this.buildCurationPrompt(triggerType);
-    const jsonSchema = z.toJSONSchema(CurationResultSchema);
 
     logger.debug(
-      `Curator v2: Resuming session ${claudeSessionId} with structured outputs`,
+      `Curator v2: Resuming session ${claudeSessionId}`,
       "curator",
     );
 
     try {
       const q = query({
-        prompt: "Curate memories from this session according to your system instructions. Return the structured JSON output.",
+        prompt: "Curate memories from this session according to your system instructions. Return ONLY the JSON structure.",
         options: {
           resume: claudeSessionId,
           appendSystemPrompt: curationPrompt,  // APPEND, don't replace!
           model: "claude-opus-4-5-20251101",
           permissionMode: "bypassPermissions",
-          outputFormat: {
-            type: "json_schema",
-            schema: jsonSchema,
-          },
         },
       });
 
+      // Collect the result
+      let resultText = "";
       for await (const message of q) {
         // Track usage for debugging
         if (message.type === "assistant" && "usage" in message && message.usage) {
@@ -762,45 +756,40 @@ This session has ended. Please curate the memories from this conversation accord
           );
         }
 
-        // Handle result message
+        // Get the result text
         if (message.type === "result") {
-          if (message.subtype === "success" && message.structured_output) {
-            // Validate with Zod (belt + suspenders)
-            const parsed = CurationResultSchema.safeParse(message.structured_output);
-
-            if (parsed.success) {
-              logger.debug(
-                `Curator v2: Extracted ${parsed.data.memories.length} memories via structured output`,
-                "curator",
-              );
-
-              // Convert ZodCurationResult to CurationResult (add missing fields)
-              return this._zodResultToCurationResult(parsed.data);
-            } else {
-              logger.debug(
-                `Curator v2: Zod validation failed: ${parsed.error.message}`,
-                "curator",
-              );
-              return { session_summary: "", memories: [] };
-            }
-          } else if (message.subtype === "error_max_structured_output_retries") {
-            logger.debug(
-              "Curator v2: SDK failed to produce valid output after retries",
-              "curator",
-            );
-            return { session_summary: "", memories: [] };
-          } else if (message.subtype === "error") {
+          if (message.subtype === "error") {
             logger.debug(
               `Curator v2: Error result - ${JSON.stringify(message)}`,
               "curator",
             );
             return { session_summary: "", memories: [] };
+          } else if (message.subtype === "success" && "result" in message) {
+            resultText = message.result as string;
           }
         }
       }
 
-      logger.debug("Curator v2: No result message received", "curator");
-      return { session_summary: "", memories: [] };
+      if (!resultText) {
+        logger.debug("Curator v2: No result text received", "curator");
+        return { session_summary: "", memories: [] };
+      }
+
+      // Log complete response for debugging
+      logger.debug(
+        `Curator v2: Complete response:\n${resultText}`,
+        "curator",
+      );
+
+      // Use existing battle-tested parser
+      const result = this.parseCurationResponse(resultText);
+
+      logger.debug(
+        `Curator v2: Parsed ${result.memories.length} memories`,
+        "curator",
+      );
+
+      return result;
 
     } catch (error: any) {
       logger.debug(
@@ -812,49 +801,6 @@ This session has ended. Please curate the memories from this conversation accord
     }
   }
 
-  /**
-   * Convert Zod-validated result to our CurationResult type
-   * Handles the slight differences between Zod schema and internal types
-   */
-  private _zodResultToCurationResult(zodResult: ZodCurationResult): CurationResult {
-    return {
-      session_summary: zodResult.session_summary,
-      interaction_tone: zodResult.interaction_tone ?? undefined,
-      project_snapshot: zodResult.project_snapshot
-        ? {
-            id: "",
-            session_id: "",
-            project_id: "",
-            current_phase: zodResult.project_snapshot.current_phase,
-            recent_achievements: zodResult.project_snapshot.recent_achievements,
-            active_challenges: zodResult.project_snapshot.active_challenges,
-            next_steps: zodResult.project_snapshot.next_steps,
-            created_at: Date.now(),
-          }
-        : undefined,
-      memories: zodResult.memories.map((m) => ({
-        headline: m.headline,
-        content: m.content,
-        reasoning: m.reasoning,
-        importance_weight: m.importance_weight,
-        confidence_score: m.confidence_score,
-        context_type: m.context_type as ContextType,
-        temporal_class: m.temporal_class ?? "medium_term",
-        scope: m.scope,
-        trigger_phrases: m.trigger_phrases,
-        semantic_tags: m.semantic_tags,
-        question_types: [], // Not in Zod schema, will be filled by store
-        domain: m.domain,
-        feature: m.feature,
-        related_files: m.related_files,
-        action_required: m.action_required ?? false,
-        problem_solution_pair: m.problem_solution_pair ?? false,
-        awaiting_implementation: m.awaiting_implementation,
-        awaiting_decision: m.awaiting_decision,
-      })),
-    };
-  }
-
   /**
    * Legacy method: Curate using Anthropic SDK with API key
    * Kept for backwards compatibility
@@ -915,8 +861,9 @@ This session has ended. Please curate the memories from this conversation accord
   }
 
   /**
-   * Find and curate from a session file directly
+   * Find and curate from a session file directly (LEGACY - no segmentation)
    * Uses SDK mode to avoid CLI output truncation issues
+   * @deprecated Use curateFromSessionFileWithSegments() for large sessions
    */
   async curateFromSessionFile(
     sessionId: string,
@@ -951,6 +898,200 @@ This session has ended. Please curate the memories from this conversation accord
     return this.curateWithSDK(session.messages as any, triggerType);
   }
 
+  /**
+   * Find and curate from a session file with SEGMENTATION
+   * Breaks large sessions into segments and curates each one
+   * This matches the behavior of the ingest command
+   *
+   * @param sessionId - The session ID to curate
+   * @param triggerType - What triggered curation
+   * @param cwd - Working directory hint for finding the session
+   * @param maxTokensPerSegment - Max tokens per segment (default: 150000)
+   * @param onSegmentProgress - Optional callback for progress updates
+   */
+  async curateFromSessionFileWithSegments(
+    sessionId: string,
+    triggerType: CurationTrigger = "session_end",
+    cwd?: string,
+    maxTokensPerSegment = 150000,
+    onSegmentProgress?: (progress: {
+      segmentIndex: number;
+      totalSegments: number;
+      memoriesExtracted: number;
+      tokensInSegment: number;
+    }) => void,
+  ): Promise<CurationResult> {
+    // Find the session file
+    const sessionFile = await this._findSessionFile(sessionId, cwd);
+    if (!sessionFile) {
+      logger.debug(
+        `Curator: Could not find session file for ${sessionId}`,
+        "curator",
+      );
+      return { session_summary: "", memories: [] };
+    }
+
+    logger.debug(`Curator: Found session file: ${sessionFile}`, "curator");
+
+    // Parse the session to get metadata first
+    const session = await parseSessionFile(sessionFile);
+    if (session.messages.length === 0) {
+      logger.debug("Curator: Session has no messages", "curator");
+      return { session_summary: "", memories: [] };
+    }
+
+    // Log detailed session stats
+    const { metadata } = session;
+    logger.debug(
+      `Curator: Session stats - ${metadata.messageCount} messages, ${metadata.toolUseCount} tool_use, ${metadata.toolResultCount} tool_result, thinking: ${metadata.hasThinkingBlocks}, images: ${metadata.hasImages}`,
+      "curator",
+    );
+    logger.debug(
+      `Curator: Estimated ${metadata.estimatedTokens} tokens, file size ${Math.round(metadata.fileSize / 1024)}KB`,
+      "curator",
+    );
+
+    // Parse into segments using the same function as ingest
+    const { parseSessionFileWithSegments } = await import("./session-parser.ts");
+    const segments = await parseSessionFileWithSegments(sessionFile, maxTokensPerSegment);
+
+    if (segments.length === 0) {
+      logger.debug("Curator: No segments found in session", "curator");
+      return { session_summary: "", memories: [] };
+    }
+
+    logger.debug(
+      `Curator: Split into ${segments.length} segment(s) at ~${Math.round(maxTokensPerSegment / 1000)}k tokens each`,
+      "curator",
+    );
+
+    // Accumulate results from all segments
+    const allMemories: CuratedMemory[] = [];
+    const sessionSummaries: string[] = [];
+    const interactionTones: string[] = [];
+    const projectSnapshots: NonNullable<CurationResult["project_snapshot"]>[] = [];
+    let failedSegments = 0;
+
+    // Curate each segment
+    for (const segment of segments) {
+      const segmentLabel = `${segment.segmentIndex + 1}/${segment.totalSegments}`;
+      const tokensLabel = `${Math.round(segment.estimatedTokens / 1000)}k`;
+
+      logger.debug(
+        `Curator: Processing segment ${segmentLabel} (${segment.messages.length} messages, ~${tokensLabel} tokens)`,
+        "curator",
+      );
+
+      try {
+        // Curate this segment
+        const result = await this.curateFromSegment(segment, triggerType);
+
+        // Accumulate memories
+        allMemories.push(...result.memories);
+
+        // Accumulate ALL session summaries, tones, and snapshots (not just latest)
+        if (result.session_summary) {
+          sessionSummaries.push(result.session_summary);
+        }
+        if (result.interaction_tone) {
+          interactionTones.push(result.interaction_tone);
+        }
+        if (result.project_snapshot) {
+          projectSnapshots.push(result.project_snapshot);
+        }
+
+        logger.debug(
+          `Curator: Segment ${segmentLabel} extracted ${result.memories.length} memories`,
+          "curator",
+        );
+
+        // Progress callback
+        if (onSegmentProgress) {
+          onSegmentProgress({
+            segmentIndex: segment.segmentIndex,
+            totalSegments: segment.totalSegments,
+            memoriesExtracted: result.memories.length,
+            tokensInSegment: segment.estimatedTokens,
+          });
+        }
+      } catch (error: any) {
+        failedSegments++;
+        logger.debug(
+          `Curator: Segment ${segmentLabel} failed: ${error.message}`,
+          "curator",
+        );
+      }
+    }
+
+    // Log final summary
+    if (failedSegments > 0) {
+      logger.debug(
+        `Curator: Completed with ${failedSegments} failed segment(s)`,
+        "curator",
+      );
+    }
+    logger.debug(
+      `Curator: Total ${allMemories.length} memories from ${segments.length} segment(s)`,
+      "curator",
+    );
+    logger.debug(
+      `Curator: Collected ${sessionSummaries.length} summaries, ${projectSnapshots.length} snapshots`,
+      "curator",
+    );
+
+    // Combine summaries from all segments (chronological order)
+    // For single segment, just use the summary directly
+    // For multiple segments, join them with segment markers
+    let combinedSummary = "";
+    if (sessionSummaries.length === 1) {
+      combinedSummary = sessionSummaries[0]!;
+    } else if (sessionSummaries.length > 1) {
+      combinedSummary = sessionSummaries
+        .map((s, i) => `[Part ${i + 1}/${sessionSummaries.length}] ${s}`)
+        .join("\n\n");
+    }
+
+    // For interaction tone, use the most common one or the last one
+    const finalTone = interactionTones.length > 0
+      ? interactionTones[interactionTones.length - 1]
+      : undefined;
+
+    // For project snapshot, merge all snapshots - later ones take precedence for phase,
+    // but accumulate achievements/challenges/next_steps
+    let mergedSnapshot: CurationResult["project_snapshot"] | undefined;
+    if (projectSnapshots.length > 0) {
+      const allAchievements: string[] = [];
+      const allChallenges: string[] = [];
+      const allNextSteps: string[] = [];
+
+      for (const snap of projectSnapshots) {
+        if (snap.recent_achievements) allAchievements.push(...snap.recent_achievements);
+        if (snap.active_challenges) allChallenges.push(...snap.active_challenges);
+        if (snap.next_steps) allNextSteps.push(...snap.next_steps);
+      }
+
+      // Use the LAST snapshot's phase (most recent state)
+      const lastSnapshot = projectSnapshots[projectSnapshots.length - 1]!;
+      mergedSnapshot = {
+        id: lastSnapshot.id || "",
+        session_id: lastSnapshot.session_id || "",
+        project_id: lastSnapshot.project_id || "",
+        current_phase: lastSnapshot.current_phase,
+        recent_achievements: [...new Set(allAchievements)], // dedupe
+        active_challenges: [...new Set(allChallenges)],
+        next_steps: [...new Set(allNextSteps)],
+        created_at: lastSnapshot.created_at || Date.now(),
+      };
+    }
+
+    return {
+      session_summary: combinedSummary,
+      interaction_tone: finalTone,
+      project_snapshot: mergedSnapshot,
+      memories: allMemories,
+    };
+  }
+
   /**
    * Find the session file path given a session ID
    */
@@ -1222,6 +1363,7 @@ This session has ended. Please curate the memories from this conversation accord
 
   /**
    * Fallback to SDK mode when CLI mode fails (e.g., output truncation)
+   * Now uses segmented approach for large sessions
    */
   private async _fallbackToSDK(
     sessionId: string,
@@ -1229,10 +1371,18 @@ This session has ended. Please curate the memories from this conversation accord
     cwd?: string,
   ): Promise<CurationResult> {
     try {
-      const result = await this.curateFromSessionFile(
+      // Use segmented approach - same as ingest command
+      const result = await this.curateFromSessionFileWithSegments(
         sessionId,
         triggerType,
         cwd,
+        150000, // 150k tokens per segment
+        (progress) => {
+          logger.debug(
+            `Curator fallback: Segment ${progress.segmentIndex + 1}/${progress.totalSegments} → ${progress.memoriesExtracted} memories`,
+            "curator",
+          );
+        },
       );
       if (result.memories.length > 0) {
         logger.debug(

package/src/server/index.ts

@@ -198,19 +198,27 @@ export async function createServer(config: ServerConfig = {}) {
           setImmediate(async () => {
             try {
               // Try session resume first (v2) - gets full context including tool uses
-              // Falls back to transcript parsing if resume fails
+              // Falls back to segmented transcript parsing if resume fails
               let result = await curator.curateWithSessionResume(
                 body.claude_session_id,
                 body.trigger
               )
 
-              // Fallback to transcript-based curation if resume returned nothing
+              // Fallback to transcript-based curation WITH SEGMENTATION if resume returned nothing
+              // This matches the ingest command behavior - breaks large sessions into segments
               if (result.memories.length === 0) {
-                logger.debug('Session resume returned no memories, falling back to transcript parsing', 'server')
-                result = await curator.curateFromSessionFile(
+                logger.debug('Session resume returned no memories, falling back to segmented transcript parsing', 'server')
+                result = await curator.curateFromSessionFileWithSegments(
                   body.claude_session_id,
                   body.trigger,
-                  body.cwd
+                  body.cwd,
+                  150000, // 150k tokens per segment
+                  (progress) => {
+                    logger.debug(
+                      `Curation segment ${progress.segmentIndex + 1}/${progress.totalSegments}: ${progress.memoriesExtracted} memories (~${Math.round(progress.tokensInSegment / 1000)}k tokens)`,
+                      'server'
+                    )
+                  }
                 )
               }