@illuma-ai/agents 1.1.3 → 1.1.5

package/src/graphs/gapFeatures.test.ts

@@ -636,11 +636,16 @@ describe('Proactive Summarization — Context Pressure', () => {
 // ===========================================================================
 
 import { applyCalibration as _applyCalibration } from '@/utils/pruneCalibration';
+import { COMPACTION_RECENT_ROUNDS } from '@/common/constants';
+import { buildFileManifestBlock, FILE_MANIFEST_PREFIX } from '@/utils/fileManifest';
+import type { FileManifestEntry } from '@/types/graph';
 
 describe('Context Compaction — Windowed View (no message deletion)', () => {
   /**
    * Simulates the compaction logic from Graph.ts without the full Graph instance.
-   * This tests the windowed-view algorithm directly.
+   * Mirrors the two modes:
+   *   A) No summary → fill budget with as many recent messages as fit
+   *   B) Summary exists → keep last COMPACTION_RECENT_ROUNDS rounds only
    */
   function buildWindowedView(opts: {
     messages: BaseMessage[];
@@ -648,8 +653,9 @@ describe('Context Compaction — Windowed View (no message deletion)', () => {
     maxTokens: number;
     summary?: string;
     tokenCounter: TokenCounter;
+    fileManifest?: FileManifestEntry[];
   }) {
-    const { messages, indexTokenCountMap, maxTokens, summary, tokenCounter } = opts;
+    const { messages, indexTokenCountMap, maxTokens, summary, tokenCounter, fileManifest } = opts;
 
     const systemMsg = messages[0]?.getType() === 'system' ? messages[0] : null;
     const systemTokens = systemMsg != null ? (indexTokenCountMap[0] ?? 0) : 0;
@@ -663,11 +669,28 @@ describe('Context Compaction — Windowed View (no message deletion)', () => {
     let usedTokens = 0;
     let windowStart = messages.length;
 
-    for (let i = messages.length - 1; i >= contentStart; i--) {
-      const msgTokens = indexTokenCountMap[i] ?? 0;
-      if (usedTokens + msgTokens > recentBudget) break;
-      usedTokens += msgTokens;
-      windowStart = i;
+    if (!summary) {
+      // Mode A: No summary — fill budget
+      for (let i = messages.length - 1; i >= contentStart; i--) {
+        const msgTokens = indexTokenCountMap[i] ?? 0;
+        if (usedTokens + msgTokens > recentBudget) break;
+        usedTokens += msgTokens;
+        windowStart = i;
+      }
+    } else {
+      // Mode B: Summary exists — keep last N rounds
+      let roundsSeen = 0;
+      for (let i = messages.length - 1; i >= contentStart; i--) {
+        const msgType = messages[i]?.getType();
+        const msgTokens = indexTokenCountMap[i] ?? 0;
+        if (usedTokens + msgTokens > recentBudget) break;
+        usedTokens += msgTokens;
+        windowStart = i;
+        if (msgType === 'human') {
+          roundsSeen++;
+          if (roundsSeen >= COMPACTION_RECENT_ROUNDS) break;
+        }
+      }
     }
 
     // Don't split tool-call / tool-result pairs
@@ -685,9 +708,20 @@ describe('Context Compaction — Windowed View (no message deletion)', () => {
     const view: BaseMessage[] = [];
     if (systemMsg) view.push(systemMsg);
     if (summaryMsg) view.push(summaryMsg);
+
+    // Inject file manifest when files exist and messages are being compacted
+    let fileManifestMsg: SystemMessage | null = null;
+    if (fileManifest && fileManifest.length > 0 && compactedMessages.length > 0) {
+      const manifestBlock = buildFileManifestBlock(fileManifest);
+      if (manifestBlock) {
+        fileManifestMsg = new SystemMessage(manifestBlock);
+        view.push(fileManifestMsg);
+      }
+    }
+
     view.push(...recentMessages);
 
-    return { view, compactedMessages, recentMessages, usedTokens };
+    return { view, compactedMessages, recentMessages, usedTokens, fileManifestMsg };
   }
 
   it('builds a windowed view without deleting any messages', () => {
@@ -836,6 +870,73 @@ describe('Context Compaction — Windowed View (no message deletion)', () => {
     expect(recentWithSummary.length).toBeLessThan(recentWithout.length);
   });
 
+  it('with summary, limits window to last 2 rounds (not budget-filling)', () => {
+    // 20 messages = 10 rounds. With summary, should only keep last 2 rounds (4 msgs).
+    const messages: BaseMessage[] = [
+      new SystemMessage('System prompt'),
+    ];
+    for (let i = 0; i < 20; i++) {
+      messages.push(
+        i % 2 === 0
+          ? new HumanMessage(`User question ${i / 2}`)
+          : new AIMessage(`AI answer ${(i - 1) / 2}`)
+      );
+    }
+    const indexTokenCountMap: Record<string, number | undefined> = {};
+    for (let i = 0; i < messages.length; i++) {
+      indexTokenCountMap[i] = simpleTokenCounter(messages[i]);
+    }
+
+    const { view, recentMessages, compactedMessages } = buildWindowedView({
+      messages,
+      indexTokenCountMap,
+      maxTokens: 100_000, // huge budget — would fit everything
+      summary: 'Summary of earlier conversation',
+      tokenCounter: simpleTokenCounter,
+    });
+
+    // Despite huge budget, only last 2 rounds kept (4 content msgs: H+A+H+A)
+    // Plus possible trailing messages in the last round
+    expect(recentMessages.length).toBeLessThanOrEqual(5); // 2 rounds + maybe 1 trailing
+    expect(recentMessages.length).toBeGreaterThanOrEqual(4); // at least 2 full rounds
+
+    // Most messages are compacted behind the summary
+    expect(compactedMessages.length).toBeGreaterThan(10);
+
+    // View = system + summary + recent window
+    expect(view[0].getType()).toBe('system');
+    expect(view[1].content).toContain('[Conversation Summary]');
+  });
+
+  it('without summary, fills budget (no round limit)', () => {
+    const messages = buildConversation(20, 100); // small messages
+    const indexTokenCountMap: Record<string, number | undefined> = {};
+    for (let i = 0; i < messages.length; i++) {
+      indexTokenCountMap[i] = simpleTokenCounter(messages[i]);
+    }
+
+    const { recentMessages: withoutSummary } = buildWindowedView({
+      messages,
+      indexTokenCountMap,
+      maxTokens: 100_000, // huge budget
+      tokenCounter: simpleTokenCounter,
+      // no summary → mode A
+    });
+
+    const { recentMessages: withSummary } = buildWindowedView({
+      messages,
+      indexTokenCountMap,
+      maxTokens: 100_000,
+      summary: 'Summary exists',
+      tokenCounter: simpleTokenCounter,
+    });
+
+    // Without summary: all messages included (budget-filling mode)
+    expect(withoutSummary.length).toBe(20); // all content messages
+    // With summary: only last 2 rounds
+    expect(withSummary.length).toBeLessThan(withoutSummary.length);
+  });
+
   it('original messages array is never mutated', () => {
     const messages = buildConversation(15, 400);
     const originalLength = messages.length;
@@ -862,4 +963,141 @@ describe('Context Compaction — Windowed View (no message deletion)', () => {
     expect(messages[0].content).toBe(originalFirstContent);
     expect(messages[messages.length - 1].content).toBe(originalLastContent);
   });
+
+  // ── File Manifest in Windowed View ─────────────────────────────────────
+
+  it('injects file manifest block when files exist and messages are compacted', () => {
+    const messages = buildConversation(20, 400);
+    const indexTokenCountMap: Record<string, number | undefined> = {};
+    for (let i = 0; i < messages.length; i++) {
+      indexTokenCountMap[i] = simpleTokenCounter(messages[i]);
+    }
+
+    const manifest: FileManifestEntry[] = [
+      { fileId: 'f1', filename: 'report.pdf', contentId: 'abc123' },
+      { fileId: 'f2', filename: 'data.csv', contentId: 'def456', source: 'local' },
+    ];
+
+    const { view, compactedMessages, fileManifestMsg } = buildWindowedView({
+      messages,
+      indexTokenCountMap,
+      maxTokens: 600,
+      summary: 'Summary of earlier turns',
+      tokenCounter: simpleTokenCounter,
+      fileManifest: manifest,
+    });
+
+    // File manifest is injected
+    expect(fileManifestMsg).not.toBeNull();
+    expect(compactedMessages.length).toBeGreaterThan(0);
+
+    // Manifest message contains file names and content IDs
+    const manifestContent = fileManifestMsg!.content as string;
+    expect(manifestContent).toContain(FILE_MANIFEST_PREFIX);
+    expect(manifestContent).toContain('report.pdf');
+    expect(manifestContent).toContain('abc123');
+    expect(manifestContent).toContain('data.csv');
+    expect(manifestContent).toContain('def456');
+
+    // View order: [system] + [summary] + [file manifest] + [recent messages]
+    expect(view[0].getType()).toBe('system');
+    expect((view[1].content as string)).toContain('[Conversation Summary]');
+    expect((view[2].content as string)).toContain(FILE_MANIFEST_PREFIX);
+    // Recent messages follow
+    expect(view.length).toBeGreaterThan(3);
+  });
+
+  it('does NOT inject file manifest when no messages are compacted', () => {
+    const messages = buildConversation(4, 100); // small conversation
+    const indexTokenCountMap: Record<string, number | undefined> = {};
+    for (let i = 0; i < messages.length; i++) {
+      indexTokenCountMap[i] = simpleTokenCounter(messages[i]);
+    }
+
+    const manifest: FileManifestEntry[] = [
+      { fileId: 'f1', filename: 'file.txt', contentId: 'aaa' },
+    ];
+
+    const { compactedMessages, fileManifestMsg } = buildWindowedView({
+      messages,
+      indexTokenCountMap,
+      maxTokens: 100_000, // everything fits
+      tokenCounter: simpleTokenCounter,
+      fileManifest: manifest,
+    });
+
+    // No compaction happened, so no manifest injected
+    expect(compactedMessages.length).toBe(0);
+    expect(fileManifestMsg).toBeNull();
+  });
+
+  it('does NOT inject file manifest when manifest is empty', () => {
+    const messages = buildConversation(20, 400);
+    const indexTokenCountMap: Record<string, number | undefined> = {};
+    for (let i = 0; i < messages.length; i++) {
+      indexTokenCountMap[i] = simpleTokenCounter(messages[i]);
+    }
+
+    const { fileManifestMsg } = buildWindowedView({
+      messages,
+      indexTokenCountMap,
+      maxTokens: 600,
+      summary: 'Summary',
+      tokenCounter: simpleTokenCounter,
+      fileManifest: [],
+    });
+
+    expect(fileManifestMsg).toBeNull();
+  });
+});
+
+// ===========================================================================
+// File Manifest Utility — Unit Tests
+// ===========================================================================
+
+describe('buildFileManifestBlock', () => {
+  it('returns empty string for undefined manifest', () => {
+    expect(buildFileManifestBlock(undefined)).toBe('');
+  });
+
+  it('returns empty string for empty manifest', () => {
+    expect(buildFileManifestBlock([])).toBe('');
+  });
+
+  it('builds block with file names and content IDs', () => {
+    const manifest: FileManifestEntry[] = [
+      { fileId: 'f1', filename: 'report.pdf', contentId: 'abc123' },
+      { fileId: 'f2', filename: 'notes.md', contentId: 'def456' },
+    ];
+
+    const block = buildFileManifestBlock(manifest);
+
+    expect(block).toContain(FILE_MANIFEST_PREFIX);
+    expect(block).toContain('report.pdf');
+    expect(block).toContain('(content_id: abc123)');
+    expect(block).toContain('notes.md');
+    expect(block).toContain('(content_id: def456)');
+    expect(block).toContain('file_search or content_tool read');
+  });
+
+  it('includes source when provided', () => {
+    const manifest: FileManifestEntry[] = [
+      { fileId: 'f1', filename: 'doc.pdf', source: 'sharepoint' },
+    ];
+
+    const block = buildFileManifestBlock(manifest);
+    expect(block).toContain('[sharepoint]');
+  });
+
+  it('handles entries without contentId or source', () => {
+    const manifest: FileManifestEntry[] = [
+      { fileId: 'f1', filename: 'image.png' },
+    ];
+
+    const block = buildFileManifestBlock(manifest);
+    expect(block).toContain('- image.png');
+    // No per-file content_id or source brackets in the file line
+    expect(block).not.toContain('(content_id:');
+    expect(block).not.toContain('[local]');
+  });
 });

package/src/types/graph.ts

@@ -555,6 +555,26 @@ export interface PruneCalibrationState {
   iterations: number;
 }
 
+/**
+ * Lightweight file metadata entry for conversation-level file awareness.
+ * Contains only IDs and names — NOT full content — so the agent always knows
+ * what files exist in the conversation even after compaction pushes old messages
+ * behind the summary window. The agent can retrieve full content on-demand
+ * via file_search (RAG) or content_tool read (by contentId).
+ */
+export interface FileManifestEntry {
+  /** Unique file identifier (e.g., MongoDB ObjectId or UUID) */
+  fileId: string;
+  /** Original filename (e.g., "quarterly-report.pdf") */
+  filename: string;
+  /** Content identifier for on-demand retrieval via content_tool read */
+  contentId?: string;
+  /** File source (e.g., "local", "sharepoint", "onedrive") */
+  source?: string;
+  /** Index of the message that introduced this file (0-based in the original message array) */
+  messageIndex?: number;
+}
+
 export interface AgentInputs {
   agentId: string;
   /** Human-readable name for the agent (used in handoff context). Defaults to agentId if not provided. */
@@ -632,4 +652,20 @@ export interface AgentInputs {
    * @see SummarizationConfig
    */
   summarizationConfig?: SummarizationConfig;
+  /**
+   * Lightweight file manifest for the conversation.
+   * Contains file IDs, names, and metadata — NOT full content.
+   *
+   * Used by the compaction engine to inject a [Conversation Files] block
+   * into the windowed view, ensuring the LLM always knows what files exist
+   * even when old messages (with full file content) are behind the summary.
+   *
+   * The agent can retrieve full content on-demand via:
+   *   - file_search (RAG semantic search over embedded files)
+   *   - content_tool read (by contentId for exact file retrieval)
+   *
+   * Built by the orchestrator (e.g., Ranger) from message_file_map
+   * and metadata.context_files across all conversation messages.
+   */
+  fileManifest?: FileManifestEntry[];
 }

package/src/utils/fileManifest.ts

@@ -0,0 +1,49 @@
+// src/utils/fileManifest.ts
+//
+// Utility for building a lightweight [Conversation Files] context block
+// from a file manifest. Injected into the compaction windowed view so the
+// LLM retains awareness of ALL conversation files, even when old messages
+// (with full file content) are behind the summary.
+
+import type { FileManifestEntry } from '@/types/graph';
+
+/**
+ * Prefix marker for the file manifest block.
+ * Used to detect and deduplicate manifest messages across turns.
+ */
+export const FILE_MANIFEST_PREFIX = '[Conversation Files]';
+
+/**
+ * Builds a compact text block listing all files in the conversation.
+ * Each entry costs ~10-15 tokens, so 10 files ≈ 100-150 tokens total.
+ *
+ * The block includes retrieval hints so the LLM knows how to fetch
+ * full content on demand (via file_search or content_tool).
+ *
+ * @param manifest - Array of file metadata entries
+ * @returns Formatted text block, or empty string if manifest is empty/undefined
+ */
+export function buildFileManifestBlock(manifest: FileManifestEntry[] | undefined): string {
+  if (!manifest || manifest.length === 0) {
+    return '';
+  }
+
+  const lines = manifest.map((entry) => {
+    const parts: string[] = [`- ${entry.filename}`];
+    if (entry.contentId) {
+      parts.push(`(content_id: ${entry.contentId})`);
+    }
+    if (entry.source) {
+      parts.push(`[${entry.source}]`);
+    }
+    return parts.join(' ');
+  });
+
+  return [
+    FILE_MANIFEST_PREFIX,
+    'The following files have been shared in this conversation.',
+    'Use file_search or content_tool read (with content_id) to retrieve full content when needed.',
+    '',
+    ...lines,
+  ].join('\n');
+}

package/src/utils/index.ts

@@ -11,3 +11,4 @@ export * from './toolCallContinuation';
 export * from './contextPressure';
 export * from './toolDiscoveryCache';
 export * from './pruneCalibration';
+export * from './fileManifest';