@lincoln504/pi-research 0.1.13 → 1.0.1

package/src/utils/shared-links.ts

@@ -1,123 +1,228 @@
 /**
- * Shared Links Pool Management
+ * Shared Links Registry
  *
- * Centralized deterministic link state management system.
- * Single source of truth: global allScrapedLinks array.
- * No per-session state - everything flows through global state.
+ * Provides a global registry of links scraped across all researchers
+ * in a session to prevent redundant work.
  */
 
-import * as crypto from 'node:crypto';
+import { normalizeUrl } from './url-utils.ts';
+export { normalizeUrl };
+import { logger } from '../logger.ts';
+import { safeUnref } from './safe-unref.ts';
+import { truncateWithMarker } from './text-utils.ts';
+import { MAX_SCRAPE_CONTENT_CHARS_PER_DOC } from '../constants.ts';
+import { randomUUID } from 'node:crypto';
+
+const sessionLinks = new Map<string, Set<string>>();
+const sessionScrapedContent = new Map<string, Map<string, string>>();
+// Per-researcher scrape tracking: researchId → researcherId → Set<normalizedUrl>
+const researcherScrapes = new Map<string, Map<string, Set<string>>>();
+
+// FIX (Issue 12): Track creation timestamps for orphaned-entry cleanup.
+const sessionTimestamps = new Map<string, number>();
+const SESSION_MAX_AGE_MS = 2 * 60 * 60 * 1000; // 2 hours
+// FIX (#20): Cap per-session cached content to prevent unbounded memory growth
+const MAX_CACHED_CONTENT_PER_SESSION = 500;
+
+// Periodic cleanup of orphaned sessions (every 15 minutes)
+let cleanupInterval: NodeJS.Timeout | null = null;
+
+function startCleanupTimer(): void {
+  if (cleanupInterval) return;
+  cleanupInterval = setInterval(() => {
+    const now = Date.now();
+    let cleaned = 0;
+    for (const [researchId, createdAt] of sessionTimestamps.entries()) {
+      if (now - createdAt > SESSION_MAX_AGE_MS) {
+        sessionLinks.delete(researchId);
+        sessionScrapedContent.delete(researchId);
+        sessionTimestamps.delete(researchId);
+        researcherScrapes.delete(researchId);
+        cleaned++;
+      }
+    }
+    if (cleaned > 0) {
+      logger.debug(`[Shared Links] Cleaned up ${cleaned} orphaned session(s) older than ${SESSION_MAX_AGE_MS / 60000} minutes`);
+    }
+  }, 15 * 60 * 1000);
+  if (cleanupInterval) safeUnref(cleanupInterval);
+}
+
+// Start on module load
+startCleanupTimer();
 
 /**
- * Generate a 4-character alphanumeric hash
+ * Cache the raw markdown of a scraped URL during a research session.
+ * This is used later to embed the raw text alongside the agent's summary.
  */
-function generate4CharHash(): string {
-  // Use randomBytes for Node.js environments, fallback to Math.random for test environments
-  let hash: string;
-  try {
-    const bytes = crypto.randomBytes(2);
-    hash = bytes.toString('hex').substring(0, 4);
-  } catch {
-    // Fallback for test environments where crypto.randomBytes might not be available
-    // Generate a valid 4-character hex string
-    const randomHex = Math.floor(Math.random() * 65536).toString(16).padStart(4, '0');
-    hash = randomHex;
-  }
-  return hash;
+export function cacheScrapedContent(researchId: string, url: string, content: string) {
+    if (!sessionScrapedContent.has(researchId)) {
+        sessionScrapedContent.set(researchId, new Map());
+    }
+    // Refresh on every write so the orphan-cleanup TTL is last-activity based, not
+    // creation based — otherwise a research session running longer than the 2h
+    // SESSION_MAX_AGE_MS has its dedup/content pools wiped mid-run.
+    sessionTimestamps.set(researchId, Date.now());
+    const cache = sessionScrapedContent.get(researchId)!;
+    const key = normalizeUrl(url);
+    // FIX (#20): Evict oldest entry only when at capacity AND inserting a NEW key.
+    // Re-caching an existing URL must not evict an unrelated entry (net size loss).
+    if (!cache.has(key) && cache.size >= MAX_CACHED_CONTENT_PER_SESSION) {
+        const firstKey = cache.keys().next().value;
+        if (firstKey !== undefined) cache.delete(firstKey);
+    }
+    // Byte-bound each entry: the cache was previously only COUNT-capped (500
+    // entries), so 500 × multi-MB documents could grow unbounded in memory —
+    // and cached entries are re-served verbatim into researcher LLM sessions
+    // (scrape duplicate path) and embedded by the knowledge-store writer.
+    cache.set(key, truncateWithMarker(content, MAX_SCRAPE_CONTENT_CHARS_PER_DOC));
 }
 
 /**
- * Generate a unique session ID with 4-char hash
- * @param baseId Base identifier (e.g., request ID)
- * @returns Unique session ID (e.g., "abc123-x9k2")
+ * Retrieve the cached raw markdown for a scraped URL.
  */
-export function generateSessionId(baseId: string): string {
-  const hash = generate4CharHash();
-  return `${baseId}-${hash}`;
+export function getCachedScrapedContent(researchId: string, url: string): string | undefined {
+    return sessionScrapedContent.get(researchId)?.get(normalizeUrl(url));
 }
 
 /**
- * Normalize a URL for comparison (lowercase, remove trailing slash)
+ * Generate a unique research ID based on the parent Pi session.
  */
-export function normalizeUrl(url: string): string {
-  try {
-    const normalized = url.toLowerCase().trim();
-    // Remove trailing slash except for root
-    if (normalized.length > 1 && normalized.endsWith('/')) {
-      return normalized.slice(0, -1);
-    }
-    return normalized;
-  } catch {
-    return url.toLowerCase().trim();
-  }
+export function generateSessionId(piSessionId: string): string {
+    return `${piSessionId}-${randomUUID().replace(/-/g, '').substring(0, 8)}`;
 }
 
 /**
- * Deduplicate a list of URLs against already-scraped URLs.
- * Uses normalized comparison for accuracy.
- * 
- * @param urls - URLs to check
- * @param alreadyScraped - Array of already-scraped URLs
- * @returns Object with kept URLs and removed duplicates
+ * Register links as scraped for a specific session.
  */
-export function deduplicateUrls(urls: string[], alreadyScraped: string[]): {
-  kept: string[];
-  duplicates: string[];
-} {
-  const normalizedAlreadyScraped = new Set(
-    alreadyScraped.map(normalizeUrl)
-  );
-
-  const kept: string[] = [];
-  const duplicates: string[] = [];
-
-  for (const url of urls) {
-    const normalized = normalizeUrl(url);
-    if (normalizedAlreadyScraped.has(normalized)) {
-      duplicates.push(url);
-    } else {
-      kept.push(url);
+export function registerScrapedLinks(researchId: string, links: string[]) {
+    if (!sessionLinks.has(researchId)) {
+        sessionLinks.set(researchId, new Set());
     }
-  }
+    sessionTimestamps.set(researchId, Date.now()); // last-activity TTL refresh
+    const pool = sessionLinks.get(researchId)!;
+    links.forEach(l => pool.add(normalizeUrl(l)));
+}
+
+/**
+ * Get all scraped links for a session.
+ */
+export function getScrapedLinks(researchId: string): string[] {
+    return Array.from(sessionLinks.get(researchId) || []);
+}
+
+/**
+ * Deduplicate a list of candidate URLs against already scraped links.
+ */
+export function deduplicateUrls(urls: string[], researchId: string): { kept: string[], duplicates: string[] } {
+    const pool = sessionLinks.get(researchId) || new Set();
+    const kept: string[] = [];
+    const duplicates: string[] = [];
+
+    urls.forEach(url => {
+        const normalized = normalizeUrl(url);
+        if (pool.has(normalized)) {
+            duplicates.push(url); // Keep original URL for logging/display
+        } else {
+            kept.push(url);
+        }
+    });
+
+    return { kept, duplicates };
+}
 
-  return { kept, duplicates };
+
+/**
+ * Cleanup session data.
+ */
+export function cleanupSharedLinks(researchId: string) {
+    sessionLinks.delete(researchId);
+    sessionScrapedContent.delete(researchId);
+    sessionTimestamps.delete(researchId);
+    researcherScrapes.delete(researchId);
+    logger.debug(`[Shared Links] Cleaned up session: ${researchId}`);
 }
 
 /**
- * Format all completed reports into a shared links section for researcher prompts.
- * Extracts links from CITED LINKS and SCRAPE CANDIDATES sections.
+ * Cleanup ALL session data and stop the background timer.
+ * Used during system shutdown to prevent memory leaks.
  */
-export function formatSharedLinksFromState(aspects: Record<string, { id: string; query: string; report?: string }>): string {
-  const completed = Object.values(aspects).filter(a => a.report);
-  if (completed.length === 0) return '';
-
-  let output = '## Shared Links from Previous Research\n\n';
-  output += 'The following links have already been examined by your siblings:\n\n';
-
-  for (const aspect of completed) {
-    const response = aspect.report!;
-    output += `### Aspect: ${aspect.query} (ID: ${aspect.id})\n\n`;
-
-    // Extract CITED LINKS
-    const citedMatch = response.match(/###?\s*CITED\s*LINKS?\s*\n([\s\S]*?)(?=\n###|$)/i);
-    if (citedMatch) {
-      output += '#### CITED LINKS\n';
-      output += citedMatch[1]!.trim() + '\n\n';
+export function clearAllSharedLinks(): void {
+    if (cleanupInterval) {
+        clearInterval(cleanupInterval);
+        cleanupInterval = null;
     }
+    sessionLinks.clear();
+    sessionScrapedContent.clear();
+    sessionTimestamps.clear();
+    researcherScrapes.clear();
+    logger.debug('[Shared Links] All global state cleared');
+}
 
-    // Extract SCRAPE CANDIDATES
-    const candidatesMatch = response.match(/###?\s*SCRAPE\s*CANDIDATES?\s*\n([\s\S]*?)(?=\n###|$)/i);
-    if (candidatesMatch) {
-      output += '#### SCRAPE CANDIDATES\n';
-      output += candidatesMatch[1]!.trim() + '\n\n';
+/**
+ * Register that a specific researcher scraped a set of URLs.
+ */
+export function registerResearcherScrapes(researchId: string, researcherId: string, urls: string[]) {
+    if (!researcherScrapes.has(researchId)) {
+        researcherScrapes.set(researchId, new Map());
     }
-  }
+    sessionTimestamps.set(researchId, Date.now()); // keep this session reapable + TTL fresh
+    const map = researcherScrapes.get(researchId)!;
+    if (!map.has(researcherId)) {
+        map.set(researcherId, new Set());
+    }
+    const set = map.get(researcherId)!;
+    urls.forEach(url => set.add(normalizeUrl(url)));
+}
+
+/**
+ * Check if a specific researcher scraped a given URL.
+ */
+export function didResearcherScrape(researchId: string, researcherId: string, url: string): boolean {
+    return researcherScrapes.get(researchId)?.get(researcherId)?.has(normalizeUrl(url)) ?? false;
+}
 
-  output += '**Review these links before scraping to avoid redundancy.**\n';
-  return output;
+/**
+ * Get all URLs scraped by a specific researcher.
+ */
+export function getResearcherScrapes(researchId: string, researcherId: string): string[] {
+    return Array.from(researcherScrapes.get(researchId)?.get(researcherId) ?? []);
 }
 
 /**
- * Legacy stub for compatibility
+ * Build the Session URL Pool footer for scrape responses.
+ * Shows URLs previously scraped in the session, filtered to exclude the current batch.
+ * Capped at MAX_FOOTER_URLS (20).
  */
-export function cleanupSharedLinks(_sessionId: string): void {}
+const MAX_FOOTER_URLS = 20;
+
+export function buildSessionPoolFooter(researchId: string, currentBatchUrls: string[], researcherId?: string): string {
+    const pool = sessionLinks.get(researchId);
+    if (!pool || pool.size === 0) return '';
+
+    const currentNormalized = new Set(currentBatchUrls.map(u => normalizeUrl(u)));
+    // Also exclude URLs the current researcher already scraped in earlier batches
+    if (researcherId) {
+      const ownScrapes = researcherScrapes.get(researchId)?.get(researcherId);
+      if (ownScrapes) {
+        for (const url of ownScrapes) {
+          currentNormalized.add(url);
+        }
+      }
+    }
+    const eligible = Array.from(pool).filter(u => !currentNormalized.has(u));
+    if (eligible.length === 0) return '';
+
+    const capped = eligible.slice(-MAX_FOOTER_URLS);
+    const overflow = eligible.length - capped.length;
+
+    let footer = '\n---\n## Session URL Pool\n';
+    footer += 'Info only — URLs scraped by other researchers in this session. Do NOT use these in your report.\n';
+    for (const url of capped) {
+        footer += `- ${url}\n`;
+    }
+    if (overflow > 0) {
+        footer += `*...and ${overflow} more*\n`;
+    }
+    return footer;
+}

package/src/utils/shutdown-manager.ts

@@ -1,9 +1,24 @@
 import { logger } from '../logger.ts';
+import { safeUnref } from './safe-unref.ts';
 
 export type CleanupTask = () => Promise<void> | void;
 
+// Minimal interface matching Node.js EventEmitter and process
+interface EventEmitterLike {
+  on(event: string | symbol, listener: (...args: any[]) => void): any;
+  off?(event: string | symbol, listener: (...args: any[]) => void): any;
+  removeListener?(event: string | symbol, listener: (...args: any[]) => void): any;
+}
+
+interface EventListenerCleanup {
+  target: EventEmitterLike;
+  event: string;
+  listener: (...args: any[]) => void;
+}
+
 export class ShutdownManager {
   private tasks: CleanupTask[] = [];
+  private eventListeners: EventListenerCleanup[] = [];
   private cleanupPromise: Promise<void> | null = null;
 
   register(task: CleanupTask) {
@@ -14,6 +29,30 @@ export class ShutdownManager {
     this.tasks.push(task);
   }
 
+  registerEventListener<T extends EventEmitterLike>(
+    target: T,
+    event: string,
+    listener: (...args: any[]) => void
+  ) {
+    target.on(event, listener);
+    this.eventListeners.push({ target, event, listener });
+  }
+
+  unregisterEventListener<T extends EventEmitterLike>(
+    target: T,
+    event: string,
+    listener: (...args: any[]) => void
+  ) {
+    if (target.off) {
+      target.off(event, listener);
+    } else if (target.removeListener) {
+      target.removeListener(event, listener);
+    }
+    this.eventListeners = this.eventListeners.filter(
+      el => el.target !== target || el.event !== event || el.listener !== listener
+    );
+  }
+
   async runCleanup(reason: string): Promise<void> {
     if (this.cleanupPromise !== null) {
       return this.cleanupPromise;
@@ -46,9 +85,35 @@ export class ShutdownManager {
     try {
       await this.cleanupPromise;
     } finally {
+      // Remove all registered event listeners to allow clean process exit
+      for (const { target, event, listener } of this.eventListeners) {
+        try {
+          if (target.off) {
+            target.off(event, listener);
+          } else if (target.removeListener) {
+            target.removeListener(event, listener);
+          }
+        } catch (error) {
+          logger.error('[ShutdownManager] Error removing event listener:', error);
+        }
+      }
+      this.eventListeners = [];
       this.cleanupPromise = null;
     }
   }
+
+  /**
+   * Force exit the process after a timeout if shutdown is hanging
+   * This should be called after runCleanup() with a short delay
+   */
+  forceExitAfter(timeoutMs: number, code = 0) {
+    const timer = setTimeout(() => {
+      logger.warn(`[ShutdownManager] Forcing exit after ${timeoutMs}ms timeout`);
+      process.env['PI_PROCESS_EXITING'] = '1';
+      process.exit(code);
+    }, timeoutMs);
+    safeUnref(timer); // unref() allows Node.js to exit if this is the only timer
+  }
 }
 
 export const shutdownManager = new ShutdownManager();