@shadowforge0/aquifer-memory 0.6.0 → 0.8.0

package/pipeline/normalize/adapters/claude-code.js

@@ -0,0 +1,90 @@
+'use strict';
+
+/**
+ * Claude Code adapter — for Claude Code CLI sessions.
+ * Entry types are 'user'/'assistant' (split format: one content type per entry).
+ * Text and tool_use are separate entries, enabling narration detection via look-ahead.
+ */
+
+const { extractContent } = require('../extract');
+const { parseTimestamp } = require('../timestamp');
+const { MAX_NARRATION_CHARS } = require('../constants');
+
+module.exports = {
+    name: 'claude-code',
+
+    detect(entry) {
+        // Only count entry types that participate in normalize
+        return entry.type === 'user' || entry.type === 'assistant';
+    },
+
+    toIntermediate(entry, ctx) {
+        const { idx, rawEntries } = ctx;
+        const entryType = entry.type;
+
+        if (entryType !== 'user' && entryType !== 'assistant') {
+            return { idx, toolNames: [], adapterSkip: 'nonMessage' };
+        }
+
+        const role = entry.message?.role || entryType;
+
+        if (role === 'toolResult') {
+            return { idx, toolNames: [], adapterSkip: 'toolResult' };
+        }
+        if (role !== 'user' && role !== 'assistant') {
+            return { idx, role: null, toolNames: [], adapterSkip: 'noRole' };
+        }
+        if (entry.isMeta) {
+            return { idx, toolNames: [], adapterSkip: 'meta' };
+        }
+
+        const { text, commandName, toolNames } = extractContent(entry.message);
+
+        // CLI internal command output tags
+        if (text.includes('<local-command-caveat>') || text.includes('<local-command-stdout>') || text.includes('<local-command-stderr>')) {
+            return { idx, toolNames, adapterSkip: 'caveat' };
+        }
+
+        const isInterrupt = text.startsWith('[Request interrupted by user');
+
+        // Tool-use-only assistant entry (no visible text, only tool calls)
+        if (!text && toolNames.length > 0 && role === 'assistant') {
+            return { idx, toolNames, adapterSkip: 'toolOnly' };
+        }
+
+        // Narration detection: short text entry immediately followed by a tool_use entry.
+        // Claude Code splits text and tool_use into separate JSONL entries.
+        // A short text before a tool call is narration ("Now reading X...", "Let me check...").
+        if (role === 'assistant' && text && text.length < MAX_NARRATION_CHARS) {
+            let nextIsTool = false;
+            for (let j = idx + 1; j < rawEntries.length && j < idx + 3; j++) {
+                const ne = rawEntries[j];
+                if (ne.type === 'assistant') {
+                    const nc = ne.message?.content;
+                    if (Array.isArray(nc) && nc.some(x => x.type === 'tool_use')) nextIsTool = true;
+                    break;
+                }
+            }
+            if (nextIsTool) {
+                return { idx, toolNames, adapterSkip: 'narration' };
+            }
+        }
+
+        return {
+            idx, role, text,
+            timestamp: parseTimestamp(entry),
+            toolNames, commandName, isInterrupt,
+            adapterSkip: null,
+        };
+    },
+
+    routinePatterns: [
+        /^<task-notification>/,
+    ],
+
+    skipCommands: [
+        '/model', '/cost', '/memory', '/permissions', '/diff', '/review',
+        '/doctor', '/login', '/logout', '/mcp', '/context', '/fast',
+        '/think', '/vim', '/exit',
+    ],
+};

package/pipeline/normalize/adapters/gateway.js

@@ -0,0 +1,67 @@
+'use strict';
+
+/**
+ * Gateway adapter — for AI gateway servers that produce type='message' entries.
+ * Content blocks combine text + thinking + toolCall in a single entry.
+ * Supports channel metadata stripping (Discord, Telegram, etc.).
+ */
+
+const { extractContent } = require('../extract');
+const { parseTimestamp } = require('../timestamp');
+
+// Channel metadata prefix injected by gateway routing layers
+const METADATA_PREFIX_RE = /^(?:Conversation info \(untrusted metadata\):[\s\S]*?```\s*\n\s*)?(?:Sender \(untrusted metadata\):[\s\S]*?```\s*\n\s*)?/;
+
+function stripChannelMetadata(text) {
+    const stripped = text.replace(METADATA_PREFIX_RE, '').trim();
+    return stripped || text;
+}
+
+module.exports = {
+    name: 'gateway',
+
+    detect(entry) {
+        return entry.type === 'message';
+    },
+
+    toIntermediate(entry, ctx) {
+        const { idx } = ctx;
+
+        if (entry.type !== 'message') {
+            return { idx, toolNames: [], adapterSkip: 'nonMessage' };
+        }
+
+        const msg = entry.message;
+        const role = msg?.role;
+
+        if (role === 'toolResult') {
+            return { idx, toolNames: [], adapterSkip: 'toolResult' };
+        }
+        if (role !== 'user' && role !== 'assistant') {
+            return { idx, role: null, toolNames: [], adapterSkip: 'noRole' };
+        }
+
+        const { text, commandName, toolNames } = extractContent(msg);
+
+        let finalText = text;
+        const isInterrupt = text.startsWith('[Request interrupted by user');
+        if (role === 'user' && finalText && !isInterrupt) {
+            finalText = stripChannelMetadata(finalText);
+        }
+
+        return {
+            idx, role, text: finalText,
+            timestamp: parseTimestamp(entry),
+            toolNames, commandName, isInterrupt,
+            adapterSkip: null,
+        };
+    },
+
+    routinePatterns: [
+        /^HEARTBEAT_OK$/,
+        /^THINK_OK$/,
+        /^\[Queued messages while agent was busy\]/,
+    ],
+
+    skipCommands: [],
+};

package/pipeline/normalize/constants.js

@@ -0,0 +1,12 @@
+'use strict';
+
+// Commands that produce no conversational value — skip entirely
+const SKIP_COMMANDS = new Set(['/clear', '/compact', '/help', '/status', '/config']);
+
+// Commands that mark session boundaries — keep as boundary markers
+const RESET_COMMANDS = new Set(['/new', '/reset']);
+
+const MAX_MSG_CHARS = 8000;
+const MAX_NARRATION_CHARS = 200;
+
+module.exports = { SKIP_COMMANDS, RESET_COMMANDS, MAX_MSG_CHARS, MAX_NARRATION_CHARS };

package/pipeline/normalize/detect.js

@@ -0,0 +1,52 @@
+'use strict';
+
+const gatewayAdapter = require('./adapters/gateway');
+const claudeCodeAdapter = require('./adapters/claude-code');
+
+const ADAPTERS = [gatewayAdapter, claudeCodeAdapter];
+
+/**
+ * Auto-detect the client type from raw session entries.
+ * Samples the first 5 entries and picks the adapter with the most matches.
+ * @param {any[]} rawEntries
+ * @returns {string} Client name ('gateway' | 'claude-code')
+ * @throws {Error} If entries are empty, no adapter matches, or detection is ambiguous
+ */
+function detectClient(rawEntries) {
+    if (!rawEntries || rawEntries.length === 0) {
+        throw new Error('Cannot detect client: empty entries');
+    }
+
+    const sample = rawEntries.slice(0, Math.min(5, rawEntries.length));
+    const scores = [];
+
+    for (const adapter of ADAPTERS) {
+        const count = sample.filter(e => adapter.detect(e)).length;
+        scores.push({ name: adapter.name, count });
+    }
+    scores.sort((a, b) => b.count - a.count);
+
+    if (scores[0].count === 0) {
+        throw new Error('Cannot detect session client type. Pass opts.client explicitly.');
+    }
+    if (scores.length > 1 && scores[0].count === scores[1].count) {
+        throw new Error(`Ambiguous client detection (${scores[0].name}=${scores[0].count}, ${scores[1].name}=${scores[1].count}). Pass opts.client explicitly.`);
+    }
+
+    return scores[0].name;
+}
+
+/**
+ * Get adapter by client name.
+ * @param {string} clientType
+ * @returns {object} Adapter object
+ * @throws {Error} If client type is unknown
+ */
+function getAdapter(clientType) {
+    for (const adapter of ADAPTERS) {
+        if (adapter.name === clientType) return adapter;
+    }
+    throw new Error(`Unknown client type: "${clientType}". Known: ${ADAPTERS.map(a => a.name).join(', ')}`);
+}
+
+module.exports = { detectClient, getAdapter, ADAPTERS };

package/pipeline/normalize/extract.js

@@ -0,0 +1,49 @@
+'use strict';
+
+// Content extraction utilities shared across adapters
+
+function extractCommandName(content) {
+    const match = typeof content === 'string'
+        ? content.match(/<command-name>(\/\w+)<\/command-name>/)
+        : null;
+    return match ? match[1] : null;
+}
+
+/**
+ * Extract text, command name, and tool names from a message object.
+ * Handles both string content and content block arrays.
+ * @param {object} msg - Message object with .content field
+ * @returns {{ text: string, commandName: string|null, toolNames: string[] }}
+ */
+function extractContent(msg) {
+    if (!msg) return { text: '', commandName: null, toolNames: [] };
+    const content = msg.content;
+    let commandName = null;
+    const toolNames = [];
+
+    if (typeof content === 'string') {
+        commandName = extractCommandName(content);
+        return { text: content.trim(), commandName, toolNames };
+    }
+
+    if (Array.isArray(content)) {
+        const texts = [];
+        for (const item of content) {
+            if (item.type === 'text' && item.text) {
+                const cmd = extractCommandName(item.text);
+                if (cmd) commandName = cmd;
+                texts.push(item.text);
+            }
+            // tool_use: Claude Code / Anthropic API format
+            // toolCall: gateway / OpenAI-style format
+            if ((item.type === 'tool_use' || item.type === 'toolCall') && item.name) {
+                toolNames.push(item.name);
+            }
+        }
+        return { text: texts.join('\n').trim(), commandName, toolNames };
+    }
+
+    return { text: '', commandName, toolNames };
+}
+
+module.exports = { extractContent, extractCommandName };

package/pipeline/normalize/index.js

@@ -0,0 +1,129 @@
+'use strict';
+
+const { SKIP_COMMANDS, RESET_COMMANDS, MAX_MSG_CHARS } = require('./constants');
+const { detectClient, getAdapter } = require('./detect');
+
+/**
+ * Normalize raw session entries into effective messages.
+ *
+ * Accepts raw JSONL entries from any supported client (gateway, Claude Code, etc.)
+ * and produces a clean, uniform array of conversational messages suitable for
+ * summarization, embedding, and recall.
+ *
+ * @param {any[]} rawEntries - Raw JSONL entries from a session file
+ * @param {object} [opts]
+ * @param {string} [opts.client] - Client type: 'gateway' | 'claude-code'. Auto-detected if omitted.
+ * @param {number} [opts.idleGapMs] - Idle gap threshold for boundary detection (default: 2 hours)
+ * @returns {{ normalized: object[], skipStats: object, boundaries: object[], toolsUsed: string[] }}
+ */
+function normalizeSession(rawEntries, opts = {}) {
+    if (!rawEntries || rawEntries.length === 0) {
+        return {
+            normalized: [],
+            skipStats: { total: 0, nonMessage: 0, noRole: 0, meta: 0, caveat: 0,
+                empty: 0, toolOnly: 0, narration: 0, toolResult: 0, routine: 0, command: 0 },
+            boundaries: [],
+            toolsUsed: [],
+        };
+    }
+
+    const idleGapMs = opts.idleGapMs || 2 * 60 * 60 * 1000;
+
+    // 1. Select adapter
+    const clientType = opts.client || detectClient(rawEntries);
+    const adapter = getAdapter(clientType);
+
+    // 2. Merge adapter-specific constants with shared constants
+    const allSkipCommands = new Set([...SKIP_COMMANDS, ...(adapter.skipCommands || [])]);
+    const allRoutinePatterns = [...(adapter.routinePatterns || [])];
+
+    // 3. Main loop: adapter.toIntermediate → shared filter → collect
+    const normalized = [];
+    const skipStats = { total: 0, nonMessage: 0, noRole: 0, meta: 0, caveat: 0,
+        empty: 0, toolOnly: 0, narration: 0, toolResult: 0, routine: 0, command: 0 };
+    const toolsUsed = new Set();
+
+    for (let idx = 0; idx < rawEntries.length; idx++) {
+        skipStats.total++;
+        const parsed = adapter.toIntermediate(rawEntries[idx], { idx, rawEntries });
+
+        // Collect tool names even from skipped entries
+        if (parsed.toolNames?.length) {
+            for (const tn of parsed.toolNames) toolsUsed.add(tn);
+        }
+
+        // Adapter-determined skip
+        if (parsed.adapterSkip) {
+            if (!(parsed.adapterSkip in skipStats)) {
+                throw new Error(`Unknown adapterSkip reason: "${parsed.adapterSkip}" from ${clientType} adapter`);
+            }
+            skipStats[parsed.adapterSkip]++;
+            continue;
+        }
+
+        // Shared: invalid role
+        if (!parsed.role || (parsed.role !== 'user' && parsed.role !== 'assistant')) {
+            skipStats.noRole++;
+            continue;
+        }
+
+        // Shared: empty text (but keep interrupts)
+        if (!parsed.text && !parsed.isInterrupt) {
+            skipStats.empty++;
+            continue;
+        }
+
+        // Shared: routine patterns
+        if (!parsed.isInterrupt && parsed.text && allRoutinePatterns.some(re => re.test(parsed.text.trim()))) {
+            skipStats.routine++;
+            continue;
+        }
+
+        // Shared: skip commands
+        if (parsed.commandName && allSkipCommands.has(parsed.commandName)) {
+            skipStats.command++;
+            continue;
+        }
+
+        // Shared: truncate + reset command handling
+        const isResetCommand = !!(parsed.commandName && RESET_COMMANDS.has(parsed.commandName));
+        let finalText = isResetCommand ? '' : (parsed.text || '');
+        if (finalText.length > MAX_MSG_CHARS) {
+            finalText = finalText.slice(0, MAX_MSG_CHARS) + '\n[truncated]';
+        }
+
+        const msg = {
+            idx: parsed.idx,
+            role: parsed.role,
+            timestamp: parsed.timestamp,
+            text: finalText,
+            commandName: parsed.commandName || null,
+            isResetCommand,
+        };
+        if (parsed.isInterrupt) msg.isInterrupt = true;
+
+        normalized.push(msg);
+    }
+
+    // 4. Boundary detection
+    const boundaries = [];
+    for (let i = 0; i < normalized.length; i++) {
+        const cur = normalized[i];
+        const prev = i > 0 ? normalized[i - 1] : null;
+
+        if (cur.isResetCommand) {
+            boundaries.push({ type: 'command', at_index: i, reason: cur.commandName });
+        }
+
+        if (prev?.timestamp && cur.timestamp) {
+            const gapMs = new Date(cur.timestamp).getTime() - new Date(prev.timestamp).getTime();
+            if (gapMs > idleGapMs) {
+                boundaries.push({ type: 'idle_gap', at_index: i, gap_minutes: Math.round(gapMs / 60000) });
+            }
+        }
+    }
+
+    return { normalized, skipStats, boundaries, toolsUsed: [...toolsUsed] };
+}
+
+module.exports = { normalizeSession, detectClient };

package/pipeline/normalize/timestamp.js

@@ -0,0 +1,33 @@
+'use strict';
+
+/**
+ * Parse timestamp from a raw session entry.
+ * Handles multiple formats: ISO string (outer), epoch ms number (inner).
+ * Unified across all adapters to ensure consistent boundary detection.
+ * @param {object} entry - Raw session entry
+ * @returns {string|null} ISO8601 string or null
+ */
+function parseTimestamp(entry) {
+    // Outer timestamp (ISO string) — common in CLI-based clients
+    const outerTs = entry.timestamp;
+    if (typeof outerTs === 'string') {
+        const d = new Date(outerTs);
+        if (!isNaN(d.getTime())) return d.toISOString();
+    }
+
+    // Inner timestamp (epoch ms) — common in gateway/server-side clients
+    const innerTs = entry.message?.timestamp;
+    if (typeof innerTs === 'number') {
+        return new Date(innerTs).toISOString();
+    }
+
+    // Inner timestamp can also be ISO string
+    if (typeof innerTs === 'string') {
+        const d = new Date(innerTs);
+        if (!isNaN(d.getTime())) return d.toISOString();
+    }
+
+    return null;
+}
+
+module.exports = { parseTimestamp };

package/pipeline/rerank.js

@@ -0,0 +1,161 @@
+'use strict';
+
+const { httpRequest, withRetry } = require('./_http');
+
+// ---------------------------------------------------------------------------
+// Custom adapter
+// ---------------------------------------------------------------------------
+
+function validateResults(results) {
+  return results.filter(r =>
+    r && typeof r.index === 'number' && Number.isFinite(r.index)
+      && typeof r.score === 'number' && Number.isFinite(r.score)
+  );
+}
+
+function createCustomReranker(config) {
+  const fn = config.fn;
+  if (!fn) throw new Error('fn is required for custom reranker');
+
+  return {
+    async rerank(query, documents, opts = {}) {
+      if (!query || !documents || documents.length === 0) return [];
+      const topN = opts.topN || documents.length;
+      const results = await fn({ query, documents, topN });
+      if (!Array.isArray(results)) throw new Error('Custom reranker fn must return an array');
+      return validateResults(results).sort((a, b) => b.score - a.score);
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// TEI adapter (HuggingFace Text Embeddings Inference)
+// ---------------------------------------------------------------------------
+
+function createTEIReranker(config) {
+  const baseUrl = (config.teiBaseUrl || config.baseUrl || 'http://localhost:8080').replace(/\/+$/, '');
+  const timeout = config.timeout || 2000;
+  const maxRetries = config.maxRetries ?? 1;
+  const initialBackoffMs = config.initialBackoffMs || 250;
+
+  return {
+    async rerank(query, documents, opts = {}) {
+      if (!query || !documents || documents.length === 0) return [];
+
+      const result = await withRetry(
+        () => httpRequest(`${baseUrl}/rerank`, {
+          method: 'POST',
+          headers: { 'Content-Type': 'application/json' },
+          timeout,
+        }, { query, texts: documents, raw_scores: false }),
+        { maxRetries, initialBackoffMs },
+      );
+
+      // TEI returns array of { index, score }
+      const arr = Array.isArray(result) ? result : [];
+      return validateResults(arr.map(r => ({ index: r.index, score: r.score })))
+        .sort((a, b) => b.score - a.score);
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Jina adapter
+// ---------------------------------------------------------------------------
+
+function createJinaReranker(config) {
+  const apiKey = config.jinaApiKey;
+  if (!apiKey) throw new Error('jinaApiKey is required for Jina reranker');
+
+  const model = config.jinaModel || 'jina-reranker-v2-base-multilingual';
+  const baseUrl = (config.jinaBaseUrl || 'https://api.jina.ai/v1/rerank').replace(/\/+$/, '');
+  const timeout = config.timeout || 2000;
+  const maxRetries = config.maxRetries ?? 1;
+  const initialBackoffMs = config.initialBackoffMs || 250;
+
+  return {
+    async rerank(query, documents, opts = {}) {
+      if (!query || !documents || documents.length === 0) return [];
+      const topN = opts.topN || documents.length;
+
+      const result = await withRetry(
+        () => httpRequest(baseUrl, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${apiKey}`,
+          },
+          timeout,
+        }, { model, query, documents, top_n: topN }),
+        { maxRetries, initialBackoffMs },
+      );
+
+      // Jina returns { results: [{ index, relevance_score }] }
+      const arr = result.results || [];
+      return validateResults(arr.map(r => ({ index: r.index, score: r.relevance_score })))
+        .sort((a, b) => b.score - a.score);
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// OpenRouter adapter (Cohere rerank etc. via OpenRouter)
+// ---------------------------------------------------------------------------
+
+function createOpenRouterReranker(config) {
+  const apiKey = config.openrouterApiKey || config.apiKey;
+  if (!apiKey) throw new Error('openrouterApiKey is required for OpenRouter reranker');
+
+  const model = config.model || 'cohere/rerank-v3.5';
+  const baseUrl = (config.openrouterBaseUrl || 'https://openrouter.ai/api/v1/rerank').replace(/\/+$/, '');
+  const timeout = config.timeout || 5000;
+  const maxRetries = config.maxRetries ?? 1;
+  const initialBackoffMs = config.initialBackoffMs || 250;
+
+  return {
+    async rerank(query, documents, opts = {}) {
+      if (!query || !documents || documents.length === 0) return [];
+      const topN = opts.topN || documents.length;
+
+      const result = await withRetry(
+        () => httpRequest(baseUrl, {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+            'Authorization': `Bearer ${apiKey}`,
+          },
+          timeout,
+        }, { model, query, documents, top_n: topN }),
+        { maxRetries, initialBackoffMs },
+      );
+
+      // OpenRouter returns { results: [{ index, relevance_score }] }
+      const arr = result.results || [];
+      return validateResults(arr.map(r => ({ index: r.index, score: r.relevance_score })))
+        .sort((a, b) => b.score - a.score);
+    },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+
+function createReranker(config = {}) {
+  const provider = config.provider || 'custom';
+
+  switch (provider) {
+    case 'custom':
+      return createCustomReranker(config);
+    case 'tei':
+      return createTEIReranker(config);
+    case 'jina':
+      return createJinaReranker(config);
+    case 'openrouter':
+      return createOpenRouterReranker(config);
+    default:
+      throw new Error(`Unknown rerank provider: ${provider}`);
+  }
+}
+
+module.exports = { createReranker };