@shadowforge0/aquifer-memory 0.2.0

package/pipeline/embed.js

@@ -0,0 +1,230 @@
+'use strict';
+
+const http = require('http');
+const https = require('https');
+
+// ---------------------------------------------------------------------------
+// HTTP helpers
+// ---------------------------------------------------------------------------
+
+function httpRequest(url, options, body) {
+  return new Promise((resolve, reject) => {
+    const parsedUrl = new URL(url);
+    const transport = parsedUrl.protocol === 'https:' ? https : http;
+
+    // M8 fix: settled flag to prevent double-settle on timeout race
+    let settled = false;
+    const finish = (fn, val) => { if (!settled) { settled = true; fn(val); } };
+
+    const req = transport.request(parsedUrl, options, (res) => {
+      const chunks = [];
+      res.on('data', (chunk) => chunks.push(chunk));
+      res.on('end', () => {
+        if (timer) clearTimeout(timer);
+        const raw = Buffer.concat(chunks).toString();
+        if (res.statusCode < 200 || res.statusCode >= 300) {
+          finish(reject, new Error(`HTTP ${res.statusCode}: ${raw.slice(0, 500)}`));
+          return;
+        }
+        try {
+          finish(resolve, JSON.parse(raw));
+        } catch (e) {
+          finish(reject, new Error(`Invalid JSON response: ${raw.slice(0, 200)}`));
+        }
+      });
+    });
+
+    const timer = options.timeout
+      ? setTimeout(() => { req.destroy(); finish(reject, new Error('Request timeout')); }, options.timeout)
+      : null;
+
+    req.on('error', (err) => { if (timer) clearTimeout(timer); finish(reject, err); });
+    if (body) req.write(JSON.stringify(body));
+    req.end();
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Retry wrapper
+// ---------------------------------------------------------------------------
+
+async function withRetry(fn, { maxRetries = 3, initialBackoffMs = 2000 } = {}) {
+  let lastErr;
+  for (let attempt = 0; attempt < maxRetries; attempt++) {
+    try {
+      return await fn();
+    } catch (err) {
+      lastErr = err;
+      if (attempt < maxRetries - 1) {
+        const delay = initialBackoffMs * Math.pow(2, attempt);
+        await new Promise(r => setTimeout(r, delay));
+      }
+    }
+  }
+  throw lastErr;
+}
+
+// ---------------------------------------------------------------------------
+// Ollama adapter
+// ---------------------------------------------------------------------------
+
+function createOllamaEmbedder(config) {
+  const url = config.ollamaUrl || 'http://localhost:11434';
+  const model = config.model || 'bge-m3';
+  const chunkSize = config.chunkSize || 32;
+  const timeout = config.timeout || 120000;
+  const maxRetries = config.maxRetries || 3;
+  const initialBackoffMs = config.initialBackoffMs || 2000;
+  let detectedDim = null;
+
+  async function embedBatchRaw(texts) {
+    const allEmbeddings = [];
+
+    for (let i = 0; i < texts.length; i += chunkSize) {
+      const chunk = texts.slice(i, i + chunkSize);
+      const result = await withRetry(
+        () => httpRequest(
+          `${url}/api/embed`,
+          {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            timeout,
+          },
+          { model, input: chunk }
+        ),
+        { maxRetries, initialBackoffMs }
+      );
+
+      const embeddings = result.embeddings || [];
+      allEmbeddings.push(...embeddings);
+
+      if (!detectedDim && embeddings.length > 0 && embeddings[0]) {
+        detectedDim = embeddings[0].length;
+      }
+    }
+
+    return allEmbeddings;
+  }
+
+  return {
+    embed: async (text) => {
+      const results = await embedBatchRaw([text]);
+      return results[0] || [];
+    },
+    embedBatch: async (texts) => {
+      if (!texts || texts.length === 0) return [];
+      return embedBatchRaw(texts);
+    },
+    get dim() { return detectedDim; },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// OpenAI adapter
+// ---------------------------------------------------------------------------
+
+function createOpenAIEmbedder(config) {
+  const apiKey = config.openaiApiKey;
+  if (!apiKey) throw new Error('openaiApiKey is required for OpenAI embedder');
+
+  const model = config.openaiModel || 'text-embedding-3-small';
+  const dimensions = config.openaiDimensions || 1536;
+  const maxRetries = config.maxRetries || 3;
+  const initialBackoffMs = config.initialBackoffMs || 2000;
+  const timeout = config.timeout || 120000;
+
+  const chunkSize = config.chunkSize || 100; // M7: batch chunking for OpenAI
+
+  async function embedBatchRaw(texts) {
+    const allEmbeddings = [];
+    for (let i = 0; i < texts.length; i += chunkSize) {
+      const chunk = texts.slice(i, i + chunkSize);
+      const result = await withRetry(
+        () => httpRequest(
+          'https://api.openai.com/v1/embeddings',
+          {
+            method: 'POST',
+            headers: {
+              'Content-Type': 'application/json',
+              'Authorization': `Bearer ${apiKey}`,
+            },
+            timeout,
+          },
+          { model, input: chunk, dimensions }
+        ),
+        { maxRetries, initialBackoffMs }
+      );
+
+      const data = result.data || [];
+      data.sort((a, b) => a.index - b.index);
+      allEmbeddings.push(...data.map(d => d.embedding));
+    }
+    return allEmbeddings;
+  }
+
+  return {
+    embed: async (text) => {
+      const results = await embedBatchRaw([text]);
+      return results[0] || [];
+    },
+    embedBatch: async (texts) => {
+      if (!texts || texts.length === 0) return [];
+      return embedBatchRaw(texts);
+    },
+    get dim() { return dimensions; },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Custom adapter
+// ---------------------------------------------------------------------------
+
+function createCustomEmbedder(config) {
+  const fn = config.fn;
+  if (!fn) throw new Error('fn is required for custom embedder');
+
+  let detectedDim = null;
+
+  return {
+    embed: async (text) => {
+      const results = await fn([text]);
+      const vec = results[0] || [];
+      if (!detectedDim && vec.length > 0) detectedDim = vec.length;
+      return vec;
+    },
+    embedBatch: async (texts) => {
+      if (!texts || texts.length === 0) return [];
+      const results = await fn(texts);
+      if (!detectedDim && results.length > 0 && results[0]) {
+        detectedDim = results[0].length;
+      }
+      return results;
+    },
+    get dim() { return detectedDim; },
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Factory
+// ---------------------------------------------------------------------------
+
+function createEmbedder(config = {}) {
+  const provider = config.provider || 'ollama';
+
+  switch (provider) {
+    case 'ollama':
+      return createOllamaEmbedder(config);
+    case 'openai':
+      return createOpenAIEmbedder(config);
+    case 'custom':
+      return createCustomEmbedder(config);
+    default:
+      throw new Error(`Unknown embedding provider: ${provider}`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Exports
+// ---------------------------------------------------------------------------
+
+module.exports = { createEmbedder };

package/pipeline/extract-entities.js

@@ -0,0 +1,73 @@
+'use strict';
+
+const { parseEntityOutput } = require('../core/entity');
+
+// ---------------------------------------------------------------------------
+// defaultEntityPrompt
+// ---------------------------------------------------------------------------
+
+function defaultEntityPrompt(messages, opts = {}) {
+  const conversation = messages
+    .map(m => `[${m.role}] ${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}`)
+    .join('\n');
+
+  return `Extract named entities from the following conversation.
+
+Output in this exact format (one entity per block, separated by ---):
+
+[ENTITIES]
+name: <display name, original casing>
+type: <person|project|concept|tool|metric|org|place|event|doc|task|topic|other>
+aliases: <comma-separated alternative names, or empty>
+---
+
+Rules:
+- Only extract entities discussed substantively (not just mentioned in passing)
+- Normalize aliases (e.g., "React.js" and "React" are aliases)
+- Choose the most specific type
+- Minimum 0, maximum 15 entities
+- If no entities found, output only: [ENTITIES]\n(none)
+
+Example:
+[ENTITIES]
+name: PostgreSQL
+type: tool
+aliases: Postgres, PG
+---
+name: Alice Chen
+type: person
+aliases: Alice
+---
+
+---
+CONVERSATION:
+${conversation}`;
+}
+
+// ---------------------------------------------------------------------------
+// extractEntities
+// ---------------------------------------------------------------------------
+
+async function extractEntities(messages, {
+  llmFn,
+  promptFn,
+} = {}) {
+  if (!llmFn) return [];
+
+  const buildPrompt = promptFn || defaultEntityPrompt;
+
+  try {
+    const prompt = buildPrompt(messages, {});
+    const response = await llmFn(prompt);
+    return parseEntityOutput(response);
+  } catch (err) {
+    // LLM failure: return empty, never throw
+    return [];
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Exports
+// ---------------------------------------------------------------------------
+
+module.exports = { defaultEntityPrompt, extractEntities, parseEntityOutput };

package/pipeline/summarize.js

@@ -0,0 +1,245 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// defaultSummarizePrompt
+// ---------------------------------------------------------------------------
+
+function defaultSummarizePrompt(messages, opts = {}) {
+  const conversation = messages
+    .map(m => `[${m.role}] ${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}`)
+    .join('\n');
+
+  let entitySection = '';
+  if (opts.mergeEntities) {
+    entitySection = `
+
+Also extract named entities from this conversation. Output them after the summary in this exact format:
+
+[ENTITIES]
+name: <display name, original casing>
+type: <person|project|concept|tool|metric|org|place|event|doc|task|topic|other>
+aliases: <comma-separated alternative names, or empty>
+---
+(repeat for each entity)
+
+Rules for entities:
+- Only extract entities that are discussed substantively (not just mentioned in passing)
+- Normalize aliases (e.g., "React.js" and "React" are aliases of the same entity)
+- Choose the most specific type that fits
+- Minimum 0, maximum 15 entities per session`;
+  }
+
+  return `Summarize the following conversation concisely. Focus on:
+1. What was discussed (main topics)
+2. What was decided or concluded
+3. What actions were taken or planned
+4. Any unresolved questions or open loops
+
+Output a structured summary in this exact format (follow spacing and prefixes precisely):
+
+TITLE: <one-line title>
+OVERVIEW: <2-3 sentence overview>
+TOPICS:
+- <topic name>: <brief summary>
+DECISIONS:
+- <decision>: <reason>
+OPEN_LOOPS:
+- <unresolved item>
+IMPORTANT_FACTS:
+- <key fact>
+
+Example:
+TITLE: Database migration strategy discussion
+OVERVIEW: Team discussed migrating from MySQL to PostgreSQL. Decided to use pgloader for data transfer. Timeline set for next sprint.
+TOPICS:
+- Migration tooling: Evaluated pgloader vs custom scripts, chose pgloader for reliability
+- Schema changes: Need to convert ENUM types to CHECK constraints
+DECISIONS:
+- Use pgloader: Handles type conversion automatically, proven in production
+OPEN_LOOPS:
+- Need to benchmark query performance on new schema
+IMPORTANT_FACTS:
+- Current DB size is 45GB with 12M rows in largest table
+${entitySection}
+
+---
+CONVERSATION:
+${conversation}`;
+}
+
+// ---------------------------------------------------------------------------
+// parseStructuredSummary — extract fields from LLM output
+// ---------------------------------------------------------------------------
+
+function _parseStructuredSummary(text) {
+  if (!text) return null;
+
+  const result = {
+    title: '',
+    overview: '',
+    topics: [],
+    decisions: [],
+    open_loops: [],
+    important_facts: [],
+  };
+
+  // Extract TITLE
+  const titleMatch = text.match(/^TITLE:\s*(.+)/m);
+  if (titleMatch) result.title = titleMatch[1].trim();
+
+  // Extract OVERVIEW
+  const overviewMatch = text.match(/^OVERVIEW:\s*([\s\S]*?)(?=\n(?:TOPICS|DECISIONS|OPEN_LOOPS|IMPORTANT_FACTS|\[ENTITIES\])|$)/m);
+  if (overviewMatch) result.overview = overviewMatch[1].trim();
+
+  // Extract TOPICS
+  const topicsMatch = text.match(/^TOPICS:\s*\n([\s\S]*?)(?=\n(?:DECISIONS|OPEN_LOOPS|IMPORTANT_FACTS|\[ENTITIES\])|$)/m);
+  if (topicsMatch) {
+    const lines = topicsMatch[1].split('\n').filter(l => l.trim().startsWith('-'));
+    for (const line of lines) {
+      const cleaned = line.replace(/^\s*-\s*/, '').trim();
+      const colonIdx = cleaned.indexOf(':');
+      if (colonIdx > 0) {
+        result.topics.push({
+          name: cleaned.slice(0, colonIdx).trim(),
+          summary: cleaned.slice(colonIdx + 1).trim(),
+        });
+      } else {
+        result.topics.push({ name: cleaned, summary: '' });
+      }
+    }
+  }
+
+  // Extract DECISIONS
+  const decisionsMatch = text.match(/^DECISIONS:\s*\n([\s\S]*?)(?=\n(?:OPEN_LOOPS|IMPORTANT_FACTS|\[ENTITIES\])|$)/m);
+  if (decisionsMatch) {
+    const lines = decisionsMatch[1].split('\n').filter(l => l.trim().startsWith('-'));
+    for (const line of lines) {
+      const cleaned = line.replace(/^\s*-\s*/, '').trim();
+      const colonIdx = cleaned.indexOf(':');
+      if (colonIdx > 0) {
+        result.decisions.push({
+          decision: cleaned.slice(0, colonIdx).trim(),
+          reason: cleaned.slice(colonIdx + 1).trim(),
+        });
+      } else {
+        result.decisions.push({ decision: cleaned, reason: '' });
+      }
+    }
+  }
+
+  // Extract OPEN_LOOPS
+  const openLoopsMatch = text.match(/^OPEN_LOOPS:\s*\n([\s\S]*?)(?=\n(?:IMPORTANT_FACTS|\[ENTITIES\])|$)/m);
+  if (openLoopsMatch) {
+    const lines = openLoopsMatch[1].split('\n').filter(l => l.trim().startsWith('-'));
+    for (const line of lines) {
+      result.open_loops.push({ item: line.replace(/^\s*-\s*/, '').trim() });
+    }
+  }
+
+  // Extract IMPORTANT_FACTS
+  const factsMatch = text.match(/^IMPORTANT_FACTS:\s*\n([\s\S]*?)(?=\n\[ENTITIES\]|$)/m);
+  if (factsMatch) {
+    const lines = factsMatch[1].split('\n').filter(l => l.trim().startsWith('-'));
+    for (const line of lines) {
+      const fact = line.replace(/^\s*-\s*/, '').trim();
+      if (fact) result.important_facts.push(fact);
+    }
+  }
+
+  return result;
+}
+
+// ---------------------------------------------------------------------------
+// extractiveFallback
+// ---------------------------------------------------------------------------
+
+function extractiveFallback(messages) {
+  const userMsgs = (messages || []).filter(m => m.role === 'user');
+  const texts = userMsgs.map(m => {
+    if (typeof m.content === 'string') return m.content.trim();
+    if (Array.isArray(m.content)) {
+      return m.content
+        .filter(p => p.type === 'text')
+        .map(p => p.text)
+        .join('\n')
+        .trim();
+    }
+    return '';
+  }).filter(Boolean);
+
+  let selected;
+  if (texts.length <= 6) {
+    selected = texts;
+  } else {
+    const head = texts.slice(0, 3);
+    const tail = texts.slice(-3);
+    // Dedupe: if any tail item is already in head, skip it
+    const headSet = new Set(head);
+    selected = [...head, ...tail.filter(t => !headSet.has(t))];
+  }
+
+  const joined = selected.join('\n---\n').slice(0, 2000);
+
+  return {
+    summaryText: joined,
+    structuredSummary: null,
+    entityRaw: null,
+    isExtractive: true,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// summarize
+// ---------------------------------------------------------------------------
+
+async function summarize(messages, {
+  llmFn,
+  promptFn,
+  mergeEntities = false,
+} = {}) {
+  if (!llmFn) {
+    return extractiveFallback(messages);
+  }
+
+  const buildPrompt = promptFn || defaultSummarizePrompt;
+
+  try {
+    const prompt = buildPrompt(messages, { mergeEntities });
+    const response = await llmFn(prompt);
+
+    // Parse structured fields
+    const structuredSummary = _parseStructuredSummary(response);
+
+    // Extract entity section if present
+    let entityRaw = null;
+    if (mergeEntities) {
+      const idx = response.indexOf('[ENTITIES]');
+      if (idx !== -1) {
+        entityRaw = response.slice(idx);
+      }
+    }
+
+    // M6 fix: strip [ENTITIES] section from summaryText before storage
+    let cleanSummary = response;
+    if (mergeEntities) {
+      const idx = response.indexOf('[ENTITIES]');
+      if (idx !== -1) cleanSummary = response.slice(0, idx).trim();
+    }
+
+    return {
+      summaryText: cleanSummary,
+      structuredSummary,
+      entityRaw,
+      isExtractive: false,
+    };
+  } catch (err) {
+    // LLM failure: fall back to extractive
+    return extractiveFallback(messages);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Exports
+// ---------------------------------------------------------------------------
+
+module.exports = { defaultSummarizePrompt, summarize, extractiveFallback };