@shadowforge0/aquifer-memory 1.0.3 → 1.3.0

package/consumers/shared/autodetect.js

@@ -0,0 +1,64 @@
+'use strict';
+
+const { Pool } = require('pg');
+
+const DEFAULT_PG_URL = 'postgresql://aquifer:aquifer@localhost:5432/aquifer';
+const DEFAULT_OLLAMA_URL = 'http://localhost:11434';
+
+async function probePostgres(url, { timeoutMs = 1500 } = {}) {
+  const pool = new Pool({
+    connectionString: url,
+    connectionTimeoutMillis: timeoutMs,
+    max: 1,
+  });
+  try {
+    await pool.query('SELECT 1');
+    return true;
+  } catch {
+    return false;
+  } finally {
+    try { await pool.end(); } catch { /* ignore */ }
+  }
+}
+
+async function probeOllama(baseUrl, { timeoutMs = 1500 } = {}) {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), timeoutMs);
+  try {
+    const res = await fetch(`${baseUrl.replace(/\/$/, '')}/api/tags`, {
+      signal: controller.signal,
+    });
+    return res.ok;
+  } catch {
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function autodetectForQuickstart(env, probes = {}) {
+  const probePg = probes.probePostgres || probePostgres;
+  const probeOll = probes.probeOllama || probeOllama;
+  const detected = {};
+
+  const hasDb = env.DATABASE_URL || env.AQUIFER_DB_URL;
+  if (!hasDb && await probePg(DEFAULT_PG_URL)) {
+    detected.DATABASE_URL = DEFAULT_PG_URL;
+  }
+
+  const hasEmbed = env.EMBED_PROVIDER
+    || (env.AQUIFER_EMBED_BASE_URL && env.AQUIFER_EMBED_MODEL);
+  if (!hasEmbed && await probeOll(DEFAULT_OLLAMA_URL)) {
+    detected.EMBED_PROVIDER = 'ollama';
+  }
+
+  return detected;
+}
+
+module.exports = {
+  autodetectForQuickstart,
+  probePostgres,
+  probeOllama,
+  DEFAULT_PG_URL,
+  DEFAULT_OLLAMA_URL,
+};

package/consumers/shared/entity-parser.js

@@ -0,0 +1,119 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// Entity section parser — shared across consumers.
+//
+// Parses LLM output lines of the form:
+//   ENTITY: <name> | <type> | <alias1, alias2, ...>
+//   RELATION: <src> | <dst>
+//
+// Returns { entities, relations } ready for Aquifer entityParseFn.
+// Dedups, normalizes names via Aquifer's normalizeEntityName, and drops noise
+// entities (generic roles, pure-numeric, file paths, CLI flags, etc.).
+//
+// Consumers that use a different ENTITIES prompt format should write their own
+// parser — this one is for the ENTITY:/RELATION: line protocol.
+// ---------------------------------------------------------------------------
+
+// Import directly from core/entity to avoid a circular dep with top-level
+// index.js, which itself re-exports parseEntitySection from here.
+const { normalizeEntityName } = require('../../core/entity');
+
+const VALID_ENTITY_TYPES = new Set([
+  'person', 'project', 'concept', 'tool', 'metric',
+  'org', 'place', 'event', 'doc', 'task', 'topic', 'other',
+]);
+
+const ENTITY_STOPLIST = new Set([
+  // Role generics
+  '助理', '使用者', '用戶', 'assistant', 'user', 'agent', 'agents', '我',
+  // Too broad
+  'api', 'db', 'llm', 'cli', 'bash', 'diff', 'bug', 'config',
+  'extensions', 'hooks', 'cron', 'manifest', 'index.js', 'node.js',
+  // Common noise
+  'ok', 'timeout', 'error', 'test', 'cache', 'token',
+  '登入狀態', '授權提示', 'chat_id', 'promise.race',
+]);
+
+const CODE_EXT_RE = /\.(js|ts|jsx|tsx|mjs|cjs|sh|py|sql|md|json|yml|yaml|css|html|vue|svelte|go|rs|rb|php|java|kt|c|cpp|h|toml|ini|cfg|conf|lock|env|proto)$/i;
+const PATH_RE = /^[.\/~].*\//;
+const DOTFILE_RE = /^\.[a-z][a-z0-9._-]*$/i;
+
+function isNoiseEntity(normalizedName, rawName) {
+  if (ENTITY_STOPLIST.has(normalizedName)) return true;
+  if (/^\d+[秒分時天日月年kKgG%]/.test(rawName)) return true;
+  if (/^\d{2,}[mM]/.test(rawName)) return true;
+  if (/^\d+錯誤/.test(rawName)) return true;
+  if (/^\d{10,}$/.test(rawName)) return true;
+  if (normalizedName.length < 2) return true;
+  if (PATH_RE.test(rawName)) return true;
+  if (DOTFILE_RE.test(rawName)) return true;
+  if (CODE_EXT_RE.test(rawName)) return true;
+  if (/^--?\w/.test(rawName)) return true;
+  return false;
+}
+
+function splitFields(line) {
+  if (line.includes('|')) return line.split('|').map(s => s.trim());
+  if (line.includes('\t')) return line.split('\t').map(s => s.trim());
+  return [line.trim()];
+}
+
+function parseEntitySection(text, opts = {}) {
+  if (!text || typeof text !== 'string') return { entities: [], relations: [] };
+
+  const maxEntities = Number.isFinite(opts.maxEntities) ? opts.maxEntities : 10;
+  const maxRelations = Number.isFinite(opts.maxRelations) ? opts.maxRelations : 15;
+
+  const entityMap = new Map();
+  const relationSet = new Set();
+  const relations = [];
+
+  for (const rawLine of text.split('\n')) {
+    const line = rawLine.trim();
+    if (!line) continue;
+
+    if (/^ENTITY:/i.test(line)) {
+      if (entityMap.size >= maxEntities) continue;
+      const fields = splitFields(line.replace(/^ENTITY:\s*/i, ''));
+      const rawName = (fields[0] || '').trim().slice(0, 200);
+      if (!rawName) continue;
+      const normalizedName = normalizeEntityName(rawName);
+      if (!normalizedName || entityMap.has(normalizedName)) continue;
+      if (isNoiseEntity(normalizedName, rawName)) continue;
+      const rawType = (fields[1] || '').toLowerCase().trim();
+      const type = VALID_ENTITY_TYPES.has(rawType) ? rawType : 'other';
+      const rawAliases = fields[2] || '';
+      const aliases = (rawAliases && rawAliases !== '-')
+        ? rawAliases.split(',').map(a => a.trim().slice(0, 200)).filter(a => a && a !== '-')
+        : [];
+      entityMap.set(normalizedName, { name: rawName, normalizedName, type, aliases });
+    } else if (/^RELATION:/i.test(line)) {
+      if (relations.length >= maxRelations) continue;
+      const fields = splitFields(line.replace(/^RELATION:\s*/i, ''));
+      const src = (fields[0] || '').trim();
+      const dst = (fields[1] || '').trim();
+      if (!src || !dst) continue;
+      const ns = normalizeEntityName(src);
+      const nd = normalizeEntityName(dst);
+      if (!ns || !nd || ns === nd) continue;
+      const pairKey = ns < nd ? `${ns}|||${nd}` : `${nd}|||${ns}`;
+      if (relationSet.has(pairKey)) continue;
+      relationSet.add(pairKey);
+      relations.push({ src, dst });
+    }
+  }
+
+  const filteredRelations = relations.filter(r =>
+    entityMap.has(normalizeEntityName(r.src)) && entityMap.has(normalizeEntityName(r.dst))
+  );
+
+  return { entities: [...entityMap.values()], relations: filteredRelations };
+}
+
+module.exports = {
+  parseEntitySection,
+  isNoiseEntity,
+  VALID_ENTITY_TYPES,
+  ENTITY_STOPLIST,
+};

package/consumers/shared/ingest.js

@@ -0,0 +1,148 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// Shared ingest flow — the standard "received session → Aquifer" pipeline.
+//
+// All three host adapters (OpenClaw before_reset, Claude Code afterburn,
+// OpenCode backfill) do the same three things:
+//   1. Normalize raw entries to commit-ready shape
+//   2. commit() the messages + metadata
+//   3. enrich() if enough user turns, else skip()
+// With dedup on (agentId, sessionId) so the same hook firing twice is safe.
+//
+// runIngest() centralizes this. Host adapters pass in their raw entries, the
+// adapter name, and an optional postProcess callback for persona side effects.
+// ---------------------------------------------------------------------------
+
+const { normalizeMessages } = require('./normalize');
+
+const RECENT_CAP = 200;
+const RECENT_TTL_MS = 30 * 60 * 1000;
+
+function evictStale(dedupMap, now = Date.now()) {
+    if (!dedupMap || dedupMap.size <= RECENT_CAP) return;
+    const cutoff = now - RECENT_TTL_MS;
+    for (const [k, ts] of dedupMap) {
+        if (ts < cutoff) dedupMap.delete(k);
+    }
+}
+
+/**
+ * Run the standard commit-then-enrich flow for a single session.
+ *
+ * @param {object} opts
+ * @param {object} opts.aquifer — Aquifer instance
+ * @param {string} opts.sessionId
+ * @param {string} opts.agentId
+ * @param {string} [opts.source] — caller-provided source tag (e.g. 'openclaw', 'cc', 'opencode')
+ * @param {string} [opts.sessionKey] — passed through to commit()
+ * @param {any[]} opts.rawEntries — host-native session entries
+ * @param {'gateway'|'cc'|'claude-code'|'preNormalized'} [opts.adapter]
+ *        'preNormalized' means rawEntries already matches normalizeMessages output
+ *        (used by OpenCode which reads SQLite directly).
+ * @param {object} [opts.preNormalized] — { messages, userCount, ... } ready to commit,
+ *        required when adapter === 'preNormalized'
+ * @param {number} [opts.minUserMessages=3] — enrich threshold
+ * @param {boolean} [opts.enrich=true] — when false, commit only; don't enrich or skip.
+ *        Useful for pull-style ingest (OpenCode) where enrichment runs later.
+ * @param {Map} [opts.dedupMap] — Map<key, timestamp>; same session won't process twice within TTL
+ * @param {Set} [opts.inFlight] — Set<key>; concurrent firings are guarded
+ * @param {function} [opts.postProcess] — forwarded to enrich()
+ * @param {function} [opts.summaryFn] — forwarded to enrich()
+ * @param {function} [opts.entityParseFn] — forwarded to enrich()
+ * @param {object} [opts.logger] — { info, warn }
+ * @returns {Promise<{status:string, normalized:any[]|null, counts:object|null, enrichResult:object|null, skipReason?:string}>}
+ */
+async function runIngest(opts = {}) {
+    const {
+        aquifer, sessionId, agentId, source, sessionKey,
+        rawEntries, adapter, preNormalized,
+        minUserMessages = 3,
+        enrich = true,
+        dedupMap = null, inFlight = null,
+        postProcess = null, summaryFn = null, entityParseFn = null,
+        logger = console,
+    } = opts;
+
+    if (!aquifer) throw new Error('aquifer is required');
+    if (!sessionId) throw new Error('sessionId is required');
+    if (!agentId) throw new Error('agentId is required');
+
+    const dedupKey = `${agentId}:${sessionId}`;
+    if (dedupMap && dedupMap.has(dedupKey)) {
+        return { status: 'dedup', normalized: null, counts: null, enrichResult: null, skipReason: 'recent' };
+    }
+    if (inFlight && inFlight.has(dedupKey)) {
+        return { status: 'dedup', normalized: null, counts: null, enrichResult: null, skipReason: 'in_flight' };
+    }
+    if (inFlight) inFlight.add(dedupKey);
+
+    try {
+        // 1. Normalize
+        let norm;
+        if (adapter === 'preNormalized') {
+            if (!preNormalized) throw new Error('preNormalized adapter requires opts.preNormalized');
+            norm = preNormalized;
+        } else {
+            norm = normalizeMessages(rawEntries, { adapter });
+        }
+
+        if (norm.userCount === 0) {
+            return { status: 'skipped_empty', normalized: norm.messages, counts: norm, enrichResult: null, skipReason: 'no_user_messages' };
+        }
+
+        // 2. Commit
+        await aquifer.commit(sessionId, norm.messages, {
+            agentId,
+            source: source || adapter || 'api',
+            sessionKey: sessionKey || null,
+            model: norm.model,
+            tokensIn: norm.tokensIn,
+            tokensOut: norm.tokensOut,
+            startedAt: norm.startedAt,
+            lastMessageAt: norm.lastMessageAt,
+        });
+        if (logger && logger.info) logger.info(`[aquifer-ingest] committed ${sessionId} (${norm.messages.length} msgs, user=${norm.userCount})`);
+
+        // 3. Enrich or skip (unless caller opts out — then commit only)
+        let enrichResult = null;
+        if (!enrich) {
+            if (dedupMap) { dedupMap.set(dedupKey, Date.now()); evictStale(dedupMap); }
+            return { status: 'committed_only', normalized: norm.messages, counts: norm, enrichResult: null };
+        }
+        if (norm.userCount >= minUserMessages) {
+            try {
+                enrichResult = await aquifer.enrich(sessionId, {
+                    agentId,
+                    summaryFn: summaryFn || undefined,
+                    entityParseFn: entityParseFn || undefined,
+                    postProcess: postProcess || undefined,
+                });
+                if (logger && logger.info) {
+                    logger.info(`[aquifer-ingest] enriched ${sessionId} (turns=${enrichResult.turnsEmbedded}, entities=${enrichResult.entitiesFound})`);
+                }
+            } catch (enrichErr) {
+                if (logger && logger.warn) logger.warn(`[aquifer-ingest] enrich failed for ${sessionId}: ${enrichErr.message}`);
+                // Commit already succeeded — don't rethrow
+            }
+        } else {
+            try {
+                await aquifer.skip(sessionId, { agentId, reason: `user_count=${norm.userCount} < min=${minUserMessages}` });
+            } catch (skipErr) {
+                if (logger && logger.warn) logger.warn(`[aquifer-ingest] skip failed for ${sessionId}: ${skipErr.message}`);
+            }
+            return { status: 'skipped_short', normalized: norm.messages, counts: norm, enrichResult: null, skipReason: `user_count=${norm.userCount}` };
+        }
+
+        if (dedupMap) {
+            dedupMap.set(dedupKey, Date.now());
+            evictStale(dedupMap);
+        }
+
+        return { status: 'ok', normalized: norm.messages, counts: norm, enrichResult };
+    } finally {
+        if (inFlight) inFlight.delete(dedupKey);
+    }
+}
+
+module.exports = { runIngest };

package/consumers/shared/llm-autodetect.js

@@ -0,0 +1,137 @@
+'use strict';
+
+// Aquifer v1.2.0: LLM provider autodetect from env for install-and-go.
+//
+// Precedence:
+//   1. config.llm.fn (explicit function — host supplies)
+//   2. AQUIFER_LLM_PROVIDER env + provider-specific api key + optional model
+//
+// We do NOT silently pick a provider from multiple keys (ambiguous). Hosts
+// must opt in by setting AQUIFER_LLM_PROVIDER explicitly when they want env
+// autowiring.
+//
+// Two response shapes in flight:
+//   - Anthropic-shape: { content: [{ type:'text', text:'...' }] }
+//     Used by: minimax, opencode
+//   - OpenAI-shape:    { choices:[{ message:{ content:'...' } }] }
+//     Used by: openai, openrouter
+
+const { createLlmFn } = require('./llm');
+
+const ANTHROPIC_PROVIDERS = {
+  minimax: {
+    envKey: 'MINIMAX_API_KEY',
+    baseUrl: 'https://api.minimax.io/anthropic/v1/messages',
+    defaultModel: 'MiniMax-M2.7',
+    extraHeaders: { 'anthropic-version': '2023-06-01' },
+  },
+  opencode: {
+    envKey: 'OPENCODE_API_KEY',
+    baseUrl: 'https://opencode.ai/zen/go/v1/messages',
+    defaultModel: 'minimax-m2.5',
+    extraHeaders: { 'anthropic-version': '2023-06-01' },
+  },
+};
+
+const OPENAI_PROVIDERS = {
+  openai: {
+    envKey: 'OPENAI_API_KEY',
+    baseUrl: 'https://api.openai.com/v1',
+    defaultModel: 'gpt-4o-mini',
+  },
+  openrouter: {
+    envKey: 'OPENROUTER_API_KEY',
+    baseUrl: 'https://openrouter.ai/api/v1',
+    defaultModel: 'openai/gpt-4o-mini',
+  },
+};
+
+function createAnthropicShapeFn({ baseUrl, apiKey, model, extraHeaders, timeoutMs, maxTokens }) {
+  const timeout = timeoutMs || 120000;
+  const mt = maxTokens || 4096;
+  return async function llmFn(prompt) {
+    const controller = new AbortController();
+    const timer = setTimeout(() => controller.abort(), timeout);
+    try {
+      const res = await fetch(baseUrl, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${apiKey}`,
+          ...(extraHeaders || {}),
+        },
+        body: JSON.stringify({
+          model,
+          messages: [{ role: 'user', content: prompt }],
+          max_tokens: mt,
+        }),
+        signal: controller.signal,
+      });
+      if (!res.ok) {
+        const body = await res.text().catch(() => '');
+        const err = new Error(`LLM ${res.status}: ${body.slice(0, 200).replace(/[\n\r]/g, ' ')}`);
+        err.statusCode = res.status;
+        throw err;
+      }
+      const data = await res.json();
+      let raw = '';
+      if (data.content && Array.isArray(data.content)) {
+        raw = data.content.map((c) => c.text || '').join('');
+      } else if (data.choices && Array.isArray(data.choices)) {
+        raw = data.choices[0]?.message?.content || '';
+      }
+      return raw.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+    } finally {
+      clearTimeout(timer);
+    }
+  };
+}
+
+function resolveLlmFn(llmConfig, env) {
+  if (llmConfig && typeof llmConfig.fn === 'function') {
+    return llmConfig.fn;
+  }
+  const provider = env.AQUIFER_LLM_PROVIDER;
+  if (!provider) return null;
+
+  const model = env.AQUIFER_LLM_MODEL || null;
+  const timeoutMs = env.AQUIFER_LLM_TIMEOUT ? Number(env.AQUIFER_LLM_TIMEOUT) : undefined;
+
+  if (ANTHROPIC_PROVIDERS[provider]) {
+    const p = ANTHROPIC_PROVIDERS[provider];
+    const apiKey = env[p.envKey];
+    if (!apiKey) {
+      throw new Error(`AQUIFER_LLM_PROVIDER=${provider} requires ${p.envKey}`);
+    }
+    return createAnthropicShapeFn({
+      baseUrl: p.baseUrl,
+      apiKey,
+      model: model || p.defaultModel,
+      extraHeaders: p.extraHeaders,
+      timeoutMs,
+    });
+  }
+
+  if (OPENAI_PROVIDERS[provider]) {
+    const p = OPENAI_PROVIDERS[provider];
+    const apiKey = env[p.envKey];
+    if (!apiKey) {
+      throw new Error(`AQUIFER_LLM_PROVIDER=${provider} requires ${p.envKey}`);
+    }
+    return createLlmFn({
+      baseUrl: p.baseUrl,
+      model: model || p.defaultModel,
+      apiKey,
+      timeoutMs,
+    });
+  }
+
+  throw new Error(
+    `AQUIFER_LLM_PROVIDER=${provider} not supported. Valid: ${[
+      ...Object.keys(ANTHROPIC_PROVIDERS),
+      ...Object.keys(OPENAI_PROVIDERS),
+    ].join(', ')}`
+  );
+}
+
+module.exports = { resolveLlmFn };

package/consumers/shared/normalize.js

@@ -0,0 +1,129 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// Shared normalize — turns raw host entries into commit-ready messages plus
+// session-level metadata. Wraps pipeline/normalize so consumers don't each
+// reinvent their own role/content extraction.
+//
+// Supported adapters: 'gateway' | 'cc' (alias of 'claude-code'). The OpenCode
+// consumer reads from SQLite and constructs the output shape directly; it is
+// not expected to route through here.
+//
+// Output shape is the one commit() + enrich() expect:
+//   { messages:[{role,content,timestamp}], userCount, assistantCount,
+//     model, tokensIn, tokensOut, startedAt, lastMessageAt,
+//     skipStats, boundaries, toolsUsed }
+// ---------------------------------------------------------------------------
+
+const { normalizeSession } = require('../../pipeline/normalize');
+
+const ADAPTER_ALIASES = {
+    'cc': 'claude-code',
+    'claude-code': 'claude-code',
+    'gateway': 'gateway',
+};
+
+function resolveAdapter(adapter) {
+    if (!adapter) return null;  // auto-detect
+    const name = ADAPTER_ALIASES[adapter];
+    if (!name) {
+        throw new Error(`Unknown adapter: "${adapter}". Supported: gateway, cc (alias claude-code).`);
+    }
+    return name;
+}
+
+function extractRawMeta(rawEntries) {
+    let model = null;
+    let tokensIn = 0;
+    let tokensOut = 0;
+
+    for (const entry of rawEntries || []) {
+        if (!entry || typeof entry !== 'object') continue;
+        const msg = entry.message || entry;
+        if (msg && typeof msg === 'object') {
+            if (msg.model && !model) model = msg.model;
+            if (msg.usage) {
+                tokensIn += msg.usage.input_tokens || msg.usage.input || 0;
+                tokensOut += msg.usage.output_tokens || msg.usage.output || 0;
+            }
+        }
+    }
+
+    return { model, tokensIn, tokensOut };
+}
+
+/**
+ * Normalize raw host entries to Aquifer-commit shape + session metadata.
+ *
+ * @param {any[]} rawEntries
+ * @param {object} [opts]
+ * @param {'gateway'|'cc'|'claude-code'} [opts.adapter] — host adapter; auto-detected if omitted
+ * @returns {{
+ *   messages: {role:string,content:string,timestamp:string|null}[],
+ *   userCount: number, assistantCount: number,
+ *   model: string|null, tokensIn: number, tokensOut: number,
+ *   startedAt: string|null, lastMessageAt: string|null,
+ *   skipStats: object, boundaries: object[], toolsUsed: string[]
+ * }}
+ */
+function normalizeMessages(rawEntries, opts = {}) {
+    const safeEntries = Array.isArray(rawEntries) ? rawEntries : [];
+
+    if (safeEntries.length === 0) {
+        return {
+            messages: [],
+            userCount: 0,
+            assistantCount: 0,
+            model: null,
+            tokensIn: 0,
+            tokensOut: 0,
+            startedAt: null,
+            lastMessageAt: null,
+            skipStats: { total: 0, nonMessage: 0, noRole: 0, meta: 0, caveat: 0,
+                empty: 0, toolOnly: 0, narration: 0, toolResult: 0, routine: 0, command: 0 },
+            boundaries: [],
+            toolsUsed: [],
+        };
+    }
+
+    const client = resolveAdapter(opts.adapter);
+    const { normalized, skipStats, boundaries, toolsUsed } = normalizeSession(
+        safeEntries,
+        client ? { client } : {},
+    );
+
+    const messages = normalized.map(m => ({
+        role: m.role,
+        content: m.text || '',
+        timestamp: m.timestamp || null,
+    }));
+
+    let userCount = 0, assistantCount = 0;
+    let startedAt = null, lastMessageAt = null;
+    for (const m of messages) {
+        if (m.role === 'user') userCount++;
+        else if (m.role === 'assistant') assistantCount++;
+        if (m.timestamp) {
+            if (!startedAt) startedAt = m.timestamp;
+            lastMessageAt = m.timestamp;
+        }
+    }
+
+    const { model, tokensIn, tokensOut } = extractRawMeta(safeEntries);
+
+    return {
+        messages,
+        userCount,
+        assistantCount,
+        model,
+        tokensIn,
+        tokensOut,
+        startedAt,
+        lastMessageAt,
+        skipStats,
+        boundaries,
+        toolsUsed,
+    };
+}
+
+module.exports = { normalizeMessages, extractRawMeta };