@shadowforge0/aquifer-memory 1.0.3 → 1.3.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shadowforge0/aquifer-memory",
-  "version": "1.0.3",
+  "version": "1.3.0",
   "description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph. MCP server, CLI, and library API.",
   "main": "index.js",
   "files": [
@@ -9,6 +9,9 @@
     "pipeline/",
     "schema/",
     "consumers/",
+    "consumers/miranda/",
+    "consumers/default/",
+    "consumers/openclaw-ext/",
     "docs/",
     "scripts/"
   ],
@@ -20,8 +23,17 @@
     "./consumers/mcp": "./consumers/mcp.js",
     "./consumers/openclaw-plugin": "./consumers/openclaw-plugin.js",
     "./consumers/opencode": "./consumers/opencode.js",
+    "./consumers/claude-code": "./consumers/claude-code.js",
+    "./consumers/miranda": "./consumers/miranda/index.js",
+    "./consumers/default": "./consumers/default/index.js",
+    "./consumers/openclaw-ext": "./consumers/openclaw-ext/index.js",
     "./consumers/shared/config": "./consumers/shared/config.js",
-    "./consumers/shared/factory": "./consumers/shared/factory.js"
+    "./consumers/shared/factory": "./consumers/shared/factory.js",
+    "./consumers/shared/entity-parser": "./consumers/shared/entity-parser.js",
+    "./consumers/shared/normalize": "./consumers/shared/normalize.js",
+    "./consumers/shared/ingest": "./consumers/shared/ingest.js",
+    "./consumers/shared/recall-format": "./consumers/shared/recall-format.js",
+    "./consumers/shared/llm-autodetect": "./consumers/shared/llm-autodetect.js"
   },
   "repository": {
     "type": "git",
@@ -32,11 +44,20 @@
     "url": "https://github.com/shadowforge0/aquifer/issues"
   },
   "author": "shadowforge0",
+  "scripts": {
+    "test": "node --test test/*.test.js",
+    "test:integration": "node --test test/integration.test.js",
+    "lint": "eslint index.js core/*.js consumers/*.js consumers/shared/*.js consumers/miranda/*.js consumers/miranda/prompts/*.js consumers/default/*.js consumers/default/prompts/*.js consumers/openclaw-ext/*.js pipeline/*.js pipeline/consolidation/*.js scripts/*.js test/*.js",
+    "prepare": "git config core.hooksPath .githooks 2>/dev/null || true"
+  },
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
     "pg": "^8.13.0",
     "zod": "^3.25.76"
   },
+  "devDependencies": {
+    "eslint": "^9.0.0"
+  },
   "engines": {
     "node": ">=18.0.0"
   },

package/pipeline/_http.js

@@ -28,7 +28,7 @@ function httpRequest(url, options, body) {
         }
         try {
           finish(resolve, JSON.parse(raw));
-        } catch (e) {
+        } catch {
           finish(reject, new Error(`Invalid JSON response: ${raw.slice(0, 200)}`));
         }
       });

package/pipeline/consolidation/apply.js

@@ -0,0 +1,176 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// Consolidation apply — executes a batch of fact-lifecycle actions in one tx.
+//
+// Actions (each object in the array):
+//   { action: 'promote',   factId }                          candidate → active
+//   { action: 'create',    subject, statement, importance? } new active fact
+//   { action: 'update',    factId, statement }               refresh active statement
+//   { action: 'confirm',   factId }                          bump last_confirmed_at
+//   { action: 'stale',     factId }                          active → stale
+//   { action: 'discard',   factId }                          candidate → archived
+//   { action: 'merge',     factId, targetId }                candidate archived, target confirmed
+//   { action: 'supersede', factId, targetId }                active → superseded by target
+//
+// All mutations scoped to (tenantId, agentId). The caller is responsible for
+// providing a normalizer for subject_key (fall back to raw subject if absent).
+// ---------------------------------------------------------------------------
+
+function qi(identifier) { return `"${identifier}"`; }
+
+async function applyConsolidation(pool, {
+  actions,
+  agentId,
+  sessionId,
+  schema,
+  tenantId = 'default',
+  normalizeSubject = null,
+  recapOverview = '',
+} = {}) {
+  if (!pool) throw new Error('pool is required');
+  if (!schema) throw new Error('schema is required');
+  if (!agentId) throw new Error('agentId is required');
+  if (!Array.isArray(actions)) throw new Error('actions must be an array');
+
+  const tbl = `${qi(schema)}.facts`;
+  const summary = {
+    promote: 0, create: 0, update: 0, confirm: 0,
+    stale: 0, discard: 0, merge: 0, supersede: 0,
+    skipped: 0,
+  };
+
+  if (actions.length === 0) return summary;
+
+  const client = await pool.connect();
+  try {
+    await client.query('BEGIN');
+
+    for (const act of actions) {
+      switch (act.action) {
+        case 'promote': {
+          const r = await client.query(
+            `UPDATE ${tbl} SET status = 'active', last_confirmed_at = now()
+             WHERE id = $1 AND status = 'candidate' AND agent_id = $2 AND tenant_id = $3`,
+            [act.factId, agentId, tenantId],
+          );
+          summary.promote += r.rowCount;
+          if (r.rowCount === 0) summary.skipped++;
+          break;
+        }
+
+        case 'create': {
+          const subjectLabel = act.subject ? String(act.subject).slice(0, 200) : '';
+          const subjectKey = normalizeSubject ? normalizeSubject(subjectLabel) : subjectLabel.trim().toLowerCase();
+          if (!subjectKey) { summary.skipped++; break; }
+          const statement = act.statement ? String(act.statement).slice(0, 2000) : '';
+          if (!statement) { summary.skipped++; break; }
+          const importance = Number.isFinite(act.importance) ? act.importance : 7;
+          const evidence = JSON.stringify([{
+            type: 'session_ref',
+            session_id: sessionId || null,
+            excerpt: (recapOverview || '').slice(0, 200),
+          }]);
+          const r = await client.query(
+            `INSERT INTO ${tbl}
+             (tenant_id, subject_key, subject_label, statement, status, importance,
+              source_session_id, agent_id, evidence)
+             VALUES ($1, $2, $3, $4, 'active', $5, $6, $7, $8::jsonb)
+             ON CONFLICT DO NOTHING`,
+            [tenantId, subjectKey, subjectLabel, statement, importance, sessionId || null, agentId, evidence],
+          );
+          summary.create += r.rowCount;
+          if (r.rowCount === 0) summary.skipped++;
+          break;
+        }
+
+        case 'update': {
+          const statement = act.statement ? String(act.statement).slice(0, 2000) : '';
+          if (!statement) { summary.skipped++; break; }
+          const r = await client.query(
+            `UPDATE ${tbl} SET statement = $1, last_confirmed_at = now()
+             WHERE id = $2 AND status = 'active' AND agent_id = $3 AND tenant_id = $4`,
+            [statement, act.factId, agentId, tenantId],
+          );
+          summary.update += r.rowCount;
+          if (r.rowCount === 0) summary.skipped++;
+          break;
+        }
+
+        case 'confirm': {
+          const r = await client.query(
+            `UPDATE ${tbl} SET last_confirmed_at = now()
+             WHERE id = $1 AND status = 'active' AND agent_id = $2 AND tenant_id = $3`,
+            [act.factId, agentId, tenantId],
+          );
+          summary.confirm += r.rowCount;
+          if (r.rowCount === 0) summary.skipped++;
+          break;
+        }
+
+        case 'stale': {
+          const r = await client.query(
+            `UPDATE ${tbl} SET status = 'stale'
+             WHERE id = $1 AND status = 'active' AND agent_id = $2 AND tenant_id = $3`,
+            [act.factId, agentId, tenantId],
+          );
+          summary.stale += r.rowCount;
+          if (r.rowCount === 0) summary.skipped++;
+          break;
+        }
+
+        case 'discard': {
+          const r = await client.query(
+            `UPDATE ${tbl} SET status = 'archived'
+             WHERE id = $1 AND status = 'candidate' AND agent_id = $2 AND tenant_id = $3`,
+            [act.factId, agentId, tenantId],
+          );
+          summary.discard += r.rowCount;
+          if (r.rowCount === 0) summary.skipped++;
+          break;
+        }
+
+        case 'merge': {
+          const r1 = await client.query(
+            `UPDATE ${tbl} SET last_confirmed_at = now()
+             WHERE id = $1 AND status = 'active' AND tenant_id = $2`,
+            [act.targetId, tenantId],
+          );
+          const r2 = await client.query(
+            `UPDATE ${tbl} SET status = 'archived'
+             WHERE id = $1 AND status = 'candidate' AND tenant_id = $2`,
+            [act.factId, tenantId],
+          );
+          summary.merge += Math.min(r1.rowCount, r2.rowCount);
+          if (r1.rowCount === 0 || r2.rowCount === 0) summary.skipped++;
+          break;
+        }
+
+        case 'supersede': {
+          const r = await client.query(
+            `UPDATE ${tbl} SET status = 'superseded', superseded_by = $1
+             WHERE id = $2 AND status = 'active' AND tenant_id = $3`,
+            [act.targetId, act.factId, tenantId],
+          );
+          summary.supersede += r.rowCount;
+          if (r.rowCount === 0) summary.skipped++;
+          break;
+        }
+
+        default:
+          summary.skipped++;
+      }
+    }
+
+    await client.query('COMMIT');
+  } catch (err) {
+    await client.query('ROLLBACK').catch(() => {});
+    throw err;
+  } finally {
+    client.release();
+  }
+
+  return summary;
+}
+
+module.exports = { applyConsolidation };

package/pipeline/consolidation/index.js

@@ -0,0 +1,21 @@
+'use strict';
+
+// ---------------------------------------------------------------------------
+// Consolidation pipeline
+//
+// Mechanics only — Aquifer ships the 8-action apply + schema. The prompt and
+// output parser stay in consumers (they're persona-specific: different agents
+// want different wording, language, and action vocabulary extensions).
+//
+// Typical flow in a consumer:
+//
+//   const output = await llmFn(consumerBuildPrompt({...}));
+//   const { actions } = consumerParse(output);
+//   await aquifer.consolidate(sessionId, { actions, agentId });
+//
+// aquifer.consolidate() is defined in core/aquifer.js and delegates here.
+// ---------------------------------------------------------------------------
+
+const { applyConsolidation } = require('./apply');
+
+module.exports = { applyConsolidation };

package/pipeline/extract-entities.js

@@ -6,7 +6,7 @@ const { parseEntityOutput } = require('../core/entity');
 // defaultEntityPrompt
 // ---------------------------------------------------------------------------
 
-function defaultEntityPrompt(messages, opts = {}) {
+function defaultEntityPrompt(messages) {
   const conversation = messages
     .map(m => `[${m.role}] ${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}`)
     .join('\n');
@@ -60,7 +60,7 @@ async function extractEntities(messages, {
     const prompt = buildPrompt(messages, {});
     const response = await llmFn(prompt);
     return parseEntityOutput(response);
-  } catch (err) {
+  } catch {
     // LLM failure: return empty, never throw
     return [];
   }

package/pipeline/rerank.js

@@ -39,7 +39,7 @@ function createTEIReranker(config) {
   const initialBackoffMs = config.initialBackoffMs || 250;
 
   return {
-    async rerank(query, documents, opts = {}) {
+    async rerank(query, documents, _opts = {}) {
       if (!query || !documents || documents.length === 0) return [];
 
       const result = await withRetry(

package/pipeline/summarize.js

@@ -206,6 +206,9 @@ async function summarize(messages, {
   try {
     const prompt = buildPrompt(messages, { mergeEntities });
     const response = await llmFn(prompt);
+    if (typeof response !== 'string' || response.trim() === '') {
+      return extractiveFallback(messages);
+    }
 
     // Parse structured fields
     const structuredSummary = _parseStructuredSummary(response);
@@ -232,7 +235,7 @@ async function summarize(messages, {
       entityRaw,
       isExtractive: false,
     };
-  } catch (err) {
+  } catch {
     // LLM failure: fall back to extractive
     return extractiveFallback(messages);
   }

package/schema/001-base.sql

@@ -43,27 +43,6 @@ CREATE INDEX IF NOT EXISTS idx_sessions_processing_status
   ON ${schema}.sessions (processing_status)
   WHERE processing_status IN ('pending', 'processing');
 
--- =========================================================================
--- Session segments: conversation boundary metadata
--- =========================================================================
-CREATE TABLE IF NOT EXISTS ${schema}.session_segments (
-  id                  BIGSERIAL    PRIMARY KEY,
-  session_row_id      BIGINT       NOT NULL REFERENCES ${schema}.sessions(id) ON DELETE CASCADE,
-  segment_no          INT          NOT NULL,
-  start_msg_idx       INT,
-  end_msg_idx         INT,
-  started_at          TIMESTAMPTZ,
-  ended_at            TIMESTAMPTZ,
-  raw_msg_count       INT          NOT NULL DEFAULT 0,
-  effective_msg_count INT          NOT NULL DEFAULT 0,
-  boundary_type       TEXT,
-  boundary_meta       JSONB        NOT NULL DEFAULT '{}',
-  UNIQUE (session_row_id, segment_no)
-);
-
-CREATE INDEX IF NOT EXISTS idx_session_segments_row
-  ON ${schema}.session_segments (session_row_id);
-
 -- =========================================================================
 -- Session summaries: LLM-generated or extractive summaries
 -- =========================================================================
@@ -78,8 +57,6 @@ CREATE TABLE IF NOT EXISTS ${schema}.session_summaries (
   message_count            INT          NOT NULL DEFAULT 0,
   user_message_count       INT          NOT NULL DEFAULT 0,
   assistant_message_count  INT          NOT NULL DEFAULT 0,
-  boundary_count           INT          NOT NULL DEFAULT 0,
-  fresh_tail_count         INT          NOT NULL DEFAULT 0,
   started_at               TIMESTAMPTZ,
   ended_at                 TIMESTAMPTZ,
   summary_text             TEXT,
@@ -92,6 +69,23 @@ CREATE TABLE IF NOT EXISTS ${schema}.session_summaries (
   updated_at               TIMESTAMPTZ  NOT NULL DEFAULT now()
 );
 
+-- Cleanup legacy segment-era schema artifacts so migrate() converges old installs.
+-- Wrapped because the implicit sequence on session_segments can be referenced from
+-- other schemas (e.g. bench/staging created via CREATE TABLE LIKE), which would
+-- otherwise hard-fail the migration. Operators get a NOTICE and must decouple
+-- dependents themselves before the table will actually drop.
+DO $$
+BEGIN
+  BEGIN
+    DROP TABLE IF EXISTS ${schema}.session_segments;
+  EXCEPTION
+    WHEN dependent_objects_still_exist THEN
+      RAISE NOTICE '[aquifer] skipped session_segments drop: %; decouple cross-schema dependents and re-run migrate to complete cleanup', SQLERRM;
+  END;
+END$$;
+ALTER TABLE ${schema}.session_summaries DROP COLUMN IF EXISTS boundary_count;
+ALTER TABLE ${schema}.session_summaries DROP COLUMN IF EXISTS fresh_tail_count;
+
 CREATE INDEX IF NOT EXISTS idx_summaries_tenant
   ON ${schema}.session_summaries (tenant_id);
 
@@ -105,6 +99,27 @@ CREATE INDEX IF NOT EXISTS idx_summaries_embedding
   ON ${schema}.session_summaries (session_row_id)
   WHERE embedding IS NOT NULL;
 
+-- HNSW approximate nearest-neighbor index for cosine-distance vector search.
+-- Without this, ORDER BY embedding <=> $vec degrades to seq scan at scale.
+-- Requires pgvector >= 0.5.0. HNSW cannot build on an empty unsized `vector`
+-- column (can't infer dim), so we defer on failure — re-running migrate()
+-- after the first insert will finish the job.
+DO $$
+BEGIN
+  BEGIN
+    EXECUTE 'CREATE INDEX IF NOT EXISTS idx_summaries_embedding_hnsw ON ${schema}.session_summaries USING hnsw (embedding vector_cosine_ops)';
+  EXCEPTION
+    WHEN invalid_parameter_value THEN
+      RAISE NOTICE '[aquifer] HNSW index on session_summaries.embedding deferred; re-run migrate() after the first embedded row';
+    WHEN feature_not_supported THEN
+      RAISE NOTICE '[aquifer] HNSW not available on this pgvector; upgrade to >= 0.5.0 for index-accelerated vector search';
+    WHEN out_of_memory THEN
+      RAISE WARNING '[aquifer] HNSW build on session_summaries.embedding ran out of memory; raise maintenance_work_mem and re-run migrate()';
+    WHEN program_limit_exceeded THEN
+      RAISE WARNING '[aquifer] HNSW build on session_summaries.embedding exceeded an internal limit; inspect pgvector logs';
+  END;
+END$$;
+
 -- FTS trigger: auto-update search_tsv on INSERT/UPDATE
 CREATE OR REPLACE FUNCTION ${schema}.session_summaries_search_tsv_update()
 RETURNS trigger
@@ -158,8 +173,12 @@ $$;
 DROP TRIGGER IF EXISTS trg_session_summaries_search_tsv
   ON ${schema}.session_summaries;
 
+-- Trigger fires on input-column changes only. search_text is a trigger output
+-- (derived from structured_summary + summary_text) and listing it here was
+-- redundant — PG's BEFORE semantics already prevent the assignment inside the
+-- trigger body from re-firing the trigger.
 CREATE TRIGGER trg_session_summaries_search_tsv
-  BEFORE INSERT OR UPDATE OF summary_text, structured_summary, search_text
+  BEFORE INSERT OR UPDATE OF summary_text, structured_summary
   ON ${schema}.session_summaries
   FOR EACH ROW
   EXECUTE FUNCTION ${schema}.session_summaries_search_tsv_update();
@@ -189,3 +208,21 @@ CREATE INDEX IF NOT EXISTS idx_turn_emb_session_row
 
 CREATE INDEX IF NOT EXISTS idx_turn_emb_tenant_agent
   ON ${schema}.turn_embeddings (tenant_id, agent_id, source);
+
+-- HNSW approximate nearest-neighbor index for turn-level vector search.
+-- See notes on session_summaries.embedding HNSW above.
+DO $$
+BEGIN
+  BEGIN
+    EXECUTE 'CREATE INDEX IF NOT EXISTS idx_turn_emb_embedding_hnsw ON ${schema}.turn_embeddings USING hnsw (embedding vector_cosine_ops)';
+  EXCEPTION
+    WHEN invalid_parameter_value THEN
+      RAISE NOTICE '[aquifer] HNSW index on turn_embeddings.embedding deferred; re-run migrate() after the first embedded row';
+    WHEN feature_not_supported THEN
+      RAISE NOTICE '[aquifer] HNSW not available on this pgvector; upgrade to >= 0.5.0 for index-accelerated vector search';
+    WHEN out_of_memory THEN
+      RAISE WARNING '[aquifer] HNSW build on turn_embeddings.embedding ran out of memory; raise maintenance_work_mem and re-run migrate()';
+    WHEN program_limit_exceeded THEN
+      RAISE WARNING '[aquifer] HNSW build on turn_embeddings.embedding exceeded an internal limit; inspect pgvector logs';
+  END;
+END$$;

package/schema/002-entities.sql

@@ -28,10 +28,24 @@ CREATE TABLE IF NOT EXISTS ${schema}.entities (
   last_seen_at    TIMESTAMPTZ  NOT NULL DEFAULT now()
 );
 
--- Migration: add entity_scope if missing (idempotent)
--- For upgrades: backfill from agent_id so existing data keeps its scope
+-- Migration: add entity_scope if missing (idempotent, scope-corruption-safe).
+-- For upgrades: backfill from agent_id ONLY on the first run of this migration,
+-- detected via the column still being NULL-able. Once SET NOT NULL below fires,
+-- subsequent runs skip the backfill so operator-assigned 'default' values are
+-- never clobbered.
 ALTER TABLE ${schema}.entities ADD COLUMN IF NOT EXISTS entity_scope TEXT DEFAULT 'default';
-UPDATE ${schema}.entities SET entity_scope = agent_id WHERE entity_scope IS NULL OR entity_scope = 'default';
+DO $$
+BEGIN
+  IF EXISTS (
+    SELECT 1 FROM information_schema.columns
+    WHERE table_schema = '${schema}' AND table_name = 'entities'
+      AND column_name = 'entity_scope' AND is_nullable = 'YES'
+  ) THEN
+    UPDATE ${schema}.entities
+      SET entity_scope = agent_id
+      WHERE entity_scope IS NULL OR entity_scope = 'default';
+  END IF;
+END$$;
 ALTER TABLE ${schema}.entities ALTER COLUMN entity_scope SET NOT NULL;
 
 -- Unique constraint: entity identity is (tenant, name, scope)