@shadowforge0/aquifer-memory 1.2.1 → 1.5.8

package/core/storage.js

@@ -59,7 +59,7 @@ async function upsertSession(pool, {
       (tenant_id, session_id, session_key, agent_id, source, messages,
        msg_count, user_count, assistant_count, model, tokens_in, tokens_out,
        started_at, ended_at, last_message_at, processing_status)
-    VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,now(),$14,'pending')
+    VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,COALESCE($13,now()),COALESCE($14,now()),$14,'pending')
     ON CONFLICT (tenant_id, agent_id, session_id) DO UPDATE SET
       session_key = EXCLUDED.session_key,
       source = COALESCE(EXCLUDED.source, ${qi(schema)}.sessions.source),
@@ -71,7 +71,7 @@ async function upsertSession(pool, {
       tokens_in = EXCLUDED.tokens_in,
       tokens_out = EXCLUDED.tokens_out,
       started_at = COALESCE(EXCLUDED.started_at, ${qi(schema)}.sessions.started_at),
-      ended_at = now(),
+      ended_at = COALESCE(EXCLUDED.last_message_at, ${qi(schema)}.sessions.ended_at),
       last_message_at = COALESCE(EXCLUDED.last_message_at, ${qi(schema)}.sessions.last_message_at),
       processing_status = 'pending',
       processing_error = NULL
@@ -223,9 +223,13 @@ async function searchSessions(pool, query, {
   dateFrom,
   dateTo,
   limit = 20,
+  ftsConfig = 'simple',
 } = {}) {
   const clampedLimit = Math.max(1, Math.min(100, limit));
 
+  // Whitelist tsconfig to prevent injection
+  const cfg = (ftsConfig === 'zhcfg' || ftsConfig === 'simple') ? ftsConfig : 'simple';
+
   // Normalize agentId/agentIds
   const agentIds = rawAgentIds && rawAgentIds.length > 0
     ? rawAgentIds
@@ -237,7 +241,7 @@ async function searchSessions(pool, query, {
   // Primary: trigram ILIKE on search_text (works for CJK + Latin)
   // Fallback: tsvector FTS (for installations without search_text populated)
   const where = [
-    `(ss.search_text ILIKE '%' || $1 || '%' OR ss.search_tsv @@ plainto_tsquery('simple', $2))`,
+    `(ss.search_text ILIKE '%' || $1 || '%' OR ss.search_tsv @@ plainto_tsquery('${cfg}', $2))`,
     `s.tenant_id = $3`,
   ];
   const params = [likeQuery, query, tenantId];
@@ -276,10 +280,15 @@ async function searchSessions(pool, query, {
       ss.trust_score,
       CASE WHEN ss.search_text IS NOT NULL
         THEN similarity(ss.search_text, $2)
-        ELSE ts_rank(ss.search_tsv, plainto_tsquery('simple', $2))
+        ELSE ts_rank(ss.search_tsv, plainto_tsquery('${cfg}', $2))
       END AS fts_rank
     FROM ${qi(schema)}.sessions s
-    LEFT JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
+    -- INNER JOIN: the WHERE clause references ss.search_text / ss.search_tsv,
+    -- which a LEFT JOIN would leave NULL for unenriched sessions — filtering
+    -- them out. Be explicit: FTS recall is a SUMMARIZED-sessions search. Raw
+    -- unenriched sessions don't participate. Named searchSessions for historic
+    -- reasons; semantically it is search-over-enriched-sessions.
+    JOIN ${qi(schema)}.session_summaries ss ON ss.session_row_id = s.id
     WHERE ${where.join(' AND ')}
     ORDER BY
       COALESCE(ss.search_text ILIKE '%' || $1 || '%', FALSE) DESC,
@@ -512,6 +521,73 @@ async function searchTurnEmbeddings(pool, {
   return { rows: fallback.rows };
 }
 
+// ---------------------------------------------------------------------------
+// searchSummaryEmbeddings — pgvector cosine search on session_summaries.embedding
+// ---------------------------------------------------------------------------
+
+async function searchSummaryEmbeddings(pool, {
+  schema,
+  tenantId,
+  queryVec,
+  agentId,
+  agentIds: rawAgentIds,
+  source,
+  dateFrom,
+  dateTo,
+  candidateSessionIds,
+  limit = 15,
+} = {}) {
+  const where = ['s.tenant_id = $1'];
+  const params = [tenantId];
+
+  params.push(`[${queryVec.join(',')}]`);
+  const vecPos = params.length;
+
+  const agentIds = rawAgentIds && rawAgentIds.length > 0
+    ? rawAgentIds
+    : (agentId ? [agentId] : null);
+
+  if (dateFrom) {
+    params.push(dateFrom);
+    where.push(`s.started_at::date >= $${params.length}::date`);
+  }
+  if (dateTo) {
+    params.push(dateTo);
+    where.push(`s.started_at::date <= $${params.length}::date`);
+  }
+  if (agentIds) {
+    params.push(agentIds);
+    where.push(`s.agent_id = ANY($${params.length})`);
+  }
+  if (source) {
+    params.push(source);
+    where.push(`s.source = $${params.length}`);
+  }
+  if (candidateSessionIds && candidateSessionIds.length > 0) {
+    params.push(candidateSessionIds);
+    where.push(`s.session_id = ANY($${params.length})`);
+  }
+
+  params.push(limit);
+
+  const result = await pool.query(
+    `SELECT
+      s.id, s.session_id, s.agent_id, s.source, s.started_at, s.last_message_at,
+      ss.summary_text, ss.structured_summary, ss.access_count, ss.last_accessed_at,
+      ss.trust_score,
+      (ss.embedding <=> $${vecPos}::vector) AS distance
+    FROM ${qi(schema)}.session_summaries ss
+    JOIN ${qi(schema)}.sessions s ON s.id = ss.session_row_id
+    WHERE ss.embedding IS NOT NULL
+      AND ${where.join(' AND ')}
+    ORDER BY distance ASC
+    LIMIT $${params.length}`,
+    params
+  );
+
+  return { rows: result.rows };
+}
+
 // ---------------------------------------------------------------------------
 // recordFeedback — explicit trust feedback with audit trail
 // ---------------------------------------------------------------------------
@@ -605,5 +681,6 @@ module.exports = {
   extractUserTurns,
   upsertTurnEmbeddings,
   searchTurnEmbeddings,
+  searchSummaryEmbeddings,
   recordFeedback,
 };

package/core/timeline.js

@@ -0,0 +1,152 @@
+'use strict';
+
+// aq.timeline.* — append-only event log capability.
+//
+// Spec: aquifer-completion §10 timeline. Fixed event shape
+// (occurred_at / source / session_ref / category / text / metadata),
+// consumer-owned category vocabulary. idempotency_key UNIQUE across the
+// table; appends with a duplicate key are a safe no-op and return the
+// existing row.
+
+const crypto = require('crypto');
+const { AqError, ok, err } = require('./errors');
+
+const DEFAULT_PROFILE = Object.freeze({
+  id: 'anon',
+  version: 0,
+  schemaHash: 'pending',
+});
+
+function resolveProfile(profile) {
+  if (!profile) return DEFAULT_PROFILE;
+  return {
+    id: profile.id || DEFAULT_PROFILE.id,
+    version: Number.isInteger(profile.version) ? profile.version : DEFAULT_PROFILE.version,
+    schemaHash: profile.schemaHash || DEFAULT_PROFILE.schemaHash,
+  };
+}
+
+function defaultIdempotencyKey({ tenantId, agentId, occurredAt, category, text }) {
+  return crypto.createHash('sha256')
+    .update(`${tenantId}:${agentId}:${occurredAt}:${category}:${text}`)
+    .digest('hex');
+}
+
+function toNumber(v) {
+  if (v === null || v === undefined) return null;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : null;
+}
+
+function mapRow(row) {
+  if (!row) return null;
+  return {
+    eventId: toNumber(row.id),
+    occurredAt: row.occurred_at,
+    source: row.source,
+    sessionRef: row.session_ref,
+    category: row.category,
+    text: row.text,
+    metadata: row.metadata || {},
+  };
+}
+
+function createTimeline({ pool, schema, defaultTenantId }) {
+  async function append(input) {
+    try {
+      if (!input || typeof input !== 'object') {
+        return err('AQ_INVALID_INPUT', 'append requires an input object');
+      }
+      if (!input.agentId) return err('AQ_INVALID_INPUT', 'agentId is required');
+      if (!input.occurredAt) return err('AQ_INVALID_INPUT', 'occurredAt is required');
+      if (!input.source) return err('AQ_INVALID_INPUT', 'source is required');
+      if (!input.category) return err('AQ_INVALID_INPUT', 'category is required');
+      if (!input.text || typeof input.text !== 'string') {
+        return err('AQ_INVALID_INPUT', 'text is required');
+      }
+
+      const tenantId = input.tenantId || defaultTenantId || 'default';
+      const agentId = input.agentId;
+      const profile = resolveProfile(input.profile);
+      const idempotencyKey = input.idempotencyKey
+        || defaultIdempotencyKey({
+          tenantId, agentId,
+          occurredAt: input.occurredAt,
+          category: input.category,
+          text: input.text,
+        });
+
+      // Idempotent append: on conflict, fall back to SELECT so the caller
+      // always gets the canonical row (the row that *won* the insert).
+      const insertResult = await pool.query(
+        `INSERT INTO ${schema}.timeline_events (
+           tenant_id, agent_id, occurred_at, source, session_ref,
+           category, text, metadata, source_session_id,
+           consumer_profile_id, consumer_profile_version, consumer_schema_hash,
+           idempotency_key
+         ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+         ON CONFLICT (idempotency_key) DO NOTHING
+         RETURNING *`,
+        [
+          tenantId, agentId, input.occurredAt, input.source,
+          input.sessionRef || null, input.category, input.text,
+          input.metadata || {}, input.sessionId || null,
+          profile.id, profile.version, profile.schemaHash,
+          idempotencyKey,
+        ],
+      );
+      let row = insertResult.rows[0];
+      if (!row) {
+        const existing = await pool.query(
+          `SELECT * FROM ${schema}.timeline_events WHERE idempotency_key = $1`,
+          [idempotencyKey],
+        );
+        row = existing.rows[0];
+      }
+      const mapped = mapRow(row);
+      return ok({ eventId: mapped.eventId, event: mapped });
+    } catch (e) {
+      if (e instanceof AqError) return err(e);
+      return err('AQ_INTERNAL', e.message, { cause: e });
+    }
+  }
+
+  async function list(input = {}) {
+    try {
+      if (!input.agentId) return err('AQ_INVALID_INPUT', 'agentId is required');
+      const tenantId = input.tenantId || defaultTenantId || 'default';
+      const limit = Math.min(Math.max(input.limit || 50, 1), 500);
+
+      const params = [tenantId, input.agentId];
+      let where = 'tenant_id = $1 AND agent_id = $2';
+      if (Array.isArray(input.categories) && input.categories.length > 0) {
+        params.push(input.categories);
+        where += ` AND category = ANY($${params.length})`;
+      }
+      if (input.since) {
+        params.push(input.since);
+        where += ` AND occurred_at >= $${params.length}`;
+      }
+      if (input.until) {
+        params.push(input.until);
+        where += ` AND occurred_at <= $${params.length}`;
+      }
+      params.push(limit);
+
+      const { rows } = await pool.query(
+        `SELECT * FROM ${schema}.timeline_events
+          WHERE ${where}
+          ORDER BY occurred_at DESC, id DESC
+          LIMIT $${params.length}`,
+        params,
+      );
+      return ok({ rows: rows.map(mapRow) });
+    } catch (e) {
+      return err('AQ_INTERNAL', e.message, { cause: e });
+    }
+  }
+
+  return { append, list };
+}
+
+module.exports = { createTimeline };

package/index.js

@@ -5,6 +5,8 @@ const { createEmbedder } = require('./pipeline/embed');
 const { createReranker } = require('./pipeline/rerank');
 const { normalizeEntityName } = require('./core/entity');
 const { parseEntitySection } = require('./consumers/shared/entity-parser');
+const { AqError, ok, err, asResult, isKnownCode, KNOWN_CODES } = require('./core/errors');
+const { MCP_SERVER_NAME, MCP_TOOL_MANIFEST, getManifest, writeManifestFile } = require('./core/mcp-manifest');
 
 module.exports = {
   createAquifer,
@@ -12,4 +14,16 @@ module.exports = {
   createReranker,
   normalizeEntityName,
   parseEntitySection,
+  // Completion-capability error envelope (P1 foundation).
+  AqError,
+  ok,
+  err,
+  asResult,
+  isKnownCode,
+  KNOWN_CODES,
+  // MCP tool manifest — canonical for gateway in-process + CC cross-process.
+  MCP_SERVER_NAME,
+  MCP_TOOL_MANIFEST,
+  getMcpManifest: getManifest,
+  writeMcpManifestFile: writeManifestFile,
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shadowforge0/aquifer-memory",
-  "version": "1.2.1",
+  "version": "1.5.8",
   "description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph. MCP server, CLI, and library API.",
   "main": "index.js",
   "files": [

package/pipeline/extract-state-changes.js

@@ -0,0 +1,205 @@
+'use strict';
+
+// Extract temporal state-change facts from session content.
+//
+// Input: message array + entity context (name→id map) + LLM.
+// Output: array of change objects ready to feed entity-state.applyChanges().
+//
+// Strict rules baked into the prompt:
+//   - Only "已發生 / past-tense / 完成式" transitions — reject tentative
+//     ("I might / I was thinking about / let's consider").
+//   - Must have explicit time anchor ("on 2026-04-18", "as of today",
+//     "this morning") — tag to session started_at if only "now".
+//   - attribute must be stable snake_case path (version.stable,
+//     editor.preference, runtime.node.version).
+//   - value must be JSON-serialisable (strings, numbers, bools, nested OK).
+//   - confidence ∈ [0,1]; default 0.7, any < threshold is dropped by caller.
+
+const ATTRIBUTE_RE = /^[a-z][a-z0-9_]*(\.[a-z][a-z0-9_]*)*$/;
+
+function defaultStateChangePrompt(messages, ctx = {}) {
+  const conversation = messages
+    .map(m => `[${m.role}] ${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}`)
+    .join('\n');
+  const entityList = ctx.entities && ctx.entities.length
+    ? ctx.entities.map(e => `  - "${e.name}" (id=${e.id})`).join('\n')
+    : '  (no entities resolved yet)';
+  const sessionTime = ctx.sessionStartedAt || new Date().toISOString();
+
+  return `You extract TEMPORAL STATE-CHANGE FACTS from a conversation.
+A state change means "this specific attribute of this specific entity CHANGED its value at a specific moment."
+
+## Strict rules
+
+1. Only extract CHANGES — not first-observations, not opinions, not preferences merely mentioned.
+2. Only PAST-TENSE / COMPLETED transitions. Reject tentative language:
+   - REJECT: "I might try", "I was thinking about", "let's consider", "maybe", "probably", "planning to"
+   - ACCEPT: "I upgraded", "switched to", "changed to", "升到", "改成", "換成", "現在用", "已經改"
+3. Must have explicit TIME ANCHOR — exact date, "today", "this morning", "as of", "自 X 起"
+   If only implicit "now", use ctx.sessionStartedAt as valid_from.
+4. attribute MUST be a STABLE snake_case path (lowercase, dots as separators):
+   - GOOD: version.stable, editor.preference, runtime.node.version, indexing.pgvector.strategy
+   - BAD: "Version Stable", "My Editor", "editor-pref"
+5. Each change MUST match an entity in the list below by entity_name (exact match preferred, alias OK).
+   If no matching entity exists, DROP the change silently.
+6. value must be JSON-serialisable. Wrap scalars plain (e.g. "1.3.0"), objects as {key: v}.
+
+## Output
+
+Emit ONE JSON object, no prose, no code fence, no commentary:
+
+{
+  "state_changes": [
+    {
+      "entity_name": "<must match list>",
+      "attribute": "<snake_case.dotted.path>",
+      "value": <any JSON>,
+      "valid_from": "<ISO8601 timestamp>",
+      "time_anchor_text": "<the phrase that anchors the time>",
+      "evidence_text": "<the sentence that states the change, <= 240 chars>",
+      "confidence": <0..1>
+    }
+  ]
+}
+
+If no changes, output: {"state_changes": []}
+
+## Entities in scope
+
+${entityList}
+
+## Session started at: ${sessionTime}
+
+## Conversation
+
+${conversation}
+`;
+}
+
+// Attempt to recover a JSON object from LLM output — some models wrap in
+// code fences, some prepend "Here is the JSON:" etc. Tolerant but strict
+// about the resulting shape.
+function extractJsonBlock(text) {
+  if (!text || typeof text !== 'string') return null;
+  // Strip triple-backtick fences if present.
+  let s = text.trim();
+  const fenceMatch = s.match(/```(?:json)?\s*([\s\S]*?)```/);
+  if (fenceMatch) s = fenceMatch[1].trim();
+  // Take the substring from the first { to the last }.
+  const first = s.indexOf('{');
+  const last = s.lastIndexOf('}');
+  if (first < 0 || last < first) return null;
+  const candidate = s.slice(first, last + 1);
+  try {
+    return JSON.parse(candidate);
+  } catch {
+    return null;
+  }
+}
+
+// Normalize one raw LLM-emitted change. Returns { entityName, ... } with the
+// human-facing name intact — resolution to entity_id happens in the caller
+// (enrich) after entity upsert, so state extraction itself doesn't need
+// a populated id lookup.
+function normalizeChange(raw, ctx) {
+  if (!raw || typeof raw !== 'object') return null;
+  const name = typeof raw.entity_name === 'string' ? raw.entity_name.trim() : null;
+  if (!name) return null;
+
+  // If a scope whitelist is passed, reject names not on it (case-insensitive).
+  if (ctx.scopeNames && !ctx.scopeNames.has(name.toLowerCase())) return null;
+
+  const attribute = typeof raw.attribute === 'string' ? raw.attribute.trim() : '';
+  if (!ATTRIBUTE_RE.test(attribute)) return null;
+
+  if (raw.value === undefined) return null;  // explicit null is OK
+
+  const validFromDate = new Date(raw.valid_from || raw.validFrom || ctx.sessionStartedAt);
+  if (!Number.isFinite(validFromDate.getTime())) return null;
+
+  let confidence = raw.confidence;
+  if (typeof confidence !== 'number' || !Number.isFinite(confidence)) confidence = 0.7;
+  if (confidence < 0) confidence = 0;
+  if (confidence > 1) confidence = 1;
+
+  const evidenceText = typeof raw.evidence_text === 'string' ? raw.evidence_text.slice(0, 240) : '';
+
+  return {
+    entityName: name,
+    attribute,
+    value: raw.value,
+    validFrom: validFromDate.toISOString(),
+    evidenceText,
+    confidence,
+    source: 'llm',
+    evidenceSessionId: ctx.evidenceSessionId || null,
+    sessionRowId: ctx.sessionRowId ?? null,
+  };
+}
+
+async function extractStateChanges(messages, {
+  llmFn,
+  promptFn,
+  entities = [],            // [{id, name, aliases?: []}]
+  sessionStartedAt,
+  evidenceSessionId,
+  sessionRowId,
+  confidenceThreshold = 0.7,
+  timeoutMs = 10000,
+  maxOutputTokens = 600,
+  logger,
+} = {}) {
+  if (!llmFn) return { changes: [], warnings: ['no_llm'] };
+  if (!entities.length) return { changes: [], warnings: ['no_entities_in_scope'] };
+
+  // Build case-insensitive name whitelist (entity name + aliases).
+  const scopeNames = new Set();
+  for (const e of entities) {
+    if (!e || !e.name) continue;
+    scopeNames.add(String(e.name).toLowerCase());
+    for (const a of (e.aliases || [])) {
+      if (typeof a === 'string') scopeNames.add(a.toLowerCase());
+    }
+  }
+
+  const buildPrompt = promptFn || defaultStateChangePrompt;
+  const prompt = buildPrompt(messages, { entities, sessionStartedAt });
+
+  const warnings = [];
+  let rawResponse;
+  try {
+    // Simple timeout wrapper — llmFn signature in this repo is (prompt) => string.
+    rawResponse = await Promise.race([
+      llmFn(prompt, { maxTokens: maxOutputTokens }),
+      new Promise((_, rej) => setTimeout(() => rej(new Error('llm_timeout')), timeoutMs)),
+    ]);
+  } catch (e) {
+    if (logger && logger.warn) logger.warn(`[extract-state-changes] llm call failed: ${e.message}`);
+    return { changes: [], warnings: [`llm_error: ${e.message}`] };
+  }
+
+  const parsed = extractJsonBlock(rawResponse);
+  if (!parsed || !Array.isArray(parsed.state_changes)) {
+    if (logger && logger.warn) logger.warn(`[extract-state-changes] malformed output, dropping batch`);
+    return { changes: [], warnings: ['malformed_json'] };
+  }
+
+  const ctx = { scopeNames, sessionStartedAt, evidenceSessionId, sessionRowId };
+  const changes = [];
+  let dropped = 0;
+  for (const raw of parsed.state_changes) {
+    const n = normalizeChange(raw, ctx);
+    if (!n) { dropped++; continue; }
+    if (n.confidence < confidenceThreshold) { dropped++; continue; }
+    changes.push(n);
+  }
+  if (dropped > 0) warnings.push(`dropped_${dropped}_invalid_or_low_confidence`);
+  return { changes, warnings };
+}
+
+module.exports = {
+  defaultStateChangePrompt,
+  extractJsonBlock,
+  normalizeChange,
+  extractStateChanges,
+};