@shadowforge0/aquifer-memory 1.6.0 → 1.8.0

package/core/storage.js

@@ -1,6 +1,14 @@
 'use strict';
 
 const crypto = require('crypto');
+const {
+  upsertCheckpointRun,
+  updateCheckpointRunStatus,
+  listCheckpointRuns,
+  upsertCheckpointRunSources,
+  listCheckpointRunSources,
+} = require('./storage-checkpoints');
+const { publicPlaceholderSummarySql } = require('./public-session-filter');
 
 // C1: quote identifier for SQL safety
 function qi(identifier) { return `"${identifier}"`; }
@@ -49,7 +57,6 @@ const FINALIZATION_MODES = new Set([
   'afterburn',
   'manual',
 ]);
-
 function requireField(obj, field) {
   if (!obj || obj[field] === undefined || obj[field] === null || obj[field] === '') {
     throw new Error(`${field} is required`);
@@ -60,6 +67,19 @@ function toJson(value, fallback) {
   return JSON.stringify(value === undefined ? fallback : value);
 }
 
+function stableJson(value) {
+  if (value === null || value === undefined) return JSON.stringify(null);
+  if (Array.isArray(value)) return `[${value.map(stableJson).join(',')}]`;
+  if (typeof value === 'object') {
+    return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${stableJson(value[key])}`).join(',')}}`;
+  }
+  return JSON.stringify(value);
+}
+
+function hashStable(value) {
+  return crypto.createHash('sha256').update(stableJson(value)).digest('hex');
+}
+
 // ---------------------------------------------------------------------------
 // upsertSession
 // ---------------------------------------------------------------------------
@@ -270,6 +290,7 @@ async function searchSessions(pool, query, {
   const where = [
     `(ss.search_text ILIKE '%' || $1 || '%' OR ss.search_tsv @@ plainto_tsquery('${cfg}', $2))`,
     `s.tenant_id = $3`,
+    `NOT ${publicPlaceholderSummarySql('ss')}`,
   ];
   const params = [likeQuery, query, tenantId];
 
@@ -372,21 +393,34 @@ async function upsertSessionFinalization(pool, input = {}, { schema, tenantId: d
   const status = normalizeFinalizationStatus(input.status || 'pending');
   const mode = normalizeFinalizationMode(input.mode || 'handoff');
   const phase = input.phase || 'curated_memory_v1';
+  const candidateEnvelope = input.candidateEnvelope || input.candidate_envelope || {};
+  const candidateEnvelopeHash = input.candidateEnvelopeHash
+    || input.candidate_envelope_hash
+    || (candidateEnvelope && Object.keys(candidateEnvelope).length > 0 ? hashStable(candidateEnvelope) : null);
+  const candidateEnvelopeVersion = input.candidateEnvelopeVersion
+    || input.candidate_envelope_version
+    || candidateEnvelope.version
+    || null;
+  const coverage = input.coverage || {};
   const preserveTerminal = `${finalizationTerminalSql(qi(schema) + '.session_finalizations')}
           AND ${qi(schema)}.session_finalizations.status <> EXCLUDED.status`;
   const result = await pool.query(
     `INSERT INTO ${qi(schema)}.session_finalizations (
        tenant_id, session_row_id, source, host, agent_id, session_id,
        transcript_hash, phase, mode, status, finalizer_model, scope_kind,
-       scope_key, context_key, topic_key, summary_row_id, memory_result,
+       scope_key, context_key, topic_key, scope_id, scope_snapshot,
+       summary_row_id, memory_result,
        summary_text, structured_summary, human_review_text, session_start_text,
-       error, metadata, claimed_at, finalized_at
+       candidate_envelope, candidate_envelope_hash, candidate_envelope_version,
+       coverage, error, metadata, claimed_at, finalized_at
      )
      VALUES (
        $1,$2,$3,COALESCE($4,'codex'),$5,$6,$7,COALESCE($8,'curated_memory_v1'),
        $9,$10,$11,$12,$13,$14,$15,$16,COALESCE($17::jsonb,'{}'::jsonb),
-       $18,COALESCE($19::jsonb,'{}'::jsonb),$20,$21,
-       $22,COALESCE($23::jsonb,'{}'::jsonb),$24,$25
+       $18,COALESCE($19::jsonb,'{}'::jsonb),
+       $20,COALESCE($21::jsonb,'{}'::jsonb),$22,$23,
+       COALESCE($24::jsonb,'{}'::jsonb),$25,$26,COALESCE($27::jsonb,'{}'::jsonb),
+       $28,COALESCE($29::jsonb,'{}'::jsonb),$30,$31
      )
      ON CONFLICT (tenant_id, source, agent_id, session_id, transcript_hash, phase)
      DO UPDATE SET
@@ -435,6 +469,16 @@ async function upsertSessionFinalization(pool, input = {}, { schema, tenantId: d
          THEN ${qi(schema)}.session_finalizations.topic_key
          ELSE COALESCE(EXCLUDED.topic_key, ${qi(schema)}.session_finalizations.topic_key)
        END,
+       scope_id = CASE
+         WHEN ${preserveTerminal}
+         THEN ${qi(schema)}.session_finalizations.scope_id
+         ELSE COALESCE(EXCLUDED.scope_id, ${qi(schema)}.session_finalizations.scope_id)
+       END,
+       scope_snapshot = CASE
+         WHEN ${preserveTerminal}
+         THEN ${qi(schema)}.session_finalizations.scope_snapshot
+         ELSE COALESCE(NULLIF(EXCLUDED.scope_snapshot, '{}'::jsonb), ${qi(schema)}.session_finalizations.scope_snapshot)
+       END,
        summary_row_id = CASE
          WHEN ${preserveTerminal}
          THEN ${qi(schema)}.session_finalizations.summary_row_id
@@ -465,6 +509,26 @@ async function upsertSessionFinalization(pool, input = {}, { schema, tenantId: d
          THEN ${qi(schema)}.session_finalizations.session_start_text
          ELSE COALESCE(EXCLUDED.session_start_text, ${qi(schema)}.session_finalizations.session_start_text)
        END,
+       candidate_envelope = CASE
+         WHEN ${preserveTerminal}
+         THEN ${qi(schema)}.session_finalizations.candidate_envelope
+         ELSE COALESCE(NULLIF(EXCLUDED.candidate_envelope, '{}'::jsonb), ${qi(schema)}.session_finalizations.candidate_envelope)
+       END,
+       candidate_envelope_hash = CASE
+         WHEN ${preserveTerminal}
+         THEN ${qi(schema)}.session_finalizations.candidate_envelope_hash
+         ELSE COALESCE(EXCLUDED.candidate_envelope_hash, ${qi(schema)}.session_finalizations.candidate_envelope_hash)
+       END,
+       candidate_envelope_version = CASE
+         WHEN ${preserveTerminal}
+         THEN ${qi(schema)}.session_finalizations.candidate_envelope_version
+         ELSE COALESCE(EXCLUDED.candidate_envelope_version, ${qi(schema)}.session_finalizations.candidate_envelope_version)
+       END,
+       coverage = CASE
+         WHEN ${preserveTerminal}
+         THEN ${qi(schema)}.session_finalizations.coverage
+         ELSE COALESCE(NULLIF(EXCLUDED.coverage, '{}'::jsonb), ${qi(schema)}.session_finalizations.coverage)
+       END,
        error = CASE
          WHEN ${preserveTerminal}
          THEN ${qi(schema)}.session_finalizations.error
@@ -507,12 +571,18 @@ async function upsertSessionFinalization(pool, input = {}, { schema, tenantId: d
       input.scopeKey || null,
       input.contextKey || null,
       input.topicKey || null,
+      input.scopeId || input.scope_id || null,
+      toJson(input.scopeSnapshot || input.scope_snapshot, {}),
       input.summaryRowId || null,
       toJson(input.memoryResult, {}),
       input.summaryText || null,
       toJson(input.structuredSummary, {}),
       input.humanReviewText || null,
       input.sessionStartText || null,
+      toJson(candidateEnvelope, {}),
+      candidateEnvelopeHash,
+      candidateEnvelopeVersion,
+      toJson(coverage, {}),
       input.error || null,
       toJson(input.metadata, {}),
       input.claimedAt || (status === 'processing' ? new Date().toISOString() : null),
@@ -638,6 +708,33 @@ function candidateText(candidate = {}) {
   return '';
 }
 
+const EXPLICIT_EVIDENCE_PAYLOAD_KEYS = [
+  'evidenceText',
+  'evidence_text',
+  'evidenceExcerpt',
+  'evidence_excerpt',
+  'sourceText',
+  'source_text',
+  'quote',
+  'evidenceItems',
+  'evidence_items',
+  'evidenceTexts',
+  'evidence_texts',
+];
+
+function candidatePayload(candidate = {}) {
+  if (!candidate || typeof candidate !== 'object') return {};
+  const base = candidate.payload && typeof candidate.payload === 'object'
+    ? { ...candidate.payload }
+    : { ...candidate };
+  for (const key of EXPLICIT_EVIDENCE_PAYLOAD_KEYS) {
+    if (candidate[key] !== undefined && base[key] === undefined) {
+      base[key] = candidate[key];
+    }
+  }
+  return base;
+}
+
 async function upsertFinalizationCandidates(pool, rows = [], input = {}, { schema, tenantId: defaultTenantId } = {}) {
   if (!Array.isArray(rows) || rows.length === 0) return [];
   requireField(input, 'finalizationId');
@@ -649,14 +746,21 @@ async function upsertFinalizationCandidates(pool, rows = [], input = {}, { schem
     const memory = row.memory || {};
     const backingFact = row.backingFact || {};
     const evidenceRefs = candidate.evidenceRefs || candidate.evidence_refs || [];
+    const candidateHash = row.candidateHash || row.candidate_hash || hashStable({
+      action: row.action || 'skipped',
+      reason: row.reason || null,
+      memoryType: candidate.memoryType || candidate.memory_type || memory.memory_type || memory.memoryType || null,
+      canonicalKey: candidate.canonicalKey || candidate.canonical_key || memory.canonical_key || memory.canonicalKey || null,
+      payload: candidatePayload(candidate),
+    });
     const result = await pool.query(
       `INSERT INTO ${qi(schema)}.finalization_candidates (
          tenant_id, finalization_id, session_id, candidate_index, action, reason,
          memory_type, canonical_key, summary, payload, provenance,
-         memory_record_id, fact_assertion_id
+         memory_record_id, fact_assertion_id, candidate_hash
        )
        VALUES (
-         $1,$2,$3,$4,$5,$6,$7,$8,$9,COALESCE($10::jsonb,'{}'::jsonb),COALESCE($11::jsonb,'{}'::jsonb),$12,$13
+         $1,$2,$3,$4,$5,$6,$7,$8,$9,COALESCE($10::jsonb,'{}'::jsonb),COALESCE($11::jsonb,'{}'::jsonb),$12,$13,$14
        )
        ON CONFLICT (tenant_id, finalization_id, candidate_index)
        DO UPDATE SET
@@ -669,6 +773,7 @@ async function upsertFinalizationCandidates(pool, rows = [], input = {}, { schem
          provenance = COALESCE(NULLIF(EXCLUDED.provenance, '{}'::jsonb), ${qi(schema)}.finalization_candidates.provenance),
          memory_record_id = COALESCE(EXCLUDED.memory_record_id, ${qi(schema)}.finalization_candidates.memory_record_id),
          fact_assertion_id = COALESCE(EXCLUDED.fact_assertion_id, ${qi(schema)}.finalization_candidates.fact_assertion_id),
+         candidate_hash = COALESCE(EXCLUDED.candidate_hash, ${qi(schema)}.finalization_candidates.candidate_hash),
          updated_at = now()
        RETURNING *`,
       [
@@ -681,10 +786,11 @@ async function upsertFinalizationCandidates(pool, rows = [], input = {}, { schem
         candidate.memoryType || candidate.memory_type || memory.memory_type || memory.memoryType || null,
         candidate.canonicalKey || candidate.canonical_key || memory.canonical_key || memory.canonicalKey || null,
         candidateText(candidate) || candidateText(memory) || null,
-        toJson(candidate.payload || candidate, {}),
+        toJson(candidatePayload(candidate), {}),
         toJson({ evidenceRefs }, {}),
         memory.id || memory.memory_id || null,
         backingFact.id || memory.backing_fact_id || null,
+        candidateHash,
       ]
     );
     out.push(result.rows[0] || null);
@@ -839,6 +945,7 @@ async function searchTurnEmbeddings(pool, {
     params.push(source);
     where.push(`s.source = $${params.length}`);
   }
+  where.push(`NOT ${publicPlaceholderSummarySql('ss')}`);
 
   params.push(`[${queryVec.join(',')}]`);
   const vecPos = params.length;
@@ -945,6 +1052,7 @@ async function searchSummaryEmbeddings(pool, {
     params.push(candidateSessionIds);
     where.push(`s.session_id = ANY($${params.length})`);
   }
+  where.push(`NOT ${publicPlaceholderSummarySql('ss')}`);
 
   params.push(limit);
 
@@ -1130,6 +1238,11 @@ module.exports = {
   getSessionFinalization,
   updateSessionFinalizationStatus,
   listSessionFinalizations,
+  upsertCheckpointRun,
+  updateCheckpointRunStatus,
+  listCheckpointRuns,
+  upsertCheckpointRunSources,
+  listCheckpointRunSources,
   upsertFinalizationCandidates,
   extractUserTurns,
   upsertTurnEmbeddings,

package/docs/getting-started.md

@@ -83,9 +83,15 @@ For first rollout, keep `AQUIFER_MEMORY_SERVING_MODE=legacy`. Switch to `curated
 | Start MCP server | `npx aquifer mcp` |
 | Search memory | `npx aquifer recall "auth middleware"` |
 | Plan curated compaction | `npx aquifer compact --cadence daily --period-start 2026-04-27T00:00:00Z --period-end 2026-04-28T00:00:00Z` |
+| Generate a timer synthesis prompt | `npx aquifer operator compaction daily --include-synthesis-prompt --json` |
+| Apply reviewed timer synthesis candidates | `npx aquifer operator compaction daily --synthesis-summary-file /tmp/timer-summary.json --apply --promote-candidates --json` |
 | Show stats | `npx aquifer stats` |
 | Enrich pending sessions | `npx aquifer backfill` |
 
+Timer synthesis is an operator-reviewed candidate workflow. The prompt output
+and summary JSON do not become active curated memory unless the apply step is
+run with `--promote-candidates`.
+
 The default public serving mode is `legacy`. To test scoped curated memory serving, set `AQUIFER_MEMORY_SERVING_MODE=curated` plus `AQUIFER_MEMORY_ACTIVE_SCOPE_KEY` or `AQUIFER_MEMORY_ACTIVE_SCOPE_PATH`. Rollback is config-only: set the serving mode back to `legacy` and restart the MCP/CLI process.
 
 ## If something fails

package/docs/setup.md

@@ -51,10 +51,25 @@ Aquifer reads configuration from three sources (in priority order):
 
 Default public serving mode is `legacy`. Opt into `curated` only when you want `session_recall` and `session_bootstrap` to read active curated memory. `evidence_recall` remains the explicit audit/debug lane in both modes, and rollback is just setting env or config back to `legacy`.
 
+Backend profiles are explicit. `postgres` is the full backend and remains required for semantic recall, migrations, curated memory, and operator workflows. `local` is a zero-config starter profile with JSON-file persistence, raw session writes, lexical recall, bootstrap, stats, and export. It is intentionally degraded and does not create embeddings or run operator workflows:
+
+```bash
+AQUIFER_BACKEND=local npx aquifer backend-info --json
+```
+
 ### Example config file
 
 ```json
 {
+  "storage": {
+    "backend": "postgres",
+    "postgres": {
+      "url": "postgresql://aquifer:aquifer@localhost:5432/aquifer"
+    },
+    "local": {
+      "path": ".aquifer/aquifer.local.json"
+    }
+  },
   "db": {
     "url": "postgresql://aquifer:aquifer@localhost:5432/aquifer"
   },
@@ -74,6 +89,7 @@ Default public serving mode is `legacy`. Opt into `curated` only when you want `
 
 ```bash
 export DATABASE_URL="postgresql://aquifer:aquifer@localhost:5432/aquifer"
+export AQUIFER_BACKEND="postgres"
 export AQUIFER_EMBED_BASE_URL="http://localhost:11434/v1"
 export AQUIFER_EMBED_MODEL="bge-m3"
 export AQUIFER_MEMORY_SERVING_MODE="legacy"
@@ -97,6 +113,12 @@ export AQUIFER_MEMORY_SERVING_MODE="legacy"
 # export AQUIFER_MEMORY_SERVING_MODE="curated"
 # export AQUIFER_MEMORY_ACTIVE_SCOPE_KEY="project:aquifer"
 # export AQUIFER_MEMORY_ACTIVE_SCOPE_PATH="global,project:aquifer"
+
+# Optional Codex active-session checkpoint heartbeat policy.
+# Command flags still take precedence over these env vars.
+# export AQUIFER_CODEX_CHECKPOINT_CHECK_INTERVAL_MINUTES="10"
+# export AQUIFER_CODEX_CHECKPOINT_EVERY_MESSAGES="20"
+# export AQUIFER_CODEX_CHECKPOINT_QUIET_MS="3000"
 ```
 
 Copy `.env.example` from the repo root for a full annotated list.
@@ -213,6 +235,45 @@ Do **not** use the OpenClaw plugin (`consumers/openclaw-plugin.js`) for tool del
 
 Curated serving rollback is config-only: set `AQUIFER_MEMORY_SERVING_MODE=legacy` and restart the MCP/CLI process. No destructive database rollback is required.
 
+## Operator compaction and timer synthesis
+
+Compaction jobs are operator-safe by default. A dry-run plans lifecycle updates
+and candidate output without writing active memory:
+
+```bash
+npx aquifer operator compaction daily --include-synthesis-prompt --json
+```
+
+If an operator or external model reviews that prompt and returns timer synthesis
+JSON, attach it back to the plan with:
+
+```bash
+npx aquifer operator compaction daily \
+  --synthesis-summary-file /tmp/timer-summary.json \
+  --apply \
+  --promote-candidates \
+  --json
+```
+
+The summary file must match the normal structured summary shape, for example:
+
+```json
+{
+  "summaryText": "Reviewed timer synthesis.",
+  "structuredSummary": {
+    "states": [
+      { "state": "The reviewed state that should continue into current memory." }
+    ],
+    "decisions": [],
+    "open_loops": []
+  }
+}
+```
+
+Without `--promote-candidates`, synthesis output is recorded as candidate
+ledger material only. The prompt and summary file are producer material; active
+curated memory still requires the explicit promotion gate.
+
 ## Release verification gates
 
 For the publish-surface checks:
@@ -222,13 +283,15 @@ node --test test/package-surface.test.js test/mcp-manifest.test.js
 npm pack --dry-run --json
 ```
 
-For the real DB-backed MCP integration gate:
+For the real DB-backed release gate:
 
 ```bash
-AQUIFER_TEST_DB_URL="postgresql://..." node --test test/consumer-mcp.integration.test.js
+AQUIFER_TEST_DB_URL="postgresql://..." npm run test:release:db
 ```
 
-That DB-backed test is the release proof that the stdio MCP server, current MCP manifest, and PostgreSQL path still line up on a live database.
+That DB-backed test is the release proof that the stdio MCP server, CLI
+consumer, Codex finalization serving path, current MCP manifest, and PostgreSQL
+path still line up on a live database.
 
 ## Troubleshooting
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@shadowforge0/aquifer-memory",
-  "version": "1.6.0",
+  "version": "1.8.0",
   "description": "PG-native long-term memory for AI agents. Turn-level embedding, hybrid RRF ranking, optional knowledge graph. MCP server, CLI, and library API.",
   "main": "index.js",
   "files": [
@@ -15,6 +15,8 @@
     "consumers/mcp.js",
     "consumers/claude-code.js",
     "consumers/codex.js",
+    "consumers/codex-active-checkpoint.js",
+    "consumers/codex-current-memory.js",
     "consumers/codex-handoff.js",
     "consumers/openclaw-plugin.js",
     "consumers/opencode.js",
@@ -26,6 +28,8 @@
     "docs/setup.md",
     ".env.example",
     "scripts/backfill-canonical-key.js",
+    "scripts/codex-checkpoint-commands.js",
+    "scripts/codex-checkpoint-runtime.js",
     "scripts/diagnose-fts-zh.js",
     "scripts/diagnose-vector.js",
     "scripts/codex-recovery.js",
@@ -67,9 +71,9 @@
   "scripts": {
     "test": "node --test test/*.test.js",
     "test:integration": "node --test test/integration.test.js",
-    "test:release:package": "node --test test/package-surface.test.js test/mcp-manifest.test.js",
-    "test:release:db": "node -e \"if (!process.env.AQUIFER_TEST_DB_URL) { console.error('AQUIFER_TEST_DB_URL is required for test:release:db'); process.exit(1); }\" && node --test test/consumer-mcp.integration.test.js test/consumer-cli.integration.test.js test/codex-finalization-serving.integration.test.js",
-    "lint": "eslint index.js core/*.js consumers/cli.js consumers/mcp.js consumers/claude-code.js consumers/codex.js consumers/codex-handoff.js consumers/openclaw-plugin.js consumers/opencode.js consumers/shared/*.js consumers/default/*.js consumers/default/prompts/*.js consumers/openclaw-ext/*.js pipeline/*.js pipeline/consolidation/*.js scripts/*.js test/*.js",
+    "test:release:package": "node --test test/package-surface.test.js test/mcp-manifest.test.js test/local-backend.test.js test/scope-attribution.test.js test/v1-checkpoint-ledger-schema.test.js test/v1-finalization-envelope-schema.test.js test/v1-evidence-items.test.js test/v1-curated-semantic-recall.test.js test/session-checkpoints.test.js test/session-checkpoint-producer.test.js test/session-checkpoint-planner.test.js test/storage-checkpoint-ranges.test.js test/v1-serving-cutover.test.js test/v1-current-memory-contract.test.js test/v1-scope-inheritance.golden.test.js test/v1-bootstrap-determinism.test.js test/consumer-codex.test.js test/codex-recovery-script.test.js test/codex-handoff.test.js",
+    "test:release:db": "node -e \"if (!process.env.AQUIFER_TEST_DB_URL) { console.error('AQUIFER_TEST_DB_URL is required for test:release:db'); process.exit(1); }\" && node --test test/v1-evidence-items.test.js test/consumer-mcp.integration.test.js test/consumer-cli.integration.test.js test/codex-finalization-serving.integration.test.js",
+    "lint": "eslint index.js core/*.js core/backends/*.js consumers/cli.js consumers/mcp.js consumers/claude-code.js consumers/codex.js consumers/codex-active-checkpoint.js consumers/codex-current-memory.js consumers/codex-handoff.js consumers/openclaw-plugin.js consumers/opencode.js consumers/shared/*.js consumers/default/*.js consumers/default/prompts/*.js consumers/openclaw-ext/*.js pipeline/*.js pipeline/consolidation/*.js scripts/*.js test/*.js",
     "hooks:install": "git config core.hooksPath .githooks"
   },
   "dependencies": {