@shadowforge0/aquifer-memory 1.9.0 → 1.9.1

package/core/aquifer.js

@@ -14,11 +14,243 @@ const { createMemoryServingRuntime } = require('./memory-serving');
 const { createLegacyBootstrap } = require('./legacy-bootstrap');
 const { buildRerankDocument, resolveEmbedFn, shouldAutoRerank } = require('./recall-runtime');
 const { filterPublicPlaceholderSessionRows } = require('./public-session-filter');
+const { buildBacklogProductStatus, buildReadinessSurface } = require('./interface');
 
 // ---------------------------------------------------------------------------
 // createAquifer
 // ---------------------------------------------------------------------------
 
+const ACTIONABLE_PENDING_STATUSES = new Set(['pending', 'failed']);
+const TERMINAL_FINALIZATION_STATUSES = ['finalized', 'skipped', 'declined', 'deferred'];
+
+function normalizePendingWorkLimit(value, fallback = 100, max = 500) {
+  const parsed = parseInt(value, 10);
+  if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
+  return Math.min(parsed, max);
+}
+
+function normalizePendingWorkStatuses(value) {
+  const input = Array.isArray(value)
+    ? value
+    : String(value || '').split(',');
+  const statuses = input.map(item => String(item || '').trim()).filter(Boolean);
+  const normalized = statuses.length > 0 ? statuses : Array.from(ACTIONABLE_PENDING_STATUSES);
+  for (const status of normalized) {
+    if (!ACTIONABLE_PENDING_STATUSES.has(status)) {
+      throw new Error(`pending work status must be one of: ${Array.from(ACTIONABLE_PENDING_STATUSES).join(', ')}`);
+    }
+  }
+  return [...new Set(normalized)];
+}
+
+function normalizePendingWorkAction(value) {
+  const action = String(value || 'inspect').trim();
+  if (!['inspect', 'backfill', 'skip'].includes(action)) {
+    throw new Error('pending work action must be inspect, backfill, or skip');
+  }
+  return action;
+}
+
+function pluralize(count, singular, plural = `${singular}s`) {
+  return count === 1 ? singular : plural;
+}
+
+function normalizePendingWorkRow(row = {}) {
+  return {
+    sessionId: row.session_id,
+    sessionKey: row.session_key || null,
+    source: row.source || 'api',
+    agentId: row.agent_id,
+    status: row.processing_status,
+    processingError: row.processing_error || null,
+    startedAt: row.started_at || null,
+    lastMessageAt: row.last_message_at || null,
+    msgCount: row.msg_count || 0,
+    userCount: row.user_count || 0,
+    model: row.model || null,
+  };
+}
+
+function normalizePendingWorkGroup(row = {}) {
+  return {
+    source: row.source || 'api',
+    agentId: row.agent_id,
+    status: row.processing_status,
+    count: row.count || 0,
+    earliest: row.earliest || null,
+    latest: row.latest || null,
+    msgCount: row.msg_count || 0,
+    userCount: row.user_count || 0,
+  };
+}
+
+function buildPendingWorkGuidance(groups = []) {
+  const statuses = new Set(groups.map(group => group.status));
+  const guidance = [];
+  if (statuses.has('pending')) {
+    guidance.push({
+      status: 'pending',
+      recommendedAction: 'backfill',
+      alternateAction: 'skip',
+      reason: 'Pending sessions can be enriched; skip is reserved for deterministic policy suppression.',
+    });
+  }
+  if (statuses.has('failed')) {
+    guidance.push({
+      status: 'failed',
+      recommendedAction: 'backfill',
+      alternateAction: 'inspect_error',
+      reason: 'Failed sessions are retryable work; session skip only mutates rows still in pending status.',
+    });
+  }
+  return guidance;
+}
+
+function buildPendingWorkPlan(samples = [], action = 'inspect', groups = []) {
+  const totalByStatus = {};
+  for (const group of groups || []) {
+    totalByStatus[group.status] = (totalByStatus[group.status] || 0) + (group.count || 0);
+  }
+  const plan = {
+    action,
+    dryRunOnly: true,
+    allowed: 0,
+    blocked: 0,
+    changes: [],
+    notes: [],
+  };
+  if (action === 'inspect') {
+    plan.notes.push('No lifecycle changes planned.');
+    return plan;
+  }
+  if (action === 'backfill') {
+    plan.allowed = Object.values(totalByStatus).reduce((sum, count) => sum + count, 0);
+  } else if (action === 'skip') {
+    plan.allowed = totalByStatus.pending || 0;
+    plan.blocked = totalByStatus.failed || 0;
+  }
+  for (const row of samples) {
+    if (action === 'backfill') {
+      plan.changes.push({
+        sessionId: row.sessionId,
+        source: row.source,
+        agentId: row.agentId,
+        status: row.status,
+        operation: 'enrich',
+        allowed: true,
+      });
+      continue;
+    }
+    if (action === 'skip' && row.status === 'pending') {
+      plan.changes.push({
+        sessionId: row.sessionId,
+        source: row.source,
+        agentId: row.agentId,
+        status: row.status,
+        operation: 'mark_session_skipped',
+        allowed: true,
+      });
+      continue;
+    }
+    plan.changes.push({
+      sessionId: row.sessionId,
+      source: row.source,
+      agentId: row.agentId,
+      status: row.status,
+      operation: action,
+      allowed: false,
+      reason: action === 'skip'
+        ? 'skip only applies to pending sessions; failed sessions should be retried or suppressed through an explicit terminal policy'
+        : 'unsupported action',
+    });
+  }
+  if (action === 'skip' && plan.blocked > 0) {
+    plan.notes.push(`${plan.blocked} failed row(s) are intentionally blocked from skip. Filter to --status pending before applying skip.`);
+  }
+  return plan;
+}
+
+function buildPendingWorkDecision(groups = []) {
+  const statusCounts = {};
+  for (const group of groups || []) {
+    statusCounts[group.status] = (statusCounts[group.status] || 0) + (group.count || 0);
+  }
+  const publicStatus = buildBacklogProductStatus({ statusCounts });
+  return {
+    status: publicStatus.code === 'clear'
+      ? 'clear'
+      : publicStatus.code === 'recoverable'
+        ? 'recoverable'
+        : 'attention',
+    summary: publicStatus.headline,
+    nextStep: publicStatus.action,
+    statusCounts,
+    publicStatus,
+  };
+}
+
+function buildReadiness(stats = {}) {
+  const servingMode = stats.serving?.mode || 'legacy';
+  const activeScopePath = Array.isArray(stats.serving?.activeScopePath)
+    ? stats.serving.activeScopePath
+    : [];
+  const memoryRecords = stats.memoryRecords || {};
+  const pendingTotal = stats.pendingSessions?.total || 0;
+  const hasHistoricalMemory = (stats.summaries || 0) > 0 || (stats.turnEmbeddings || 0) > 0;
+  const currentEnabled = servingMode === 'curated';
+  const activeCurrentRows = memoryRecords.available ? (memoryRecords.active || 0) : 0;
+  const currentReady = currentEnabled && activeCurrentRows > 0;
+  const currentEmpty = currentEnabled && activeCurrentRows === 0;
+  const activeScopeReady = currentEnabled && activeScopePath.length > 0 && !(activeScopePath.length === 1 && activeScopePath[0] === 'global');
+
+  const checks = [
+    {
+      key: 'backend',
+      status: 'ready',
+      label: 'Service',
+      message: 'Aquifer is online.',
+    },
+    {
+      key: 'historical_memory',
+      status: hasHistoricalMemory ? 'ready' : 'empty',
+      label: 'Saved content',
+      message: hasHistoricalMemory
+        ? 'Saved content is available.'
+        : 'No saved content is available yet.',
+    },
+    {
+      key: 'current_memory',
+      status: currentReady ? 'ready' : currentEmpty ? 'empty' : 'not_enabled',
+      label: 'Memory',
+      message: currentReady
+        ? `Memory has ${activeCurrentRows} active row(s).`
+        : currentEmpty
+          ? 'Memory is enabled, but no active rows are available yet.'
+          : 'Memory is still getting ready.',
+    },
+    {
+      key: 'active_scope',
+      status: activeScopeReady ? 'ready' : currentEnabled ? 'attention' : 'not_enabled',
+      label: 'App link',
+      message: activeScopeReady
+        ? `Memory is linked through ${activeScopePath.join(' > ')}.`
+        : currentEnabled
+          ? 'Memory is not linked to this app yet.'
+          : 'Memory link is not active yet.',
+    },
+    {
+      key: 'backlog',
+      status: pendingTotal > 0 ? 'attention' : 'ready',
+      label: 'Saved content',
+      message: pendingTotal > 0
+        ? `${pendingTotal} saved-content ${pluralize(pendingTotal, 'item')} still need attention.`
+        : 'Saved content is ready.',
+    },
+  ];
+
+  return buildReadinessSurface(stats, { checks });
+}
+
 function createAquifer(config = {}) {
   const backendKind = normalizeBackendKind(config.backend?.kind || config.storage?.backend || 'postgres');
   if (backendKind !== 'postgres') {
@@ -681,7 +913,10 @@ function createAquifer(config = {}) {
       const [lexicalRows, embeddingRows] = await Promise.all([
         runLexical ? aquifer.memory.recall(query, {
           ...scopedOpts,
-          ftsConfig: migrationRuntime.getFtsConfig(),
+          // memory_records.search_tsv is generated with the stable simple config
+          // in schema/007, so curated memory_recall must query with the same
+          // config even when legacy session search selected a zh tokenizer.
+          ftsConfig: 'simple',
         }) : Promise.resolve([]),
         runVector ? aquifer.memory.recallViaMemoryEmbeddings(queryVec, scopedOpts) : Promise.resolve([]),
       ]);
@@ -1163,22 +1398,24 @@ function createAquifer(config = {}) {
     async skip(sessionId, opts = {}) {
       const agentId = opts.agentId || 'agent';
       const reason = opts.reason || null;
+      const source = opts.source || null;
       // Atomic CAS: only skip if still pending (avoids race with concurrent enrich)
       const result = await pool.query(
         `UPDATE ${qi(schema)}.sessions
         SET processing_status = 'skipped', processing_error = $1
         WHERE session_id = $2 AND agent_id = $3 AND tenant_id = $4
+          AND ($5::text IS NULL OR source = $5)
           AND processing_status = 'pending'
         RETURNING id`,
-        [reason, sessionId, agentId, tenantId]
+        [reason, sessionId, agentId, tenantId, source]
       );
       if (result.rows.length === 0) {
         // Check if session exists at all
-        const existing = await storage.getSession(pool, sessionId, agentId, {}, { schema, tenantId });
+        const existing = await storage.getSession(pool, sessionId, agentId, { source: source || undefined }, { schema, tenantId });
         if (!existing) throw new Error(`Session not found: ${sessionId} (agentId=${agentId})`);
         return null; // exists but not pending — no-op
       }
-      return { id: result.rows[0].id, sessionId, agentId, status: 'skipped' };
+      return { id: result.rows[0].id, sessionId, agentId, source, status: 'skipped' };
     },
 
     // --- public config accessor ---
@@ -1361,7 +1598,7 @@ function createAquifer(config = {}) {
         /* session_finalizations table may not exist on older installs */
       }
 
-      return {
+      const stats = {
         backendKind,
         backendProfile: backendInfo.profile,
         serving: {
@@ -1381,17 +1618,145 @@ function createAquifer(config = {}) {
         earliest: timeRange.rows[0]?.earliest || null,
         latest: timeRange.rows[0]?.latest || null,
       };
+      return {
+        ...stats,
+        readiness: buildReadiness(stats),
+      };
+    },
+
+    async getPendingWork(opts = {}) {
+      const limit = normalizePendingWorkLimit(opts.limit, 100, 500);
+      const statuses = normalizePendingWorkStatuses(opts.statuses || opts.status);
+      const action = normalizePendingWorkAction(opts.action || opts.plan);
+      const filters = {
+        source: opts.source || null,
+        agentId: opts.agentId || null,
+        statuses,
+        limit,
+      };
+
+      const params = [tenantId, statuses];
+      const where = [
+        's.tenant_id = $1',
+        's.processing_status = ANY($2::text[])',
+      ];
+      if (filters.source) {
+        params.push(filters.source);
+        where.push(`s.source = $${params.length}`);
+      }
+      if (filters.agentId) {
+        params.push(filters.agentId);
+        where.push(`s.agent_id = $${params.length}`);
+      }
+
+      const terminalExclusion = `NOT EXISTS (
+        SELECT 1
+        FROM ${qi(schema)}.session_finalizations f
+        WHERE f.tenant_id = s.tenant_id
+          AND f.session_id = s.session_id
+          AND f.agent_id = s.agent_id
+          AND f.source = s.source
+          AND f.status = ANY($${params.length + 1}::text[])
+      )`;
+      const runQueries = async (excludeTerminal = true) => {
+        const queryParams = excludeTerminal ? [...params, TERMINAL_FINALIZATION_STATUSES] : params;
+        const whereSql = [...where, ...(excludeTerminal ? [terminalExclusion] : [])].join(' AND ');
+        const limitParam = queryParams.length + 1;
+        const [groupsResult, samplesResult] = await Promise.all([
+          pool.query(
+            `SELECT
+               COALESCE(s.source, 'api') AS source,
+               s.agent_id,
+               s.processing_status,
+               COUNT(*)::int AS count,
+               MIN(s.started_at) AS earliest,
+               MAX(s.started_at) AS latest,
+               COALESCE(SUM(s.msg_count), 0)::int AS msg_count,
+               COALESCE(SUM(s.user_count), 0)::int AS user_count
+             FROM ${qi(schema)}.sessions s
+             WHERE ${whereSql}
+             GROUP BY COALESCE(s.source, 'api'), s.agent_id, s.processing_status
+             ORDER BY COUNT(*) DESC, latest DESC`,
+            queryParams,
+          ),
+          pool.query(
+            `SELECT
+               s.session_id,
+               s.session_key,
+               COALESCE(s.source, 'api') AS source,
+               s.agent_id,
+               s.processing_status,
+               s.processing_error,
+               s.started_at,
+               s.last_message_at,
+               s.msg_count,
+               s.user_count,
+               s.model
+             FROM ${qi(schema)}.sessions s
+             WHERE ${whereSql}
+             ORDER BY s.started_at DESC
+             LIMIT $${limitParam}`,
+            [...queryParams, limit],
+          ),
+        ]);
+        return { groupsResult, samplesResult, terminalFiltering: excludeTerminal };
+      };
+
+      let result;
+      try {
+        result = await runQueries(true);
+      } catch (err) {
+        if (err?.code !== '42P01') throw err;
+        result = await runQueries(false);
+      }
+
+      const groups = result.groupsResult.rows.map(normalizePendingWorkGroup);
+      const samples = result.samplesResult.rows.map(normalizePendingWorkRow);
+      const total = groups.reduce((sum, group) => sum + group.count, 0);
+      const decision = buildPendingWorkDecision(groups);
+      return {
+        available: true,
+        generatedAt: new Date().toISOString(),
+        terminalFiltering: result.terminalFiltering,
+        filters,
+        total,
+        status: decision.status,
+        statusCounts: decision.statusCounts,
+        summary: decision.summary,
+        nextStep: decision.nextStep,
+        publicStatus: decision.publicStatus,
+        groups,
+        samples,
+        guidance: buildPendingWorkGuidance(groups),
+        plan: buildPendingWorkPlan(samples, action, groups),
+      };
     },
 
     async getPendingSessions(opts = {}) {
-      const limit = opts.limit !== undefined ? opts.limit : 100;
+      const limit = normalizePendingWorkLimit(opts.limit, 100, 500);
+      const statuses = normalizePendingWorkStatuses(opts.statuses || opts.status);
+      const params = [tenantId, statuses];
+      const where = [
+        's.tenant_id = $1',
+        's.processing_status = ANY($2::text[])',
+      ];
+      if (opts.source) {
+        params.push(opts.source);
+        where.push(`s.source = $${params.length}`);
+      }
+      if (opts.agentId) {
+        params.push(opts.agentId);
+        where.push(`s.agent_id = $${params.length}`);
+      }
       let result;
       try {
+        const terminalParam = params.length + 1;
+        const limitParam = params.length + 2;
         result = await pool.query(
-          `SELECT s.session_id, s.agent_id, s.processing_status
+          `SELECT s.session_id, s.session_key, s.source, s.agent_id, s.processing_status,
+                  s.processing_error, s.started_at, s.last_message_at, s.msg_count, s.user_count, s.model
            FROM ${qi(schema)}.sessions s
-           WHERE s.tenant_id = $1
-             AND s.processing_status IN ('pending', 'failed')
+           WHERE ${where.join(' AND ')}
              AND NOT EXISTS (
                SELECT 1
                FROM ${qi(schema)}.session_finalizations f
@@ -1399,22 +1764,23 @@ function createAquifer(config = {}) {
                  AND f.session_id = s.session_id
                  AND f.agent_id = s.agent_id
                  AND f.source = s.source
-                 AND f.status IN ('finalized', 'skipped', 'declined', 'deferred')
+                 AND f.status = ANY($${terminalParam}::text[])
              )
            ORDER BY s.started_at DESC
-           LIMIT $2`,
-          [tenantId, limit]
+           LIMIT $${limitParam}`,
+          [...params, TERMINAL_FINALIZATION_STATUSES, limit]
         );
       } catch (err) {
         if (err?.code !== '42P01') throw err;
+        const limitParam = params.length + 1;
         result = await pool.query(
-          `SELECT session_id, agent_id, processing_status
-          FROM ${qi(schema)}.sessions
-          WHERE tenant_id = $1
-            AND processing_status IN ('pending', 'failed')
+          `SELECT session_id, session_key, source, agent_id, processing_status,
+                  processing_error, started_at, last_message_at, msg_count, user_count, model
+          FROM ${qi(schema)}.sessions s
+          WHERE ${where.join(' AND ')}
           ORDER BY started_at DESC
-          LIMIT $2`,
-          [tenantId, limit]
+          LIMIT $${limitParam}`,
+          [...params, limit]
         );
       }
       return result.rows;
@@ -1499,8 +1865,7 @@ function createAquifer(config = {}) {
   aquifer.consolidation = createConsolidation({ pool, schema: qSchema, defaultTenantId: tenantId });
   aquifer.bundles = createBundles({ pool, schema: qSchema, defaultTenantId: tenantId });
   // entityState materialises in schema/005-entity-state-history.sql, gated on
-  // entitiesEnabled (it FK-references entities). Drop-clean — see
-  // scripts/drop-entity-state-history.sql.
+  // entitiesEnabled because it FK-references entities.
   aquifer.entityState = createEntityState({ pool, schema: qSchema, defaultTenantId: tenantId });
   // insights materialises in schema/006-insights.sql. No FK from elsewhere
   // into this table; DROP CASCADE is clean. See scripts/drop-insights.sql.

package/core/backends/local.js

@@ -3,6 +3,7 @@
 const fs = require('fs/promises');
 const path = require('path');
 const { backendCapabilities, unsupportedCapabilityError } = require('./capabilities');
+const { buildBacklogProductStatus, buildReadinessSurface } = require('../interface');
 
 function emptyStore() {
   const now = new Date().toISOString();
@@ -420,7 +421,7 @@ function createLocalAquifer(config = {}) {
         counts[status] = (counts[status] || 0) + 1;
       }
       const dates = sessions.map(s => s.startedAt).filter(Boolean).sort();
-      return {
+      const stats = {
         backendKind: 'local',
         backendProfile: capabilities.profile,
         serving: {
@@ -454,6 +455,64 @@ function createLocalAquifer(config = {}) {
         degraded: true,
         capabilities: capabilities.capabilities,
       };
+      return {
+        ...stats,
+        readiness: buildReadinessSurface(stats, {
+          checks: [
+            {
+              key: 'backend',
+              status: 'ready',
+              label: 'Service',
+              message: 'Aquifer is online.',
+            },
+            {
+              key: 'historical_memory',
+              status: sessions.length > 0 ? 'ready' : 'empty',
+              label: 'Saved content',
+              message: sessions.length > 0
+                ? 'Saved local content is available.'
+                : 'No saved content is available yet.',
+            },
+            {
+              key: 'current_memory',
+              status: 'not_supported',
+              label: 'Memory',
+              message: 'Complete memory requires full storage.',
+            },
+          ],
+        }),
+      };
+    },
+    async getPendingWork(opts = {}) {
+      const publicStatus = buildBacklogProductStatus({ statusCounts: {} });
+      return {
+        available: true,
+        generatedAt: new Date().toISOString(),
+        terminalFiltering: false,
+        filters: {
+          source: opts.source || null,
+          agentId: opts.agentId || null,
+          statuses: opts.status ? [String(opts.status)] : ['pending', 'failed'],
+          limit: Math.max(1, Math.min(500, opts.limit || 100)),
+        },
+        total: 0,
+        status: 'clear',
+        statusCounts: {},
+        summary: 'All saved content is ready.',
+        nextStep: 'No action required.',
+        publicStatus,
+        groups: [],
+        samples: [],
+        guidance: [],
+        plan: {
+          action: opts.action || opts.plan || 'inspect',
+          dryRunOnly: true,
+          allowed: 0,
+          blocked: 0,
+          changes: [],
+          notes: ['Local backend has no asynchronous pending/failed session backlog.'],
+        },
+      };
     },
     async getPendingSessions() {
       return [];