@shadowforge0/aquifer-memory 1.2.1 → 1.5.8

package/core/consolidation.js

@@ -0,0 +1,340 @@
+'use strict';
+
+// aq.consolidation.* — session-level multi-phase orchestration.
+//
+// Spec: aquifer-completion §3 consolidationOrchestration. State lives in
+// sessions.consolidation_phases JSONB keyed by phase name. 10 phases cover
+// the post-session pipeline: summary_extract, entity_extract, fact_extract,
+// fact_consolidation, narrative_refresh, decision_write, handoff_write,
+// session_state_write, timeline_write, artifact_dispatch.
+//
+// Status vocabulary: pending|claimed|running|succeeded|failed|skipped.
+// State transitions (enforced by transitionPhase):
+//   pending  → claimed
+//   claimed  → running|failed|skipped|claimed (stale reclaim only)
+//   running  → succeeded|failed|claimed (stale reclaim only)
+//   failed   → claimed (retry)
+//   succeeded|skipped → non-terminal requires forceReplay=true
+//
+// Advisory lock (pg_advisory_xact_lock on session_row_id) wraps the
+// read-modify-write in a transaction so two workers can't claim the same
+// session phase simultaneously.
+
+const crypto = require('crypto');
+const { AqError, ok, err } = require('./errors');
+
+const PHASES = Object.freeze([
+  'summary_extract',
+  'entity_extract',
+  'fact_extract',
+  'fact_consolidation',
+  'narrative_refresh',
+  'decision_write',
+  'handoff_write',
+  'session_state_write',
+  'timeline_write',
+  'artifact_dispatch',
+]);
+const PHASE_SET = new Set(PHASES);
+
+const STATUSES = Object.freeze([
+  'pending', 'claimed', 'running', 'succeeded', 'failed', 'skipped',
+]);
+const STATUS_SET = new Set(STATUSES);
+
+const TERMINAL = new Set(['succeeded', 'skipped']);
+
+// Valid transitions. Caller must specify fromStatus to guard against races.
+const VALID_TRANSITIONS = {
+  pending: new Set(['claimed']),
+  claimed: new Set(['running', 'failed', 'skipped', 'claimed']),
+  running: new Set(['succeeded', 'failed', 'claimed']),
+  failed: new Set(['claimed']),
+  succeeded: new Set([]),
+  skipped: new Set([]),
+};
+
+function toNumber(v) {
+  if (v === null || v === undefined) return null;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : null;
+}
+
+function emptyPhaseState() {
+  return { status: 'pending', attempts: 0 };
+}
+
+function fillDefaults(phasesJson) {
+  const out = { ...(phasesJson || {}) };
+  for (const phase of PHASES) {
+    if (!out[phase]) out[phase] = emptyPhaseState();
+  }
+  return out;
+}
+
+function isStale(phaseState, staleAfterSeconds) {
+  if (phaseState.status !== 'claimed' && phaseState.status !== 'running') return false;
+  const startedAt = phaseState.startedAt;
+  if (!startedAt) return true;
+  const ageMs = Date.now() - new Date(startedAt).getTime();
+  return ageMs > staleAfterSeconds * 1000;
+}
+
+function newClaimToken() {
+  return crypto.randomBytes(12).toString('hex');
+}
+
+function advisoryLockKey(sessionRowId) {
+  // Map bigint-ish id to signed int4 range for pg_advisory_xact_lock.
+  const id = Number(sessionRowId);
+  return (id ^ 0x9e3779b9) & 0x7fffffff;
+}
+
+function createConsolidation({ pool, schema, defaultTenantId }) {
+
+  async function claimNext(input = {}) {
+    try {
+      if (!input.workerId) return err('AQ_INVALID_INPUT', 'workerId is required');
+      const tenantId = input.tenantId || defaultTenantId || 'default';
+      const phases = Array.isArray(input.phases) && input.phases.length > 0
+        ? input.phases.filter(p => PHASE_SET.has(p))
+        : PHASES;
+      if (phases.length === 0) {
+        return err('AQ_INVALID_INPUT', 'phases filter produced empty list');
+      }
+      const staleAfterSeconds = Number.isFinite(input.staleAfterSeconds)
+        ? Math.max(10, input.staleAfterSeconds)
+        : 600;
+
+      // Look at candidate sessions with at least one non-terminal phase,
+      // then iterate under advisory lock to claim atomically.
+      const candidates = await pool.query(
+        `SELECT id AS session_row_id, session_id, agent_id, processing_status,
+                consolidation_phases
+           FROM ${schema}.sessions
+          WHERE tenant_id = $1
+          ORDER BY id ASC
+          LIMIT 200`,
+        [tenantId],
+      );
+
+      for (const row of candidates.rows) {
+        const current = fillDefaults(row.consolidation_phases);
+        let targetPhase = null;
+        for (const p of phases) {
+          const st = current[p];
+          if (st.status === 'pending' || st.status === 'failed') {
+            targetPhase = p; break;
+          }
+          if ((st.status === 'claimed' || st.status === 'running')
+              && isStale(st, staleAfterSeconds)) {
+            targetPhase = p; break;
+          }
+        }
+        if (!targetPhase) continue;
+
+        const client = await pool.connect();
+        try {
+          await client.query('BEGIN');
+          await client.query('SELECT pg_advisory_xact_lock($1)',
+            [advisoryLockKey(row.session_row_id)]);
+
+          // Re-read under lock — another worker may have claimed in between.
+          const { rows: freshRows } = await client.query(
+            `SELECT consolidation_phases FROM ${schema}.sessions WHERE id = $1`,
+            [row.session_row_id],
+          );
+          const fresh = fillDefaults(freshRows[0] && freshRows[0].consolidation_phases);
+          const freshState = fresh[targetPhase];
+          const eligible =
+            freshState.status === 'pending'
+            || freshState.status === 'failed'
+            || ((freshState.status === 'claimed' || freshState.status === 'running')
+                && isStale(freshState, staleAfterSeconds));
+          if (!eligible) {
+            await client.query('ROLLBACK');
+            client.release();
+            continue;
+          }
+
+          const claimToken = newClaimToken();
+          const attempts = (freshState.attempts || 0) + 1;
+          fresh[targetPhase] = {
+            ...freshState,
+            status: 'claimed',
+            claimToken,
+            workerId: input.workerId,
+            startedAt: new Date().toISOString(),
+            finishedAt: null,
+            attempts,
+            errorCode: null,
+            errorMessage: null,
+          };
+
+          await client.query(
+            `UPDATE ${schema}.sessions SET consolidation_phases = $1
+              WHERE id = $2`,
+            [JSON.stringify(fresh), row.session_row_id],
+          );
+          await client.query('COMMIT');
+          client.release();
+
+          return ok({
+            session: {
+              sessionRowId: toNumber(row.session_row_id),
+              sessionId: row.session_id,
+              agentId: row.agent_id,
+              processingStatus: row.processing_status,
+              phases: fresh,
+            },
+            claimToken,
+            claimedPhase: targetPhase,
+          });
+        } catch (e) {
+          await client.query('ROLLBACK').catch(() => {});
+          client.release();
+          throw e;
+        }
+      }
+
+      return ok({ session: null, claimToken: null, claimedPhase: null });
+    } catch (e) {
+      if (e instanceof AqError) return err(e);
+      return err('AQ_INTERNAL', e.message, { cause: e });
+    }
+  }
+
+  async function transitionPhase(input = {}) {
+    try {
+      if (!input.sessionId) return err('AQ_INVALID_INPUT', 'sessionId is required');
+      if (!input.phase || !PHASE_SET.has(input.phase)) {
+        return err('AQ_INVALID_INPUT', `phase must be one of ${PHASES.join(', ')}`);
+      }
+      if (!input.fromStatus || !STATUS_SET.has(input.fromStatus)) {
+        return err('AQ_INVALID_INPUT', 'valid fromStatus is required');
+      }
+      if (!input.toStatus || !STATUS_SET.has(input.toStatus)) {
+        return err('AQ_INVALID_INPUT', 'valid toStatus is required');
+      }
+      const tenantId = input.tenantId || defaultTenantId || 'default';
+
+      const sessionRow = await pool.query(
+        `SELECT id FROM ${schema}.sessions WHERE tenant_id = $1 AND session_id = $2`,
+        [tenantId, input.sessionId],
+      );
+      if (sessionRow.rowCount === 0) {
+        return err('AQ_NOT_FOUND', `session ${input.sessionId} not found`);
+      }
+      const sessionRowId = sessionRow.rows[0].id;
+
+      const client = await pool.connect();
+      try {
+        await client.query('BEGIN');
+        await client.query('SELECT pg_advisory_xact_lock($1)',
+          [advisoryLockKey(sessionRowId)]);
+
+        const { rows } = await client.query(
+          `SELECT consolidation_phases FROM ${schema}.sessions WHERE id = $1`,
+          [sessionRowId],
+        );
+        const phases = fillDefaults(rows[0] && rows[0].consolidation_phases);
+        const current = phases[input.phase];
+
+        // Guard: fromStatus must match current.
+        if (current.status !== input.fromStatus) {
+          await client.query('ROLLBACK');
+          return err('AQ_PHASE_CLAIM_CONFLICT',
+            `phase ${input.phase} currently ${current.status}, not ${input.fromStatus}`);
+        }
+
+        // Guard: claimToken must match when transitioning from claimed/running.
+        if ((input.fromStatus === 'claimed' || input.fromStatus === 'running')
+            && input.claimToken && current.claimToken !== input.claimToken) {
+          await client.query('ROLLBACK');
+          return err('AQ_PHASE_CLAIM_CONFLICT',
+            `claimToken mismatch for phase ${input.phase}`);
+        }
+
+        // Validate transition.
+        const allowed = VALID_TRANSITIONS[input.fromStatus] || new Set();
+        if (!allowed.has(input.toStatus)) {
+          // Terminal → non-terminal requires forceReplay.
+          const leavingTerminal = TERMINAL.has(input.fromStatus);
+          if (!(leavingTerminal && input.forceReplay === true)) {
+            await client.query('ROLLBACK');
+            return err('AQ_PHASE_TRANSITION_INVALID',
+              `cannot transition ${input.phase} from ${input.fromStatus} to ${input.toStatus}`);
+          }
+        }
+
+        const next = { ...current, status: input.toStatus };
+        if (input.toStatus === 'running') {
+          next.startedAt = current.startedAt || new Date().toISOString();
+        }
+        if (input.toStatus === 'succeeded' || input.toStatus === 'failed'
+            || input.toStatus === 'skipped') {
+          next.finishedAt = new Date().toISOString();
+          if (input.toStatus === 'succeeded' || input.toStatus === 'skipped') {
+            next.errorCode = null;
+            next.errorMessage = null;
+          }
+        }
+        if (input.toStatus === 'failed' && input.error) {
+          next.errorCode = input.error.code || 'AQ_INTERNAL';
+          next.errorMessage = input.error.message || '';
+        }
+        if (input.retryAfter) next.retryAfter = input.retryAfter;
+        if (input.idempotencyKey) next.idempotencyKey = input.idempotencyKey;
+        if (input.outputRef) next.outputRef = { ...(current.outputRef || {}), ...input.outputRef };
+
+        phases[input.phase] = next;
+
+        await client.query(
+          `UPDATE ${schema}.sessions SET consolidation_phases = $1 WHERE id = $2`,
+          [JSON.stringify(phases), sessionRowId],
+        );
+        await client.query('COMMIT');
+
+        return ok({
+          sessionId: input.sessionId,
+          phase: input.phase,
+          state: next,
+        });
+      } catch (e) {
+        await client.query('ROLLBACK').catch(() => {});
+        throw e;
+      } finally {
+        client.release();
+      }
+    } catch (e) {
+      if (e instanceof AqError) return err(e);
+      return err('AQ_INTERNAL', e.message, { cause: e });
+    }
+  }
+
+  async function getState(input = {}) {
+    try {
+      if (!input.sessionId) return err('AQ_INVALID_INPUT', 'sessionId is required');
+      const tenantId = input.tenantId || defaultTenantId || 'default';
+      const { rows } = await pool.query(
+        `SELECT processing_status, consolidation_phases
+           FROM ${schema}.sessions
+          WHERE tenant_id = $1 AND session_id = $2`,
+        [tenantId, input.sessionId],
+      );
+      if (rows.length === 0) {
+        return err('AQ_NOT_FOUND', `session ${input.sessionId} not found`);
+      }
+      return ok({
+        processingStatus: rows[0].processing_status,
+        phases: fillDefaults(rows[0].consolidation_phases),
+      });
+    } catch (e) {
+      return err('AQ_INTERNAL', e.message, { cause: e });
+    }
+  }
+
+  return { claimNext, transitionPhase, getState, PHASES, STATUSES };
+}
+
+module.exports = { createConsolidation, PHASES, STATUSES };

package/core/decisions.js

@@ -0,0 +1,164 @@
+'use strict';
+
+// aq.decisions.* — append-only decision log capability.
+//
+// Spec: aquifer-completion §9 decisionLog. status vocabulary
+// (proposed/committed/reversed) enforced both at API layer (fast reject)
+// and by DB CHECK constraint (defense in depth). reversal is implemented
+// by appending a new 'reversed' decision and optionally pointing
+// reversed_by_decision_id; Aquifer doesn't auto-compute the chain.
+
+const crypto = require('crypto');
+const { AqError, ok, err } = require('./errors');
+
+const DEFAULT_PROFILE = Object.freeze({
+  id: 'anon',
+  version: 0,
+  schemaHash: 'pending',
+});
+
+const VALID_STATUSES = new Set(['proposed', 'committed', 'reversed']);
+
+function resolveProfile(profile) {
+  if (!profile) return DEFAULT_PROFILE;
+  return {
+    id: profile.id || DEFAULT_PROFILE.id,
+    version: Number.isInteger(profile.version) ? profile.version : DEFAULT_PROFILE.version,
+    schemaHash: profile.schemaHash || DEFAULT_PROFILE.schemaHash,
+  };
+}
+
+function toNumber(v) {
+  if (v === null || v === undefined) return null;
+  const n = Number(v);
+  return Number.isFinite(n) ? n : null;
+}
+
+function defaultIdempotencyKey({ tenantId, agentId, sessionId, payload }) {
+  return crypto.createHash('sha256')
+    .update(`${tenantId}:${agentId}:${sessionId}:${JSON.stringify(payload)}`)
+    .digest('hex');
+}
+
+function mapRow(row) {
+  if (!row) return null;
+  return {
+    decisionId: toNumber(row.id),
+    sessionId: row.source_session_id,
+    agentId: row.agent_id,
+    status: row.status,
+    decisionText: row.decision_text,
+    reasonText: row.reason_text,
+    payload: row.payload || {},
+    metadata: row.metadata || {},
+    decidedAt: row.decided_at,
+    reversedByDecisionId: toNumber(row.reversed_by_decision_id),
+    createdAt: row.created_at,
+  };
+}
+
+function createDecisions({ pool, schema, defaultTenantId }) {
+  async function append(input) {
+    try {
+      if (!input || typeof input !== 'object') {
+        return err('AQ_INVALID_INPUT', 'append requires an input object');
+      }
+      if (!input.agentId) return err('AQ_INVALID_INPUT', 'agentId is required');
+      if (!input.sessionId) return err('AQ_INVALID_INPUT', 'sessionId is required');
+      if (!input.payload || typeof input.payload !== 'object') {
+        return err('AQ_INVALID_INPUT', 'payload is required');
+      }
+      const tenantId = input.tenantId || defaultTenantId || 'default';
+      const agentId = input.agentId;
+      const sessionId = input.sessionId;
+      const payload = input.payload;
+      const profile = resolveProfile(input.profile);
+
+      const status = payload.status || 'committed';
+      if (!VALID_STATUSES.has(status)) {
+        return err('AQ_INVALID_INPUT',
+          `status must be one of ${Array.from(VALID_STATUSES).join(', ')}`);
+      }
+      const decisionText = typeof payload.decision === 'string'
+        ? payload.decision
+        : (typeof payload.decision_text === 'string' ? payload.decision_text : null);
+      if (!decisionText) {
+        return err('AQ_INVALID_INPUT', 'payload.decision (or decision_text) is required');
+      }
+      const reasonText = typeof payload.reason === 'string'
+        ? payload.reason
+        : (typeof payload.reason_text === 'string' ? payload.reason_text : null);
+
+      const idempotencyKey = input.idempotencyKey
+        || defaultIdempotencyKey({ tenantId, agentId, sessionId, payload });
+
+      const insertResult = await pool.query(
+        `INSERT INTO ${schema}.decisions (
+           tenant_id, agent_id, source_session_id,
+           consumer_profile_id, consumer_profile_version, consumer_schema_hash,
+           idempotency_key, payload, status, decision_text, reason_text,
+           decided_at, metadata
+         ) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11,
+                   COALESCE($12::timestamptz, now()), $13)
+         ON CONFLICT (idempotency_key) DO NOTHING
+         RETURNING *`,
+        [
+          tenantId, agentId, sessionId,
+          profile.id, profile.version, profile.schemaHash,
+          idempotencyKey, JSON.stringify(payload), status,
+          decisionText, reasonText,
+          input.decidedAt || null,
+          JSON.stringify(payload.metadata || {}),
+        ],
+      );
+      let row = insertResult.rows[0];
+      if (!row) {
+        const existing = await pool.query(
+          `SELECT * FROM ${schema}.decisions WHERE idempotency_key = $1`,
+          [idempotencyKey],
+        );
+        row = existing.rows[0];
+      }
+      const mapped = mapRow(row);
+      return ok({ decisionId: mapped.decisionId, payload: mapped.payload });
+    } catch (e) {
+      if (e instanceof AqError) return err(e);
+      return err('AQ_INTERNAL', e.message, { cause: e });
+    }
+  }
+
+  async function list(input = {}) {
+    try {
+      if (!input.agentId) return err('AQ_INVALID_INPUT', 'agentId is required');
+      const tenantId = input.tenantId || defaultTenantId || 'default';
+      const limit = Math.min(Math.max(input.limit || 50, 1), 500);
+
+      const params = [tenantId, input.agentId];
+      let where = 'tenant_id = $1 AND agent_id = $2';
+      if (Array.isArray(input.statuses) && input.statuses.length > 0) {
+        params.push(input.statuses);
+        where += ` AND status = ANY($${params.length})`;
+      }
+      if (input.sessionId) {
+        params.push(input.sessionId);
+        where += ` AND source_session_id = $${params.length}`;
+      }
+      params.push(limit);
+
+      const { rows } = await pool.query(
+        `SELECT * FROM ${schema}.decisions
+          WHERE ${where}
+          ORDER BY decided_at DESC, id DESC
+          LIMIT $${params.length}`,
+        params,
+      );
+      return ok({ rows: rows.map(mapRow) });
+    } catch (e) {
+      return err('AQ_INTERNAL', e.message, { cause: e });
+    }
+  }
+
+  return { append, list };
+}
+
+module.exports = { createDecisions };