@graphorin/memory 0.6.0 → 0.7.0

package/src/consolidator/idempotency.ts

@@ -0,0 +1,27 @@
+/**
+ * Idempotency cursor helpers. The runtime reads the persisted state
+ * row directly via {@link ConsolidatorMemoryStoreExt.getState}; this
+ * file centralises the pure helpers around the cursor advance.
+ *
+ * @packageDocumentation
+ */
+
+import type { SessionMessageRecord } from '../internal/storage-adapter.js';
+
+/**
+ * Advance the cursor to the highest sequence number observed in the
+ * batch. Returns `null` when the batch is empty.
+ *
+ * @stable
+ */
+export function tipMessageId(batch: ReadonlyArray<SessionMessageRecord>): string | null {
+  if (batch.length === 0) return null;
+  let bestIdx = 0;
+  for (let i = 1; i < batch.length; i++) {
+    const candidate = batch[i];
+    const champion = batch[bestIdx];
+    if (candidate === undefined || champion === undefined) continue;
+    if (candidate.sequence > champion.sequence) bestIdx = i;
+  }
+  return batch[bestIdx]?.id ?? null;
+}

package/src/consolidator/index.ts

@@ -0,0 +1,269 @@
+/**
+ * `@graphorin/memory/consolidator` - Phase 10c runtime that turns
+ * raw conversation turns into long-lived facts and episodes. The
+ * module ships:
+ *
+ * - {@link createConsolidator} - the production factory that wires
+ *   triggers, the budget envelope, the wait-then-defer lock, the
+ *   idempotent cursor, the DLQ, and the three phases (light /
+ *   standard / deep) into a single `Consolidator` handle.
+ * - {@link createConsolidatorPlaceholder} - kept for back-compat
+ *   with consumers that wired the no-op stub during Phase 10a; it
+ *   honours the same shape but does no background work.
+ * - Pure helpers (`noise-filter`, `decay`, `budget`, `lock`, `dlq`,
+ *   `idempotency`, `triggers`) and the per-phase orchestrators -
+ *   exported so tests + the standalone server (Phase 14) can compose
+ *   them without re-implementing the algorithm.
+ *
+ * The `Consolidator` interface is **stable** - Phase 14 wires the
+ * runtime into the lifecycle hooks, and Phase 15 surfaces every
+ * method through `graphorin consolidator …` CLI commands.
+ *
+ * @packageDocumentation
+ */
+
+import { DEFAULT_SALIENCE_WEIGHTS } from './decay.js';
+import type { Consolidator } from './runtime.js';
+import type {
+  ConsolidatorConfig,
+  ConsolidatorPhase,
+  ConsolidatorStatus,
+  ConsolidatorTier,
+  ConsolidatorTriggerSpec,
+  PhaseListener,
+  PhaseOutcome,
+} from './types.js';
+
+export {
+  type BudgetCheck,
+  type BudgetSnapshot,
+  BudgetTracker,
+  type BudgetTrackerOptions,
+  bucketStart,
+  nextBucketStart,
+} from './budget.js';
+export {
+  DEFAULT_DECAY_ARCHIVE_THRESHOLD,
+  DEFAULT_DECAY_TAU_DAYS,
+  DEFAULT_SALIENCE_WEIGHTS,
+  NEUTRAL_IMPORTANCE,
+  retention,
+  type SalienceWeights,
+  salience,
+  selectForCapacityEviction,
+  shouldArchive,
+} from './decay.js';
+export {
+  classifyError,
+  describeError,
+  nextBackoffMs,
+} from './dlq.js';
+export {
+  BudgetExceededError,
+  CustomTierMisconfiguredError,
+  ProviderNotConfiguredError,
+} from './errors.js';
+export { tipMessageId } from './idempotency.js';
+export { LockManager, type LockManagerOptions, type LockOutcome } from './lock.js';
+export {
+  applyNoiseFilters,
+  type NoiseFilterPreset,
+  type NoiseFilterResult,
+} from './noise-filter.js';
+export { type DeepPhaseDeps, runDeepPhase } from './phases/deep.js';
+export {
+  buildInductionRequest,
+  checkSuccessCriteria,
+  createProviderWorkflowInducer,
+  DEFAULT_INDUCTION_MAX_TOKENS,
+  INDUCTION_SYSTEM_PROMPT,
+  type InducedProcedure,
+  MAX_PROCEDURE_STEPS,
+  MAX_TRAJECTORY_STEPS_SHOWN,
+  normalizeInducedProcedure,
+  parseInducedProcedure,
+  runWorkflowInduction,
+  type Trajectory,
+  type TrajectoryStep,
+  trajectoryFromRunState,
+  type VerificationResult,
+  type WorkflowInducer,
+  type WorkflowInductionOptions,
+} from './phases/induce.js';
+export {
+  type LightPhaseDeps,
+  runLightPhase,
+} from './phases/light.js';
+export {
+  parseExtraction,
+  runStandardPhase,
+  type StandardPhaseDeps,
+} from './phases/standard.js';
+export { type Consolidator, createConsolidator } from './runtime.js';
+export {
+  type ConsolidatorCatchupPolicy,
+  type RegisterTriggersOptions,
+  type RegisterTriggersResult,
+  registerConsolidatorTriggers,
+  type SchedulerLike,
+  type TriggerDeclarationLike,
+} from './scheduler.js';
+export {
+  type ParsedTrigger,
+  parseTriggerSpec,
+  reasonFromTrigger,
+} from './triggers.js';
+export type {
+  ConsolidatorBudgetSnapshot,
+  ConsolidatorCeilings,
+  ConsolidatorConfig,
+  ConsolidatorLastRuns,
+  ConsolidatorPhase,
+  ConsolidatorStatus,
+  ConsolidatorTier,
+  ConsolidatorTriggerReason,
+  ConsolidatorTriggerSpec,
+  CreateConsolidatorOptions,
+  OnBudgetExceed,
+  PhaseListener,
+  PhaseOutcome,
+} from './types.js';
+export { CONSOLIDATOR_TIER_DEFAULTS } from './types.js';
+
+/**
+ * Build a no-op consolidator that honours the full {@link Consolidator}
+ * interface but performs no background work. Useful for consumers
+ * that want the typed shape (e.g., unit tests of higher tiers) but
+ * do not pay the runtime cost of triggers / locking / DLQ.
+ *
+ * Phase 10c's {@link createConsolidator} is the production factory.
+ *
+ * @stable
+ */
+export function createConsolidatorPlaceholder(
+  options: {
+    readonly triggers?: ReadonlyArray<ConsolidatorTriggerSpec>;
+    readonly tier?: ConsolidatorTier;
+  } = {},
+): Consolidator {
+  let running = false;
+  let paused = false;
+  let tier: ConsolidatorTier = options.tier ?? 'free';
+  const triggers = Object.freeze([
+    // Matches the real runtime default (MCON-4): idle drives light/standard,
+    // the daily cron makes the deep phase reachable; turn:N is consumer-emitted
+    // so it is not a scheduler default.
+    ...(options.triggers ?? (['idle:5m', 'cron:0 4 * * *'] as ConsolidatorTriggerSpec[])),
+  ]) as ReadonlyArray<ConsolidatorTriggerSpec>;
+  const listeners = new Set<PhaseListener>();
+
+  const config: ConsolidatorConfig = Object.freeze({
+    triggers,
+    tier,
+    phases: Object.freeze(['light' as ConsolidatorPhase]),
+    ceilings: {
+      maxTokensPerDay: 0,
+      maxCostPerDay: 0,
+      maxConcurrentRuns: 1,
+      maxRunDurationMs: 5 * 60 * 1000,
+      cooldownMs: 60_000,
+    },
+    onExceed: 'pause',
+    cheapModel: null,
+    deepModel: null,
+    budgetResetSemantics: 'utc',
+    noiseFilters: Object.freeze(['default' as const]),
+    lockWaitMs: 30_000,
+    decayTauDays: 7,
+    decayArchiveThreshold: 0.05,
+    decayCapacity: null,
+    salienceWeights: DEFAULT_SALIENCE_WEIGHTS,
+    maxStandardBatchSize: 50,
+    maxTranscriptChars: 60_000,
+    maxDeepConflictsPerRun: 20,
+    dlqMaxRetries: 5,
+    dlqBaseBackoffMs: 60_000,
+    dlqMaxBackoffMs: 60 * 60 * 1000,
+    formEpisodes: false,
+    importanceScoring: false,
+    autoPromoteExtraction: false,
+    reflection: false,
+    importanceThreshold: 3,
+    reflectionMaxQuestions: 3,
+    reflectionMaxQuarantinedInsights: 100,
+    contextualRetrieval: 'late-chunk',
+    learnedContext: false,
+    learnedContextMaxChars: 1200,
+  });
+
+  const status = async (): Promise<ConsolidatorStatus> =>
+    Object.freeze({
+      tier,
+      triggers,
+      phases: config.phases,
+      running,
+      paused,
+      pendingConflicts: 0,
+      queueDepth: 0,
+      dlqSize: 0,
+      deferredRuns: 0,
+      emptyExtractions: 0,
+      lastRuns: Object.freeze({}),
+      budget: {
+        tokensUsedToday: 0,
+        costUsedToday: 0,
+        tokensRemaining: 0,
+        costRemaining: 0,
+        resetAt: new Date(Date.now() + 86_400_000).toISOString(),
+      },
+      budgetRemaining: {
+        tokens: 0,
+        costUsd: 0,
+      },
+    });
+
+  return {
+    async start() {
+      running = true;
+    },
+    async stop() {
+      running = false;
+    },
+    async trigger() {
+      return null;
+    },
+    async fireNow(): Promise<PhaseOutcome | null> {
+      return null;
+    },
+    async setTier(next: ConsolidatorTier) {
+      tier = next;
+    },
+    async pause() {
+      paused = true;
+    },
+    async resume() {
+      paused = false;
+    },
+    recordExternalSpend(): void {
+      // Placeholder - no budget to record against.
+    },
+    onPhaseFinished(listener) {
+      listeners.add(listener);
+      return () => listeners.delete(listener);
+    },
+    config() {
+      return config;
+    },
+    async registerWithScheduler() {
+      // No-op consolidator: nothing to schedule.
+      return Object.freeze({ registered: Object.freeze([]), skipped: Object.freeze([]) });
+    },
+    isFree() {
+      return tier === 'free';
+    },
+    async drainDlq() {
+      return 0;
+    },
+    status,
+  };
+}

package/src/consolidator/lock.ts

@@ -0,0 +1,141 @@
+/**
+ * Wait-then-defer lock manager. The lock state lives in the storage
+ * layer's `consolidator_state` row so concurrent processes (lib +
+ * server) coordinate through SQLite. The manager waits up to
+ * `lockWaitMs` for the lock to clear; if the wait expires it returns
+ * `'deferred'` so the trigger is recorded but no phase runs.
+ *
+ * The lock honours DEC-114 ("memory does not block the agent loop")
+ * by capping the wait - the agent loop never observes a memory
+ * blocker.
+ *
+ * @packageDocumentation
+ */
+
+import type { SessionScope } from '@graphorin/core';
+import type { ConsolidatorMemoryStoreExt } from '../internal/storage-adapter.js';
+
+/** Outcome surfaced by {@link LockManager.acquire}. */
+export type LockOutcome =
+  | { readonly kind: 'acquired'; readonly runId: string }
+  | { readonly kind: 'deferred'; readonly heldBy: string | null };
+
+/** Options accepted by {@link LockManager}. */
+export interface LockManagerOptions {
+  readonly store: ConsolidatorMemoryStoreExt | null;
+  readonly waitMs: number;
+  readonly maxRunDurationMs: number;
+  readonly now?: () => number;
+  readonly sleep?: (ms: number) => Promise<void>;
+  readonly randomId?: () => string;
+}
+
+const DEFAULT_POLL_MS = 100;
+
+/** @stable */
+export class LockManager {
+  readonly #store: ConsolidatorMemoryStoreExt | null;
+  readonly #waitMs: number;
+  readonly #maxAge: number;
+  readonly #now: () => number;
+  readonly #sleep: (ms: number) => Promise<void>;
+  readonly #randomId: () => string;
+  // Per-process fallback when the storage adapter does not expose a
+  // consolidator surface (lib mode + in-memory test doubles).
+  readonly #localLocks = new Map<string, { runId: string; acquiredAt: number }>();
+
+  constructor(opts: LockManagerOptions) {
+    this.#store = opts.store;
+    this.#waitMs = Math.max(0, opts.waitMs);
+    this.#maxAge = Math.max(0, opts.maxRunDurationMs);
+    this.#now = opts.now ?? Date.now;
+    this.#sleep =
+      opts.sleep ??
+      ((ms): Promise<void> => new Promise((resolve) => globalThis.setTimeout(resolve, ms)));
+    this.#randomId =
+      opts.randomId ??
+      ((): string => {
+        const a = Math.floor(Math.random() * 1e9).toString(36);
+        const b = Math.floor(Math.random() * 1e9).toString(36);
+        return `run_${a}${b}`;
+      });
+  }
+
+  /**
+   * Try to acquire the per-scope lock. Returns the run id on success
+   * or `'deferred'` after the wait window elapses.
+   *
+   * @stable
+   */
+  async acquire(scope: SessionScope): Promise<LockOutcome> {
+    const runId = this.#randomId();
+    const start = this.#now();
+    const pollMs = Math.min(DEFAULT_POLL_MS, Math.max(10, this.#waitMs / 5 || DEFAULT_POLL_MS));
+    while (true) {
+      const ok = await this.#tryAcquire(scope, runId);
+      if (ok) return { kind: 'acquired', runId };
+      const elapsed = this.#now() - start;
+      if (elapsed >= this.#waitMs) {
+        const heldBy = await this.#heldBy(scope);
+        return { kind: 'deferred', heldBy };
+      }
+      await this.#sleep(pollMs);
+    }
+  }
+
+  /**
+   * Release the lock. Idempotent - releasing an already-released
+   * lock is a no-op.
+   *
+   * @stable
+   */
+  async release(scope: SessionScope, runId: string): Promise<void> {
+    if (this.#store !== null) {
+      await this.#store.releaseLock(scope, runId);
+    }
+    const key = scopeKey(scope);
+    const local = this.#localLocks.get(key);
+    if (local !== undefined && local.runId === runId) {
+      this.#localLocks.delete(key);
+    }
+  }
+
+  async #tryAcquire(scope: SessionScope, runId: string): Promise<boolean> {
+    const now = this.#now();
+    if (this.#store !== null) {
+      return this.#store.acquireLock(scope, runId, now, this.#maxAge);
+    }
+    // Fall back to per-process map.
+    const key = scopeKey(scope);
+    const existing = this.#localLocks.get(key);
+    if (existing === undefined) {
+      this.#localLocks.set(key, { runId, acquiredAt: now });
+      return true;
+    }
+    if (existing.runId === runId) return true;
+    if (this.#maxAge > 0 && now - existing.acquiredAt > this.#maxAge) {
+      this.#localLocks.set(key, { runId, acquiredAt: now });
+      return true;
+    }
+    return false;
+  }
+
+  async #heldBy(scope: SessionScope): Promise<string | null> {
+    if (this.#store !== null) {
+      const state = await this.#store.getState(scope);
+      return state?.activeLockHeldBy ?? null;
+    }
+    const local = this.#localLocks.get(scopeKey(scope));
+    return local?.runId ?? null;
+  }
+}
+
+/**
+ * Canonical string key for a consolidation scope. The lock manager and
+ * the runtime's per-scope dispatch bookkeeping (W-142) must agree on
+ * this shape: the lock serialises work per scope, so any state keyed by
+ * `scopeKey` holds at most one live entry per scope.
+ */
+export function scopeKey(scope: SessionScope): string {
+  return `${scope.userId}|${scope.sessionId ?? ''}|${scope.agentId ?? ''}`;
+}

package/src/consolidator/noise-filter.ts

@@ -0,0 +1,235 @@
+/**
+ * Built-in noise filter pipeline. Runs before LLM extraction in the
+ * standard phase + every light-phase invocation. The defaults are
+ * heuristic-only (no LLM); adapters may extend the pipeline through
+ * `noiseFilters: [...customFilters]` once the public surface is wired
+ * (post-MVP - DEC-134 carve-out).
+ *
+ * @packageDocumentation
+ */
+
+import type { Message } from '@graphorin/core';
+import type { SessionMessageRecord } from '../internal/storage-adapter.js';
+
+/**
+ * Bundle level surfaced by {@link applyNoiseFilters}. The runtime
+ * uses these to log + report `noise_filtered_count` per phase.
+ *
+ * @stable
+ */
+export type NoiseFilterPreset = 'default' | 'minimal' | 'none';
+
+const MIN_LENGTH_DEFAULT = 10;
+const MIN_LENGTH_MINIMAL = 4;
+const MAX_LENGTH = 10_000;
+const STOP_WORD_RATIO = 0.8;
+const STOP_WORDS = new Set([
+  'a',
+  'an',
+  'and',
+  'are',
+  'as',
+  'at',
+  'be',
+  'but',
+  'by',
+  'can',
+  'do',
+  'does',
+  'for',
+  'from',
+  'had',
+  'has',
+  'have',
+  'he',
+  'her',
+  'his',
+  'i',
+  'if',
+  'in',
+  'is',
+  'it',
+  'its',
+  'me',
+  'my',
+  'no',
+  'not',
+  'of',
+  'on',
+  'or',
+  'our',
+  'she',
+  'so',
+  'that',
+  'the',
+  'their',
+  'them',
+  'they',
+  'this',
+  'to',
+  'too',
+  'us',
+  'was',
+  'we',
+  'were',
+  'what',
+  'when',
+  'where',
+  'which',
+  'who',
+  'will',
+  'with',
+  'would',
+  'yes',
+  'you',
+  'your',
+]);
+
+/**
+ * Result returned by {@link applyNoiseFilters}.
+ *
+ * @stable
+ */
+export interface NoiseFilterResult {
+  readonly kept: ReadonlyArray<SessionMessageRecord>;
+  readonly droppedCount: number;
+  readonly droppedByReason: Readonly<Record<string, number>>;
+}
+
+/**
+ * Run the configured filter chain against the supplied messages.
+ * Filters short-circuit at the first hit so the cheapest checks
+ * always run first.
+ *
+ * @stable
+ */
+export function applyNoiseFilters(
+  messages: ReadonlyArray<SessionMessageRecord>,
+  presets: ReadonlyArray<NoiseFilterPreset> = ['default'],
+): NoiseFilterResult {
+  if (presets.length === 0 || presets.every((p) => p === 'none')) {
+    return { kept: messages, droppedCount: 0, droppedByReason: Object.freeze({}) };
+  }
+  const minimal = presets.includes('minimal') && !presets.includes('default');
+  const minLength = minimal ? MIN_LENGTH_MINIMAL : MIN_LENGTH_DEFAULT;
+  const droppedByReason: Record<string, number> = {};
+  const kept: SessionMessageRecord[] = [];
+  const recentTexts: string[] = [];
+
+  for (const record of messages) {
+    const reason = classify(record.message, recentTexts, minLength, minimal);
+    if (reason === null) {
+      kept.push(record);
+      const text = renderText(record.message);
+      if (text.length > 0) {
+        recentTexts.push(text);
+        if (recentTexts.length > 5) recentTexts.shift();
+      }
+    } else {
+      droppedByReason[reason] = (droppedByReason[reason] ?? 0) + 1;
+    }
+  }
+  return {
+    kept,
+    droppedCount: messages.length - kept.length,
+    droppedByReason: Object.freeze({ ...droppedByReason }),
+  };
+}
+
+function classify(
+  message: Message,
+  recentTexts: ReadonlyArray<string>,
+  minLength: number,
+  minimal: boolean,
+): string | null {
+  if (message.role === 'system') return 'role-system';
+  if (message.role === 'tool' && minimal) return null;
+  const text = renderText(message);
+  const trimmed = text.trim();
+  if (trimmed.length === 0) return 'empty';
+  if (trimmed.length < minLength) return 'too-short';
+  if (trimmed.length > MAX_LENGTH) return 'too-long';
+  if (looksLikeCodeOrJson(trimmed)) return 'code-block';
+  if (!minimal && stopWordRatio(trimmed) > STOP_WORD_RATIO) return 'stop-word-ratio';
+  if (!minimal && isNearDuplicate(trimmed, recentTexts)) return 'near-duplicate';
+  return null;
+}
+
+/**
+ * Render a message as a single text blob for the heuristic filters.
+ *
+ * @internal
+ */
+export function renderText(message: Message): string {
+  if (message.role === 'system') return message.content;
+  const c = message.content;
+  if (typeof c === 'string') return c;
+  let out = '';
+  for (const part of c) {
+    if (part.type === 'text' || part.type === 'reasoning') {
+      out += `${part.text} `;
+    }
+  }
+  return out.trim();
+}
+
+function looksLikeCodeOrJson(text: string): boolean {
+  if (text.startsWith('```')) return true;
+  const trimmed = text.trim();
+  if (
+    (trimmed.startsWith('{') && trimmed.endsWith('}')) ||
+    (trimmed.startsWith('[') && trimmed.endsWith(']'))
+  ) {
+    try {
+      JSON.parse(trimmed);
+      return true;
+    } catch {
+      // Not actually JSON - fall through.
+    }
+  }
+  return false;
+}
+
+function stopWordRatio(text: string): number {
+  const tokens = text
+    .toLowerCase()
+    .split(/[^a-z0-9]+/u)
+    .filter((t) => t.length > 0);
+  if (tokens.length === 0) return 0;
+  let stop = 0;
+  for (const t of tokens) if (STOP_WORDS.has(t)) stop += 1;
+  return stop / tokens.length;
+}
+
+function isNearDuplicate(text: string, recent: ReadonlyArray<string>): boolean {
+  const normalised = text.replace(/\s+/gu, ' ').toLowerCase().trim();
+  for (const prior of recent) {
+    const priorNorm = prior.replace(/\s+/gu, ' ').toLowerCase().trim();
+    if (priorNorm.length === 0) continue;
+    if (priorNorm === normalised) return true;
+    const similarity = trigramSimilarity(priorNorm, normalised);
+    if (similarity >= 0.9) return true;
+  }
+  return false;
+}
+
+function trigramSimilarity(a: string, b: string): number {
+  const setA = trigramSet(a);
+  const setB = trigramSet(b);
+  if (setA.size === 0 || setB.size === 0) return 0;
+  let inter = 0;
+  for (const t of setA) if (setB.has(t)) inter += 1;
+  return inter / Math.max(setA.size, setB.size);
+}
+
+function trigramSet(text: string): Set<string> {
+  const out = new Set<string>();
+  if (text.length < 3) {
+    out.add(text);
+    return out;
+  }
+  for (let i = 0; i < text.length - 2; i++) {
+    out.add(text.slice(i, i + 3));
+  }
+  return out;
+}