@etiquekit/etq 1.0.12 → 1.0.14

package/packages/control/src/memory/store.ts

@@ -0,0 +1,284 @@
+import { Database } from 'bun:sqlite';
+import { mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+import type { CompactMemoryResult, MemoryCandidate, RecallCapsule, RejectedMemoryCandidate } from './types';
+
+export interface MemoryStore {
+  dbPath: string;
+  close: () => void;
+  reset: () => void;
+  stats: () => MemoryStoreStats;
+  insertCandidate: (candidate: MemoryCandidate) => void;
+  markCandidate: (candidateId: string, status: 'promoted' | 'rejected', reasons: string[]) => void;
+  insertCapsule: (capsule: RecallCapsule, searchText?: string) => void;
+  search: (query: string, limit: number) => CompactMemoryResult[];
+  show: (capsuleId: string) => RecallCapsule | null;
+  stale: (limit: number) => CompactMemoryResult[];
+  rejected: (limit: number) => RejectedMemoryCandidate[];
+}
+
+export interface MemoryStoreStats {
+  candidates: {
+    total: number;
+    pending: number;
+    promoted: number;
+    rejected: number;
+  };
+  recall_capsules: number;
+}
+
+interface CapsuleRow {
+  capsule_id: string;
+  title: string;
+  category: string;
+  confidence: string;
+  stale_status: string;
+  evidence_count: number;
+  capsule_json: string;
+}
+
+interface CandidateRow {
+  candidate_id: string;
+  title: string;
+  category: string;
+  risk_flags_json: string;
+  rejection_reasons_json: string;
+  candidate_json: string;
+}
+
+function hasColumn(db: Database, table: string, column: string): boolean {
+  const rows = db.query<{ name: string }, []>(`PRAGMA table_info(${table})`).all();
+  return rows.some((row) => row.name === column);
+}
+
+function addColumnIfMissing(db: Database, table: string, column: string, ddl: string): void {
+  if (!hasColumn(db, table, column)) {
+    db.exec(`ALTER TABLE ${table} ADD COLUMN ${ddl}`);
+  }
+}
+
+function ensureFtsSchema(db: Database): void {
+  const columns = db.query<{ name: string }, []>('PRAGMA table_info(memory_capsule_fts)').all();
+  if (columns.length > 0 && !columns.some((row) => row.name === 'search_text')) {
+    db.exec('DROP TABLE memory_capsule_fts;');
+  }
+  db.exec(`
+    CREATE VIRTUAL TABLE IF NOT EXISTS memory_capsule_fts
+      USING fts5(capsule_id UNINDEXED, title, summary, category, evidence_refs_text, search_text);
+  `);
+}
+
+function tokenizeQuery(query: string): string[] {
+  return [...new Set(query.toLowerCase().split(/[^a-z0-9_]+/).filter((part) => part.length >= 2))];
+}
+
+function escapeFtsToken(token: string): string {
+  return token.replace(/"/g, '""');
+}
+
+function ftsExpression(tokens: string[], joiner: 'AND' | 'OR'): string {
+  return tokens.map((token) => `"${escapeFtsToken(token)}"*`).join(` ${joiner} `);
+}
+
+function appendUniqueRows(target: CapsuleRow[], rows: CapsuleRow[], limit: number): void {
+  const seen = new Set(target.map((row) => row.capsule_id));
+  for (const row of rows) {
+    if (seen.has(row.capsule_id)) continue;
+    target.push(row);
+    seen.add(row.capsule_id);
+    if (target.length >= limit) return;
+  }
+}
+
+export function openMemoryStore(dbPath: string): MemoryStore {
+  mkdirSync(dirname(dbPath), { recursive: true });
+  const db = new Database(dbPath);
+  db.exec(`
+    PRAGMA journal_mode = WAL;
+    PRAGMA busy_timeout = 5000;
+    CREATE TABLE IF NOT EXISTS memory_candidates (
+      candidate_id TEXT PRIMARY KEY,
+      category TEXT NOT NULL,
+      title TEXT NOT NULL,
+      summary TEXT NOT NULL,
+      risk_flags_json TEXT NOT NULL,
+      candidate_json TEXT NOT NULL,
+      promotion_status TEXT NOT NULL DEFAULT 'pending',
+      rejection_reasons_json TEXT NOT NULL DEFAULT '[]',
+      indexed_at TEXT NOT NULL
+    );
+    CREATE TABLE IF NOT EXISTS recall_capsules (
+      capsule_id TEXT PRIMARY KEY,
+      title TEXT NOT NULL,
+      category TEXT NOT NULL,
+      summary TEXT NOT NULL,
+      confidence TEXT NOT NULL,
+      stale_status TEXT NOT NULL,
+      evidence_count INTEGER NOT NULL,
+      evidence_refs_text TEXT NOT NULL DEFAULT '',
+      capsule_json TEXT NOT NULL,
+      updated_at TEXT NOT NULL
+    );
+  `);
+  addColumnIfMissing(db, 'memory_candidates', 'promotion_status', "promotion_status TEXT NOT NULL DEFAULT 'pending'");
+  addColumnIfMissing(db, 'memory_candidates', 'rejection_reasons_json', "rejection_reasons_json TEXT NOT NULL DEFAULT '[]'");
+  addColumnIfMissing(db, 'recall_capsules', 'evidence_refs_text', "evidence_refs_text TEXT NOT NULL DEFAULT ''");
+  addColumnIfMissing(db, 'recall_capsules', 'search_text', "search_text TEXT NOT NULL DEFAULT ''");
+  ensureFtsSchema(db);
+
+  const mapCompact = (row: CapsuleRow): CompactMemoryResult => ({
+    capsule_id: row.capsule_id,
+    title: row.title,
+    category: row.category as CompactMemoryResult['category'],
+    confidence: row.confidence as CompactMemoryResult['confidence'],
+    stale_status: row.stale_status as CompactMemoryResult['stale_status'],
+    evidence_count: row.evidence_count,
+  });
+
+  return {
+    dbPath,
+    close: () => db.close(),
+    reset: () => {
+      db.exec('DELETE FROM memory_candidates; DELETE FROM recall_capsules;');
+      db.exec('DELETE FROM memory_capsule_fts;');
+    },
+    stats: () => {
+      const count = (sql: string): number => db.query<{ count: number }, []>(sql).get()?.count ?? 0;
+      return {
+        candidates: {
+          total: count('SELECT COUNT(*) AS count FROM memory_candidates'),
+          pending: count("SELECT COUNT(*) AS count FROM memory_candidates WHERE promotion_status = 'pending'"),
+          promoted: count("SELECT COUNT(*) AS count FROM memory_candidates WHERE promotion_status = 'promoted'"),
+          rejected: count("SELECT COUNT(*) AS count FROM memory_candidates WHERE promotion_status = 'rejected'"),
+        },
+        recall_capsules: count('SELECT COUNT(*) AS count FROM recall_capsules'),
+      };
+    },
+    insertCandidate: (candidate) => {
+      db.query(`
+        INSERT OR REPLACE INTO memory_candidates
+          (candidate_id, category, title, summary, risk_flags_json, candidate_json, promotion_status, rejection_reasons_json, indexed_at)
+        VALUES (?, ?, ?, ?, ?, ?, 'pending', '[]', ?)
+      `).run(
+        candidate.candidate_id,
+        candidate.category,
+        candidate.title,
+        candidate.summary,
+        JSON.stringify(candidate.risk_flags),
+        JSON.stringify(candidate),
+        new Date().toISOString(),
+      );
+    },
+    markCandidate: (candidateId, status, reasons) => {
+      db.query(`
+        UPDATE memory_candidates
+        SET promotion_status = ?, rejection_reasons_json = ?
+        WHERE candidate_id = ?
+      `).run(status, JSON.stringify(reasons), candidateId);
+    },
+    insertCapsule: (capsule, searchText = '') => {
+      const evidenceRefsText = capsule.evidence_refs.map((ref) => `${ref.type}:${ref.ref}`).join(' ');
+      db.query(`
+        INSERT OR REPLACE INTO recall_capsules
+          (capsule_id, title, category, summary, confidence, stale_status, evidence_count, evidence_refs_text, search_text, capsule_json, updated_at)
+        VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `).run(
+        capsule.capsule_id,
+        capsule.title,
+        capsule.category,
+        capsule.summary,
+        capsule.confidence,
+        capsule.staleness.status,
+        capsule.evidence_refs.length,
+        evidenceRefsText,
+        searchText,
+        JSON.stringify(capsule),
+        capsule.updated_at,
+      );
+      db.query('DELETE FROM memory_capsule_fts WHERE capsule_id = ?').run(capsule.capsule_id);
+      db.query(`
+        INSERT INTO memory_capsule_fts (capsule_id, title, summary, category, evidence_refs_text, search_text)
+        VALUES (?, ?, ?, ?, ?, ?)
+      `).run(capsule.capsule_id, capsule.title, capsule.summary, capsule.category, evidenceRefsText, searchText);
+    },
+    search: (query, limit) => {
+      const tokens = tokenizeQuery(query);
+      if (tokens.length === 0) return [];
+      const hasIdentifierToken = tokens.some((token) => token.includes('_'));
+      const rows: CapsuleRow[] = [];
+      const runFts = (remaining: number): CapsuleRow[] => db.query<CapsuleRow, [string, number]>(`
+        SELECT c.capsule_id, c.title, c.category, c.confidence, c.stale_status, c.evidence_count, c.capsule_json
+        FROM memory_capsule_fts
+        JOIN recall_capsules c ON c.capsule_id = memory_capsule_fts.capsule_id
+        WHERE memory_capsule_fts MATCH ?
+        ORDER BY bm25(memory_capsule_fts, 0.0, 3.0, 1.5, 1.0, 2.0, 0.6), c.updated_at DESC, c.capsule_id ASC
+        LIMIT ?
+      `).all(ftsExpression(tokens, 'AND'), remaining);
+
+      if (!hasIdentifierToken) {
+        try {
+          appendUniqueRows(rows, runFts(limit), limit);
+        } catch {
+          // Fall through to the tokenized LIKE fallback below. FTS should normally
+          // handle sanitized tokens, but fallback keeps existing databases usable.
+        }
+      }
+
+      if (rows.length < limit) {
+        const clauses = tokens.map(() => [
+          'lower(title) LIKE ?',
+          'lower(summary) LIKE ?',
+          'lower(category) LIKE ?',
+          'lower(evidence_refs_text) LIKE ?',
+          'lower(search_text) LIKE ?',
+        ].join(' OR '));
+        const params = tokens.flatMap((token) => Array(5).fill(`%${token}%`));
+        const fallbackRows = db.query<CapsuleRow, [...string[], number]>(`
+          SELECT capsule_id, title, category, confidence, stale_status, evidence_count, capsule_json
+          FROM recall_capsules
+          WHERE ${clauses.map((clause) => `(${clause})`).join(' AND ')}
+          ORDER BY updated_at DESC, capsule_id ASC
+          LIMIT ?
+        `).all(...params, limit - rows.length);
+        appendUniqueRows(rows, fallbackRows, limit);
+      }
+      return rows.map(mapCompact);
+    },
+    show: (capsuleId) => {
+      const row = db.query<{ capsule_json: string }, [string]>(
+        'SELECT capsule_json FROM recall_capsules WHERE capsule_id = ?',
+      ).get(capsuleId);
+      return row ? JSON.parse(row.capsule_json) as RecallCapsule : null;
+    },
+    stale: (limit) => {
+      const rows = db.query<CapsuleRow, [number]>(`
+        SELECT capsule_id, title, category, confidence, stale_status, evidence_count, capsule_json
+        FROM recall_capsules
+        WHERE stale_status != 'current'
+        ORDER BY updated_at DESC, capsule_id ASC
+        LIMIT ?
+      `).all(limit);
+      return rows.map(mapCompact);
+    },
+    rejected: (limit) => {
+      const rows = db.query<CandidateRow, [number]>(`
+        SELECT candidate_id, title, category, risk_flags_json, rejection_reasons_json, candidate_json
+        FROM memory_candidates
+        WHERE promotion_status = 'rejected'
+        ORDER BY indexed_at DESC, candidate_id ASC
+        LIMIT ?
+      `).all(limit);
+      return rows.map((row) => {
+        const candidate = JSON.parse(row.candidate_json) as MemoryCandidate;
+        return {
+          candidate_id: row.candidate_id,
+          title: row.title,
+          category: row.category as RejectedMemoryCandidate['category'],
+          risk_flags: JSON.parse(row.risk_flags_json),
+          rejection_reasons: JSON.parse(row.rejection_reasons_json),
+          source_refs: candidate.source_refs,
+        };
+      });
+    },
+  };
+}

package/packages/control/src/memory/types.ts

@@ -0,0 +1,146 @@
+export const MEMORY_CATEGORIES = [
+  'decision',
+  'pattern',
+  'risk',
+  'mistake',
+  'playback_candidate',
+  'onboarding',
+  'adapter_note',
+] as const;
+
+export type MemoryCategory = (typeof MEMORY_CATEGORIES)[number];
+
+export const EVIDENCE_REF_TYPES = [
+  'ledger',
+  'receipt',
+  'playback',
+  'event',
+  'doc',
+  'bus',
+  'commit',
+] as const;
+
+export type EvidenceRefType = (typeof EVIDENCE_REF_TYPES)[number];
+
+export interface EvidenceRef {
+  type: EvidenceRefType;
+  ref: string;
+}
+
+export const MEMORY_CONFIDENCE = ['high', 'medium', 'low'] as const;
+export type MemoryConfidence = (typeof MEMORY_CONFIDENCE)[number];
+
+export const STALENESS_STATUS = ['current', 'watch', 'stale'] as const;
+export type StalenessStatus = (typeof STALENESS_STATUS)[number];
+
+export const RECOMMENDED_ACTION_TYPES = [
+  'none',
+  'read_before_work',
+  'create_ledger_entry',
+  'run_playback',
+  'update_docs',
+  'open_review',
+] as const;
+
+export type RecommendedActionType = (typeof RECOMMENDED_ACTION_TYPES)[number];
+
+export const MEMORY_RISK_FLAGS = [
+  'no_durable_evidence',
+  'stale',
+  'contains_private_context',
+  'contradicts_current_ledger',
+  'contains_raw_prompt',
+  'contains_raw_tool_output',
+  'contains_source_snippet',
+  'contains_local_path',
+  'contains_hostname',
+  'contains_credential',
+  'confidence_low_for_shared_export',
+] as const;
+
+export type MemoryRiskFlag = (typeof MEMORY_RISK_FLAGS)[number];
+
+export const ACTIVE_MEMORY_ADAPTERS = ['local_importer', 'manual'] as const;
+
+export const RESERVED_MEMORY_ADAPTERS = [
+  'claude_memory',
+  'claude_mem',
+  'mempalace',
+  'vector_search',
+] as const;
+
+export const MEMORY_ADAPTERS = [
+  ...ACTIVE_MEMORY_ADAPTERS,
+  ...RESERVED_MEMORY_ADAPTERS,
+] as const;
+
+export type MemoryAdapter = (typeof MEMORY_ADAPTERS)[number];
+
+export interface MemoryCandidate {
+  candidate_id: string;
+  adapter: MemoryAdapter;
+  query: string;
+  category: MemoryCategory;
+  title: string;
+  summary: string;
+  source_refs: EvidenceRef[];
+  confidence: MemoryConfidence;
+  risk_flags: MemoryRiskFlag[];
+}
+
+export interface RecallCapsule {
+  schema: 'recall-capsule.v0';
+  capsule_id: string;
+  created_at: string;
+  updated_at: string;
+  scope: {
+    org_id?: string;
+    workspace_id?: string;
+    repo_ref?: string;
+    lane_ref?: string;
+  };
+  category: MemoryCategory;
+  title: string;
+  summary: string;
+  why_it_matters: string;
+  evidence_refs: EvidenceRef[];
+  confidence: MemoryConfidence;
+  staleness: {
+    status: StalenessStatus;
+    stale_if?: string;
+  };
+  recommended_action: {
+    type: RecommendedActionType;
+    ref?: string;
+  };
+  authority_boundary: {
+    can_authorize_work: false;
+    can_close_task: false;
+    can_replace_receipt: false;
+  };
+}
+
+export interface MemoryIndexResult {
+  candidates: number;
+  promoted: number;
+  rejected: number;
+  capsuleIds: string[];
+}
+
+export interface CompactMemoryResult {
+  capsule_id: string;
+  title: string;
+  category: MemoryCategory;
+  confidence: MemoryConfidence;
+  stale_status: StalenessStatus;
+  evidence_count: number;
+}
+
+export interface RejectedMemoryCandidate {
+  candidate_id: string;
+  title: string;
+  category: MemoryCategory;
+  risk_flags: MemoryRiskFlag[];
+  rejection_reasons: MemoryRiskFlag[];
+  source_refs: EvidenceRef[];
+}