@soleri/core 2.4.0 → 2.5.0

package/src/cognee/sync-manager.ts

@@ -0,0 +1,389 @@
+/**
+ * CogneeSyncManager — queued, resilient sync between the Vault SQLite DB and Cognee.
+ *
+ * Maintains a persistent queue (`cognee_sync_queue`) so that ingestions, updates,
+ * and deletions survive process restarts. Drain is idempotent and retry-safe.
+ * Health-flip detection auto-drains when Cognee comes back online.
+ *
+ * Ported from Salvador MCP's battle-tested cognee-sync module.
+ */
+
+import { createHash } from 'node:crypto';
+import type { PersistenceProvider } from '../persistence/types.js';
+import type { CogneeClient } from './client.js';
+import type { IntelligenceEntry } from '../intelligence/types.js';
+
+// ─── Types ──────────────────────────────────────────────────────────
+
+export type SyncOp = 'ingest' | 'update' | 'delete';
+export type SyncStatus = 'pending' | 'processing' | 'completed' | 'failed';
+
+export interface SyncQueueItem {
+  id: number;
+  op: SyncOp;
+  entryId: string;
+  dataset: string;
+  contentHash: string | null;
+  status: SyncStatus;
+  attempts: number;
+  error: string | null;
+  createdAt: number;
+  processedAt: number | null;
+}
+
+export interface SyncManagerStats {
+  pending: number;
+  processing: number;
+  completed: number;
+  failed: number;
+  queueSize: number;
+  lastDrainAt: number | null;
+}
+
+// ─── Constants ──────────────────────────────────────────────────────
+
+const MAX_BATCH = 10;
+const MAX_RETRIES = 3;
+
+// ─── CogneeSyncManager ─────────────────────────────────────────────
+
+export class CogneeSyncManager {
+  private db: PersistenceProvider;
+  private cognee: CogneeClient;
+  private dataset: string;
+  private lastDrainAt: number | null = null;
+  private drainTimer: ReturnType<typeof setInterval> | null = null;
+  private wasAvailable: boolean = false;
+
+  constructor(db: PersistenceProvider, cognee: CogneeClient, dataset: string) {
+    this.db = db;
+    this.cognee = cognee;
+    this.dataset = dataset;
+    this.initSchema();
+    this.wasAvailable = cognee.isAvailable;
+  }
+
+  // ─── Schema ────────────────────────────────────────────────────
+
+  private initSchema(): void {
+    this.db.execSql(`
+      CREATE TABLE IF NOT EXISTS cognee_sync_queue (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        op TEXT NOT NULL,
+        entry_id TEXT NOT NULL,
+        dataset TEXT NOT NULL,
+        content_hash TEXT,
+        status TEXT NOT NULL DEFAULT 'pending',
+        attempts INTEGER NOT NULL DEFAULT 0,
+        error TEXT,
+        created_at INTEGER NOT NULL DEFAULT (unixepoch()),
+        processed_at INTEGER
+      )
+    `);
+
+    // Add cognee_ingested_hash column to entries table for reconciliation.
+    // ALTER TABLE ... ADD COLUMN is a no-op error when the column already exists.
+    try {
+      this.db.run('ALTER TABLE entries ADD COLUMN cognee_ingested_hash TEXT');
+    } catch {
+      // Column already exists — expected on subsequent runs.
+    }
+  }
+
+  // ─── Content hashing ──────────────────────────────────────────
+
+  /**
+   * SHA-256 of the serialized entry fields, truncated to 16 hex characters.
+   * Deterministic for identical content — used to detect drift.
+   */
+  static contentHash(entry: IntelligenceEntry): string {
+    const payload = JSON.stringify({
+      id: entry.id,
+      type: entry.type,
+      domain: entry.domain,
+      title: entry.title,
+      severity: entry.severity,
+      description: entry.description,
+      context: entry.context ?? null,
+      example: entry.example ?? null,
+      counterExample: entry.counterExample ?? null,
+      why: entry.why ?? null,
+      tags: entry.tags,
+      appliesTo: entry.appliesTo ?? [],
+    });
+    return createHash('sha256').update(payload).digest('hex').slice(0, 16);
+  }
+
+  // ─── Enqueue ──────────────────────────────────────────────────
+
+  /**
+   * Add an operation to the sync queue.
+   *
+   * @param op       The operation type (ingest | update | delete).
+   * @param entryId  The vault entry ID.
+   * @param entry    Optional entry for hash computation. If omitted, hash is null.
+   */
+  enqueue(op: SyncOp, entryId: string, entry?: IntelligenceEntry): void {
+    const contentHash = entry ? CogneeSyncManager.contentHash(entry) : null;
+    this.db.run(
+      `INSERT INTO cognee_sync_queue (op, entry_id, dataset, content_hash)
+       VALUES (@op, @entryId, @dataset, @contentHash)`,
+      {
+        op,
+        entryId,
+        dataset: this.dataset,
+        contentHash,
+      },
+    );
+  }
+
+  // ─── Drain ────────────────────────────────────────────────────
+
+  /**
+   * Process up to MAX_BATCH pending items from the queue.
+   * Returns the number of items successfully processed.
+   *
+   * If Cognee is not available, returns 0 without touching the queue.
+   */
+  async drain(): Promise<number> {
+    if (!this.cognee.isAvailable) return 0;
+
+    // Claim a batch: move pending → processing
+    const items = this.db.all<Record<string, unknown>>(
+      `SELECT * FROM cognee_sync_queue
+       WHERE status = 'pending' AND dataset = @dataset
+       ORDER BY created_at ASC
+       LIMIT @limit`,
+      { dataset: this.dataset, limit: MAX_BATCH },
+    );
+
+    if (items.length === 0) return 0;
+
+    let processed = 0;
+
+    for (const raw of items) {
+      const item = rowToQueueItem(raw);
+
+      // Mark as processing
+      this.db.run(
+        `UPDATE cognee_sync_queue SET status = 'processing', attempts = attempts + 1 WHERE id = @id`,
+        { id: item.id },
+      );
+
+      try {
+        if (item.op === 'ingest' || item.op === 'update') {
+          const entry = this.readEntry(item.entryId);
+          if (!entry) {
+            // Entry was deleted from vault before we could sync — mark completed
+            this.markCompleted(item.id);
+            processed++;
+            continue;
+          }
+
+          const result = await this.cognee.addEntries([entry]);
+          if (result.added === 0) {
+            throw new Error('Cognee addEntries returned 0 added');
+          }
+
+          // Update the ingested hash on the entries table
+          const hash = CogneeSyncManager.contentHash(entry);
+          this.db.run(`UPDATE entries SET cognee_ingested_hash = @hash WHERE id = @id`, {
+            hash,
+            id: item.entryId,
+          });
+
+          this.markCompleted(item.id);
+          processed++;
+        } else if (item.op === 'delete') {
+          // deleteEntries may not exist yet — graceful degradation
+          const client = this.cognee as unknown as Record<string, unknown>;
+          if (typeof client.deleteEntries === 'function') {
+            await (client.deleteEntries as (ids: string[]) => Promise<unknown>)([item.entryId]);
+          }
+          // Clear the ingested hash (entry may already be gone from entries table)
+          this.db.run(`UPDATE entries SET cognee_ingested_hash = NULL WHERE id = @id`, {
+            id: item.entryId,
+          });
+          this.markCompleted(item.id);
+          processed++;
+        }
+      } catch (err) {
+        const errorMsg = err instanceof Error ? err.message : String(err);
+        const attempts = item.attempts + 1; // Already incremented above
+        if (attempts >= MAX_RETRIES) {
+          this.db.run(
+            `UPDATE cognee_sync_queue SET status = 'failed', error = @error, processed_at = unixepoch() WHERE id = @id`,
+            { id: item.id, error: errorMsg },
+          );
+        } else {
+          // Back to pending for retry
+          this.db.run(
+            `UPDATE cognee_sync_queue SET status = 'pending', error = @error WHERE id = @id`,
+            { id: item.id, error: errorMsg },
+          );
+        }
+      }
+    }
+
+    this.lastDrainAt = Math.floor(Date.now() / 1000);
+    return processed;
+  }
+
+  // ─── Reconciliation ───────────────────────────────────────────
+
+  /**
+   * Find entries whose cognee_ingested_hash is NULL or doesn't match the
+   * current content hash. Enqueue dirty entries for re-ingestion.
+   *
+   * Returns the number of entries enqueued.
+   */
+  reconcile(): number {
+    // Get all entries that either have never been ingested or whose content changed
+    const rows = this.db.all<Record<string, unknown>>(
+      `SELECT * FROM entries WHERE cognee_ingested_hash IS NULL
+       UNION ALL
+       SELECT * FROM entries WHERE cognee_ingested_hash IS NOT NULL`,
+    );
+
+    let enqueued = 0;
+
+    for (const raw of rows) {
+      const entry = this.rowToEntry(raw);
+      const currentHash = CogneeSyncManager.contentHash(entry);
+      const ingestedHash = raw.cognee_ingested_hash as string | null;
+
+      if (ingestedHash === currentHash) continue;
+
+      // Determine op: null hash means never ingested, mismatched means update
+      const op: SyncOp = ingestedHash === null ? 'ingest' : 'update';
+
+      // Avoid duplicate pending items for the same entry
+      const existing = this.db.get<{ id: number }>(
+        `SELECT id FROM cognee_sync_queue
+         WHERE entry_id = @entryId AND dataset = @dataset AND status = 'pending'`,
+        { entryId: entry.id, dataset: this.dataset },
+      );
+
+      if (!existing) {
+        this.enqueue(op, entry.id, entry);
+        enqueued++;
+      }
+    }
+
+    return enqueued;
+  }
+
+  // ─── Stats ────────────────────────────────────────────────────
+
+  getStats(): SyncManagerStats {
+    const countByStatus = (status: SyncStatus): number => {
+      const row = this.db.get<{ count: number }>(
+        `SELECT COUNT(*) as count FROM cognee_sync_queue WHERE status = @status AND dataset = @dataset`,
+        { status, dataset: this.dataset },
+      );
+      return row?.count ?? 0;
+    };
+
+    const pending = countByStatus('pending');
+    const processing = countByStatus('processing');
+    const completed = countByStatus('completed');
+    const failed = countByStatus('failed');
+
+    return {
+      pending,
+      processing,
+      completed,
+      failed,
+      queueSize: pending + processing,
+      lastDrainAt: this.lastDrainAt,
+    };
+  }
+
+  // ─── Health-flip detection ────────────────────────────────────
+
+  /**
+   * Detects an unavailable-to-available transition on the Cognee client.
+   * When Cognee comes back online, automatically triggers a drain.
+   *
+   * Call this periodically (e.g. after each health check).
+   */
+  async checkHealthFlip(): Promise<void> {
+    const nowAvailable = this.cognee.isAvailable;
+    if (nowAvailable && !this.wasAvailable) {
+      // Cognee just came back online — drain the queue
+      await this.drain();
+    }
+    this.wasAvailable = nowAvailable;
+  }
+
+  // ─── Cleanup ──────────────────────────────────────────────────
+
+  /**
+   * Clear any periodic drain timer.
+   */
+  close(): void {
+    if (this.drainTimer) {
+      clearInterval(this.drainTimer);
+      this.drainTimer = null;
+    }
+  }
+
+  // ─── Private helpers ──────────────────────────────────────────
+
+  private markCompleted(id: number): void {
+    this.db.run(
+      `UPDATE cognee_sync_queue SET status = 'completed', processed_at = unixepoch() WHERE id = @id`,
+      { id },
+    );
+  }
+
+  /**
+   * Read an entry from the entries table by ID.
+   * Returns null if the entry doesn't exist.
+   */
+  private readEntry(id: string): IntelligenceEntry | null {
+    const row = this.db.get<Record<string, unknown>>('SELECT * FROM entries WHERE id = @id', {
+      id,
+    });
+    return row ? this.rowToEntry(row) : null;
+  }
+
+  /**
+   * Convert a raw DB row into an IntelligenceEntry.
+   */
+  private rowToEntry(row: Record<string, unknown>): IntelligenceEntry {
+    return {
+      id: row.id as string,
+      type: row.type as IntelligenceEntry['type'],
+      domain: row.domain as string,
+      title: row.title as string,
+      severity: row.severity as IntelligenceEntry['severity'],
+      description: row.description as string,
+      context: (row.context as string) ?? undefined,
+      example: (row.example as string) ?? undefined,
+      counterExample: (row.counter_example as string) ?? undefined,
+      why: (row.why as string) ?? undefined,
+      tags: JSON.parse((row.tags as string) || '[]'),
+      appliesTo: JSON.parse((row.applies_to as string) || '[]'),
+      validFrom: (row.valid_from as number) ?? undefined,
+      validUntil: (row.valid_until as number) ?? undefined,
+    };
+  }
+}
+
+// ─── Module-level helpers ─────────────────────────────────────────
+
+function rowToQueueItem(row: Record<string, unknown>): SyncQueueItem {
+  return {
+    id: row.id as number,
+    op: row.op as SyncOp,
+    entryId: row.entry_id as string,
+    dataset: row.dataset as string,
+    contentHash: (row.content_hash as string) ?? null,
+    status: row.status as SyncStatus,
+    attempts: row.attempts as number,
+    error: (row.error as string) ?? null,
+    createdAt: row.created_at as number,
+    processedAt: (row.processed_at as number) ?? null,
+  };
+}

package/src/curator/curator.ts

@@ -1,5 +1,6 @@
 import type { Vault } from '../vault/vault.js';
 import type { IntelligenceEntry } from '../intelligence/types.js';
+import type { CogneeClient } from '../cognee/client.js';
 import {
   tokenize,
   calculateTfIdf,
@@ -49,9 +50,11 @@ const DEFAULT_TAG_ALIASES: Array<[string, string]> = [
 
 export class Curator {
   private vault: Vault;
+  private cognee: CogneeClient | undefined;
 
-  constructor(vault: Vault) {
+  constructor(vault: Vault, cognee?: CogneeClient) {
     this.vault = vault;
+    this.cognee = cognee;
     this.initializeTables();
     this.seedDefaultAliases();
   }
@@ -378,6 +381,85 @@ export class Curator {
     return row ? this.rowToContradiction(row) : null;
   }
 
+  async detectContradictionsHybrid(threshold?: number): Promise<{
+    contradictions: Contradiction[];
+    cogneeAvailable: boolean;
+    method: 'hybrid' | 'tfidf-only';
+  }> {
+    const effectiveThreshold = threshold ?? DEFAULT_CONTRADICTION_THRESHOLD;
+    const entries = this.vault.list({ limit: 100000 });
+    const antipatterns = entries.filter((e) => e.type === 'anti-pattern');
+    const patterns = entries.filter((e) => e.type === 'pattern');
+
+    if (antipatterns.length === 0 || patterns.length === 0) {
+      return { contradictions: [], cogneeAvailable: false, method: 'tfidf-only' };
+    }
+
+    const vocabulary = this.buildVocabulary(entries);
+    const db = this.vault.getDb();
+    const detected: Contradiction[] = [];
+
+    const insertStmt = db.prepare(
+      `INSERT OR IGNORE INTO curator_contradictions (pattern_id, antipattern_id, similarity)
+       VALUES (?, ?, ?)`,
+    );
+
+    const cogneeAvailable = this.cognee?.isAvailable ?? false;
+
+    for (const ap of antipatterns) {
+      let candidates: IntelligenceEntry[];
+      try {
+        const searchResults = this.vault.search(ap.title, { type: 'pattern', limit: 20 });
+        candidates = searchResults.length > 0 ? searchResults.map((r) => r.entry) : patterns;
+      } catch {
+        candidates = patterns;
+      }
+
+      const apText = [ap.title, ap.description, ap.context ?? ''].join(' ');
+      const apVec = calculateTfIdf(tokenize(apText), vocabulary);
+
+      for (const pattern of candidates) {
+        const pText = [pattern.title, pattern.description, pattern.context ?? ''].join(' ');
+        const pVec = calculateTfIdf(tokenize(pText), vocabulary);
+        const tfidfScore = cosineSimilarity(apVec, pVec);
+
+        let finalScore = tfidfScore;
+        if (cogneeAvailable && this.cognee) {
+          try {
+            const cogneeResults = await this.cognee.search(`${ap.title} ${pattern.title}`, {
+              limit: 5,
+            });
+            const cogneeScore =
+              cogneeResults.length > 0
+                ? cogneeResults.reduce((sum, r) => sum + r.score, 0) / cogneeResults.length
+                : 0;
+            finalScore = 0.6 * tfidfScore + 0.4 * cogneeScore;
+          } catch {
+            finalScore = tfidfScore;
+          }
+        }
+
+        if (finalScore >= effectiveThreshold) {
+          const result = insertStmt.run(pattern.id, ap.id, finalScore);
+          if (result.changes > 0) {
+            const row = db
+              .prepare(
+                'SELECT * FROM curator_contradictions WHERE pattern_id = ? AND antipattern_id = ?',
+              )
+              .get(pattern.id, ap.id) as Record<string, unknown>;
+            detected.push(this.rowToContradiction(row));
+          }
+        }
+      }
+    }
+
+    return {
+      contradictions: detected,
+      cogneeAvailable,
+      method: cogneeAvailable ? 'hybrid' : 'tfidf-only',
+    };
+  }
+
   // ─── Grooming ───────────────────────────────────────────────────
 
   groomEntry(entryId: string): GroomResult | null {
@@ -667,9 +749,7 @@ export class Curator {
     return { recorded: true, historyId: Number(result.lastInsertRowid) };
   }
 
-  getVersionHistory(
-    entryId: string,
-  ): Array<{
+  getVersionHistory(entryId: string): Array<{
     historyId: number;
     entryId: string;
     snapshot: IntelligenceEntry;
@@ -762,9 +842,10 @@ export class Curator {
 
   // ─── Metadata Enrichment ──────────────────────────────────────
 
-  enrichMetadata(
-    entryId: string,
-  ): { enriched: boolean; changes: Array<{ field: string; before: string; after: string }> } {
+  enrichMetadata(entryId: string): {
+    enriched: boolean;
+    changes: Array<{ field: string; before: string; after: string }>;
+  } {
     const entry = this.vault.get(entryId);
     if (!entry) return { enriched: false, changes: [] };
 

package/src/errors/classify.ts

@@ -0,0 +1,102 @@
+/**
+ * Error classifier — converts any thrown value into a typed SoleriError.
+ *
+ * Maps HTTP status codes, network error codes, and message patterns
+ * to the appropriate SoleriErrorCode.
+ */
+
+import { SoleriError, SoleriErrorCode } from './types.js';
+
+interface ErrorLike {
+  message?: string;
+  status?: number;
+  statusCode?: number;
+  code?: string;
+  cause?: unknown;
+}
+
+function toErrorLike(error: unknown): ErrorLike {
+  if (error instanceof SoleriError) return error;
+  if (error instanceof Error) return error as unknown as ErrorLike;
+  if (typeof error === 'object' && error !== null) return error as ErrorLike;
+  return { message: String(error) };
+}
+
+function getHttpStatus(e: ErrorLike): number | undefined {
+  return e.status ?? e.statusCode;
+}
+
+function classifyByHttpStatus(status: number): SoleriErrorCode | undefined {
+  if (status === 401 || status === 403) return SoleriErrorCode.AUTH;
+  if (status === 404) return SoleriErrorCode.RESOURCE_NOT_FOUND;
+  if (status === 408) return SoleriErrorCode.TIMEOUT;
+  if (status === 429) return SoleriErrorCode.RATE_LIMIT;
+  if (status === 422) return SoleriErrorCode.VALIDATION;
+  if (status >= 500 && status < 600) return SoleriErrorCode.INTERNAL;
+  return undefined;
+}
+
+const NETWORK_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'ECONNRESET', 'EPIPE', 'EHOSTUNREACH']);
+const TIMEOUT_CODES = new Set(['ETIMEDOUT', 'ESOCKETTIMEDOUT', 'UND_ERR_CONNECT_TIMEOUT']);
+
+function classifyByErrorCode(code: string | undefined): SoleriErrorCode | undefined {
+  if (!code) return undefined;
+  if (NETWORK_CODES.has(code)) return SoleriErrorCode.NETWORK;
+  if (TIMEOUT_CODES.has(code)) return SoleriErrorCode.TIMEOUT;
+  return undefined;
+}
+
+const MESSAGE_PATTERNS: Array<[RegExp, SoleriErrorCode]> = [
+  [/overloaded|capacity|model.*busy/i, SoleriErrorCode.LLM_OVERLOAD],
+  [/timeout|timed?\s*out/i, SoleriErrorCode.TIMEOUT],
+  [/vault|database|sqlite/i, SoleriErrorCode.VAULT_UNREACHABLE],
+  [/invalid|validation|schema/i, SoleriErrorCode.VALIDATION],
+  [/config(uration)?|missing.*key|env/i, SoleriErrorCode.CONFIG_ERROR],
+  [/auth(entication|orization)?|forbidden|denied|unauthorized/i, SoleriErrorCode.AUTH],
+  [/not\s*found|404|no\s+such/i, SoleriErrorCode.RESOURCE_NOT_FOUND],
+  [/rate\s*limit|too\s+many\s+requests|throttl/i, SoleriErrorCode.RATE_LIMIT],
+  [/network|connect|socket|dns/i, SoleriErrorCode.NETWORK],
+];
+
+function classifyByMessage(message: string | undefined): SoleriErrorCode | undefined {
+  if (!message) return undefined;
+  for (const [pattern, code] of MESSAGE_PATTERNS) {
+    if (pattern.test(message)) return code;
+  }
+  return undefined;
+}
+
+/**
+ * Classify any thrown value into a SoleriError.
+ * If the value is already a SoleriError, returns it as-is.
+ */
+export function classifyError(error: unknown): SoleriError {
+  if (error instanceof SoleriError) return error;
+
+  const e = toErrorLike(error);
+  const originalError = error instanceof Error ? error : undefined;
+  const message = e.message ?? 'Unknown error';
+
+  // 1. HTTP status code
+  const httpStatus = getHttpStatus(e);
+  if (httpStatus !== undefined) {
+    const code = classifyByHttpStatus(httpStatus);
+    if (code)
+      return new SoleriError(message, code, { cause: originalError, context: { httpStatus } });
+  }
+
+  // 2. Node.js error code
+  const errCode = classifyByErrorCode(e.code);
+  if (errCode)
+    return new SoleriError(message, errCode, {
+      cause: originalError,
+      context: { errorCode: e.code },
+    });
+
+  // 3. Message pattern matching
+  const msgCode = classifyByMessage(message);
+  if (msgCode) return new SoleriError(message, msgCode, { cause: originalError });
+
+  // 4. Default: permanent INTERNAL
+  return new SoleriError(message, SoleriErrorCode.INTERNAL, { cause: originalError });
+}

package/src/errors/index.ts

@@ -0,0 +1,5 @@
+export { SoleriErrorCode, SoleriError, ok, err, isOk, isErr } from './types.js';
+export type { ErrorClassification, Result, SoleriErrorOptions } from './types.js';
+export { classifyError } from './classify.js';
+export { retryWithPreset, shouldRetry, getRetryDelay, RETRY_PRESETS } from './retry.js';
+export type { RetryPreset, RetryConfig, RetryOptions } from './retry.js';