@soleri/core 1.0.0 → 2.0.1

package/src/brain/brain.ts

@@ -1,11 +1,13 @@
 import type { Vault } from '../vault/vault.js';
 import type { SearchResult } from '../vault/vault.js';
 import type { IntelligenceEntry } from '../intelligence/types.js';
+import type { CogneeClient } from '../cognee/client.js';
 
 // ─── Types ───────────────────────────────────────────────────────────
 
 export interface ScoringWeights {
   semantic: number;
+  vector: number;
   severity: number;
   recency: number;
   tagOverlap: number;
@@ -14,6 +16,7 @@ export interface ScoringWeights {
 
 export interface ScoreBreakdown {
   semantic: number;
+  vector: number;
   severity: number;
   recency: number;
   tagOverlap: number;
@@ -233,12 +236,22 @@ const SEVERITY_SCORES: Record<string, number> = {
 
 const DEFAULT_WEIGHTS: ScoringWeights = {
   semantic: 0.4,
+  vector: 0.0,
   severity: 0.15,
   recency: 0.15,
   tagOverlap: 0.15,
   domainMatch: 0.15,
 };
 
+const COGNEE_WEIGHTS: ScoringWeights = {
+  semantic: 0.25,
+  vector: 0.35,
+  severity: 0.1,
+  recency: 0.1,
+  tagOverlap: 0.1,
+  domainMatch: 0.1,
+};
+
 const WEIGHT_BOUND = 0.15;
 const FEEDBACK_THRESHOLD = 30;
 const DUPLICATE_BLOCK_THRESHOLD = 0.8;
@@ -247,16 +260,18 @@ const RECENCY_HALF_LIFE_DAYS = 365;
 
 export class Brain {
   private vault: Vault;
+  private cognee: CogneeClient | undefined;
   private vocabulary: Map<string, number> = new Map();
   private weights: ScoringWeights = { ...DEFAULT_WEIGHTS };
 
-  constructor(vault: Vault) {
+  constructor(vault: Vault, cognee?: CogneeClient) {
     this.vault = vault;
+    this.cognee = cognee;
     this.rebuildVocabulary();
     this.recomputeWeights();
   }
 
-  intelligentSearch(query: string, options?: SearchOptions): RankedResult[] {
+  async intelligentSearch(query: string, options?: SearchOptions): Promise<RankedResult[]> {
     const limit = options?.limit ?? 10;
     const rawResults = this.vault.search(query, {
       domain: options?.domain,
@@ -265,6 +280,97 @@ export class Brain {
       limit: Math.max(limit * 3, 30),
     });
 
+    // Cognee vector search (parallel, with timeout fallback)
+    let cogneeScoreMap: Map<string, number> = new Map();
+    const cogneeAvailable = this.cognee?.isAvailable ?? false;
+    if (cogneeAvailable && this.cognee) {
+      try {
+        const cogneeResults = await this.cognee.search(query, { limit: Math.max(limit * 2, 20) });
+
+        // Build title → entryIds reverse index from FTS results for text-based matching.
+        // Cognee assigns its own UUIDs to chunks and may strip embedded metadata during
+        // chunking, so we need multiple strategies to cross-reference results.
+        // Multiple entries can share a title, so map to arrays of IDs.
+        const titleToIds = new Map<string, string[]>();
+        for (const r of rawResults) {
+          const key = r.entry.title.toLowerCase().trim();
+          const ids = titleToIds.get(key) ?? [];
+          ids.push(r.entry.id);
+          titleToIds.set(key, ids);
+        }
+
+        const vaultIdPattern = /\[vault-id:([^\]]+)\]/;
+        const unmatchedCogneeResults: Array<{ text: string; score: number }> = [];
+
+        for (const cr of cogneeResults) {
+          const text = cr.text ?? '';
+
+          // Strategy 1: Extract vault ID from [vault-id:XXX] prefix (if Cognee preserved it)
+          const vaultIdMatch = text.match(vaultIdPattern);
+          if (vaultIdMatch) {
+            const vaultId = vaultIdMatch[1];
+            cogneeScoreMap.set(vaultId, Math.max(cogneeScoreMap.get(vaultId) ?? 0, cr.score));
+            continue;
+          }
+
+          // Strategy 2: Match first line of chunk text against known entry titles.
+          // serializeEntry() puts the title on the first line after the [vault-id:] prefix,
+          // and Cognee's chunking typically preserves this as the chunk start.
+          const firstLine = text.split('\n')[0]?.trim().toLowerCase() ?? '';
+          const matchedIds = firstLine ? titleToIds.get(firstLine) : undefined;
+          if (matchedIds) {
+            for (const id of matchedIds) {
+              cogneeScoreMap.set(id, Math.max(cogneeScoreMap.get(id) ?? 0, cr.score));
+            }
+            continue;
+          }
+
+          // Strategy 3: Check if any known title appears as a substring in the chunk.
+          // Handles cases where the title isn't on the first line (mid-document chunks).
+          const textLower = text.toLowerCase();
+          let found = false;
+          for (const [title, ids] of titleToIds) {
+            if (title.length >= 8 && textLower.includes(title)) {
+              for (const id of ids) {
+                cogneeScoreMap.set(id, Math.max(cogneeScoreMap.get(id) ?? 0, cr.score));
+              }
+              found = true;
+              break;
+            }
+          }
+          if (!found && text.length > 0) {
+            unmatchedCogneeResults.push({ text, score: cr.score });
+          }
+        }
+
+        // Strategy 4: For Cognee-only semantic matches (not in FTS results),
+        // use the first line as a vault FTS query to find the source entry.
+        // Preserve caller filters (domain/type/severity) to avoid reintroducing
+        // entries the original query excluded.
+        for (const unmatched of unmatchedCogneeResults) {
+          const searchTerm = unmatched.text.split('\n')[0]?.trim();
+          if (!searchTerm || searchTerm.length < 3) continue;
+          const vaultHits = this.vault.search(searchTerm, {
+            domain: options?.domain,
+            type: options?.type,
+            severity: options?.severity,
+            limit: 1,
+          });
+          if (vaultHits.length > 0) {
+            const id = vaultHits[0].entry.id;
+            cogneeScoreMap.set(id, Math.max(cogneeScoreMap.get(id) ?? 0, unmatched.score));
+            // Also add to FTS results pool if not already present
+            if (!rawResults.some((r) => r.entry.id === id)) {
+              rawResults.push(vaultHits[0]);
+            }
+          }
+        }
+      } catch {
+        // Cognee failed — fall back to FTS5 only
+        cogneeScoreMap = new Map();
+      }
+    }
+
     if (rawResults.length === 0) return [];
 
     const queryTokens = tokenize(query);
@@ -272,9 +378,22 @@ export class Brain {
     const queryDomain = options?.domain;
     const now = Math.floor(Date.now() / 1000);
 
+    // Use cognee-aware weights only if at least one ranked candidate has a vector score
+    const hasVectorCandidate = rawResults.some((r) => cogneeScoreMap.has(r.entry.id));
+    const activeWeights = hasVectorCandidate ? this.getCogneeWeights() : this.weights;
+
     const ranked = rawResults.map((result) => {
       const entry = result.entry;
-      const breakdown = this.scoreEntry(entry, queryTokens, queryTags, queryDomain, now);
+      const vectorScore = cogneeScoreMap.get(entry.id) ?? 0;
+      const breakdown = this.scoreEntry(
+        entry,
+        queryTokens,
+        queryTags,
+        queryDomain,
+        now,
+        vectorScore,
+        activeWeights,
+      );
       return { entry, score: breakdown.total, breakdown };
     });
 
@@ -325,6 +444,11 @@ export class Brain {
     this.vault.add(fullEntry);
     this.updateVocabularyIncremental(fullEntry);
 
+    // Fire-and-forget Cognee sync
+    if (this.cognee?.isAvailable) {
+      this.cognee.addEntries([fullEntry]).catch(() => {});
+    }
+
     const result: CaptureResult = {
       captured: true,
       id: entry.id,
@@ -348,13 +472,39 @@ export class Brain {
     this.recomputeWeights();
   }
 
-  getRelevantPatterns(context: QueryContext): RankedResult[] {
+  async getRelevantPatterns(context: QueryContext): Promise<RankedResult[]> {
     return this.intelligentSearch(context.query, {
       domain: context.domain,
       tags: context.tags,
     });
   }
 
+  async syncToCognee(): Promise<{ synced: number; cognified: boolean }> {
+    if (!this.cognee?.isAvailable) return { synced: 0, cognified: false };
+
+    const batchSize = 1000;
+    let offset = 0;
+    let totalSynced = 0;
+
+    while (true) {
+      const batch = this.vault.list({ limit: batchSize, offset });
+      if (batch.length === 0) break;
+
+      const { added } = await this.cognee.addEntries(batch);
+      totalSynced += added;
+      offset += batch.length;
+
+      if (batch.length < batchSize) break;
+    }
+
+    if (totalSynced === 0) return { synced: 0, cognified: false };
+
+    let cognified = false;
+    const cognifyResult = await this.cognee.cognify();
+    cognified = cognifyResult.status === 'ok';
+    return { synced: totalSynced, cognified };
+  }
+
   rebuildVocabulary(): void {
     const entries = this.vault.list({ limit: 100000 });
     const docCount = entries.length;
@@ -408,7 +558,11 @@ export class Brain {
     queryTags: string[],
     queryDomain: string | undefined,
     now: number,
+    vectorScore: number = 0,
+    activeWeights?: ScoringWeights,
   ): ScoreBreakdown {
+    const w = activeWeights ?? this.weights;
+
     let semantic = 0;
     if (this.vocabulary.size > 0 && queryTokens.length > 0) {
       const entryText = [
@@ -433,14 +587,17 @@ export class Brain {
 
     const domainMatch = queryDomain && entry.domain === queryDomain ? 1.0 : 0;
 
-    const total =
-      this.weights.semantic * semantic +
-      this.weights.severity * severity +
-      this.weights.recency * recency +
-      this.weights.tagOverlap * tagOverlap +
-      this.weights.domainMatch * domainMatch;
+    const vector = vectorScore;
 
-    return { semantic, severity, recency, tagOverlap, domainMatch, total };
+    const total =
+      w.semantic * semantic +
+      w.vector * vector +
+      w.severity * severity +
+      w.recency * recency +
+      w.tagOverlap * tagOverlap +
+      w.domainMatch * domainMatch;
+
+    return { semantic, vector, severity, recency, tagOverlap, domainMatch, total };
   }
 
   private generateTags(title: string, description: string, context?: string): string[] {
@@ -534,6 +691,10 @@ export class Brain {
     tx();
   }
 
+  private getCogneeWeights(): ScoringWeights {
+    return { ...COGNEE_WEIGHTS };
+  }
+
   private recomputeWeights(): void {
     const db = this.vault.getDb();
     const feedbackCount = (
@@ -560,7 +721,10 @@ export class Brain {
       DEFAULT_WEIGHTS.semantic + WEIGHT_BOUND,
     );
 
-    const remaining = 1.0 - newWeights.semantic;
+    // vector stays 0 in base weights (only active during hybrid search)
+    newWeights.vector = 0;
+
+    const remaining = 1.0 - newWeights.semantic - newWeights.vector;
     const otherSum =
       DEFAULT_WEIGHTS.severity +
       DEFAULT_WEIGHTS.recency +

package/src/cognee/client.ts

@@ -0,0 +1,352 @@
+import type {
+  CogneeConfig,
+  CogneeSearchResult,
+  CogneeSearchType,
+  CogneeStatus,
+  CogneeAddResult,
+  CogneeCognifyResult,
+} from './types.js';
+import type { IntelligenceEntry } from '../intelligence/types.js';
+
+// ─── Defaults ──────────────────────────────────────────────────────
+// Aligned with Salvador MCP's battle-tested Cognee integration.
+
+const DEFAULT_SERVICE_EMAIL = 'soleri-agent@cognee.dev';
+const DEFAULT_SERVICE_PASSWORD = 'soleri-cognee-local';
+
+/** Only allow default service credentials for local endpoints. */
+function isLocalUrl(url: string): boolean {
+  try {
+    const { hostname } = new URL(url);
+    return (
+      hostname === 'localhost' ||
+      hostname === '127.0.0.1' ||
+      hostname === '::1' ||
+      hostname === '0.0.0.0'
+    );
+  } catch {
+    return false;
+  }
+}
+
+const DEFAULT_CONFIG: CogneeConfig = {
+  baseUrl: 'http://localhost:8000',
+  dataset: 'vault',
+  timeoutMs: 30_000,
+  searchTimeoutMs: 120_000, // Ollama cold start can take 90s
+  healthTimeoutMs: 5_000,
+  healthCacheTtlMs: 60_000,
+  cognifyDebounceMs: 30_000,
+};
+
+// ─── CogneeClient ──────────────────────────────────────────────────
+
+export class CogneeClient {
+  private config: CogneeConfig;
+  private healthCache: { status: CogneeStatus; cachedAt: number } | null = null;
+  private accessToken: string | null = null;
+  private cognifyTimers: Map<string, ReturnType<typeof setTimeout>> = new Map();
+  private pendingDatasets: Set<string> = new Set();
+
+  constructor(config?: Partial<CogneeConfig>) {
+    this.config = { ...DEFAULT_CONFIG, ...config };
+    // Strip trailing slash
+    this.config.baseUrl = this.config.baseUrl.replace(/\/+$/, '');
+    // Pre-set token if provided
+    if (this.config.apiToken) {
+      this.accessToken = this.config.apiToken;
+    }
+  }
+
+  // ─── Health ────────────────────────────────────────────────────
+
+  get isAvailable(): boolean {
+    if (!this.healthCache) return false;
+    const age = Date.now() - this.healthCache.cachedAt;
+    if (age > this.config.healthCacheTtlMs) return false;
+    return this.healthCache.status.available;
+  }
+
+  async healthCheck(): Promise<CogneeStatus> {
+    const start = Date.now();
+    try {
+      // Cognee health endpoint is GET / (returns {"message":"Hello, World, I am alive!"})
+      const res = await globalThis.fetch(`${this.config.baseUrl}/`, {
+        signal: AbortSignal.timeout(this.config.healthTimeoutMs),
+      });
+      const latencyMs = Date.now() - start;
+      if (res.ok) {
+        const status: CogneeStatus = { available: true, url: this.config.baseUrl, latencyMs };
+        this.healthCache = { status, cachedAt: Date.now() };
+        return status;
+      }
+      const status: CogneeStatus = {
+        available: false,
+        url: this.config.baseUrl,
+        latencyMs,
+        error: `HTTP ${res.status}`,
+      };
+      this.healthCache = { status, cachedAt: Date.now() };
+      return status;
+    } catch (err) {
+      const latencyMs = Date.now() - start;
+      const status: CogneeStatus = {
+        available: false,
+        url: this.config.baseUrl,
+        latencyMs,
+        error: err instanceof Error ? err.message : String(err),
+      };
+      this.healthCache = { status, cachedAt: Date.now() };
+      return status;
+    }
+  }
+
+  // ─── Ingest ────────────────────────────────────────────────────
+
+  async addEntries(entries: IntelligenceEntry[]): Promise<CogneeAddResult> {
+    if (!this.isAvailable || entries.length === 0) return { added: 0 };
+
+    try {
+      const token = await this.ensureAuth().catch(() => null);
+
+      // Cognee /add expects multipart/form-data with files + datasetName
+      const formData = new FormData();
+      formData.append('datasetName', this.config.dataset);
+
+      for (const entry of entries) {
+        const text = this.serializeEntry(entry);
+        const blob = new Blob([text], { type: 'text/plain' });
+        formData.append('data', blob, `${entry.id}.txt`);
+      }
+
+      const headers: Record<string, string> = {};
+      if (token) headers.Authorization = `Bearer ${token}`;
+
+      const res = await globalThis.fetch(`${this.config.baseUrl}/api/v1/add`, {
+        method: 'POST',
+        headers,
+        body: formData,
+        signal: AbortSignal.timeout(this.config.timeoutMs),
+      });
+
+      if (!res.ok) return { added: 0 };
+
+      // Schedule debounced cognify (multiple rapid ingests coalesce)
+      this.scheduleCognify(this.config.dataset);
+
+      return { added: entries.length };
+    } catch {
+      return { added: 0 };
+    }
+  }
+
+  async cognify(dataset?: string): Promise<CogneeCognifyResult> {
+    if (!this.isAvailable) return { status: 'unavailable' };
+
+    try {
+      const res = await this.post('/api/v1/cognify', {
+        datasets: [dataset ?? this.config.dataset],
+      });
+
+      if (!res.ok) return { status: `error: HTTP ${res.status}` };
+      return { status: 'ok' };
+    } catch (err) {
+      return { status: `error: ${err instanceof Error ? err.message : String(err)}` };
+    }
+  }
+
+  // ─── Cognify debounce ───────────────────────────────────────────
+
+  /**
+   * Schedule a debounced cognify for a dataset.
+   * Sliding window: each call resets the timer. When it expires,
+   * cognify fires once. Prevents pipeline dedup on rapid ingests.
+   */
+  private scheduleCognify(dataset: string): void {
+    const existing = this.cognifyTimers.get(dataset);
+    if (existing) clearTimeout(existing);
+
+    this.pendingDatasets.add(dataset);
+
+    const timer = setTimeout(() => {
+      this.cognifyTimers.delete(dataset);
+      this.pendingDatasets.delete(dataset);
+      this.post('/api/v1/cognify', { datasets: [dataset] }).catch(() => {});
+    }, this.config.cognifyDebounceMs);
+
+    // Unref so the timer doesn't keep the process alive during shutdown
+    if (typeof timer === 'object' && 'unref' in timer) (timer as NodeJS.Timeout).unref();
+
+    this.cognifyTimers.set(dataset, timer);
+  }
+
+  /** Flush all pending debounced cognify calls immediately. */
+  async flushPendingCognify(): Promise<void> {
+    const datasets = [...this.pendingDatasets];
+    for (const timer of this.cognifyTimers.values()) clearTimeout(timer);
+    this.cognifyTimers.clear();
+    this.pendingDatasets.clear();
+
+    if (datasets.length === 0) return;
+
+    await Promise.allSettled(
+      datasets.map((ds) => this.post('/api/v1/cognify', { datasets: [ds] }).catch(() => {})),
+    );
+  }
+
+  /** Cancel all pending cognify calls without firing them. For test teardown. */
+  resetPendingCognify(): void {
+    for (const timer of this.cognifyTimers.values()) clearTimeout(timer);
+    this.cognifyTimers.clear();
+    this.pendingDatasets.clear();
+  }
+
+  // ─── Search ────────────────────────────────────────────────────
+
+  async search(
+    query: string,
+    opts?: { searchType?: CogneeSearchType; limit?: number },
+  ): Promise<CogneeSearchResult[]> {
+    if (!this.isAvailable) return [];
+
+    // Default to CHUNKS (pure vector similarity) — GRAPH_COMPLETION requires
+    // the LLM to produce instructor-compatible JSON which small local models
+    // (llama3.2) can't do reliably, causing infinite retries and timeouts.
+    const searchType = opts?.searchType ?? 'CHUNKS';
+    const topK = opts?.limit ?? 10;
+
+    try {
+      const res = await this.post(
+        '/api/v1/search',
+        { query, search_type: searchType, datasets: [this.config.dataset], topK },
+        this.config.searchTimeoutMs,
+      );
+
+      if (!res.ok) return [];
+
+      const data = (await res.json()) as Array<{
+        id?: string;
+        text?: string;
+        score?: number;
+        payload?: { id?: string };
+      }>;
+
+      // Position-based scoring when Cognee omits scores.
+      // Cognee returns results ordered by relevance but may not include numeric scores.
+      return data.slice(0, topK).map((item, idx) => ({
+        id: item.payload?.id ?? item.id ?? '',
+        score: item.score ?? positionScore(idx, data.length),
+        text: typeof item.text === 'string' ? item.text : String(item.text ?? ''),
+        searchType,
+      }));
+    } catch {
+      return [];
+    }
+  }
+
+  // ─── Config access ─────────────────────────────────────────────
+
+  getConfig(): Readonly<CogneeConfig> {
+    return { ...this.config };
+  }
+
+  getStatus(): CogneeStatus | null {
+    return this.healthCache?.status ?? null;
+  }
+
+  // ─── Auth ──────────────────────────────────────────────────────
+  // Auto-register + login pattern from Salvador MCP.
+  // Tries login first (account may already exist), falls back to register.
+
+  private async ensureAuth(): Promise<string> {
+    if (this.accessToken) return this.accessToken;
+
+    const email = this.config.serviceEmail ?? DEFAULT_SERVICE_EMAIL;
+    const password = this.config.servicePassword ?? DEFAULT_SERVICE_PASSWORD;
+
+    // Refuse default credentials for non-local endpoints
+    if (
+      !isLocalUrl(this.config.baseUrl) &&
+      email === DEFAULT_SERVICE_EMAIL &&
+      password === DEFAULT_SERVICE_PASSWORD
+    ) {
+      throw new Error('Explicit Cognee credentials are required for non-local endpoints');
+    }
+
+    // Try login first
+    const loginResp = await globalThis.fetch(`${this.config.baseUrl}/api/v1/auth/login`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+      body: `username=${encodeURIComponent(email)}&password=${encodeURIComponent(password)}`,
+      signal: AbortSignal.timeout(this.config.healthTimeoutMs),
+    });
+
+    if (loginResp.ok) {
+      const data = (await loginResp.json()) as { access_token: string };
+      this.accessToken = data.access_token;
+      return this.accessToken;
+    }
+
+    // Register, then retry login
+    await globalThis.fetch(`${this.config.baseUrl}/api/v1/auth/register`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify({ email, password }),
+      signal: AbortSignal.timeout(this.config.healthTimeoutMs),
+    });
+
+    const retryLogin = await globalThis.fetch(`${this.config.baseUrl}/api/v1/auth/login`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
+      body: `username=${encodeURIComponent(email)}&password=${encodeURIComponent(password)}`,
+      signal: AbortSignal.timeout(this.config.healthTimeoutMs),
+    });
+
+    if (!retryLogin.ok) {
+      throw new Error(`Cognee auth failed: HTTP ${retryLogin.status}`);
+    }
+
+    const retryData = (await retryLogin.json()) as { access_token: string };
+    this.accessToken = retryData.access_token;
+    return this.accessToken;
+  }
+
+  private async authHeaders(): Promise<Record<string, string>> {
+    try {
+      const token = await this.ensureAuth();
+      return { 'Content-Type': 'application/json', Authorization: `Bearer ${token}` };
+    } catch {
+      // Fall back to no auth (works if AUTH_REQUIRED=false)
+      return { 'Content-Type': 'application/json' };
+    }
+  }
+
+  // ─── Private helpers ───────────────────────────────────────────
+
+  private serializeEntry(entry: IntelligenceEntry): string {
+    // Prefix with vault ID so we can cross-reference search results back to vault entries.
+    // Cognee assigns its own UUIDs to chunks — the vault ID would otherwise be lost.
+    const parts = [`[vault-id:${entry.id}]`, entry.title, entry.description];
+    if (entry.context) parts.push(entry.context);
+    if (entry.tags.length > 0) parts.push(`Tags: ${entry.tags.join(', ')}`);
+    return parts.join('\n');
+  }
+
+  private async post(path: string, body: unknown, timeoutMs?: number): Promise<Response> {
+    const headers = await this.authHeaders();
+    return globalThis.fetch(`${this.config.baseUrl}${path}`, {
+      method: 'POST',
+      headers,
+      body: JSON.stringify(body),
+      signal: AbortSignal.timeout(timeoutMs ?? this.config.timeoutMs),
+    });
+  }
+}
+
+// ─── Helpers ──────────────────────────────────────────────────────
+
+/** Position-based score: first result gets ~1.0, last gets ~0.05. */
+function positionScore(index: number, total: number): number {
+  if (total <= 1) return 1.0;
+  return Math.max(0.05, 1.0 - (index / (total - 1)) * 0.95);
+}