@soleri/core 7.0.0 → 8.1.0

package/src/intake/content-classifier.ts

@@ -60,8 +60,6 @@ export async function classifyChunk(
 ): Promise<ClassifiedItem[]> {
   try {
     const result = await llm.complete({
-      provider: 'openai',
-      model: 'gpt-4o-mini',
       systemPrompt: CLASSIFICATION_PROMPT,
       userPrompt: chunkText,
       maxTokens: 4096,

package/src/intake/text-ingester.ts

@@ -0,0 +1,234 @@
+/**
+ * Text Ingester — ingest articles, transcripts, and plain text into the vault.
+ *
+ * Reuses existing content-classifier (LLM extraction) and dedup-gate (TF-IDF).
+ * No new dependencies — fetch() is built-in, HTML stripping is regex-based.
+ */
+
+import type { Vault } from '../vault/vault.js';
+import type { LLMClient } from '../llm/llm-client.js';
+import type { IntelligenceEntry } from '../intelligence/types.js';
+import type { ClassifiedItem } from './types.js';
+import { classifyChunk } from './content-classifier.js';
+import { dedupItems } from './dedup-gate.js';
+
+// ─── Types ───────────────────────────────────────────────────────────
+
+export interface IngestSource {
+  type: 'article' | 'transcript' | 'notes' | 'documentation';
+  title: string;
+  url?: string;
+  author?: string;
+}
+
+export interface IngestOptions {
+  domain?: string;
+  tags?: string[];
+  /** Max chars per chunk for LLM classification. Default 4000. */
+  chunkSize?: number;
+}
+
+export interface IngestResult {
+  source: IngestSource;
+  ingested: number;
+  duplicates: number;
+  entries: Array<{ id: string; title: string; type: string }>;
+}
+
+// ─── Constants ───────────────────────────────────────────────────────
+
+const DEFAULT_CHUNK_SIZE = 4000;
+const FETCH_TIMEOUT_MS = 15000;
+
+// ─── Class ───────────────────────────────────────────────────────────
+
+export class TextIngester {
+  private vault: Vault;
+  private llm: LLMClient | null;
+
+  constructor(vault: Vault, llm: LLMClient | null) {
+    this.vault = vault;
+    this.llm = llm;
+  }
+
+  /**
+   * Ingest a URL — fetch, strip HTML, classify, dedup, store.
+   */
+  async ingestUrl(url: string, opts?: IngestOptions): Promise<IngestResult> {
+    if (!this.llm) {
+      return { source: { type: 'article', title: url }, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    let text: string;
+    let title = url;
+    try {
+      const response = await fetch(url, {
+        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+        headers: { 'User-Agent': 'Soleri/1.0 (knowledge ingestion)' },
+      });
+      if (!response.ok) {
+        return { source: { type: 'article', title }, ingested: 0, duplicates: 0, entries: [] };
+      }
+      const html = await response.text();
+      title = extractTitle(html) ?? url;
+      text = stripHtml(html);
+    } catch {
+      return { source: { type: 'article', title }, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    if (text.length < 50) {
+      return { source: { type: 'article', title }, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    const source: IngestSource = { type: 'article', title, url };
+    return this.ingestText(text, source, opts);
+  }
+
+  /**
+   * Ingest raw text — classify, dedup, store.
+   */
+  async ingestText(
+    text: string,
+    source: IngestSource,
+    opts?: IngestOptions,
+  ): Promise<IngestResult> {
+    if (!this.llm) {
+      return { source, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    const chunkSize = opts?.chunkSize ?? DEFAULT_CHUNK_SIZE;
+    const chunks = splitIntoChunks(text, chunkSize);
+    const domain = opts?.domain ?? 'general';
+    const extraTags = opts?.tags ?? [];
+
+    // Classify all chunks
+    const allItems: ClassifiedItem[] = [];
+    for (const chunk of chunks) {
+      const items = await classifyChunk(this.llm, chunk, `${source.type}: ${source.title}`);
+      allItems.push(...items);
+    }
+
+    if (allItems.length === 0) {
+      return { source, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    // Dedup against vault
+    const dedupResults = dedupItems(allItems, this.vault);
+    const unique = dedupResults.filter((r) => !r.isDuplicate).map((r) => r.item);
+    const duplicateCount = dedupResults.filter((r) => r.isDuplicate).length;
+
+    // Build source attribution for context field
+    const attribution = buildAttribution(source);
+
+    // Store in vault
+    const entries: IntelligenceEntry[] = unique.map((item, i) => ({
+      id: `ingest-${source.type}-${Date.now()}-${i}-${Math.random().toString(36).slice(2, 6)}`,
+      type: mapType(item.type),
+      domain,
+      title: item.title,
+      description: item.description,
+      severity: mapSeverity(item.severity),
+      tags: [...(item.tags ?? []), ...extraTags, 'ingested', source.type],
+      context: attribution,
+      origin: 'user' as const,
+    }));
+
+    if (entries.length > 0) {
+      this.vault.seed(entries);
+    }
+
+    return {
+      source,
+      ingested: entries.length,
+      duplicates: duplicateCount,
+      entries: entries.map((e) => ({ id: e.id, title: e.title, type: e.type })),
+    };
+  }
+
+  /**
+   * Ingest multiple items in sequence.
+   */
+  async ingestBatch(
+    items: Array<{ text: string; source: IngestSource; opts?: IngestOptions }>,
+  ): Promise<IngestResult[]> {
+    const results: IngestResult[] = [];
+    for (const item of items) {
+      const result = await this.ingestText(item.text, item.source, item.opts);
+      results.push(result);
+    }
+    return results;
+  }
+}
+
+// ─── Helpers ─────────────────────────────────────────────────────────
+
+function stripHtml(html: string): string {
+  return (
+    html
+      // Remove script and style blocks
+      .replace(/<script[\s\S]*?<\/script>/gi, '')
+      .replace(/<style[\s\S]*?<\/style>/gi, '')
+      // Remove nav, header, footer, aside
+      .replace(/<(nav|header|footer|aside)[\s\S]*?<\/\1>/gi, '')
+      // Remove all HTML tags
+      .replace(/<[^>]+>/g, ' ')
+      // Decode common entities
+      .replace(/&amp;/g, '&')
+      .replace(/&lt;/g, '<')
+      .replace(/&gt;/g, '>')
+      .replace(/&quot;/g, '"')
+      .replace(/&#39;/g, "'")
+      .replace(/&nbsp;/g, ' ')
+      // Collapse whitespace
+      .replace(/\s+/g, ' ')
+      .trim()
+  );
+}
+
+function extractTitle(html: string): string | null {
+  const match = html.match(/<title[^>]*>(.*?)<\/title>/i);
+  if (match) {
+    return match[1].replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').trim();
+  }
+  return null;
+}
+
+function splitIntoChunks(text: string, chunkSize: number): string[] {
+  if (text.length <= chunkSize) return [text];
+
+  const chunks: string[] = [];
+  let start = 0;
+  while (start < text.length) {
+    let end = start + chunkSize;
+    // Try to break at a sentence boundary
+    if (end < text.length) {
+      const lastPeriod = text.lastIndexOf('. ', end);
+      if (lastPeriod > start + chunkSize * 0.5) {
+        end = lastPeriod + 2;
+      }
+    }
+    chunks.push(text.slice(start, end).trim());
+    start = end;
+  }
+  return chunks.filter((c) => c.length > 0);
+}
+
+function buildAttribution(source: IngestSource): string {
+  const parts = [`Source: ${source.type}`];
+  if (source.title) parts.push(`Title: ${source.title}`);
+  if (source.url) parts.push(`URL: ${source.url}`);
+  if (source.author) parts.push(`Author: ${source.author}`);
+  return parts.join(' | ');
+}
+
+function mapType(type: string): IntelligenceEntry['type'] {
+  if (type === 'pattern') return 'pattern';
+  if (type === 'anti-pattern') return 'anti-pattern';
+  return 'rule';
+}
+
+function mapSeverity(severity: string | undefined): IntelligenceEntry['severity'] {
+  if (severity === 'critical') return 'critical';
+  if (severity === 'warning') return 'warning';
+  return 'suggestion';
+}

package/src/llm/llm-client.ts

@@ -25,8 +25,45 @@ const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions';
 // =============================================================================
 
 function loadRoutingConfig(agentId: string): RoutingConfig {
+  // Default task→model routing: cheap models for routine, powerful for reasoning.
+  // Anthropic routes use extended thinking for quality decisions when available.
+  // Agents can override via ~/.{agentId}/model-routing.json.
+  const defaultRoutes: RouteEntry[] = [
+    // OpenAI routes (default — works without Anthropic key)
+    { caller: 'quality-gate', task: 'evaluate', model: 'gpt-4o', provider: 'openai' },
+    { caller: 'classifier', task: 'classify', model: 'gpt-4o-mini', provider: 'openai' },
+    { caller: 'knowledge-synthesizer', task: 'synthesize', model: 'gpt-4o', provider: 'openai' },
+    { caller: 'content-classifier', model: 'gpt-4o-mini', provider: 'openai' },
+    { caller: 'vault-linking', task: 'evaluate-links', model: 'gpt-4o-mini', provider: 'openai' },
+    // Anthropic routes (higher quality when key available — extended thinking capable)
+    {
+      caller: 'quality-gate-anthropic',
+      task: 'evaluate',
+      model: 'claude-sonnet-4-20250514',
+      provider: 'anthropic',
+    },
+    {
+      caller: 'contradiction-evaluator',
+      task: 'evaluate',
+      model: 'claude-sonnet-4-20250514',
+      provider: 'anthropic',
+    },
+    {
+      caller: 'knowledge-synthesizer-anthropic',
+      task: 'synthesize',
+      model: 'claude-sonnet-4-20250514',
+      provider: 'anthropic',
+    },
+    {
+      caller: 'classifier-anthropic',
+      task: 'classify',
+      model: 'claude-haiku-4-5-20251001',
+      provider: 'anthropic',
+    },
+  ];
+
   const defaultConfig: RoutingConfig = {
-    routes: [],
+    routes: defaultRoutes,
     defaultOpenAIModel: 'gpt-4o-mini',
     defaultAnthropicModel: 'claude-sonnet-4-20250514',
   };
@@ -123,6 +160,8 @@ interface AnthropicClient {
   };
 }
 
+type ResolvedLLMOptions = LLMCallOptions & { model: string; provider: 'openai' | 'anthropic' };
+
 export class LLMClient {
   private openaiKeyPool: KeyPool;
   private anthropicKeyPool: KeyPool;
@@ -144,11 +183,15 @@ export class LLMClient {
 
   async complete(options: LLMCallOptions): Promise<LLMCallResult> {
     const routed = this.router.resolve(options.caller, options.task, options.model);
-    const resolvedOptions = { ...options, model: routed.model, provider: routed.provider };
+    const resolved: ResolvedLLMOptions = {
+      ...options,
+      model: options.model ?? routed.model,
+      provider: options.provider ?? routed.provider,
+    };
 
-    return resolvedOptions.provider === 'anthropic'
-      ? this.callAnthropic(resolvedOptions)
-      : this.callOpenAI(resolvedOptions);
+    return resolved.provider === 'anthropic'
+      ? this.callAnthropic(resolved)
+      : this.callOpenAI(resolved);
   }
 
   isAvailable(): { openai: boolean; anthropic: boolean } {
@@ -166,7 +209,7 @@ export class LLMClient {
   // OPENAI
   // ===========================================================================
 
-  private async callOpenAI(options: LLMCallOptions): Promise<LLMCallResult> {
+  private async callOpenAI(options: ResolvedLLMOptions): Promise<LLMCallResult> {
     const keyPool = this.openaiKeyPool.hasKeys ? this.openaiKeyPool : null;
 
     if (!keyPool) {
@@ -238,7 +281,7 @@ export class LLMClient {
   // ANTHROPIC
   // ===========================================================================
 
-  private async callAnthropic(options: LLMCallOptions): Promise<LLMCallResult> {
+  private async callAnthropic(options: ResolvedLLMOptions): Promise<LLMCallResult> {
     const client = await this.getAnthropicClient();
     if (!client) {
       throw new LLMError('Anthropic API key not configured', { retryable: false });

package/src/llm/oauth-discovery.ts

@@ -0,0 +1,151 @@
+/**
+ * OAuth Token Discovery — find Claude Code OAuth tokens on macOS and Linux.
+ *
+ * Priority:
+ * 1. ANTHROPIC_API_KEY env var (explicit, highest priority)
+ * 2. Claude Code credentials file (~/.claude/.credentials.json or similar)
+ * 3. macOS Keychain (security find-generic-password)
+ * 4. Linux GNOME Keyring (secret-tool lookup)
+ * 5. null (graceful fallback → use OpenAI or no LLM)
+ *
+ * Cached for 5 minutes to avoid repeated I/O.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir, platform } from 'node:os';
+
+// ─── Cache ───────────────────────────────────────────────────────────
+
+let cachedToken: string | null = null;
+let cacheTimestamp = 0;
+const CACHE_TTL_MS = 5 * 60 * 1000;
+
+// ─── Public API ──────────────────────────────────────────────────────
+
+/**
+ * Discover an Anthropic API token. Returns null if none found.
+ * Results cached for 5 minutes.
+ */
+export function discoverAnthropicToken(): string | null {
+  if (cachedToken && Date.now() - cacheTimestamp < CACHE_TTL_MS) {
+    return cachedToken;
+  }
+
+  const token = tryEnvVar() ?? tryCredentialsFile() ?? tryPlatformKeychain();
+
+  if (token) {
+    cachedToken = token;
+    cacheTimestamp = Date.now();
+  }
+
+  return token;
+}
+
+// ─── Discovery Methods ───────────────────────────────────────────────
+
+function tryEnvVar(): string | null {
+  return process.env.ANTHROPIC_API_KEY ?? null;
+}
+
+function tryCredentialsFile(): string | null {
+  const candidates = [
+    join(homedir(), '.claude', '.credentials.json'),
+    join(homedir(), '.claude', 'credentials.json'),
+    join(homedir(), '.config', 'claude', 'credentials.json'),
+  ];
+
+  for (const path of candidates) {
+    try {
+      if (!existsSync(path)) continue;
+      const raw = readFileSync(path, 'utf-8');
+      const parsed = JSON.parse(raw) as Record<string, unknown>;
+
+      // Claude Code OAuth format: { claudeAiOauth: { accessToken: "..." } }
+      const oauth = parsed.claudeAiOauth as Record<string, unknown> | undefined;
+      if (oauth?.accessToken && typeof oauth.accessToken === 'string') {
+        return oauth.accessToken;
+      }
+
+      // Alternative: direct token field
+      if (parsed.accessToken && typeof parsed.accessToken === 'string') {
+        return parsed.accessToken as string;
+      }
+
+      // Alternative: API key field
+      if (parsed.apiKey && typeof parsed.apiKey === 'string') {
+        return parsed.apiKey as string;
+      }
+    } catch {
+      continue;
+    }
+  }
+
+  return null;
+}
+
+function tryPlatformKeychain(): string | null {
+  const os = platform();
+
+  if (os === 'darwin') return tryMacKeychain();
+  if (os === 'linux') return tryLinuxKeyring();
+
+  return null;
+}
+
+function tryMacKeychain(): string | null {
+  try {
+    const raw = execFileSync(
+      'security',
+      ['find-generic-password', '-s', 'Claude Code-credentials', '-w'],
+      { encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] },
+    ).trim();
+
+    if (!raw) return null;
+
+    // Try JSON parse
+    try {
+      const parsed = JSON.parse(raw) as Record<string, unknown>;
+      const oauth = parsed.claudeAiOauth as Record<string, unknown> | undefined;
+      if (oauth?.accessToken && typeof oauth.accessToken === 'string') {
+        return oauth.accessToken;
+      }
+    } catch {
+      // JSON might be truncated — try regex fallback
+      const match = raw.match(/"accessToken"\s*:\s*"([^"]+)"/);
+      if (match) return match[1];
+    }
+  } catch {
+    // Keychain not available or no entry
+  }
+
+  return null;
+}
+
+function tryLinuxKeyring(): string | null {
+  try {
+    // GNOME Keyring via secret-tool
+    const token = execFileSync(
+      'secret-tool',
+      ['lookup', 'service', 'Claude Code', 'type', 'credentials'],
+      { encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] },
+    ).trim();
+
+    if (token) {
+      // May be JSON or raw token
+      try {
+        const parsed = JSON.parse(token) as Record<string, unknown>;
+        const oauth = parsed.claudeAiOauth as Record<string, unknown> | undefined;
+        if (oauth?.accessToken) return oauth.accessToken as string;
+      } catch {
+        // Treat as raw token
+        if (token.length > 20) return token;
+      }
+    }
+  } catch {
+    // secret-tool not available or no entry
+  }
+
+  return null;
+}

package/src/llm/types.ts

@@ -43,8 +43,10 @@ export class LLMError extends Error {
 }
 
 export interface LLMCallOptions {
-  provider: 'openai' | 'anthropic';
-  model: string;
+  /** Provider override. If omitted, the model router selects based on caller/task. */
+  provider?: 'openai' | 'anthropic';
+  /** Model override. If omitted, the model router selects based on caller/task. */
+  model?: string;
   systemPrompt: string;
   userPrompt: string;
   temperature?: number;

package/src/packs/pack-installer.ts

@@ -21,6 +21,7 @@ import { loadIntelligenceData } from '../intelligence/loader.js';
 import type { Vault } from '../vault/vault.js';
 import type { PluginRegistry } from '../plugins/plugin-registry.js';
 import type { PluginContext } from '../plugins/types.js';
+import type { PackRuntime } from '../domain-packs/pack-runtime.js';
 
 const MANIFEST_FILENAME = 'soleri-pack.json';
 
@@ -108,7 +109,11 @@ export class PackInstaller {
   /**
    * Install a knowledge pack from a directory.
    */
-  async install(packDir: string, runtimeCtx?: unknown): Promise<InstallResult> {
+  async install(
+    packDir: string,
+    runtimeCtx?: unknown,
+    packRuntime?: PackRuntime,
+  ): Promise<InstallResult> {
     // Validate first
     const validation = this.validate(packDir);
     if (!validation.valid || !validation.manifest) {
@@ -174,6 +179,16 @@ export class PackInstaller {
         }
 
         const ctx: PluginContext = {
+          packRuntime:
+            packRuntime ??
+            ({
+              vault: {},
+              getProject: () => undefined,
+              listProjects: () => [],
+              createCheck: () => '',
+              validateCheck: () => null,
+              validateAndConsume: () => null,
+            } as unknown as PackRuntime),
           runtime: runtimeCtx ?? {},
           manifest: pluginLoaded.manifest,
           directory: packDir,

package/src/persistence/index.ts

@@ -6,4 +6,3 @@ export type {
   FtsSearchOptions,
 } from './types.js';
 export { SQLitePersistenceProvider } from './sqlite-provider.js';
-export { PostgresPersistenceProvider, translateSql } from './postgres-provider.js';

package/src/persistence/types.ts

@@ -35,7 +35,7 @@ export interface PersistenceProvider {
   transaction<T>(fn: () => T): T;
 
   /** Identifies the backend engine. */
-  readonly backend: 'sqlite' | 'postgres';
+  readonly backend: 'sqlite';
 
   /** Full-text search abstraction. */
   ftsSearch<T = Record<string, unknown>>(
@@ -52,12 +52,8 @@ export interface PersistenceProvider {
 }
 
 export interface PersistenceConfig {
-  type: 'sqlite' | 'postgres';
+  type: 'sqlite';
   path: string;
-  /** PostgreSQL connection string. */
-  connectionString?: string;
-  /** PostgreSQL pool size. */
-  poolSize?: number;
 }
 
 export interface FtsSearchOptions {