@soleri/core 7.0.0 → 8.0.0

package/src/intake/text-ingester.ts

@@ -0,0 +1,234 @@
+/**
+ * Text Ingester — ingest articles, transcripts, and plain text into the vault.
+ *
+ * Reuses existing content-classifier (LLM extraction) and dedup-gate (TF-IDF).
+ * No new dependencies — fetch() is built-in, HTML stripping is regex-based.
+ */
+
+import type { Vault } from '../vault/vault.js';
+import type { LLMClient } from '../llm/llm-client.js';
+import type { IntelligenceEntry } from '../intelligence/types.js';
+import type { ClassifiedItem } from './types.js';
+import { classifyChunk } from './content-classifier.js';
+import { dedupItems } from './dedup-gate.js';
+
+// ─── Types ───────────────────────────────────────────────────────────
+
+export interface IngestSource {
+  type: 'article' | 'transcript' | 'notes' | 'documentation';
+  title: string;
+  url?: string;
+  author?: string;
+}
+
+export interface IngestOptions {
+  domain?: string;
+  tags?: string[];
+  /** Max chars per chunk for LLM classification. Default 4000. */
+  chunkSize?: number;
+}
+
+export interface IngestResult {
+  source: IngestSource;
+  ingested: number;
+  duplicates: number;
+  entries: Array<{ id: string; title: string; type: string }>;
+}
+
+// ─── Constants ───────────────────────────────────────────────────────
+
+const DEFAULT_CHUNK_SIZE = 4000;
+const FETCH_TIMEOUT_MS = 15000;
+
+// ─── Class ───────────────────────────────────────────────────────────
+
+export class TextIngester {
+  private vault: Vault;
+  private llm: LLMClient | null;
+
+  constructor(vault: Vault, llm: LLMClient | null) {
+    this.vault = vault;
+    this.llm = llm;
+  }
+
+  /**
+   * Ingest a URL — fetch, strip HTML, classify, dedup, store.
+   */
+  async ingestUrl(url: string, opts?: IngestOptions): Promise<IngestResult> {
+    if (!this.llm) {
+      return { source: { type: 'article', title: url }, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    let text: string;
+    let title = url;
+    try {
+      const response = await fetch(url, {
+        signal: AbortSignal.timeout(FETCH_TIMEOUT_MS),
+        headers: { 'User-Agent': 'Soleri/1.0 (knowledge ingestion)' },
+      });
+      if (!response.ok) {
+        return { source: { type: 'article', title }, ingested: 0, duplicates: 0, entries: [] };
+      }
+      const html = await response.text();
+      title = extractTitle(html) ?? url;
+      text = stripHtml(html);
+    } catch {
+      return { source: { type: 'article', title }, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    if (text.length < 50) {
+      return { source: { type: 'article', title }, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    const source: IngestSource = { type: 'article', title, url };
+    return this.ingestText(text, source, opts);
+  }
+
+  /**
+   * Ingest raw text — classify, dedup, store.
+   */
+  async ingestText(
+    text: string,
+    source: IngestSource,
+    opts?: IngestOptions,
+  ): Promise<IngestResult> {
+    if (!this.llm) {
+      return { source, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    const chunkSize = opts?.chunkSize ?? DEFAULT_CHUNK_SIZE;
+    const chunks = splitIntoChunks(text, chunkSize);
+    const domain = opts?.domain ?? 'general';
+    const extraTags = opts?.tags ?? [];
+
+    // Classify all chunks
+    const allItems: ClassifiedItem[] = [];
+    for (const chunk of chunks) {
+      const items = await classifyChunk(this.llm, chunk, `${source.type}: ${source.title}`);
+      allItems.push(...items);
+    }
+
+    if (allItems.length === 0) {
+      return { source, ingested: 0, duplicates: 0, entries: [] };
+    }
+
+    // Dedup against vault
+    const dedupResults = dedupItems(allItems, this.vault);
+    const unique = dedupResults.filter((r) => !r.isDuplicate).map((r) => r.item);
+    const duplicateCount = dedupResults.filter((r) => r.isDuplicate).length;
+
+    // Build source attribution for context field
+    const attribution = buildAttribution(source);
+
+    // Store in vault
+    const entries: IntelligenceEntry[] = unique.map((item, i) => ({
+      id: `ingest-${source.type}-${Date.now()}-${i}-${Math.random().toString(36).slice(2, 6)}`,
+      type: mapType(item.type),
+      domain,
+      title: item.title,
+      description: item.description,
+      severity: mapSeverity(item.severity),
+      tags: [...(item.tags ?? []), ...extraTags, 'ingested', source.type],
+      context: attribution,
+      origin: 'user' as const,
+    }));
+
+    if (entries.length > 0) {
+      this.vault.seed(entries);
+    }
+
+    return {
+      source,
+      ingested: entries.length,
+      duplicates: duplicateCount,
+      entries: entries.map((e) => ({ id: e.id, title: e.title, type: e.type })),
+    };
+  }
+
+  /**
+   * Ingest multiple items in sequence.
+   */
+  async ingestBatch(
+    items: Array<{ text: string; source: IngestSource; opts?: IngestOptions }>,
+  ): Promise<IngestResult[]> {
+    const results: IngestResult[] = [];
+    for (const item of items) {
+      const result = await this.ingestText(item.text, item.source, item.opts);
+      results.push(result);
+    }
+    return results;
+  }
+}
+
+// ─── Helpers ─────────────────────────────────────────────────────────
+
+function stripHtml(html: string): string {
+  return (
+    html
+      // Remove script and style blocks
+      .replace(/<script[\s\S]*?<\/script>/gi, '')
+      .replace(/<style[\s\S]*?<\/style>/gi, '')
+      // Remove nav, header, footer, aside
+      .replace(/<(nav|header|footer|aside)[\s\S]*?<\/\1>/gi, '')
+      // Remove all HTML tags
+      .replace(/<[^>]+>/g, ' ')
+      // Decode common entities
+      .replace(/&amp;/g, '&')
+      .replace(/&lt;/g, '<')
+      .replace(/&gt;/g, '>')
+      .replace(/&quot;/g, '"')
+      .replace(/&#39;/g, "'")
+      .replace(/&nbsp;/g, ' ')
+      // Collapse whitespace
+      .replace(/\s+/g, ' ')
+      .trim()
+  );
+}
+
+function extractTitle(html: string): string | null {
+  const match = html.match(/<title[^>]*>(.*?)<\/title>/i);
+  if (match) {
+    return match[1].replace(/&amp;/g, '&').replace(/&lt;/g, '<').replace(/&gt;/g, '>').trim();
+  }
+  return null;
+}
+
+function splitIntoChunks(text: string, chunkSize: number): string[] {
+  if (text.length <= chunkSize) return [text];
+
+  const chunks: string[] = [];
+  let start = 0;
+  while (start < text.length) {
+    let end = start + chunkSize;
+    // Try to break at a sentence boundary
+    if (end < text.length) {
+      const lastPeriod = text.lastIndexOf('. ', end);
+      if (lastPeriod > start + chunkSize * 0.5) {
+        end = lastPeriod + 2;
+      }
+    }
+    chunks.push(text.slice(start, end).trim());
+    start = end;
+  }
+  return chunks.filter((c) => c.length > 0);
+}
+
+function buildAttribution(source: IngestSource): string {
+  const parts = [`Source: ${source.type}`];
+  if (source.title) parts.push(`Title: ${source.title}`);
+  if (source.url) parts.push(`URL: ${source.url}`);
+  if (source.author) parts.push(`Author: ${source.author}`);
+  return parts.join(' | ');
+}
+
+function mapType(type: string): IntelligenceEntry['type'] {
+  if (type === 'pattern') return 'pattern';
+  if (type === 'anti-pattern') return 'anti-pattern';
+  return 'rule';
+}
+
+function mapSeverity(severity: string | undefined): IntelligenceEntry['severity'] {
+  if (severity === 'critical') return 'critical';
+  if (severity === 'warning') return 'warning';
+  return 'suggestion';
+}

package/src/llm/llm-client.ts

@@ -25,8 +25,45 @@ const OPENAI_API_URL = 'https://api.openai.com/v1/chat/completions';
 // =============================================================================
 
 function loadRoutingConfig(agentId: string): RoutingConfig {
+  // Default task→model routing: cheap models for routine, powerful for reasoning.
+  // Anthropic routes use extended thinking for quality decisions when available.
+  // Agents can override via ~/.{agentId}/model-routing.json.
+  const defaultRoutes: RouteEntry[] = [
+    // OpenAI routes (default — works without Anthropic key)
+    { caller: 'quality-gate', task: 'evaluate', model: 'gpt-4o', provider: 'openai' },
+    { caller: 'classifier', task: 'classify', model: 'gpt-4o-mini', provider: 'openai' },
+    { caller: 'knowledge-synthesizer', task: 'synthesize', model: 'gpt-4o', provider: 'openai' },
+    { caller: 'content-classifier', model: 'gpt-4o-mini', provider: 'openai' },
+    { caller: 'vault-linking', task: 'evaluate-links', model: 'gpt-4o-mini', provider: 'openai' },
+    // Anthropic routes (higher quality when key available — extended thinking capable)
+    {
+      caller: 'quality-gate-anthropic',
+      task: 'evaluate',
+      model: 'claude-sonnet-4-20250514',
+      provider: 'anthropic',
+    },
+    {
+      caller: 'contradiction-evaluator',
+      task: 'evaluate',
+      model: 'claude-sonnet-4-20250514',
+      provider: 'anthropic',
+    },
+    {
+      caller: 'knowledge-synthesizer-anthropic',
+      task: 'synthesize',
+      model: 'claude-sonnet-4-20250514',
+      provider: 'anthropic',
+    },
+    {
+      caller: 'classifier-anthropic',
+      task: 'classify',
+      model: 'claude-haiku-4-5-20251001',
+      provider: 'anthropic',
+    },
+  ];
+
   const defaultConfig: RoutingConfig = {
-    routes: [],
+    routes: defaultRoutes,
     defaultOpenAIModel: 'gpt-4o-mini',
     defaultAnthropicModel: 'claude-sonnet-4-20250514',
   };

package/src/llm/oauth-discovery.ts

@@ -0,0 +1,169 @@
+/**
+ * OAuth Token Discovery — find Claude Code OAuth tokens on macOS and Linux.
+ *
+ * Priority:
+ * 1. ANTHROPIC_API_KEY env var (explicit, highest priority)
+ * 2. Claude Code credentials file (~/.claude/.credentials.json or similar)
+ * 3. macOS Keychain (security find-generic-password)
+ * 4. Linux GNOME Keyring (secret-tool lookup)
+ * 5. null (graceful fallback → use OpenAI or no LLM)
+ *
+ * Cached for 5 minutes to avoid repeated I/O.
+ */
+
+import { execFileSync } from 'node:child_process';
+import { readFileSync, existsSync } from 'node:fs';
+import { join } from 'node:path';
+import { homedir, platform } from 'node:os';
+
+// ─── Cache ───────────────────────────────────────────────────────────
+
+let cachedToken: string | null = null;
+let cacheTimestamp = 0;
+const CACHE_TTL_MS = 5 * 60 * 1000;
+
+// ─── Public API ──────────────────────────────────────────────────────
+
+/**
+ * Discover an Anthropic API token. Returns null if none found.
+ * Results cached for 5 minutes.
+ */
+export function discoverAnthropicToken(): string | null {
+  if (cachedToken && Date.now() - cacheTimestamp < CACHE_TTL_MS) {
+    return cachedToken;
+  }
+
+  const token = tryEnvVar() ?? tryCredentialsFile() ?? tryPlatformKeychain();
+
+  if (token) {
+    cachedToken = token;
+    cacheTimestamp = Date.now();
+  }
+
+  return token;
+}
+
+/**
+ * Clear the cached token (for testing or rotation).
+ */
+export function resetTokenCache(): void {
+  cachedToken = null;
+  cacheTimestamp = 0;
+}
+
+/**
+ * Get discovery source info (for diagnostics).
+ */
+export function getTokenSource(): string {
+  if (process.env.ANTHROPIC_API_KEY) return 'env:ANTHROPIC_API_KEY';
+  if (tryCredentialsFile()) return 'file:credentials';
+  if (tryPlatformKeychain()) return `keychain:${platform()}`;
+  return 'none';
+}
+
+// ─── Discovery Methods ───────────────────────────────────────────────
+
+function tryEnvVar(): string | null {
+  return process.env.ANTHROPIC_API_KEY ?? null;
+}
+
+function tryCredentialsFile(): string | null {
+  const candidates = [
+    join(homedir(), '.claude', '.credentials.json'),
+    join(homedir(), '.claude', 'credentials.json'),
+    join(homedir(), '.config', 'claude', 'credentials.json'),
+  ];
+
+  for (const path of candidates) {
+    try {
+      if (!existsSync(path)) continue;
+      const raw = readFileSync(path, 'utf-8');
+      const parsed = JSON.parse(raw) as Record<string, unknown>;
+
+      // Claude Code OAuth format: { claudeAiOauth: { accessToken: "..." } }
+      const oauth = parsed.claudeAiOauth as Record<string, unknown> | undefined;
+      if (oauth?.accessToken && typeof oauth.accessToken === 'string') {
+        return oauth.accessToken;
+      }
+
+      // Alternative: direct token field
+      if (parsed.accessToken && typeof parsed.accessToken === 'string') {
+        return parsed.accessToken as string;
+      }
+
+      // Alternative: API key field
+      if (parsed.apiKey && typeof parsed.apiKey === 'string') {
+        return parsed.apiKey as string;
+      }
+    } catch {
+      continue;
+    }
+  }
+
+  return null;
+}
+
+function tryPlatformKeychain(): string | null {
+  const os = platform();
+
+  if (os === 'darwin') return tryMacKeychain();
+  if (os === 'linux') return tryLinuxKeyring();
+
+  return null;
+}
+
+function tryMacKeychain(): string | null {
+  try {
+    const raw = execFileSync(
+      'security',
+      ['find-generic-password', '-s', 'Claude Code-credentials', '-w'],
+      { encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] },
+    ).trim();
+
+    if (!raw) return null;
+
+    // Try JSON parse
+    try {
+      const parsed = JSON.parse(raw) as Record<string, unknown>;
+      const oauth = parsed.claudeAiOauth as Record<string, unknown> | undefined;
+      if (oauth?.accessToken && typeof oauth.accessToken === 'string') {
+        return oauth.accessToken;
+      }
+    } catch {
+      // JSON might be truncated — try regex fallback
+      const match = raw.match(/"accessToken"\s*:\s*"([^"]+)"/);
+      if (match) return match[1];
+    }
+  } catch {
+    // Keychain not available or no entry
+  }
+
+  return null;
+}
+
+function tryLinuxKeyring(): string | null {
+  try {
+    // GNOME Keyring via secret-tool
+    const token = execFileSync(
+      'secret-tool',
+      ['lookup', 'service', 'Claude Code', 'type', 'credentials'],
+      { encoding: 'utf-8', timeout: 5000, stdio: ['pipe', 'pipe', 'pipe'] },
+    ).trim();
+
+    if (token) {
+      // May be JSON or raw token
+      try {
+        const parsed = JSON.parse(token) as Record<string, unknown>;
+        const oauth = parsed.claudeAiOauth as Record<string, unknown> | undefined;
+        if (oauth?.accessToken) return oauth.accessToken as string;
+      } catch {
+        // Treat as raw token
+        if (token.length > 20) return token;
+      }
+    }
+  } catch {
+    // secret-tool not available or no entry
+  }
+
+  return null;
+}

package/src/planning/evidence-collector.ts

@@ -0,0 +1,247 @@
+/**
+ * Evidence Collector — cross-references plan tasks against git reality.
+ *
+ * Runs `git diff` to find what actually changed, then matches file changes
+ * against planned tasks to produce an evidence-based drift report.
+ */
+
+import { execFileSync } from 'node:child_process';
+import type { Plan, PlanTask } from './planner.js';
+
+export interface FileChange {
+  path: string;
+  status: 'added' | 'modified' | 'deleted' | 'renamed';
+}
+
+export interface GitTaskEvidence {
+  taskId: string;
+  taskTitle: string;
+  plannedStatus: string;
+  matchedFiles: FileChange[];
+  verdict: 'DONE' | 'PARTIAL' | 'MISSING' | 'SKIPPED';
+}
+
+export interface UnplannedChange {
+  file: FileChange;
+  possibleReason: string;
+}
+
+export interface EvidenceReport {
+  planId: string;
+  planObjective: string;
+  accuracy: number;
+  evidenceSources: string[];
+  taskEvidence: GitTaskEvidence[];
+  unplannedChanges: UnplannedChange[];
+  missingWork: GitTaskEvidence[];
+  summary: string;
+}
+
+/**
+ * Collect git diff evidence for a plan.
+ *
+ * @param plan - The plan to verify
+ * @param projectPath - Project root (must be a git repo)
+ * @param baseBranch - Compare against this branch (default: 'main')
+ */
+export function collectGitEvidence(
+  plan: Plan,
+  projectPath: string,
+  baseBranch: string = 'main',
+): EvidenceReport {
+  const fileChanges = getGitDiff(projectPath, baseBranch);
+  const taskEvidence: GitTaskEvidence[] = [];
+  const matchedFiles = new Set<string>();
+
+  for (const task of plan.tasks) {
+    const matches = findMatchingFiles(task, fileChanges);
+    for (const m of matches) matchedFiles.add(m.path);
+
+    const verdict = determineVerdict(task, matches);
+    taskEvidence.push({
+      taskId: task.id,
+      taskTitle: task.title,
+      plannedStatus: task.status,
+      matchedFiles: matches,
+      verdict,
+    });
+  }
+
+  const unplannedChanges: UnplannedChange[] = fileChanges
+    .filter((f) => !matchedFiles.has(f.path))
+    .map((f) => ({
+      file: f,
+      possibleReason: inferReason(f),
+    }));
+
+  const missingWork = taskEvidence.filter((te) => te.verdict === 'MISSING');
+
+  const totalTasks = taskEvidence.length;
+  const doneTasks = taskEvidence.filter((te) => te.verdict === 'DONE').length;
+  const partialTasks = taskEvidence.filter((te) => te.verdict === 'PARTIAL').length;
+  const skippedTasks = taskEvidence.filter((te) => te.verdict === 'SKIPPED').length;
+  const accuracy =
+    totalTasks > 0
+      ? Math.round(((doneTasks + partialTasks * 0.5 + skippedTasks * 0.25) / totalTasks) * 100)
+      : 100;
+
+  const summary = buildSummary(
+    totalTasks,
+    doneTasks,
+    partialTasks,
+    missingWork.length,
+    unplannedChanges.length,
+  );
+
+  return {
+    planId: plan.id,
+    planObjective: plan.objective,
+    accuracy,
+    evidenceSources: ['git'],
+    taskEvidence,
+    unplannedChanges,
+    missingWork,
+    summary,
+  };
+}
+
+function getGitDiff(projectPath: string, baseBranch: string): FileChange[] {
+  try {
+    const currentBranch = execFileSync('git', ['rev-parse', '--abbrev-ref', 'HEAD'], {
+      cwd: projectPath,
+      encoding: 'utf-8',
+      timeout: 5000,
+    }).trim();
+
+    const diffTarget = currentBranch === baseBranch ? 'HEAD~10' : baseBranch;
+
+    let output: string;
+    try {
+      output = execFileSync('git', ['diff', '--name-status', `${diffTarget}...HEAD`], {
+        cwd: projectPath,
+        encoding: 'utf-8',
+        timeout: 10000,
+      });
+    } catch {
+      output = execFileSync('git', ['diff', '--name-status', 'HEAD~5'], {
+        cwd: projectPath,
+        encoding: 'utf-8',
+        timeout: 10000,
+      });
+    }
+
+    return output
+      .trim()
+      .split('\n')
+      .filter((line) => line.length > 0)
+      .map(parseGitDiffLine)
+      .filter((f): f is FileChange => f !== null);
+  } catch {
+    return [];
+  }
+}
+
+function parseGitDiffLine(line: string): FileChange | null {
+  const match = line.match(/^([AMDRC])\d*\t(.+?)(?:\t(.+))?$/);
+  if (!match) return null;
+
+  const statusChar = match[1];
+  const path = match[3] ?? match[2];
+
+  const statusMap: Record<string, FileChange['status']> = {
+    A: 'added',
+    M: 'modified',
+    D: 'deleted',
+    R: 'renamed',
+    C: 'added',
+  };
+
+  return { path, status: statusMap[statusChar] ?? 'modified' };
+}
+
+function findMatchingFiles(task: PlanTask, files: FileChange[]): FileChange[] {
+  const keywords = extractKeywords(task.title + ' ' + task.description);
+  if (keywords.length === 0) return [];
+
+  return files.filter((f) => {
+    const pathLower = f.path.toLowerCase();
+    return keywords.some((kw) => pathLower.includes(kw));
+  });
+}
+
+function extractKeywords(text: string): string[] {
+  const stopWords = new Set([
+    'the',
+    'and',
+    'for',
+    'with',
+    'that',
+    'this',
+    'from',
+    'into',
+    'add',
+    'create',
+    'implement',
+    'update',
+    'fix',
+    'remove',
+    'delete',
+    'new',
+    'use',
+    'should',
+    'must',
+    'will',
+    'can',
+    'all',
+    'each',
+    'when',
+    'not',
+    'are',
+    'has',
+    'have',
+    'been',
+    'was',
+  ]);
+
+  const words = text
+    .toLowerCase()
+    .replace(/[^a-z0-9\s\-_/.]/g, ' ')
+    .split(/[\s\-_/]+/)
+    .filter((w) => w.length >= 3 && !stopWords.has(w));
+
+  return [...new Set(words)];
+}
+
+function determineVerdict(task: PlanTask, matches: FileChange[]): GitTaskEvidence['verdict'] {
+  if (task.status === 'skipped') return 'SKIPPED';
+  if (matches.length === 0) return 'MISSING';
+  if (task.status === 'completed') return 'DONE';
+  if (matches.length > 0) return 'PARTIAL';
+  return 'MISSING';
+}
+
+function inferReason(file: FileChange): string {
+  const path = file.path.toLowerCase();
+  if (path.includes('index.') || path.includes('barrel')) return 'likely re-export update';
+  if (path.includes('config') || path.includes('.env')) return 'configuration change';
+  if (path.includes('test') || path.includes('spec')) return 'test file';
+  if (path.includes('package.json') || path.includes('lock')) return 'dependency update';
+  if (path.includes('readme') || path.includes('.md')) return 'documentation';
+  if (path.includes('types') || path.includes('.d.ts')) return 'type definition update';
+  return 'unplanned scope';
+}
+
+function buildSummary(
+  total: number,
+  done: number,
+  partial: number,
+  missing: number,
+  unplanned: number,
+): string {
+  const parts: string[] = [];
+  parts.push(`${done}/${total} tasks verified by git evidence`);
+  if (partial > 0) parts.push(`${partial} partially done`);
+  if (missing > 0) parts.push(`${missing} with no file evidence`);
+  if (unplanned > 0) parts.push(`${unplanned} unplanned file changes`);
+  return parts.join(', ');
+}