@soleri/core 2.4.0 → 2.5.0

package/src/errors/retry.ts

@@ -0,0 +1,132 @@
+/**
+ * Retry presets with exponential backoff + jitter.
+ *
+ * Three presets map to different execution contexts:
+ * - fast:    quick API calls, 3 attempts, short waits
+ * - normal:  standard operations, 10 attempts, moderate waits
+ * - patient: batch/pipeline work, 25 attempts, long waits
+ */
+
+import { classifyError } from './classify.js';
+import { SoleriError, type Result, ok, err } from './types.js';
+
+// ─── Types ─────────────────────────────────────────────────────────────
+
+export type RetryPreset = 'fast' | 'normal' | 'patient';
+
+export interface RetryConfig {
+  initialIntervalMs: number;
+  maxIntervalMs: number;
+  maxAttempts: number;
+  backoffMultiplier: number;
+}
+
+export interface RetryOptions {
+  onRetry?: (error: SoleriError, attempt: number, delayMs: number) => void;
+  signal?: AbortSignal;
+}
+
+// ─── Presets ───────────────────────────────────────────────────────────
+
+export const RETRY_PRESETS: Record<RetryPreset, RetryConfig> = {
+  fast: { initialIntervalMs: 1_000, maxIntervalMs: 10_000, maxAttempts: 3, backoffMultiplier: 2 },
+  normal: {
+    initialIntervalMs: 10_000,
+    maxIntervalMs: 120_000,
+    maxAttempts: 10,
+    backoffMultiplier: 2,
+  },
+  patient: {
+    initialIntervalMs: 60_000,
+    maxIntervalMs: 900_000,
+    maxAttempts: 25,
+    backoffMultiplier: 1.5,
+  },
+};
+
+// ─── Helpers ───────────────────────────────────────────────────────────
+
+/**
+ * Check if a classified error should be retried at the given attempt.
+ */
+export function shouldRetry(error: SoleriError, attempt: number, preset: RetryPreset): boolean {
+  if (!error.retryable) return false;
+  return attempt < RETRY_PRESETS[preset].maxAttempts;
+}
+
+/**
+ * Calculate retry delay with exponential backoff + jitter.
+ * Jitter adds ±25% to prevent thundering herd.
+ */
+export function getRetryDelay(attempt: number, preset: RetryPreset): number {
+  const config = RETRY_PRESETS[preset];
+  const base = config.initialIntervalMs * Math.pow(config.backoffMultiplier, attempt);
+  const capped = Math.min(base, config.maxIntervalMs);
+  // ±25% jitter
+  const jitter = capped * 0.25 * (Math.random() * 2 - 1);
+  return Math.max(0, Math.round(capped + jitter));
+}
+
+// ─── Retry Loop ────────────────────────────────────────────────────────
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve, reject) => {
+    if (signal?.aborted) {
+      reject(signal.reason ?? new Error('Aborted'));
+      return;
+    }
+    const timer = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      'abort',
+      () => {
+        clearTimeout(timer);
+        reject(signal.reason ?? new Error('Aborted'));
+      },
+      { once: true },
+    );
+  });
+}
+
+/**
+ * Retry an async operation with a named preset.
+ *
+ * - On success: returns ok(result)
+ * - On permanent/fixable error: returns err() immediately
+ * - On retryable error: retries up to maxAttempts with backoff
+ * - On exhaustion: returns err() with last error
+ */
+export async function retryWithPreset<T>(
+  fn: () => Promise<T>,
+  preset: RetryPreset,
+  options?: RetryOptions,
+): Promise<Result<T>> {
+  let lastError: SoleriError | undefined;
+
+  for (let attempt = 0; attempt < RETRY_PRESETS[preset].maxAttempts; attempt++) {
+    try {
+      const value = await fn();
+      return ok(value);
+    } catch (thrown: unknown) {
+      lastError = classifyError(thrown);
+
+      if (!shouldRetry(lastError, attempt + 1, preset)) {
+        return err(lastError);
+      }
+
+      const delay = getRetryDelay(attempt, preset);
+      options?.onRetry?.(lastError, attempt + 1, delay);
+
+      try {
+        await sleep(delay, options?.signal);
+      } catch {
+        // Aborted during sleep
+        return err(lastError);
+      }
+    }
+  }
+
+  return err(lastError ?? new SoleriError('Max retries exceeded', SoleriErrorCode.INTERNAL));
+}
+
+// Re-import for the err fallback
+import { SoleriErrorCode } from './types.js';

package/src/errors/types.ts

@@ -0,0 +1,81 @@
+/**
+ * Classified error types with typed retry presets.
+ *
+ * Every error is classified as retryable, fixable, or permanent.
+ * This lets callers decide strategy without inspecting codes.
+ */
+
+// ─── Error Codes ───────────────────────────────────────────────────────
+
+export enum SoleriErrorCode {
+  NETWORK = 'NETWORK',
+  AUTH = 'AUTH',
+  VALIDATION = 'VALIDATION',
+  TIMEOUT = 'TIMEOUT',
+  RATE_LIMIT = 'RATE_LIMIT',
+  INTERNAL = 'INTERNAL',
+  LLM_OVERLOAD = 'LLM_OVERLOAD',
+  RESOURCE_NOT_FOUND = 'RESOURCE_NOT_FOUND',
+  CONFIG_ERROR = 'CONFIG_ERROR',
+  VAULT_UNREACHABLE = 'VAULT_UNREACHABLE',
+}
+
+// ─── Classification ────────────────────────────────────────────────────
+
+export type ErrorClassification = 'retryable' | 'fixable' | 'permanent';
+
+const CLASSIFICATION_MAP: Record<SoleriErrorCode, ErrorClassification> = {
+  [SoleriErrorCode.NETWORK]: 'retryable',
+  [SoleriErrorCode.TIMEOUT]: 'retryable',
+  [SoleriErrorCode.RATE_LIMIT]: 'retryable',
+  [SoleriErrorCode.LLM_OVERLOAD]: 'retryable',
+  [SoleriErrorCode.VAULT_UNREACHABLE]: 'retryable',
+  [SoleriErrorCode.INTERNAL]: 'retryable',
+  [SoleriErrorCode.VALIDATION]: 'fixable',
+  [SoleriErrorCode.AUTH]: 'permanent',
+  [SoleriErrorCode.RESOURCE_NOT_FOUND]: 'permanent',
+  [SoleriErrorCode.CONFIG_ERROR]: 'permanent',
+};
+
+// ─── SoleriError ───────────────────────────────────────────────────────
+
+export interface SoleriErrorOptions {
+  cause?: Error;
+  context?: Record<string, unknown>;
+}
+
+export class SoleriError extends Error {
+  readonly code: SoleriErrorCode;
+  readonly classification: ErrorClassification;
+  readonly retryable: boolean;
+  readonly context?: Record<string, unknown>;
+
+  constructor(message: string, code: SoleriErrorCode, options?: SoleriErrorOptions) {
+    super(message, options?.cause ? { cause: options.cause } : undefined);
+    this.name = 'SoleriError';
+    this.code = code;
+    this.classification = CLASSIFICATION_MAP[code];
+    this.retryable = this.classification === 'retryable';
+    this.context = options?.context;
+  }
+}
+
+// ─── Result Type ───────────────────────────────────────────────────────
+
+export type Result<T, E = SoleriError> = { ok: true; value: T } | { ok: false; error: E };
+
+export function ok<T>(value: T): Result<T, never> {
+  return { ok: true, value };
+}
+
+export function err<T = never>(error: SoleriError): Result<T> {
+  return { ok: false, error };
+}
+
+export function isOk<T, E>(result: Result<T, E>): result is { ok: true; value: T } {
+  return result.ok === true;
+}
+
+export function isErr<T, E>(result: Result<T, E>): result is { ok: false; error: E } {
+  return result.ok === false;
+}

package/src/index.ts

@@ -5,6 +5,31 @@ export { loadIntelligenceData } from './intelligence/loader.js';
 // ─── Vault ───────────────────────────────────────────────────────────
 export { Vault } from './vault/vault.js';
 export type { SearchResult, VaultStats, ProjectInfo, Memory, MemoryStats } from './vault/vault.js';
+export { validatePlaybook, parsePlaybookFromEntry } from './vault/playbook.js';
+export type { Playbook, PlaybookStep, PlaybookValidationResult } from './vault/playbook.js';
+
+// ─── Playbook System (registry, matching, seeding) ─────────────────
+export {
+  getBuiltinPlaybook,
+  getAllBuiltinPlaybooks,
+  scorePlaybook,
+  mergePlaybooks,
+  matchPlaybooks,
+  seedDefaultPlaybooks,
+  playbookDefinitionToEntry,
+  entryToPlaybookDefinition,
+} from './playbooks/index.js';
+export type {
+  PlaybookTier,
+  PlaybookIntent,
+  BrainstormSection,
+  PlaybookGate,
+  PlaybookTaskTemplate,
+  PlaybookDefinition,
+  MergedPlaybook,
+  PlaybookMatchResult,
+} from './playbooks/index.js';
+export { createPlaybookOps } from './runtime/playbook-ops.js';
 
 // ─── Text Utilities ─────────────────────────────────────────────────
 export {
@@ -99,12 +124,47 @@ export type {
   CogneeCognifyResult,
 } from './cognee/types.js';
 
+// ─── Cognee Sync ──────────────────────────────────────────────────────
+export { CogneeSyncManager } from './cognee/sync-manager.js';
+export type { SyncOp, SyncStatus, SyncQueueItem, SyncManagerStats } from './cognee/sync-manager.js';
+
+// ─── Intake Pipeline ──────────────────────────────────────────────────
+export { IntakePipeline } from './intake/intake-pipeline.js';
+export { classifyChunk, VALID_TYPES, CLASSIFICATION_PROMPT } from './intake/content-classifier.js';
+export { dedupItems, DEDUP_THRESHOLD } from './intake/dedup-gate.js';
+export type {
+  IntakeConfig,
+  IntakeChunk,
+  IntakeJobRecord,
+  IntakeJobStatus,
+  IntakeChunkStatus,
+  KnowledgeType,
+  ClassifiedItem,
+  IntakePreviewResult,
+} from './intake/types.js';
+export type { DedupResult } from './intake/dedup-gate.js';
+
 // ─── Planning ────────────────────────────────────────────────────────
-export { Planner, calculateScore } from './planning/planner.js';
+export {
+  Planner,
+  calculateScore,
+  calculateDriftScore,
+  isValidTransition,
+  getValidNextStatuses,
+  shouldExpire,
+  LIFECYCLE_TRANSITIONS,
+  NON_EXPIRING_STATUSES,
+  DRIFT_WEIGHTS,
+} from './planning/planner.js';
 export type {
   PlanStatus,
   TaskStatus,
+  TaskEvidence,
+  TaskMetrics,
+  TaskDeliverable,
+  ExecutionSummary,
   PlanTask,
+  PlanDecision,
   Plan,
   PlanStore,
   DriftItem,
@@ -115,7 +175,12 @@ export type {
 } from './planning/planner.js';
 
 // ─── Plan Gap Analysis ──────────────────────────────────────────────
-export { runGapAnalysis } from './planning/gap-analysis.js';
+export {
+  runGapAnalysis,
+  createToolFeasibilityPass,
+  createFlowAlignmentPass,
+  createAntiPatternPass,
+} from './planning/gap-analysis.js';
 export type { GapAnalysisOptions, GapAnalysisPass } from './planning/gap-analysis.js';
 export {
   SEVERITY_WEIGHTS,
@@ -128,13 +193,21 @@ export {
 export type { GapSeverity, GapCategory, PlanGap } from './planning/gap-types.js';
 
 // ─── Loop ────────────────────────────────────────────────────────────
-export { LoopManager } from './loop/loop-manager.js';
+export {
+  LoopManager,
+  extractPromise,
+  detectImplicitCompletion,
+  detectAnomaly,
+} from './loop/loop-manager.js';
 export type {
   LoopMode,
   LoopConfig,
   LoopIteration,
   LoopStatus,
   LoopState,
+  LoopKnowledge,
+  LoopHistoryEntry,
+  LoopIterateDecision,
 } from './loop/types.js';
 
 // ─── LLM Types ───────────────────────────────────────────────────────
@@ -226,4 +299,42 @@ export { createCaptureOps } from './runtime/capture-ops.js';
 export { createCuratorExtraOps } from './runtime/curator-extra-ops.js';
 export { createProjectOps } from './runtime/project-ops.js';
 export { createMemoryCrossProjectOps } from './runtime/memory-cross-project-ops.js';
+export { createCogneeSyncOps } from './runtime/cognee-sync-ops.js';
+export { createIntakeOps } from './runtime/intake-ops.js';
 export type { AgentRuntimeConfig, AgentRuntime } from './runtime/types.js';
+
+// ─── Errors ────────────────────────────────────────────────────────────
+export {
+  SoleriErrorCode,
+  SoleriError,
+  ok,
+  err,
+  isOk,
+  isErr,
+  classifyError,
+  retryWithPreset,
+  shouldRetry,
+  getRetryDelay,
+  RETRY_PRESETS,
+} from './errors/index.js';
+export type {
+  ErrorClassification,
+  Result,
+  SoleriErrorOptions,
+  RetryPreset,
+  RetryConfig as SoleriRetryConfig,
+  RetryOptions,
+} from './errors/index.js';
+
+// ─── Persistence ───────────────────────────────────────────────────────
+export { SQLitePersistenceProvider } from './persistence/index.js';
+export type {
+  PersistenceProvider,
+  PersistenceParams,
+  RunResult,
+  PersistenceConfig,
+} from './persistence/index.js';
+
+// ─── Prompts ───────────────────────────────────────────────────────────
+export { TemplateManager, parseVariables, resolveIncludes } from './prompts/index.js';
+export type { PromptTemplate, TemplateVariable, RenderOptions } from './prompts/index.js';

package/src/intake/content-classifier.ts

@@ -0,0 +1,146 @@
+// ─── Content Classifier — LLM-based knowledge extraction ────────────────────
+//
+// Takes a text chunk and uses an LLM to classify it into structured
+// knowledge items. Graceful degradation: returns [] on any error.
+
+import type { LLMClient } from '../llm/llm-client.js';
+import type { ClassifiedItem, KnowledgeType } from './types.js';
+
+// =============================================================================
+// CONSTANTS
+// =============================================================================
+
+export const VALID_TYPES: KnowledgeType[] = [
+  'pattern',
+  'anti-pattern',
+  'principle',
+  'concept',
+  'reference',
+  'workflow',
+  'idea',
+  'roadmap',
+];
+
+const VALID_SEVERITIES = ['critical', 'warning', 'suggestion'] as const;
+type Severity = (typeof VALID_SEVERITIES)[number];
+
+export const CLASSIFICATION_PROMPT = `You are a knowledge extraction engine. Your job is to analyze a text chunk and extract structured knowledge items from it.
+
+For each distinct piece of knowledge you identify, produce an object with these fields:
+- type: one of ${JSON.stringify(VALID_TYPES)}
+- title: concise title, max 80 characters
+- description: 2-3 sentence summary of the knowledge
+- tags: 3-5 lowercase single-word or hyphenated tags
+- severity: one of "critical", "warning", "suggestion"
+
+Rules:
+- Extract ALL meaningful knowledge items from the text.
+- Each item must be self-contained and independently useful.
+- Use "critical" for must-know items, "warning" for important gotchas, "suggestion" for nice-to-know.
+- Tags should be specific and useful for search.
+- Respond with a pure JSON array of objects. No markdown fences, no explanation, no wrapping.
+- If the text contains no extractable knowledge, respond with an empty array: []`;
+
+// =============================================================================
+// CLASSIFIER
+// =============================================================================
+
+/**
+ * Classify a text chunk into structured knowledge items using an LLM.
+ *
+ * @param llm      - LLMClient instance
+ * @param chunkText - The text to classify
+ * @param citation  - Source citation (e.g. "book.pdf, pages 12-15")
+ * @returns Classified items, or [] on any error
+ */
+export async function classifyChunk(
+  llm: LLMClient,
+  chunkText: string,
+  citation: string,
+): Promise<ClassifiedItem[]> {
+  try {
+    const result = await llm.complete({
+      provider: 'openai',
+      model: 'gpt-4o-mini',
+      systemPrompt: CLASSIFICATION_PROMPT,
+      userPrompt: chunkText,
+      maxTokens: 4096,
+      temperature: 0.3,
+      caller: 'intake',
+      task: 'classify',
+    });
+
+    const raw = parseJsonResponse(result.text);
+    if (!Array.isArray(raw)) return [];
+
+    return raw
+      .map((item: unknown) => sanitizeItem(item, citation))
+      .filter((item): item is ClassifiedItem => item !== null);
+  } catch {
+    // Graceful degradation — never throw
+    return [];
+  }
+}
+
+// =============================================================================
+// HELPERS
+// =============================================================================
+
+/**
+ * Parse a JSON response, handling potential markdown fences the LLM
+ * might include despite instructions.
+ */
+function parseJsonResponse(text: string): unknown {
+  const trimmed = text.trim();
+
+  // Strip markdown fences if present (defensive)
+  const fenceMatch = trimmed.match(/^```(?:json)?\s*\n?([\s\S]*?)\n?\s*```$/);
+  const jsonStr = fenceMatch ? fenceMatch[1] : trimmed;
+
+  return JSON.parse(jsonStr);
+}
+
+/**
+ * Validate and sanitize a single classified item.
+ * Returns null if the item is not salvageable.
+ */
+function sanitizeItem(raw: unknown, citation: string): ClassifiedItem | null {
+  if (!raw || typeof raw !== 'object') return null;
+
+  const obj = raw as Record<string, unknown>;
+
+  // Type — must be a valid KnowledgeType
+  const type = typeof obj.type === 'string' ? obj.type : '';
+  if (!VALID_TYPES.includes(type as KnowledgeType)) return null;
+
+  // Title — required, truncate to 80 chars
+  const title = typeof obj.title === 'string' ? obj.title.slice(0, 80).trim() : '';
+  if (!title) return null;
+
+  // Description — required
+  const description = typeof obj.description === 'string' ? obj.description.trim() : '';
+  if (!description) return null;
+
+  // Tags — must be array of strings, cap at 5
+  const tags = Array.isArray(obj.tags)
+    ? obj.tags
+        .filter((t): t is string => typeof t === 'string')
+        .map((t) => t.toLowerCase().trim())
+        .filter((t) => t.length > 0)
+        .slice(0, 5)
+    : [];
+
+  // Severity — default to 'suggestion' if invalid
+  const severity: Severity = VALID_SEVERITIES.includes(obj.severity as Severity)
+    ? (obj.severity as Severity)
+    : 'suggestion';
+
+  return {
+    type: type as KnowledgeType,
+    title,
+    description,
+    tags,
+    severity,
+    citation,
+  };
+}

package/src/intake/dedup-gate.ts

@@ -0,0 +1,92 @@
+// ─── Dedup Gate ───────────────────────────────────────────────────
+// TF-IDF cosine similarity check against existing vault entries.
+// Pure function: no side effects, no I/O beyond reading vault.
+
+import {
+  tokenize,
+  calculateTfIdf,
+  cosineSimilarity,
+  type SparseVector,
+} from '../text/similarity.js';
+import type { Vault } from '../vault/vault.js';
+import type { ClassifiedItem } from './types.js';
+
+export const DEDUP_THRESHOLD = 0.85;
+
+export interface DedupResult {
+  item: ClassifiedItem;
+  isDuplicate: boolean;
+  bestMatchId?: string;
+  similarity: number;
+}
+
+/**
+ * Check new items against existing vault entries for duplicates using TF-IDF cosine similarity.
+ *
+ * Builds a shared IDF vocabulary from all texts (existing + new), computes TF-IDF vectors,
+ * and marks items as duplicates when cosine similarity >= DEDUP_THRESHOLD.
+ */
+export function dedupItems(items: ClassifiedItem[], vault: Vault): DedupResult[] {
+  const existing = vault.exportAll().entries;
+
+  // Fast path: nothing in vault — everything is new
+  if (existing.length === 0) {
+    return items.map((item) => ({
+      item,
+      isDuplicate: false,
+      similarity: 0,
+    }));
+  }
+
+  // ── Build texts for vocabulary ──────────────────────────────────
+  const existingTexts = existing.map((e) => `${e.title} ${e.description}`);
+  const newTexts = items.map((i) => `${i.title} ${i.description}`);
+  const allTexts = [...existingTexts, ...newTexts];
+  const totalDocs = allTexts.length;
+
+  // ── Count document frequency per term ───────────────────────────
+  const docFreq = new Map<string, number>();
+  for (const text of allTexts) {
+    const terms = new Set(tokenize(text));
+    for (const term of terms) {
+      docFreq.set(term, (docFreq.get(term) ?? 0) + 1);
+    }
+  }
+
+  // ── Build IDF vocabulary ────────────────────────────────────────
+  const vocabulary = new Map<string, number>();
+  for (const [term, df] of docFreq) {
+    vocabulary.set(term, Math.log((totalDocs + 1) / (df + 1)) + 1);
+  }
+
+  // ── Compute TF-IDF vectors for existing entries ─────────────────
+  const existingVectors: Array<{ id: string; vec: SparseVector }> = existing.map((entry, idx) => ({
+    id: entry.id,
+    vec: calculateTfIdf(tokenize(existingTexts[idx]), vocabulary),
+  }));
+
+  // ── Score each new item against all existing entries ────────────
+  return items.map((item, idx) => {
+    const itemVec = calculateTfIdf(tokenize(newTexts[idx]), vocabulary);
+
+    let bestSimilarity = 0;
+    let bestMatchId: string | undefined;
+
+    for (const { id, vec } of existingVectors) {
+      const sim = cosineSimilarity(itemVec, vec);
+      if (sim > bestSimilarity) {
+        bestSimilarity = sim;
+        bestMatchId = id;
+      }
+    }
+
+    const isDuplicate = bestSimilarity >= DEDUP_THRESHOLD;
+
+    return {
+      item,
+      isDuplicate,
+      bestMatchId: isDuplicate ? bestMatchId : undefined,
+      similarity: bestSimilarity,
+    };
+  });
+}