@od-oneapp/ai-platform 0.1.0

package/src/grounding/rag/pipeline.ts

@@ -0,0 +1,602 @@
+/**
+ * @fileoverview Multi-model research pipeline
+ *
+ * Three-stage research pipeline for high-quality, verified research:
+ * 1. Perplexity: Context expansion (grounded notes + citations)
+ * 2. Sonnet 4.5: Structured reasoning (claims, assumptions, gaps)
+ * 3. GPT-5: Verification (fact-check, tighten citations)
+ *
+ * Benefits:
+ * - Cuts hallucinations via grounded notes
+ * - Readable traces (notes → reasoning → verification)
+ * - Tunable research depth (fast/balanced/deep)
+ * - Circuit breaker protection for fault tolerance
+ *
+ * @module @od-oneapp/ai-platform/research/pipeline
+ */
+
+import { logError, logWarn } from '@repo/shared/logs';
+import { z } from 'zod';
+
+import { anthropic, openai, generateText } from '../../shared';
+
+import { CircuitBreaker, type CircuitMetrics, type CircuitState } from './circuit-breaker';
+
+import type {
+  PerplexityResponse,
+  ReasoningOutput,
+  ResearchConfig,
+  ResearchPipelineResult,
+  ResearchTier,
+  VerificationOutput,
+} from './types';
+
+
+/**
+ * Circuit breaker instances for each pipeline stage.
+ * @internal
+ */
+const pipelineCircuits = {
+  perplexity: new CircuitBreaker({
+    name: 'research-perplexity',
+    failureThreshold: 3,
+    resetTimeout: 60000, // 1 minute
+    halfOpenRequests: 1,
+    fallbackBehavior: 'error',
+    onStateChange: (from: CircuitState, to: CircuitState) => {
+      if (to === 'open') {
+        logError('[Pipeline] Perplexity circuit opened - requests will fail until reset');
+      }
+    },
+  }),
+  sonnet: new CircuitBreaker({
+    name: 'research-sonnet',
+    failureThreshold: 3,
+    resetTimeout: 45000, // 45 seconds
+    halfOpenRequests: 1,
+    fallbackBehavior: 'error',
+  }),
+  verification: new CircuitBreaker({
+    name: 'research-verification',
+    failureThreshold: 3,
+    resetTimeout: 45000, // 45 seconds
+    halfOpenRequests: 1,
+    fallbackBehavior: 'error',
+  }),
+};
+
+/**
+ * Gets circuit breaker metrics for monitoring.
+ *
+ * @returns Array of circuit metrics for all pipeline stages.
+ * @example
+ * ```ts
+ * const metrics = getPipelineCircuitMetrics();
+ * console.log(metrics.map(m => `${m.name}: ${m.state}`));
+ * ```
+ */
+export function getPipelineCircuitMetrics(): CircuitMetrics[] {
+  return [
+    pipelineCircuits.perplexity.getMetrics(),
+    pipelineCircuits.sonnet.getMetrics(),
+    pipelineCircuits.verification.getMetrics(),
+  ];
+}
+
+/**
+ * Resets all pipeline circuit breakers.
+ * Use for testing or manual recovery.
+ */
+export function resetPipelineCircuits(): void {
+  pipelineCircuits.perplexity.reset();
+  pipelineCircuits.sonnet.reset();
+  pipelineCircuits.verification.reset();
+}
+
+/**
+ * Tier configurations for research depth.
+ * Defines token limits and modes for each tier.
+ * @internal
+ */
+const TIER_CONFIGS: Record<ResearchTier, { maxTokens: number; mode: string }> = {
+  fast: { maxTokens: 400, mode: 'concise-notes' },
+  balanced: { maxTokens: 800, mode: 'concise-notes' },
+  deep: { maxTokens: 1600, mode: 'deep-notes' },
+};
+
+/**
+ * Gets verification model from environment variable.
+ * Supports: gpt-4-turbo, o1-preview, o1-pro, gpt-5
+ * @returns Model identifier (defaults to 'gpt-4-turbo').
+ * @internal
+ */
+function getVerificationModel(): string {
+  return process.env.NEXT_AI_VERIFICATION_MODEL ?? 'gpt-4-turbo';
+}
+
+/**
+ * Step 1: Perplexity - Context expansion.
+ * Returns crisp notes + URLs only (no long prose).
+ * Bounded context keeps costs predictable.
+ * Protected by circuit breaker for fault tolerance.
+ *
+ * @param query - Research question or query.
+ * @param tier - Research depth tier (defaults to 'balanced').
+ * @returns Perplexity response with notes, citations, and usage stats.
+ * @throws {Error} If Perplexity API fails and GPT-4 Turbo fallback also fails.
+ * @throws {CircuitOpenError} If circuit breaker is open after repeated failures.
+ * @remarks Falls back to GPT-4 Turbo if Perplexity API is unavailable or times out (30s timeout).
+ * Extracts citations from both API response and inline [Source: URL] markers.
+ * Limits citations to maximum of 7 URLs.
+ * Circuit breaker opens after 3 failures and resets after 60 seconds.
+ * @example
+ * ```ts
+ * const result = await researchWithPerplexity('What is quantum computing?', 'balanced');
+ * console.log(result.notes, result.cites);
+ * ```
+ */
+export async function researchWithPerplexity(
+  query: string,
+  tier: ResearchTier = 'balanced',
+): Promise<PerplexityResponse> {
+  // Wrap the entire function in circuit breaker protection
+  return pipelineCircuits.perplexity.execute(async () => {
+    return researchWithPerplexityInternal(query, tier);
+  });
+}
+
+/**
+ * Internal implementation of Perplexity research.
+ * @internal
+ */
+async function researchWithPerplexityInternal(
+  query: string,
+  tier: ResearchTier,
+): Promise<PerplexityResponse> {
+  const config = TIER_CONFIGS[tier];
+
+  // Try real Perplexity API first
+  if (process.env.PPLX_API_KEY) {
+    const controller = new AbortController();
+    const timeoutId = setTimeout(() => controller.abort(), 30000); // 30s timeout
+
+    try {
+      const response = await fetch('https://api.perplexity.ai/chat/completions', {
+        method: 'POST',
+        signal: controller.signal,
+        headers: {
+          Authorization: `Bearer ${process.env.PPLX_API_KEY}`,
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          model: 'llama-3.1-sonar-large-128k-online',
+          messages: [
+            {
+              role: 'system',
+              content: `You are a research assistant. Provide concise, citation-dense notes.
+Format:
+- Use bullet points
+- Include specific facts and figures
+- Cite sources as [Source: URL]
+- Keep to ${config.maxTokens} tokens max
+- ${config.mode === 'deep-notes' ? 'Provide comprehensive coverage' : 'Be concise'}`,
+            },
+            { role: 'user', content: query },
+          ],
+          max_tokens: config.maxTokens,
+          return_citations: true,
+          temperature: 0.3,
+        }),
+      });
+
+      clearTimeout(timeoutId);
+
+      if (!response.ok) {
+        const errorBody = await response.text().catch(() => 'Unable to read error body');
+        throw new Error(
+          `Perplexity API error: ${response.status} ${response.statusText} - ${errorBody}`,
+        );
+      }
+
+      const data = (await response.json()) as {
+        citations?: string[];
+        choices?: Array<{ message?: { content?: string } }>;
+        usage?: { prompt_tokens?: number; completion_tokens?: number };
+      };
+
+      // Extract citations from response
+      const cites: string[] = data.citations ?? [];
+
+      // Also extract inline citations from content
+      const content = data.choices?.[0]?.message?.content ?? '';
+      const citationRegex = /\[Source: (https?:\/\/[^\]]+)\]/g;
+      let match;
+      while ((match = citationRegex.exec(content)) !== null) {
+        const url = match[1];
+        if (url && !cites.includes(url)) {
+          cites.push(url);
+        }
+      }
+
+      return {
+        notes: content,
+        cites: cites.slice(0, 7), // Max 7 citations
+        usage: {
+          inputTokens: data.usage?.prompt_tokens ?? 0,
+          outputTokens: data.usage?.completion_tokens ?? 0,
+        },
+      };
+    } catch (error) {
+      clearTimeout(timeoutId); // Cleanup timeout to prevent memory leak
+      // Handle specific error types
+      if (error instanceof Error) {
+        if (error.name === 'AbortError') {
+          logWarn('[Perplexity] Request timed out after 30s, falling back to GPT-4 Turbo');
+        } else {
+          logWarn('[Perplexity] API call failed, falling back to GPT-4 Turbo', {
+            error: error.message,
+          });
+        }
+      } else {
+        logWarn('[Perplexity] Unknown error, falling back to GPT-4 Turbo', {
+          error: String(error),
+        });
+      }
+      // Fall through to simulation
+    }
+  }
+
+  // Fallback: Simulate with GPT-4 Turbo
+  const result = await generateText({
+    model: openai('gpt-4-turbo'),
+    prompt: `You are a research assistant. Provide concise, citation-dense notes for: "${query}"
+
+Format:
+- Use bullet points
+- Include specific facts and figures
+- Cite sources as [Source: URL]
+- Keep to ${config.maxTokens} tokens max
+- ${config.mode === 'deep-notes' ? 'Provide comprehensive coverage' : 'Be concise'}
+
+Return only the notes with citations.`,
+    maxRetries: 2,
+  });
+
+  // Extract citations from notes
+  const citationRegex = /\[Source: (https?:\/\/[^\]]+)\]/g;
+  const cites: string[] = [];
+  let match;
+  while ((match = citationRegex.exec(result.text)) !== null) {
+    const url = match[1];
+    if (url && !cites.includes(url)) {
+      cites.push(url);
+    }
+  }
+
+  return {
+    notes: result.text,
+    cites: cites.slice(0, 7), // Max 7 citations
+    usage: {
+      inputTokens: result.usage.inputTokens ?? 0,
+      outputTokens: result.usage.outputTokens ?? 0,
+    },
+  };
+}
+
+/**
+ * Schema for Sonnet reasoning output.
+ * @internal
+ */
+const reasoningSchema = z.object({
+  claims: z.array(
+    z.object({
+      statement: z.string(),
+      supportedBy: z.array(z.string()).describe('Citation URLs'),
+      confidence: z.enum(['high', 'medium', 'low']),
+    }),
+  ),
+  assumptions: z.array(z.string()).describe('Underlying assumptions made'),
+  gaps: z
+    .array(
+      z.object({
+        description: z.string(),
+        sources: z.array(z.string()).describe('Conflicting sources if any'),
+      }),
+    )
+    .describe('Knowledge gaps or conflicts'),
+  answerDraft: z.string().describe('Draft answer based on notes'),
+});
+
+/**
+ * Step 2: Sonnet 4.5 - Structured reasoning.
+ * Reasons over ONLY the provided notes.
+ * Outputs: claims[], assumptions[], gaps[], answerDraft.
+ * Protected by circuit breaker for fault tolerance.
+ *
+ * @param input - Reasoning input containing question, notes, and citations.
+ * @param input.question - Research question.
+ * @param input.notes - Research notes from Perplexity.
+ * @param input.cites - Citation URLs.
+ * @returns Reasoning output with claims, assumptions, gaps, and draft answer.
+ * @throws {Error} If JSON extraction or parsing fails, or if model call fails.
+ * @throws {CircuitOpenError} If circuit breaker is open after repeated failures.
+ * @remarks Uses Claude 3.5 Sonnet with temperature 0.3 for consistent reasoning.
+ * Strictly enforces using only provided notes (no external knowledge).
+ * Returns structured JSON matching the reasoning schema.
+ * Circuit breaker opens after 3 failures and resets after 45 seconds.
+ * @example
+ * ```ts
+ * const reasoning = await reasonWithSonnet({
+ *   question: 'What is quantum computing?',
+ *   notes: perplexityResult.notes,
+ *   cites: perplexityResult.cites
+ * });
+ * ```
+ */
+export async function reasonWithSonnet(input: {
+  question: string;
+  notes: string;
+  cites: string[];
+}): Promise<ReasoningOutput> {
+  return pipelineCircuits.sonnet.execute(async () => {
+    return reasonWithSonnetInternal(input);
+  });
+}
+
+/**
+ * Internal implementation of Sonnet reasoning.
+ * @internal
+ */
+async function reasonWithSonnetInternal(input: {
+  question: string;
+  notes: string;
+  cites: string[];
+}): Promise<ReasoningOutput> {
+  const systemPrompt = `You are a rigorous analyst.
+
+CRITICAL RULES:
+1. Use ONLY the provided notes and citations
+2. Do not add external knowledge
+3. Explicitly mark any assumptions
+4. Flag gaps where citations conflict or are insufficient
+5. Output structured JSON matching the schema
+
+Your goal is to produce verifiable, citation-backed reasoning.`;
+
+  const scratchpad = `## QUESTION
+${input.question}
+
+## NOTES
+${input.notes}
+
+## CITATIONS
+${input.cites.map((c, i) => `[${i + 1}] ${c}`).join('\n')}
+
+## TASK
+Analyze the notes and produce structured reasoning. Return JSON only.`;
+
+  const result = await generateText({
+    model: anthropic('claude-3-5-sonnet-20241022'),
+    system: systemPrompt,
+    prompt: scratchpad,
+    temperature: 0.3,
+    maxRetries: 2,
+  });
+
+  // Extract JSON from response
+  const jsonMatch = result.text.match(/\{[\s\S]*\}/);
+  if (!jsonMatch) {
+    throw new Error('Failed to extract JSON from Sonnet response');
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(jsonMatch[0]);
+  } catch (parseError) {
+    throw new Error(
+      `Failed to parse JSON from Sonnet: ${parseError instanceof Error ? parseError.message : 'Unknown error'}`,
+    );
+  }
+
+  const validated = reasoningSchema.parse(parsed);
+
+  return {
+    ...validated,
+    usage: {
+      inputTokens: result.usage.inputTokens ?? 0,
+      outputTokens: result.usage.outputTokens ?? 0,
+    },
+  };
+}
+
+/**
+ * Schema for GPT-5 verification output.
+ * @internal
+ */
+const verificationSchema = z.object({
+  finalAnswer: z.string().describe('Verified, tightened answer'),
+  flags: z
+    .array(
+      z.object({
+        claim: z.string(),
+        issue: z.enum(['unverifiable', 'weak-citation', 'conflicting-sources']),
+        details: z.string(),
+      }),
+    )
+    .describe('Issues found during verification'),
+  trace: z
+    .array(
+      z.object({
+        claim: z.string(),
+        citationCheck: z.enum(['verified', 'partial', 'failed']),
+        notes: z.string().optional(),
+      }),
+    )
+    .describe('Per-claim verification trace'),
+  confidence: z.enum(['high', 'medium', 'low']).describe('Overall confidence in answer'),
+});
+
+/**
+ * Step 3: GPT-5 - Verification.
+ * Fact-checks each claim against citations.
+ * Flags unverifiable items.
+ * Returns final answer with trace.
+ * Protected by circuit breaker for fault tolerance.
+ *
+ * @param draft - Reasoning output from Sonnet stage.
+ * @param cites - Citation URLs to verify against.
+ * @returns Verification output with final answer, flags, trace, and confidence.
+ * @throws {Error} If JSON extraction or parsing fails, or if model call fails.
+ * @throws {CircuitOpenError} If circuit breaker is open after repeated failures.
+ * @remarks Uses verification model from NEXT_AI_VERIFICATION_MODEL env var (defaults to 'gpt-4-turbo').
+ * Supports: gpt-4-turbo, o1-preview, o1-pro, gpt-5.
+ * Uses temperature 0.2 for strict verification.
+ * Returns verification trace for each claim with citation check results.
+ * Circuit breaker opens after 3 failures and resets after 45 seconds.
+ * @example
+ * ```ts
+ * const verification = await verifyWithGPT5(reasoning, cites);
+ * console.log(verification.finalAnswer, verification.confidence);
+ * ```
+ */
+export async function verifyWithGPT5(
+  draft: ReasoningOutput,
+  cites: string[],
+): Promise<VerificationOutput> {
+  return pipelineCircuits.verification.execute(async () => {
+    return verifyWithGPT5Internal(draft, cites);
+  });
+}
+
+/**
+ * Internal implementation of GPT-5 verification.
+ * @internal
+ */
+async function verifyWithGPT5Internal(
+  draft: ReasoningOutput,
+  cites: string[],
+): Promise<VerificationOutput> {
+  const verificationModel = getVerificationModel();
+
+  const systemPrompt = `You are ${verificationModel}, a verification model.
+
+TASKS:
+1. Check each claim against the provided citations
+2. Flag claims that lack strong citation support
+3. Identify conflicting sources
+4. Produce a tightened final answer
+5. Return structured JSON with verification trace
+
+VERIFICATION RULES:
+- "verified": Claim directly supported by citation
+- "partial": Citation provides weak or indirect support
+- "failed": No citation support found
+
+Be strict but fair. Better to flag uncertainty than claim certainty.`;
+
+  const verificationInput = {
+    draft: {
+      claims: draft.claims,
+      assumptions: draft.assumptions,
+      gaps: draft.gaps,
+      answerDraft: draft.answerDraft,
+    },
+    citations: cites.map((c, i) => ({ id: i + 1, url: c })),
+  };
+
+  const result = await generateText({
+    model: openai(verificationModel),
+    system: systemPrompt,
+    prompt: `Verify this reasoning:\n\n${JSON.stringify(verificationInput, null, 2)}\n\nReturn JSON only.`,
+    temperature: 0.2,
+    maxRetries: 2,
+  });
+
+  // Extract JSON from response
+  const jsonMatch = result.text.match(/\{[\s\S]*\}/);
+  if (!jsonMatch) {
+    throw new Error('Failed to extract JSON from verification response');
+  }
+
+  let parsed;
+  try {
+    parsed = JSON.parse(jsonMatch[0]);
+  } catch (parseError) {
+    throw new Error(
+      `Failed to parse JSON from verification: ${parseError instanceof Error ? parseError.message : 'Unknown error'}`,
+    );
+  }
+
+  const validated = verificationSchema.parse(parsed);
+
+  return {
+    ...validated,
+    usage: {
+      inputTokens: result.usage.inputTokens ?? 0,
+      outputTokens: result.usage.outputTokens ?? 0,
+    },
+  };
+}
+
+/**
+ * Orchestrator: Full research pipeline.
+ * Executes all three steps in sequence (Perplexity → Sonnet → GPT-5).
+ * Returns final answer with full trace.
+ *
+ * @param question - Research question to answer.
+ * @param config - Research configuration (defaults to { tier: 'balanced' }).
+ * @returns Complete research pipeline result with answer, confidence, flags, and trace.
+ * @throws {Error} If any pipeline stage fails (propagates errors from individual stages).
+ * @remarks Executes stages sequentially. Each stage's output feeds into the next.
+ * Full trace includes token usage and results from all three stages.
+ * Confidence level reflects overall verification quality.
+ * @example
+ * ```ts
+ * const result = await executeResearchPipeline(
+ *   'What are the latest developments in quantum computing?',
+ *   { tier: 'deep' }
+ * );
+ * console.log(result.answer, result.confidence);
+ * ```
+ */
+export async function executeResearchPipeline(
+  question: string,
+  config: ResearchConfig = { tier: 'balanced' },
+): Promise<ResearchPipelineResult> {
+  // Step 1: Context expansion
+  const {
+    notes,
+    cites,
+    usage: perplexityUsage,
+  } = await researchWithPerplexity(question, config.tier);
+
+  // Step 2: Structured reasoning
+  const reasoning = await reasonWithSonnet({ question, notes, cites });
+
+  // Step 3: Verification
+  const verification = await verifyWithGPT5(reasoning, cites);
+
+  return {
+    answer: verification.finalAnswer,
+    confidence: verification.confidence,
+    flags: verification.flags,
+    trace: {
+      contextExpansion: {
+        notes,
+        cites,
+        usage: perplexityUsage,
+      },
+      reasoning: {
+        claims: reasoning.claims,
+        assumptions: reasoning.assumptions,
+        gaps: reasoning.gaps,
+        usage: reasoning.usage,
+      },
+      verification: {
+        trace: verification.trace,
+        flags: verification.flags,
+        usage: verification.usage,
+      },
+    },
+  };
+}