@ryanfw/prompt-orchestration-pipeline 0.0.1

package/src/llm/README.md

@@ -0,0 +1,345 @@
+# LLM Abstraction Layer
+
+This directory contains the unified LLM (Large Language Model) abstraction layer for the prompt orchestration pipeline.
+
+## Purpose
+
+The LLM layer provides a consistent interface for interacting with multiple AI providers (OpenAI, DeepSeek, Anthropic) while handling:
+
+- Provider routing and API key validation
+- Request/response formatting
+- Token usage tracking and cost calculation
+- Event-based metrics collection
+- Error handling and retries
+- Multi-turn conversation chains
+
+## Architecture
+
+```
+src/
+├── llm/
+│   ├── index.js          ← Main LLM abstraction layer (THIS FILE)
+│   └── README.md         ← This documentation
+└── providers/
+    ├── base.js           ← Base provider class
+    ├── openai.js         ← OpenAI implementation
+    ├── deepseek.js       ← DeepSeek implementation
+    └── anthropic.js      ← Anthropic implementation
+```
+
+## Core Functions
+
+### `chat(options)`
+
+Main function for LLM interactions. Supports multiple providers and models.
+
+```javascript
+import { chat } from "../llm/index.js";
+
+const response = await chat({
+  provider: "openai", // or "deepseek", "anthropic"
+  model: "gpt-5-chat-latest",
+  messages: [
+    { role: "system", content: "You are a helpful assistant" },
+    { role: "user", content: "Hello!" },
+  ],
+  temperature: 0.7,
+  maxTokens: 1000,
+  metadata: { taskId: "task-123" }, // Optional tracking data
+});
+
+console.log(response.content); // AI response text
+console.log(response.usage); // Token usage stats
+```
+
+### `complete(prompt, options)`
+
+Convenience function for simple single-turn completions.
+
+```javascript
+import { complete } from "../llm/index.js";
+
+const response = await complete("What is 2+2?", {
+  provider: "openai",
+  model: "gpt-5-chat-latest",
+});
+
+console.log(response.content); // "4"
+```
+
+### `createLLM(options)`
+
+Factory function to create a bound LLM interface with default settings.
+
+```javascript
+import { createLLM } from "../llm/index.js";
+
+const llm = createLLM({
+  defaultProvider: "openai",
+  defaultModel: "gpt-5-chat-latest",
+});
+
+// Use the bound interface
+const response = await llm.chat({
+  messages: [{ role: "user", content: "Hello!" }],
+});
+
+// Or use convenience methods
+const result = await llm.complete("What is AI?");
+```
+
+### `createChain()`
+
+Create a multi-turn conversation chain.
+
+```javascript
+import { createChain } from "../llm/index.js";
+
+const chain = createChain();
+
+chain
+  .addSystemMessage("You are a helpful math tutor")
+  .addUserMessage("What is 2+2?");
+
+const response1 = await chain.execute({ provider: "openai" });
+console.log(response1.content); // "4"
+
+chain.addUserMessage("What about 3+3?");
+const response2 = await chain.execute({ provider: "openai" });
+console.log(response2.content); // "6"
+
+// Get full conversation history
+const history = chain.getMessages();
+```
+
+### `withRetry(fn, args, maxRetries, backoffMs)`
+
+Retry wrapper with exponential backoff.
+
+```javascript
+import { withRetry, chat } from "../llm/index.js";
+
+const response = await withRetry(
+  chat,
+  [
+    {
+      provider: "openai",
+      messages: [{ role: "user", content: "Hello!" }],
+    },
+  ],
+  3, // max retries
+  1000 // initial backoff in ms
+);
+```
+
+### `parallel(fn, items, maxConcurrency)`
+
+Execute multiple LLM requests in parallel with concurrency control.
+
+```javascript
+import { parallel, chat } from "../llm/index.js";
+
+const prompts = ["What is AI?", "What is ML?", "What is DL?"];
+
+const responses = await parallel(
+  (prompt) =>
+    chat({
+      provider: "openai",
+      messages: [{ role: "user", content: prompt }],
+    }),
+  prompts,
+  5 // max concurrent requests
+);
+```
+
+## Utility Functions
+
+### `getAvailableProviders()`
+
+Check which providers have API keys configured.
+
+```javascript
+import { getAvailableProviders } from "../llm/index.js";
+
+const available = getAvailableProviders();
+// { openai: true, deepseek: false, anthropic: true }
+```
+
+### `calculateCost(provider, model, usage)`
+
+Calculate the cost of an LLM request based on token usage.
+
+```javascript
+import { calculateCost } from "../llm/index.js";
+
+const cost = calculateCost("openai", "gpt-5-chat-latest", {
+  promptTokens: 100,
+  completionTokens: 50,
+});
+
+console.log(`Cost: $${cost.toFixed(4)}`);
+```
+
+### `estimateTokens(text)`
+
+Rough estimation of token count for a text string.
+
+```javascript
+import { estimateTokens } from "../llm/index.js";
+
+const tokens = estimateTokens("Hello, world!");
+console.log(`Estimated tokens: ${tokens}`);
+```
+
+## Event System
+
+The LLM layer emits events for monitoring and metrics collection.
+
+```javascript
+import { getLLMEvents } from "../llm/index.js";
+
+const events = getLLMEvents();
+
+// Listen for request start
+events.on("llm:request:start", (data) => {
+  console.log(`Request ${data.id} started`);
+  console.log(`Provider: ${data.provider}, Model: ${data.model}`);
+});
+
+// Listen for successful completion
+events.on("llm:request:complete", (data) => {
+  console.log(`Request ${data.id} completed in ${data.duration}ms`);
+  console.log(`Tokens: ${data.totalTokens}, Cost: $${data.cost}`);
+});
+
+// Listen for errors
+events.on("llm:request:error", (data) => {
+  console.error(`Request ${data.id} failed: ${data.error}`);
+});
+```
+
+## Provider Configuration
+
+### Environment Variables
+
+```bash
+# OpenAI
+export OPENAI_API_KEY="sk-..."
+
+# DeepSeek
+export DEEPSEEK_API_KEY="..."
+
+# Anthropic
+export ANTHROPIC_API_KEY="..."
+```
+
+### Supported Models
+
+**OpenAI:**
+
+- `gpt-5-chat-latest` (default)
+- `gpt-5-chat-preview`
+- `gpt-4-turbo-preview`
+- `gpt-4`
+- `gpt-3.5-turbo`
+
+**DeepSeek:**
+
+- `deepseek-reasoner` (default)
+- `deepseek-chat`
+- `deepseek-coder`
+
+**Anthropic:**
+
+- `claude-3-opus-20240229`
+- `claude-3-sonnet-20240229`
+- `claude-3-haiku-20240307`
+
+## Error Handling
+
+The LLM layer handles various error scenarios:
+
+```javascript
+try {
+  const response = await chat({
+    provider: "openai",
+    messages: [{ role: "user", content: "Hello!" }],
+  });
+} catch (error) {
+  if (error.status === 401) {
+    console.error("Invalid API key");
+  } else if (error.message?.includes("rate limit")) {
+    console.error("Rate limit exceeded");
+  } else {
+    console.error("Request failed:", error.message);
+  }
+}
+```
+
+## Best Practices
+
+1. **Use `createLLM()` for consistent settings:**
+
+   ```javascript
+   const llm = createLLM({ defaultProvider: "openai" });
+   ```
+
+2. **Add metadata for tracking:**
+
+   ```javascript
+   await chat({
+     messages: [...],
+     metadata: { taskId: "task-123", userId: "user-456" },
+   });
+   ```
+
+3. **Use chains for multi-turn conversations:**
+
+   ```javascript
+   const chain = createChain();
+   // Maintains conversation history automatically
+   ```
+
+4. **Implement retries for reliability:**
+
+   ```javascript
+   await withRetry(chat, [options], 3, 1000);
+   ```
+
+5. **Control concurrency for batch operations:**
+   ```javascript
+   await parallel(fn, items, 5); // Max 5 concurrent requests
+   ```
+
+## Testing
+
+See `tests/llm.test.js` for comprehensive test coverage including:
+
+- Provider routing and validation
+- Token usage tracking
+- Cost calculation
+- Event emission
+- Error handling
+- Retry logic
+- Parallel execution
+
+## Migration from Legacy Code
+
+If you're migrating from the old `src/providers/index.js` (now removed):
+
+```javascript
+// OLD (removed):
+import { chat } from "../providers/index.js";
+
+// NEW (correct):
+import { chat } from "../llm/index.js";
+```
+
+All exports are compatible - this is a drop-in replacement.
+
+## Related Documentation
+
+- **Architecture:** `docs/architecture.md`
+- **Provider Implementations:** `src/providers/`
+- **Test Coverage:** `tests/llm.test.js`
+- **Provider Fix Documentation:** `docs/providers-fix.md`

package/src/llm/index.js

@@ -0,0 +1,320 @@
+import { openaiChat } from "../providers/openai.js";
+import { deepseekChat } from "../providers/deepseek.js";
+import { EventEmitter } from "node:events";
+import { getConfig } from "../core/config.js";
+
+// Global mock provider instance (for demo/testing)
+let mockProviderInstance = null;
+
+// Global event bus for LLM metrics
+const llmEvents = new EventEmitter();
+export const getLLMEvents = () => llmEvents;
+
+// Register mock provider for demo/testing
+export function registerMockProvider(provider) {
+  mockProviderInstance = provider;
+}
+
+// Check available providers
+export function getAvailableProviders() {
+  return {
+    openai: !!process.env.OPENAI_API_KEY,
+    deepseek: !!process.env.DEEPSEEK_API_KEY,
+    anthropic: !!process.env.ANTHROPIC_API_KEY,
+    mock: !!mockProviderInstance,
+  };
+}
+
+// Simple token estimation
+export function estimateTokens(text) {
+  return Math.ceil((text || "").length / 4);
+}
+
+// Calculate cost based on provider and model
+export function calculateCost(provider, model, usage) {
+  const pricing = {
+    mock: {
+      "gpt-3.5-turbo": { prompt: 0.0005, completion: 0.0015 },
+      "gpt-4": { prompt: 0.03, completion: 0.06 },
+      "gpt-4-turbo": { prompt: 0.01, completion: 0.03 },
+    },
+    openai: {
+      "gpt-5-chat-latest": { prompt: 0.015, completion: 0.06 },
+      "gpt-4": { prompt: 0.03, completion: 0.06 },
+      "gpt-4-turbo": { prompt: 0.01, completion: 0.03 },
+      "gpt-3.5-turbo": { prompt: 0.0005, completion: 0.0015 },
+    },
+    deepseek: {
+      "deepseek-reasoner": { prompt: 0.001, completion: 0.002 },
+      "deepseek-chat": { prompt: 0.0005, completion: 0.001 },
+    },
+    anthropic: {
+      "claude-3-opus": { prompt: 0.015, completion: 0.075 },
+      "claude-3-sonnet": { prompt: 0.003, completion: 0.015 },
+    },
+  };
+
+  const modelPricing = pricing[provider]?.[model];
+  if (!modelPricing || !usage) return 0;
+
+  const promptCost = ((usage.promptTokens || 0) / 1000) * modelPricing.prompt;
+  const completionCost =
+    ((usage.completionTokens || 0) / 1000) * modelPricing.completion;
+
+  return promptCost + completionCost;
+}
+
+// Main chat function - no metrics handling needed!
+export async function chat(options) {
+  const {
+    provider = "openai",
+    model,
+    messages = [],
+    temperature,
+    maxTokens,
+    metadata = {},
+    ...rest
+  } = options;
+
+  const available = getAvailableProviders();
+
+  if (!available[provider]) {
+    throw new Error(`Provider ${provider} not available. Check API keys.`);
+  }
+
+  const startTime = Date.now();
+  const requestId = `req_${Date.now()}_${Math.random().toString(36).substring(7)}`;
+
+  // Extract system and user messages
+  const systemMsg = messages.find((m) => m.role === "system")?.content || "";
+  const userMessages = messages.filter((m) => m.role === "user");
+  const userMsg = userMessages.map((m) => m.content).join("\n");
+
+  // Emit request start event
+  llmEvents.emit("llm:request:start", {
+    id: requestId,
+    provider,
+    model,
+    metadata,
+    timestamp: new Date().toISOString(),
+  });
+
+  try {
+    let response;
+    let usage;
+
+    if (provider === "mock") {
+      if (!mockProviderInstance) {
+        throw new Error(
+          "Mock provider not registered. Call registerMockProvider() first."
+        );
+      }
+
+      const result = await mockProviderInstance.chat({
+        messages,
+        model: model || "gpt-3.5-turbo",
+        temperature,
+        maxTokens,
+        ...rest,
+      });
+
+      response = {
+        content: result.content,
+        raw: result.raw,
+      };
+
+      usage = {
+        promptTokens: result.usage.prompt_tokens,
+        completionTokens: result.usage.completion_tokens,
+        totalTokens: result.usage.total_tokens,
+      };
+    } else if (provider === "openai") {
+      const result = await openaiChat({
+        messages,
+        model: model || "gpt-5-chat-latest",
+        maxTokens,
+        temperature,
+        ...rest,
+      });
+
+      response = {
+        content: typeof result === "string" ? result : JSON.stringify(result),
+        raw: result,
+      };
+
+      // Estimate tokens since GPT-5 responses API might not return usage
+      const promptTokens = estimateTokens(systemMsg + userMsg);
+      const completionTokens = estimateTokens(response.content);
+      usage = {
+        promptTokens,
+        completionTokens,
+        totalTokens: promptTokens + completionTokens,
+      };
+    } else if (provider === "deepseek") {
+      const result = await deepseekChat(
+        systemMsg,
+        userMsg,
+        model || "deepseek-reasoner"
+      );
+
+      response = {
+        content: typeof result === "string" ? result : JSON.stringify(result),
+        raw: result,
+      };
+
+      const promptTokens = estimateTokens(systemMsg + userMsg);
+      const completionTokens = estimateTokens(response.content);
+      usage = {
+        promptTokens,
+        completionTokens,
+        totalTokens: promptTokens + completionTokens,
+      };
+    } else {
+      throw new Error(`Provider ${provider} not yet implemented`);
+    }
+
+    const duration = Date.now() - startTime;
+    const cost = calculateCost(provider, model, usage);
+
+    // Emit success event with metrics
+    llmEvents.emit("llm:request:complete", {
+      id: requestId,
+      provider,
+      model,
+      duration,
+      ...usage,
+      cost,
+      metadata,
+      timestamp: new Date().toISOString(),
+    });
+
+    // Return clean response - no metrics attached!
+    return {
+      ...response,
+      usage,
+    };
+  } catch (error) {
+    const duration = Date.now() - startTime;
+
+    // Emit error event
+    llmEvents.emit("llm:request:error", {
+      id: requestId,
+      provider,
+      model,
+      duration,
+      error: error.message,
+      metadata,
+      timestamp: new Date().toISOString(),
+    });
+
+    throw error;
+  }
+}
+
+// Convenience function for simple completions
+export async function complete(prompt, options = {}) {
+  return chat({
+    ...options,
+    messages: [{ role: "user", content: prompt }],
+  });
+}
+
+// Create a chain for multi-turn conversations
+export function createChain() {
+  const messages = [];
+
+  return {
+    addSystemMessage: function (content) {
+      messages.push({ role: "system", content });
+      return this;
+    },
+
+    addUserMessage: function (content) {
+      messages.push({ role: "user", content });
+      return this;
+    },
+
+    addAssistantMessage: function (content) {
+      messages.push({ role: "assistant", content });
+      return this;
+    },
+
+    execute: async function (options = {}) {
+      const response = await chat({ ...options, messages });
+      messages.push({
+        role: "assistant",
+        content: response.content,
+      });
+      return response;
+    },
+
+    getMessages: () => [...messages],
+
+    clear: function () {
+      messages.length = 0;
+      return this;
+    },
+  };
+}
+
+// Retry wrapper
+export async function withRetry(fn, args = [], options = {}) {
+  const config = getConfig();
+  const maxRetries = options.maxRetries ?? config.llm.retryMaxAttempts;
+  const backoffMs = options.backoffMs ?? config.llm.retryBackoffMs;
+
+  let lastError;
+
+  for (let i = 0; i <= maxRetries; i++) {
+    try {
+      if (i > 0) {
+        await new Promise((r) => setTimeout(r, backoffMs * Math.pow(2, i - 1)));
+      }
+      return await fn(...args);
+    } catch (error) {
+      lastError = error;
+      // Don't retry auth errors
+      if (error.status === 401 || error.message?.includes("API key")) {
+        throw error;
+      }
+    }
+  }
+
+  throw lastError;
+}
+
+// Parallel execution with concurrency control
+export async function parallel(fn, items, maxConcurrency) {
+  const config = getConfig();
+  const concurrency = maxConcurrency ?? config.llm.maxConcurrency;
+
+  const results = [];
+  for (let i = 0; i < items.length; i += concurrency) {
+    const batch = items.slice(i, i + concurrency);
+    const batchResults = await Promise.all(batch.map((item) => fn(item)));
+    results.push(...batchResults);
+  }
+  return results;
+}
+
+// Create a bound LLM interface (no metrics handling needed!)
+export function createLLM(options = {}) {
+  const config = getConfig();
+  const defaultProvider = options.defaultProvider || config.llm.defaultProvider;
+
+  return {
+    chat: (opts) => chat({ provider: defaultProvider, ...opts }),
+    complete: (prompt, opts) =>
+      complete(prompt, { provider: defaultProvider, ...opts }),
+    createChain: () => createChain(),
+    withRetry: (opts) =>
+      withRetry(chat, [{ provider: defaultProvider, ...opts }]),
+    parallel: (requests, maxConcurrency) =>
+      parallel(
+        (req) => chat({ provider: defaultProvider, ...req }),
+        requests,
+        maxConcurrency
+      ),
+    getAvailableProviders,
+  };
+}