@almadar/llm 1.0.0

package/src/rate-limiter.ts

@@ -0,0 +1,237 @@
+/**
+ * Rate Limiter for LLM API Calls
+ *
+ * Implements token bucket algorithm with:
+ * - Configurable requests per minute/second
+ * - Automatic backoff on 429 errors
+ * - Queue for pending requests
+ *
+ * @packageDocumentation
+ */
+
+export interface RateLimiterOptions {
+  /** Maximum requests per minute (default: 60) */
+  requestsPerMinute?: number;
+  /** Maximum requests per second (default: 3) */
+  requestsPerSecond?: number;
+  /** Maximum concurrent requests (default: 5) */
+  maxConcurrent?: number;
+  /** Base delay for exponential backoff in ms (default: 1000) */
+  baseBackoffMs?: number;
+  /** Maximum backoff delay in ms (default: 60000) */
+  maxBackoffMs?: number;
+}
+
+interface QueuedRequest<T> {
+  execute: () => Promise<T>;
+  resolve: (value: T) => void;
+  reject: (error: Error) => void;
+  retryCount: number;
+}
+
+/**
+ * Rate limiter for LLM API calls using token bucket algorithm.
+ *
+ * @example
+ * ```typescript
+ * const limiter = new RateLimiter({ requestsPerMinute: 30 });
+ * const result = await limiter.execute(() => llm.invoke(messages));
+ * ```
+ */
+export class RateLimiter {
+  private requestsPerMinute: number;
+  private requestsPerSecond: number;
+  private maxConcurrent: number;
+  private baseBackoffMs: number;
+  private maxBackoffMs: number;
+
+  private minuteTokens: number;
+  private secondTokens: number;
+  private activeRequests: number = 0;
+  private queue: QueuedRequest<unknown>[] = [];
+  private lastMinuteReset: number = Date.now();
+  private lastSecondReset: number = Date.now();
+  private processing: boolean = false;
+  private currentBackoffMs: number = 0;
+
+  constructor(options: RateLimiterOptions = {}) {
+    this.requestsPerMinute = options.requestsPerMinute ?? 60;
+    this.requestsPerSecond = options.requestsPerSecond ?? 3;
+    this.maxConcurrent = options.maxConcurrent ?? 5;
+    this.baseBackoffMs = options.baseBackoffMs ?? 1000;
+    this.maxBackoffMs = options.maxBackoffMs ?? 60000;
+
+    this.minuteTokens = this.requestsPerMinute;
+    this.secondTokens = this.requestsPerSecond;
+  }
+
+  async execute<T>(fn: () => Promise<T>, _maxRetries: number = 3): Promise<T> {
+    return new Promise<T>((resolve, reject) => {
+      this.queue.push({
+        execute: fn as () => Promise<unknown>,
+        resolve: resolve as (value: unknown) => void,
+        reject,
+        retryCount: 0,
+      });
+      this.processQueue();
+    });
+  }
+
+  getStatus(): {
+    queueLength: number;
+    activeRequests: number;
+    minuteTokens: number;
+    secondTokens: number;
+    backoffMs: number;
+  } {
+    return {
+      queueLength: this.queue.length,
+      activeRequests: this.activeRequests,
+      minuteTokens: this.minuteTokens,
+      secondTokens: this.secondTokens,
+      backoffMs: this.currentBackoffMs,
+    };
+  }
+
+  reset(): void {
+    this.minuteTokens = this.requestsPerMinute;
+    this.secondTokens = this.requestsPerSecond;
+    this.activeRequests = 0;
+    this.queue = [];
+    this.currentBackoffMs = 0;
+    this.lastMinuteReset = Date.now();
+    this.lastSecondReset = Date.now();
+  }
+
+  private async processQueue(): Promise<void> {
+    if (this.processing) return;
+    this.processing = true;
+
+    while (this.queue.length > 0) {
+      this.refillTokens();
+
+      if (!this.canMakeRequest()) {
+        const waitTime = this.getWaitTime();
+        await this.sleep(waitTime);
+        continue;
+      }
+
+      if (this.currentBackoffMs > 0) {
+        await this.sleep(this.currentBackoffMs);
+        this.currentBackoffMs = 0;
+      }
+
+      const request = this.queue.shift();
+      if (!request) continue;
+
+      this.consumeTokens();
+      this.activeRequests++;
+
+      try {
+        const result = await request.execute();
+        request.resolve(result);
+        this.currentBackoffMs = 0;
+      } catch (error) {
+        const err = error instanceof Error ? error : new Error(String(error));
+
+        if (this.isRateLimitError(err)) {
+          this.currentBackoffMs = Math.min(
+            this.baseBackoffMs * Math.pow(2, request.retryCount),
+            this.maxBackoffMs,
+          );
+
+          console.warn(
+            `[RateLimiter] Rate limited. Backing off for ${this.currentBackoffMs}ms ` +
+              `(retry ${request.retryCount + 1})`,
+          );
+
+          if (request.retryCount < 3) {
+            this.queue.unshift({
+              ...request,
+              retryCount: request.retryCount + 1,
+            });
+          } else {
+            request.reject(
+              new Error(
+                `Rate limit exceeded after ${request.retryCount + 1} retries: ${err.message}`,
+              ),
+            );
+          }
+        } else {
+          request.reject(err);
+        }
+      } finally {
+        this.activeRequests--;
+      }
+    }
+
+    this.processing = false;
+  }
+
+  private refillTokens(): void {
+    const now = Date.now();
+    if (now - this.lastMinuteReset >= 60000) {
+      this.minuteTokens = this.requestsPerMinute;
+      this.lastMinuteReset = now;
+    }
+    if (now - this.lastSecondReset >= 1000) {
+      this.secondTokens = this.requestsPerSecond;
+      this.lastSecondReset = now;
+    }
+  }
+
+  private canMakeRequest(): boolean {
+    return (
+      this.minuteTokens > 0 &&
+      this.secondTokens > 0 &&
+      this.activeRequests < this.maxConcurrent
+    );
+  }
+
+  private consumeTokens(): void {
+    this.minuteTokens--;
+    this.secondTokens--;
+  }
+
+  private getWaitTime(): number {
+    const now = Date.now();
+    if (this.secondTokens <= 0) {
+      return Math.max(0, 1000 - (now - this.lastSecondReset));
+    }
+    if (this.minuteTokens <= 0) {
+      return Math.max(0, 60000 - (now - this.lastMinuteReset));
+    }
+    return 100;
+  }
+
+  private isRateLimitError(error: Error): boolean {
+    const message = error.message.toLowerCase();
+    return (
+      message.includes('429') ||
+      message.includes('rate limit') ||
+      message.includes('too many requests') ||
+      message.includes('quota exceeded')
+    );
+  }
+
+  private sleep(ms: number): Promise<void> {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+  }
+}
+
+// Singleton instance
+let globalRateLimiter: RateLimiter | null = null;
+
+export function getGlobalRateLimiter(
+  options?: RateLimiterOptions,
+): RateLimiter {
+  if (!globalRateLimiter) {
+    globalRateLimiter = new RateLimiter(options);
+  }
+  return globalRateLimiter;
+}
+
+export function resetGlobalRateLimiter(): void {
+  globalRateLimiter?.reset();
+  globalRateLimiter = null;
+}

package/src/structured-output.ts

@@ -0,0 +1,330 @@
+/**
+ * Structured Output Client for OpenAI
+ *
+ * Uses OpenAI's structured outputs feature (json_schema response_format)
+ * to guarantee schema compliance at generation time.
+ *
+ * The system prompt builder is injectable so consumers can provide
+ * domain-specific prompts (e.g., orbital schema references).
+ *
+ * @packageDocumentation
+ */
+
+import OpenAI from 'openai';
+import type { ChatCompletionCreateParamsNonStreaming } from 'openai/resources/chat/completions';
+import { z } from 'zod';
+import {
+  RateLimiter,
+  getGlobalRateLimiter,
+  type RateLimiterOptions,
+} from './rate-limiter.js';
+import { TokenTracker, getGlobalTokenTracker } from './token-tracker.js';
+
+// ============================================================================
+// Types
+// ============================================================================
+
+/**
+ * JSON Schema type used for OpenAI structured outputs.
+ */
+export interface JsonSchema {
+  type?: string | string[];
+  properties?: Record<string, JsonSchema>;
+  required?: string[];
+  items?: JsonSchema;
+  enum?: unknown[];
+  const?: unknown;
+  anyOf?: JsonSchema[];
+  oneOf?: JsonSchema[];
+  allOf?: JsonSchema[];
+  $ref?: string;
+  $defs?: Record<string, JsonSchema>;
+  definitions?: Record<string, JsonSchema>;
+  additionalProperties?: boolean | JsonSchema;
+  description?: string;
+  default?: unknown;
+  minItems?: number;
+  maxItems?: number;
+  minLength?: number;
+}
+
+export interface StructuredOutputOptions {
+  model?: string;
+  temperature?: number;
+  maxTokens?: number;
+  rateLimiter?: RateLimiterOptions;
+  useGlobalRateLimiter?: boolean;
+  trackTokens?: boolean;
+}
+
+export interface StructuredGenerationOptions {
+  /** User's natural language request */
+  userRequest: string;
+  /** Model to use (overrides client default) */
+  model?: string;
+  /** Temperature (overrides client default) */
+  temperature?: number;
+  /** Maximum tokens (overrides client default) */
+  maxTokens?: number;
+  /** JSON Schema for structured output */
+  jsonSchema?: JsonSchema;
+  /** Schema name for the json_schema response format */
+  schemaName?: string;
+  /** System prompt override */
+  systemPrompt?: string;
+  /** System prompt builder function (called dynamically) */
+  buildSystemPrompt?: () => string;
+  /** Additional system prompt instructions */
+  additionalInstructions?: string;
+  /** Existing context for updates (e.g., existing schema JSON) */
+  existingContext?: string;
+  /** Skip post-generation validation (default: false) */
+  skipValidation?: boolean;
+}
+
+export interface StructuredGenerationResult<T = unknown> {
+  /** Generated data (guaranteed to match JSON Schema structure) */
+  data: T;
+  /** Raw JSON string from API */
+  raw: string;
+  /** Token usage statistics */
+  usage: {
+    promptTokens: number;
+    completionTokens: number;
+    totalTokens: number;
+  };
+  /** Generation latency in milliseconds */
+  latencyMs: number;
+  /** Model used for generation */
+  model: string;
+  /** Zod validation result (if not skipped) */
+  zodValidation?: {
+    success: boolean;
+    errors?: z.ZodError['errors'];
+  };
+}
+
+export const STRUCTURED_OUTPUT_MODELS = {
+  GPT5_MINI: 'gpt-5-mini',
+  GPT4O_MINI: 'gpt-4o-mini',
+  GPT4O: 'gpt-4o',
+  GPT4O_2024_08_06: 'gpt-4o-2024-08-06',
+} as const;
+
+// ============================================================================
+// Default System Prompt
+// ============================================================================
+
+const DEFAULT_SYSTEM_PROMPT = `You are an expert application architect that generates structured schemas from natural language requirements.
+
+Generate a complete, well-structured schema based on the user's requirements. Follow the JSON Schema structure exactly.`;
+
+// ============================================================================
+// Structured Output Client
+// ============================================================================
+
+export class StructuredOutputClient {
+  private openai: OpenAI;
+  private rateLimiter: RateLimiter;
+  private tokenTracker: TokenTracker | null;
+  private defaultModel: string;
+  private defaultTemperature: number;
+  private defaultMaxTokens: number;
+
+  constructor(options: StructuredOutputOptions = {}) {
+    const apiKey = process.env.OPENAI_API_KEY;
+    if (!apiKey) {
+      throw new Error(
+        'OPENAI_API_KEY environment variable is required for StructuredOutputClient',
+      );
+    }
+
+    this.openai = new OpenAI({ apiKey });
+    this.defaultModel = options.model || STRUCTURED_OUTPUT_MODELS.GPT5_MINI;
+    this.defaultTemperature = options.temperature ?? 0.3;
+    this.defaultMaxTokens = options.maxTokens ?? 16384;
+
+    this.rateLimiter =
+      options.useGlobalRateLimiter !== false
+        ? getGlobalRateLimiter(options.rateLimiter)
+        : new RateLimiter(options.rateLimiter);
+
+    this.tokenTracker =
+      options.trackTokens !== false
+        ? getGlobalTokenTracker(this.defaultModel)
+        : null;
+
+    console.log(
+      `[StructuredOutputClient] Initialized with model: ${this.defaultModel}`,
+    );
+  }
+
+  private usesMaxCompletionTokens(model: string): boolean {
+    const m = model.toLowerCase();
+    return (
+      m.startsWith('o1') ||
+      m.startsWith('gpt-5') ||
+      m.includes('o1-') ||
+      m.includes('o3')
+    );
+  }
+
+  /**
+   * Generate structured output with guaranteed JSON Schema compliance.
+   */
+  async generate<T = unknown>(
+    options: StructuredGenerationOptions,
+  ): Promise<StructuredGenerationResult<T>> {
+    const model = options.model || this.defaultModel;
+    const temperature = options.temperature ?? this.defaultTemperature;
+    const maxTokens = options.maxTokens ?? this.defaultMaxTokens;
+    const startTime = Date.now();
+
+    const jsonSchema: JsonSchema = options.jsonSchema || {
+      type: 'object',
+      properties: {},
+      required: [],
+      additionalProperties: false,
+    };
+
+    // Build system prompt
+    let systemPrompt: string;
+    if (options.systemPrompt) {
+      systemPrompt = options.systemPrompt;
+    } else if (options.buildSystemPrompt) {
+      systemPrompt = options.buildSystemPrompt();
+    } else {
+      systemPrompt = DEFAULT_SYSTEM_PROMPT;
+    }
+
+    if (options.additionalInstructions) {
+      systemPrompt += `\n\n## Additional Instructions\n${options.additionalInstructions}`;
+    }
+
+    // Build user prompt
+    let userPrompt = options.userRequest;
+    if (options.existingContext) {
+      userPrompt += `\n\n## Existing Context\nUpdate based on the above request:\n\`\`\`json\n${options.existingContext}\n\`\`\``;
+    }
+
+    const schemaName = options.schemaName || 'structured_output';
+
+    console.log(
+      `[StructuredOutputClient] Generating with ${model}...`,
+    );
+    console.log(
+      `[StructuredOutputClient] Request: "${options.userRequest.slice(0, 80)}..."`,
+    );
+
+    const response = await this.rateLimiter.execute(async () => {
+      const isReasoningModel = this.usesMaxCompletionTokens(model);
+
+      const tokenParam = isReasoningModel
+        ? { max_completion_tokens: maxTokens }
+        : { max_tokens: maxTokens };
+
+      const tempParam = isReasoningModel ? {} : { temperature };
+
+      const params: ChatCompletionCreateParamsNonStreaming = {
+        model,
+        messages: [
+          { role: 'system', content: systemPrompt },
+          { role: 'user', content: userPrompt },
+        ],
+        response_format: {
+          type: 'json_schema',
+          json_schema: {
+            name: schemaName,
+            strict: true,
+            schema: jsonSchema as Record<string, unknown>,
+          },
+        },
+        ...tempParam,
+        ...tokenParam,
+      };
+
+      return this.openai.chat.completions.create(params);
+    });
+
+    const latencyMs = Date.now() - startTime;
+
+    const content = response.choices[0]?.message?.content;
+    if (!content) {
+      throw new Error('No content in OpenAI response');
+    }
+
+    let data: T;
+    try {
+      data = JSON.parse(content) as T;
+    } catch (error) {
+      throw new Error(`Failed to parse response JSON: ${error}`);
+    }
+
+    const usage = {
+      promptTokens: response.usage?.prompt_tokens || 0,
+      completionTokens: response.usage?.completion_tokens || 0,
+      totalTokens: response.usage?.total_tokens || 0,
+    };
+
+    if (this.tokenTracker) {
+      this.tokenTracker.addUsage(usage.promptTokens, usage.completionTokens);
+    }
+
+    console.log(
+      `[StructuredOutputClient] Generated in ${latencyMs}ms, ${usage.totalTokens} tokens`,
+    );
+
+    let zodValidation: StructuredGenerationResult['zodValidation'];
+    if (!options.skipValidation) {
+      zodValidation = { success: true };
+    }
+
+    return {
+      data,
+      raw: content,
+      usage,
+      latencyMs,
+      model,
+      zodValidation,
+    };
+  }
+
+  getModel(): string {
+    return this.defaultModel;
+  }
+
+  getRateLimiterStatus() {
+    return this.rateLimiter.getStatus();
+  }
+
+  getTokenUsage() {
+    return this.tokenTracker?.getSummary() ?? null;
+  }
+}
+
+// ============================================================================
+// Singleton Instance
+// ============================================================================
+
+let sharedClient: StructuredOutputClient | null = null;
+
+export function getStructuredOutputClient(
+  options?: StructuredOutputOptions,
+): StructuredOutputClient {
+  if (!sharedClient) {
+    sharedClient = new StructuredOutputClient(options);
+  }
+  return sharedClient;
+}
+
+export function resetStructuredOutputClient(): void {
+  sharedClient = null;
+}
+
+// ============================================================================
+// Convenience Functions
+// ============================================================================
+
+export function isStructuredOutputAvailable(): boolean {
+  return !!process.env.OPENAI_API_KEY;
+}

package/src/token-tracker.ts

@@ -0,0 +1,116 @@
+/**
+ * Token Tracker for LLM Usage
+ *
+ * Tracks token usage across multiple LLM calls for:
+ * - Cost estimation
+ * - Usage monitoring
+ * - Quota management
+ *
+ * @packageDocumentation
+ */
+
+export interface TokenUsage {
+  promptTokens: number;
+  completionTokens: number;
+  totalTokens: number;
+  callCount: number;
+}
+
+export interface TokenCost {
+  promptCostPer1K: number;
+  completionCostPer1K: number;
+}
+
+// Pricing as of 2024 (update as needed)
+const MODEL_COSTS: Record<string, TokenCost> = {
+  'gpt-4o': { promptCostPer1K: 0.005, completionCostPer1K: 0.015 },
+  'gpt-4o-mini': { promptCostPer1K: 0.00015, completionCostPer1K: 0.0006 },
+  'gpt-4-turbo': { promptCostPer1K: 0.01, completionCostPer1K: 0.03 },
+  'gpt-4': { promptCostPer1K: 0.03, completionCostPer1K: 0.06 },
+  'gpt-3.5-turbo': {
+    promptCostPer1K: 0.0005,
+    completionCostPer1K: 0.0015,
+  },
+};
+
+export class TokenTracker {
+  private model: string;
+  private usage: TokenUsage = {
+    promptTokens: 0,
+    completionTokens: 0,
+    totalTokens: 0,
+    callCount: 0,
+  };
+
+  constructor(model: string = 'gpt-4o') {
+    this.model = model;
+  }
+
+  addUsage(promptTokens: number, completionTokens: number): void {
+    this.usage.promptTokens += promptTokens;
+    this.usage.completionTokens += completionTokens;
+    this.usage.totalTokens += promptTokens + completionTokens;
+    this.usage.callCount++;
+  }
+
+  getSummary(): TokenUsage {
+    return { ...this.usage };
+  }
+
+  getEstimatedCost(): number {
+    const costs = MODEL_COSTS[this.model] || MODEL_COSTS['gpt-4o'];
+    const promptCost =
+      (this.usage.promptTokens / 1000) * costs.promptCostPer1K;
+    const completionCost =
+      (this.usage.completionTokens / 1000) * costs.completionCostPer1K;
+    return promptCost + completionCost;
+  }
+
+  getFormattedCost(): string {
+    const cost = this.getEstimatedCost();
+    return `$${cost.toFixed(4)}`;
+  }
+
+  getReport(): string {
+    const summary = this.getSummary();
+    const cost = this.getEstimatedCost();
+    return [
+      `Token Usage Report (${this.model})`,
+      `─────────────────────────────`,
+      `Calls:              ${summary.callCount}`,
+      `Prompt Tokens:      ${summary.promptTokens.toLocaleString()}`,
+      `Completion Tokens:  ${summary.completionTokens.toLocaleString()}`,
+      `Total Tokens:       ${summary.totalTokens.toLocaleString()}`,
+      `Estimated Cost:     $${cost.toFixed(4)}`,
+    ].join('\n');
+  }
+
+  reset(): void {
+    this.usage = {
+      promptTokens: 0,
+      completionTokens: 0,
+      totalTokens: 0,
+      callCount: 0,
+    };
+  }
+
+  setModel(model: string): void {
+    this.model = model;
+  }
+}
+
+// Global tracker instance
+let globalTracker: TokenTracker | null = null;
+
+export function getGlobalTokenTracker(model?: string): TokenTracker {
+  if (!globalTracker) {
+    globalTracker = new TokenTracker(model);
+  } else if (model) {
+    globalTracker.setModel(model);
+  }
+  return globalTracker;
+}
+
+export function resetGlobalTokenTracker(): void {
+  globalTracker?.reset();
+}