confused-ai-core 0.1.0

package/src/production/rate-limiter.ts

@@ -0,0 +1,287 @@
+/**
+ * Rate Limiter - Agno-style Production Throttling
+ *
+ * Token bucket algorithm for controlling request rates to external APIs.
+ * Supports:
+ * - Per-agent and per-tool rate limits
+ * - Configurable burst capacity
+ * - Backpressure modes (queue or reject)
+ * - Metrics integration
+ */
+
+import type { MetricsCollector } from '../observability/types.js';
+
+/** Rate limiter configuration */
+export interface RateLimiterConfig {
+    /** Unique name for this limiter (for metrics/logging) */
+    readonly name: string;
+    /** Maximum requests per interval (default: 60) */
+    readonly maxRequests: number;
+    /** Interval in milliseconds (default: 60000 = 1 minute) */
+    readonly intervalMs?: number;
+    /** Burst capacity beyond maxRequests (default: 10) */
+    readonly burstCapacity?: number;
+    /** Action when limit reached: 'reject' throws, 'queue' waits (default: 'reject') */
+    readonly overflowMode?: 'reject' | 'queue';
+    /** Max queue size when mode is 'queue' (default: 100) */
+    readonly maxQueueSize?: number;
+    /** Max wait time in queue before rejection in ms (default: 30000) */
+    readonly maxQueueWaitMs?: number;
+    /** Optional metrics collector */
+    readonly metrics?: MetricsCollector;
+}
+
+/** Rate limit exceeded error */
+export class RateLimitError extends Error {
+    readonly limiterName: string;
+    readonly retryAfterMs: number;
+
+    constructor(name: string, retryAfterMs: number) {
+        super(`Rate limit exceeded for '${name}'. Retry after ${retryAfterMs}ms`);
+        this.name = 'RateLimitError';
+        this.limiterName = name;
+        this.retryAfterMs = retryAfterMs;
+        Object.setPrototypeOf(this, RateLimitError.prototype);
+    }
+}
+
+/** Queued request */
+interface QueuedRequest<T> {
+    readonly fn: () => Promise<T>;
+    readonly resolve: (value: T) => void;
+    readonly reject: (error: Error) => void;
+    readonly enqueuedAt: number;
+}
+
+/**
+ * Token bucket rate limiter with optional request queuing.
+ *
+ * @example
+ * const limiter = new RateLimiter({
+ *   name: 'openai-api',
+ *   maxRequests: 100,
+ *   intervalMs: 60000, // 100 requests per minute
+ *   overflowMode: 'queue',
+ * });
+ *
+ * const result = await limiter.execute(() => openai.chat(...));
+ */
+export class RateLimiter {
+    private tokens: number;
+    private lastRefill: number;
+    private queue: QueuedRequest<unknown>[] = [];
+    private processing = false;
+
+    private readonly config: Required<Omit<RateLimiterConfig, 'metrics'>> &
+        Pick<RateLimiterConfig, 'metrics'>;
+
+    constructor(config: RateLimiterConfig) {
+        this.config = {
+            name: config.name,
+            maxRequests: config.maxRequests,
+            intervalMs: config.intervalMs ?? 60_000,
+            burstCapacity: config.burstCapacity ?? Math.ceil(config.maxRequests * 0.1),
+            overflowMode: config.overflowMode ?? 'reject',
+            maxQueueSize: config.maxQueueSize ?? 100,
+            maxQueueWaitMs: config.maxQueueWaitMs ?? 30_000,
+            metrics: config.metrics,
+        };
+
+        this.tokens = this.config.maxRequests + this.config.burstCapacity;
+        this.lastRefill = Date.now();
+    }
+
+    /** Get current available tokens */
+    getAvailableTokens(): number {
+        this.refillTokens();
+        return this.tokens;
+    }
+
+    /** Get current queue size */
+    getQueueSize(): number {
+        return this.queue.length;
+    }
+
+    /** Check if a request can be made immediately */
+    canProceed(): boolean {
+        this.refillTokens();
+        return this.tokens > 0;
+    }
+
+    /** Get time until next token available in ms */
+    getTimeUntilAvailable(): number {
+        if (this.tokens > 0) return 0;
+
+        const tokenRefillRate = this.config.intervalMs / this.config.maxRequests;
+        const timeSinceLastRefill = Date.now() - this.lastRefill;
+        return Math.max(0, tokenRefillRate - timeSinceLastRefill);
+    }
+
+    /**
+     * Execute a function through the rate limiter.
+     */
+    async execute<T>(fn: () => Promise<T>): Promise<T> {
+        this.refillTokens();
+
+        if (this.tokens > 0) {
+            this.tokens--;
+            this.recordMetric('rate_limiter_allowed', 1);
+            return fn();
+        }
+
+        if (this.config.overflowMode === 'queue') {
+            return this.enqueue(fn);
+        }
+
+        const retryAfterMs = this.getTimeUntilAvailable();
+        this.recordMetric('rate_limiter_rejected', 1);
+        throw new RateLimitError(this.config.name, retryAfterMs);
+    }
+
+    /**
+     * Try to acquire a token without executing anything.
+     * Returns true if token acquired, false otherwise.
+     */
+    tryAcquire(): boolean {
+        this.refillTokens();
+        if (this.tokens > 0) {
+            this.tokens--;
+            return true;
+        }
+        return false;
+    }
+
+    /** Get rate limiter statistics */
+    getStats(): {
+        availableTokens: number;
+        queueSize: number;
+        maxRequests: number;
+        intervalMs: number;
+    } {
+        return {
+            availableTokens: this.getAvailableTokens(),
+            queueSize: this.queue.length,
+            maxRequests: this.config.maxRequests,
+            intervalMs: this.config.intervalMs,
+        };
+    }
+
+    // --- Private methods ---
+
+    private refillTokens(): void {
+        const now = Date.now();
+        const elapsed = now - this.lastRefill;
+        const maxTokens = this.config.maxRequests + this.config.burstCapacity;
+
+        // Token refill rate: tokens per ms
+        const refillRate = this.config.maxRequests / this.config.intervalMs;
+        const tokensToAdd = elapsed * refillRate;
+
+        this.tokens = Math.min(maxTokens, this.tokens + tokensToAdd);
+        this.lastRefill = now;
+    }
+
+    private async enqueue<T>(fn: () => Promise<T>): Promise<T> {
+        if (this.queue.length >= this.config.maxQueueSize) {
+            this.recordMetric('rate_limiter_queue_full', 1);
+            throw new RateLimitError(
+                this.config.name,
+                this.getTimeUntilAvailable()
+            );
+        }
+
+        this.recordMetric('rate_limiter_queued', 1);
+
+        return new Promise<T>((resolve, reject) => {
+            this.queue.push({
+                fn: fn as () => Promise<unknown>,
+                resolve: resolve as (value: unknown) => void,
+                reject,
+                enqueuedAt: Date.now(),
+            });
+
+            this.processQueue();
+        });
+    }
+
+    private async processQueue(): Promise<void> {
+        if (this.processing) return;
+        this.processing = true;
+
+        while (this.queue.length > 0) {
+            this.refillTokens();
+
+            if (this.tokens <= 0) {
+                // Wait for a token
+                await this.waitForToken();
+                continue;
+            }
+
+            const request = this.queue.shift()!;
+            const waitTime = Date.now() - request.enqueuedAt;
+
+            if (waitTime > this.config.maxQueueWaitMs) {
+                request.reject(
+                    new RateLimitError(this.config.name, this.getTimeUntilAvailable())
+                );
+                this.recordMetric('rate_limiter_queue_timeout', 1);
+                continue;
+            }
+
+            this.tokens--;
+            this.recordMetric('rate_limiter_dequeued', 1, {
+                wait_time_ms: String(waitTime),
+            });
+
+            try {
+                const result = await request.fn();
+                request.resolve(result);
+            } catch (error) {
+                request.reject(error as Error);
+            }
+        }
+
+        this.processing = false;
+    }
+
+    private async waitForToken(): Promise<void> {
+        const waitTime = this.getTimeUntilAvailable();
+        await new Promise(resolve => setTimeout(resolve, Math.max(waitTime, 10)));
+    }
+
+    private recordMetric(
+        name: string,
+        value: number,
+        labels: Record<string, string> = {}
+    ): void {
+        this.config.metrics?.counter(`${this.config.name}.${name}`, value, {
+            limiter: this.config.name,
+            ...labels,
+        });
+    }
+}
+
+/**
+ * Create a rate limiter with defaults for OpenAI API (RPM limits).
+ */
+export function createOpenAIRateLimiter(
+    tier: 'free' | 'tier1' | 'tier2' | 'tier3' | 'tier4' | 'tier5' = 'tier1',
+    options?: Partial<RateLimiterConfig>
+): RateLimiter {
+    const tierLimits: Record<string, number> = {
+        free: 3,
+        tier1: 60,
+        tier2: 100,
+        tier3: 500,
+        tier4: 5000,
+        tier5: 10000,
+    };
+
+    return new RateLimiter({
+        name: 'openai-api',
+        maxRequests: tierLimits[tier] ?? 60,
+        intervalMs: 60_000,
+        overflowMode: 'queue',
+        ...options,
+    });
+}

package/src/production/resumable-stream.ts

@@ -0,0 +1,289 @@
+/**
+ * Resumable Streaming - VoltAgent-Style Stream Reconnection
+ *
+ * Enables clients to reconnect to in-flight streams after refresh/disconnect:
+ * - Checkpoint-based stream state persistence
+ * - Automatic resume from last position
+ * - SSE-compatible output format
+ */
+
+/** Stream checkpoint for resumption */
+export interface StreamCheckpoint {
+    /** Unique stream ID */
+    readonly streamId: string;
+    /** Position in the stream (chunk index) */
+    readonly position: number;
+    /** Accumulated content up to this point */
+    readonly accumulatedContent: string;
+    /** Tool calls seen so far */
+    readonly toolCalls: Array<{
+        id: string;
+        name: string;
+        arguments: string;
+    }>;
+    /** Stream start time */
+    readonly startedAt: Date;
+    /** Last activity time */
+    readonly lastActivityAt: Date;
+    /** Is stream complete? */
+    readonly isComplete: boolean;
+    /** Finish reason if complete */
+    readonly finishReason?: string;
+}
+
+/** Resumable stream configuration */
+export interface ResumableStreamConfig {
+    /** Maximum age of resumable streams in ms (default: 5 minutes) */
+    readonly maxAgeMs?: number;
+    /** Cleanup interval in ms (default: 1 minute) */
+    readonly cleanupIntervalMs?: number;
+    /** Maximum number of stored streams (default: 1000) */
+    readonly maxStreams?: number;
+}
+
+/** Stream chunk for SSE */
+export interface StreamChunkSSE {
+    readonly id: string;
+    readonly event: 'delta' | 'error' | 'done';
+    readonly data: {
+        type: 'text' | 'tool_call';
+        content?: string;
+        toolCall?: {
+            id: string;
+            name: string;
+            arguments: string;
+        };
+    };
+    readonly position: number;
+}
+
+/**
+ * ResumableStreamManager - manages stream checkpoints for reconnection.
+ *
+ * @example
+ * const manager = new ResumableStreamManager();
+ *
+ * // Start a stream
+ * const streamId = manager.createStream();
+ *
+ * // As chunks arrive, save checkpoints
+ * manager.saveChunk(streamId, { type: 'text', content: 'Hello' });
+ *
+ * // Client reconnects - get missed content
+ * const checkpoint = manager.getCheckpoint(streamId);
+ * const missedChunks = manager.getChunksSince(streamId, clientPosition);
+ */
+export class ResumableStreamManager {
+    private readonly streams = new Map<string, StreamCheckpoint>();
+    private readonly chunks = new Map<string, StreamChunkSSE[]>();
+    private readonly config: Required<ResumableStreamConfig>;
+    private cleanupTimer: NodeJS.Timeout | null = null;
+
+    constructor(config: ResumableStreamConfig = {}) {
+        this.config = {
+            maxAgeMs: config.maxAgeMs ?? 5 * 60 * 1000, // 5 minutes
+            cleanupIntervalMs: config.cleanupIntervalMs ?? 60 * 1000, // 1 minute
+            maxStreams: config.maxStreams ?? 1000,
+        };
+
+        this.startCleanup();
+    }
+
+    /** Create a new resumable stream */
+    createStream(): string {
+        const streamId = this.generateId();
+        const now = new Date();
+
+        // Evict if at capacity
+        if (this.streams.size >= this.config.maxStreams) {
+            this.evictOldest();
+        }
+
+        this.streams.set(streamId, {
+            streamId,
+            position: 0,
+            accumulatedContent: '',
+            toolCalls: [],
+            startedAt: now,
+            lastActivityAt: now,
+            isComplete: false,
+        });
+
+        this.chunks.set(streamId, []);
+
+        return streamId;
+    }
+
+    /** Get current checkpoint for a stream */
+    getCheckpoint(streamId: string): StreamCheckpoint | null {
+        return this.streams.get(streamId) ?? null;
+    }
+
+    /** Save a chunk to the stream */
+    saveChunk(
+        streamId: string,
+        chunk: { type: 'text'; content: string } | { type: 'tool_call'; toolCall: { id: string; name: string; arguments: string } }
+    ): StreamChunkSSE | null {
+        const checkpoint = this.streams.get(streamId);
+        if (!checkpoint) return null;
+
+        const position = checkpoint.position + 1;
+        const now = new Date();
+
+        // Create SSE chunk
+        const sseChunk: StreamChunkSSE = {
+            id: `${streamId}_${position}`,
+            event: 'delta',
+            data: chunk.type === 'text'
+                ? { type: 'text', content: chunk.content }
+                : { type: 'tool_call', toolCall: chunk.toolCall },
+            position,
+        };
+
+        // Save chunk
+        this.chunks.get(streamId)!.push(sseChunk);
+
+        // Update checkpoint
+        const newToolCalls = chunk.type === 'tool_call'
+            ? [...checkpoint.toolCalls, chunk.toolCall]
+            : checkpoint.toolCalls;
+
+        this.streams.set(streamId, {
+            ...checkpoint,
+            position,
+            accumulatedContent: checkpoint.accumulatedContent + (chunk.type === 'text' ? chunk.content : ''),
+            toolCalls: newToolCalls,
+            lastActivityAt: now,
+        });
+
+        return sseChunk;
+    }
+
+    /** Complete the stream */
+    completeStream(streamId: string, finishReason = 'stop'): void {
+        const checkpoint = this.streams.get(streamId);
+        if (!checkpoint) return;
+
+        this.streams.set(streamId, {
+            ...checkpoint,
+            isComplete: true,
+            finishReason,
+            lastActivityAt: new Date(),
+        });
+
+        // Send done event
+        this.chunks.get(streamId)?.push({
+            id: `${streamId}_done`,
+            event: 'done',
+            data: { type: 'text', content: finishReason },
+            position: checkpoint.position + 1,
+        });
+    }
+
+    /** Get all chunks since a position (for resume) */
+    getChunksSince(streamId: string, position: number): StreamChunkSSE[] {
+        const chunks = this.chunks.get(streamId);
+        if (!chunks) return [];
+
+        return chunks.filter(c => c.position > position);
+    }
+
+    /** Get all chunks for a stream */
+    getAllChunks(streamId: string): StreamChunkSSE[] {
+        return this.chunks.get(streamId) ?? [];
+    }
+
+    /** Check if stream exists and is active */
+    isStreamActive(streamId: string): boolean {
+        const checkpoint = this.streams.get(streamId);
+        return checkpoint !== undefined && !checkpoint.isComplete;
+    }
+
+    /** Delete a stream */
+    deleteStream(streamId: string): boolean {
+        this.chunks.delete(streamId);
+        return this.streams.delete(streamId);
+    }
+
+    /** Shutdown the manager */
+    shutdown(): void {
+        if (this.cleanupTimer) {
+            clearInterval(this.cleanupTimer);
+            this.cleanupTimer = null;
+        }
+    }
+
+    // --- Private ---
+
+    private generateId(): string {
+        return `stream_${Date.now()}_${Math.random().toString(36).substring(2, 11)}`;
+    }
+
+    private startCleanup(): void {
+        this.cleanupTimer = setInterval(() => {
+            this.cleanup();
+        }, this.config.cleanupIntervalMs);
+    }
+
+    private cleanup(): void {
+        const now = Date.now();
+        const maxAge = this.config.maxAgeMs;
+
+        for (const [streamId, checkpoint] of this.streams.entries()) {
+            const age = now - checkpoint.lastActivityAt.getTime();
+            if (age > maxAge) {
+                this.deleteStream(streamId);
+            }
+        }
+    }
+
+    private evictOldest(): void {
+        let oldest: StreamCheckpoint | null = null;
+        let oldestId = '';
+
+        for (const [id, checkpoint] of this.streams.entries()) {
+            if (!oldest || checkpoint.lastActivityAt < oldest.lastActivityAt) {
+                oldest = checkpoint;
+                oldestId = id;
+            }
+        }
+
+        if (oldestId) {
+            this.deleteStream(oldestId);
+        }
+    }
+}
+
+/**
+ * Format a chunk for SSE transmission
+ */
+export function formatSSE(chunk: StreamChunkSSE): string {
+    return `id: ${chunk.id}\nevent: ${chunk.event}\ndata: ${JSON.stringify(chunk.data)}\n\n`;
+}
+
+/**
+ * Create a resumable stream wrapper for async generators
+ */
+export function createResumableStream(
+    manager: ResumableStreamManager,
+    generator: AsyncGenerator<{ type: 'text'; content: string } | { type: 'tool_call'; toolCall: { id: string; name: string; arguments: string } }>
+): { streamId: string; stream: AsyncGenerator<StreamChunkSSE> } {
+    const streamId = manager.createStream();
+
+    async function* wrappedGenerator(): AsyncGenerator<StreamChunkSSE> {
+        try {
+            for await (const chunk of generator) {
+                const sseChunk = manager.saveChunk(streamId, chunk);
+                if (sseChunk) {
+                    yield sseChunk;
+                }
+            }
+            manager.completeStream(streamId);
+        } catch (error) {
+            manager.completeStream(streamId, 'error');
+            throw error;
+        }
+    }
+
+    return { streamId, stream: wrappedGenerator() };
+}

package/src/production/types.ts

@@ -0,0 +1,81 @@
+/**
+ * Production: runtime, control plane, evals for accuracy, performance, latency.
+ */
+
+import type { EntityId } from '../core/types.js';
+
+/** HTTP runtime: serve agents over HTTP (FastAPI-style; use Express/Hono/Fastify impl) */
+export interface AgentRuntime {
+    /** Start the server (e.g. listen on port) */
+    start(): Promise<void>;
+
+    /** Stop the server */
+    stop(): Promise<void>;
+
+    /** Register an agent route (e.g. POST /agents/:id/run) */
+    registerAgent?(agentId: EntityId, handler: (body: unknown) => Promise<unknown>): void;
+}
+
+/** Control plane: monitor and manage agents (optional UI backend) */
+export interface ControlPlane {
+    /** List agents */
+    listAgents(): Promise<{ id: EntityId; name: string; status?: string }[]>;
+
+    /** Get agent stats (runs, latency, errors) */
+    getAgentStats?(agentId: EntityId): Promise<ProductionAgentStats>;
+
+    /** Optional: get runs/sessions for an agent */
+    getRuns?(agentId: EntityId, options?: { limit?: number }): Promise<RunSummary[]>;
+}
+
+export interface ProductionAgentStats {
+    readonly agentId: EntityId;
+    readonly totalRuns: number;
+    readonly successCount: number;
+    readonly failureCount: number;
+    readonly avgLatencyMs: number;
+    readonly p95LatencyMs?: number;
+}
+
+export interface RunSummary {
+    readonly id: string;
+    readonly agentId: EntityId;
+    readonly sessionId?: string;
+    readonly status: string;
+    readonly latencyMs?: number;
+    readonly startedAt: Date;
+}
+
+/** Eval: accuracy, performance, latency */
+export interface EvalSuite {
+    readonly id: string;
+    readonly name: string;
+    /** Run eval and return metrics */
+    run(options?: EvalRunOptions): Promise<EvalResult>;
+}
+
+export interface EvalRunOptions {
+    readonly agentId?: EntityId;
+    readonly dataset?: EvalSample[];
+    readonly maxSamples?: number;
+}
+
+export interface EvalSample {
+    readonly id: string;
+    readonly input: string | Record<string, unknown>;
+    readonly expectedOutput?: string | Record<string, unknown>;
+    readonly metadata?: Record<string, unknown>;
+}
+
+export interface EvalResult {
+    readonly suiteId: string;
+    readonly accuracy?: number;
+    readonly latencyMs?: number;
+    readonly latencyP95Ms?: number;
+    readonly latencyP99Ms?: number;
+    readonly throughputPerMin?: number;
+    readonly errorRate?: number;
+    readonly samplesTotal: number;
+    readonly samplesPassed?: number;
+    readonly details?: Record<string, unknown>;
+}