@vidda/llm-watcher 1.0.0

package/server/dist/server/services/storage.js

@@ -0,0 +1,225 @@
+import { DatabaseSync } from 'node:sqlite';
+import path from 'node:path';
+import fs from 'node:fs';
+import os from 'node:os';
+import { fileURLToPath } from 'node:url';
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+let db = null;
+export function getDb(dbPath) {
+    if (db)
+        return db;
+    // Default to ~/.promptglass/promptglass.db
+    const homeDir = os.homedir();
+    const appDataDir = path.join(homeDir, '.promptglass');
+    const defaultPath = path.join(appDataDir, 'promptglass.db');
+    const resolvedPath = dbPath ?? process.env.DB_PATH ?? defaultPath;
+    const dir = path.dirname(resolvedPath);
+    try {
+        if (!fs.existsSync(dir)) {
+            fs.mkdirSync(dir, { recursive: true });
+        }
+        db = new DatabaseSync(resolvedPath);
+        db.exec('PRAGMA journal_mode = WAL');
+        db.exec('PRAGMA foreign_keys = ON');
+        initSchema(db);
+        return db;
+    }
+    catch (error) {
+        console.error(`Failed to initialize database with error ${error}`);
+        throw error;
+    }
+}
+function initSchema(db) {
+    db.exec('BEGIN');
+    try {
+        db.prepare(`
+    CREATE TABLE IF NOT EXISTS requests (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      mode TEXT NOT NULL CHECK(mode IN ('chat', 'observe', 'benchmark')),
+      created_at DATETIME DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')),
+      context_hash TEXT,
+      request_body TEXT,
+      response_body TEXT,
+      metrics TEXT
+    );
+  `).run();
+        db.prepare(`
+    CREATE TABLE IF NOT EXISTS settings (
+      key TEXT PRIMARY KEY,
+      value TEXT NOT NULL
+    );
+  `).run();
+        db.exec('COMMIT');
+    }
+    catch (error) {
+        db.exec('ROLLBACK');
+        throw error;
+    }
+}
+export function saveRequest(mode, requestBody, responseBody, metrics, contextHash) {
+    const db = getDb();
+    const info = db.prepare(`
+    INSERT INTO requests (mode, request_body, response_body, metrics, context_hash)
+    VALUES (?, ?, ?, ?, ?)
+  `).run(mode, JSON.stringify(requestBody), responseBody ? JSON.stringify(responseBody) : null, metrics ? JSON.stringify(metrics) : null, contextHash || null);
+    return info.lastInsertRowid;
+}
+export function updateRequest(id, responseBody, metrics) {
+    const db = getDb();
+    if (metrics) {
+        db.prepare(`
+      UPDATE requests 
+      SET response_body = ?, metrics = ?
+      WHERE id = ?
+    `).run(JSON.stringify(responseBody), JSON.stringify(metrics), id);
+    }
+    else {
+        db.prepare(`
+      UPDATE requests 
+      SET response_body = ?
+      WHERE id = ?
+    `).run(JSON.stringify(responseBody), id);
+    }
+}
+export function getRequests(limit = 50, offset = 0) {
+    const db = getDb();
+    const rows = db.prepare(`
+    SELECT * FROM requests ORDER BY created_at DESC LIMIT ? OFFSET ?
+  `).all(limit, offset);
+    return rows.map(row => ({
+        id: row.id,
+        mode: row.mode,
+        createdAt: row.created_at,
+        contextHash: row.context_hash,
+        requestBody: JSON.parse(row.request_body),
+        responseBody: row.response_body ? JSON.parse(row.response_body) : undefined,
+        metrics: row.metrics ? JSON.parse(row.metrics) : undefined,
+    }));
+}
+export function getRequestById(id) {
+    const db = getDb();
+    const row = db.prepare('SELECT * FROM requests WHERE id = ?').get(id);
+    if (!row)
+        return null;
+    return {
+        id: row.id,
+        mode: row.mode,
+        createdAt: row.created_at,
+        contextHash: row.context_hash,
+        requestBody: JSON.parse(row.request_body),
+        responseBody: row.response_body ? JSON.parse(row.response_body) : undefined,
+        metrics: row.metrics ? JSON.parse(row.metrics) : undefined,
+    };
+}
+export function deleteRequest(id) {
+    const db = getDb();
+    db.prepare('DELETE FROM requests WHERE id = ?').run(id);
+}
+export function deleteRequests(ids) {
+    if (ids.length === 0)
+        return;
+    const db = getDb();
+    const placeholders = ids.map(() => '?').join(',');
+    db.prepare(`DELETE FROM requests WHERE id IN (${placeholders})`).run(...ids);
+}
+export function saveSetting(key, value) {
+    const db = getDb();
+    db.prepare('INSERT OR REPLACE INTO settings (key, value) VALUES (?, ?)').run(key, value);
+}
+export function getSetting(key) {
+    const db = getDb();
+    const row = db.prepare('SELECT value FROM settings WHERE key = ?').get(key);
+    return row ? row.value : null;
+}
+export function closeDb() {
+    if (db) {
+        db.close();
+        db = null;
+    }
+}
+const PROFILES_KEY = 'PROXY_PROFILES';
+const ACTIVE_PROFILE_KEY = 'ACTIVE_PROXY_ID';
+function generateId() {
+    return Date.now().toString(36) + Math.random().toString(36).slice(2, 8);
+}
+export function getProxyProfiles() {
+    const raw = getSetting(PROFILES_KEY);
+    if (raw) {
+        try {
+            const parsed = JSON.parse(raw);
+            if (Array.isArray(parsed))
+                return parsed;
+        }
+        catch {
+            // fall through to migration
+        }
+    }
+    // Migration: if legacy single settings exist, convert to first profile
+    const legacyUrl = getSetting('TARGET_URL') || process.env.TARGET_URL;
+    const legacyKey = getSetting('TARGET_API_KEY') || process.env.TARGET_API_KEY;
+    if (legacyUrl) {
+        const profile = {
+            id: generateId(),
+            name: 'Default',
+            targetUrl: legacyUrl,
+            targetApiKey: legacyKey || '',
+        };
+        saveSetting(PROFILES_KEY, JSON.stringify([profile]));
+        saveSetting(ACTIVE_PROFILE_KEY, profile.id);
+        return [profile];
+    }
+    return [];
+}
+export function saveProxyProfiles(profiles) {
+    saveSetting(PROFILES_KEY, JSON.stringify(profiles));
+}
+export function getActiveProxyProfile() {
+    const profiles = getProxyProfiles();
+    if (profiles.length === 0)
+        return null;
+    const activeId = getSetting(ACTIVE_PROFILE_KEY);
+    if (activeId) {
+        const found = profiles.find((p) => p.id === activeId);
+        if (found)
+            return found;
+    }
+    return profiles[0];
+}
+export function setActiveProxyProfile(id) {
+    saveSetting(ACTIVE_PROFILE_KEY, id);
+}
+export function createProxyProfile(name, targetUrl, targetApiKey, model, inputTokenPrice, outputTokenPrice, cachedInputTokenPrice) {
+    const profiles = getProxyProfiles();
+    const profile = { id: generateId(), name, targetUrl, targetApiKey, model, inputTokenPrice, outputTokenPrice, cachedInputTokenPrice };
+    profiles.push(profile);
+    saveProxyProfiles(profiles);
+    // If first profile, auto-activate it
+    if (profiles.length === 1) {
+        setActiveProxyProfile(profile.id);
+    }
+    return profile;
+}
+export function updateProxyProfile(id, updates) {
+    const profiles = getProxyProfiles();
+    const idx = profiles.findIndex((p) => p.id === id);
+    if (idx === -1)
+        return null;
+    profiles[idx] = { ...profiles[idx], ...updates };
+    saveProxyProfiles(profiles);
+    return profiles[idx];
+}
+export function deleteProxyProfile(id) {
+    const profiles = getProxyProfiles();
+    const filtered = profiles.filter((p) => p.id !== id);
+    saveProxyProfiles(filtered);
+    // If deleted the active one, reassign
+    const activeId = getSetting(ACTIVE_PROFILE_KEY);
+    if (activeId === id) {
+        if (filtered.length > 0) {
+            setActiveProxyProfile(filtered[0].id);
+        }
+        else {
+            saveSetting(ACTIVE_PROFILE_KEY, '');
+        }
+    }
+}

package/server/dist/server/services/stream.d.ts

@@ -0,0 +1,25 @@
+import { Metrics } from '../../shared/types.js';
+export interface ProcessedStream {
+    clientStream: ReadableStream<Uint8Array>;
+    metricsPromise: Promise<{
+        metrics: Metrics;
+        responseBody: any;
+    }>;
+}
+/**
+ * Extracts a normalized token breakdown from an upstream usage payload.
+ * Reads values directly from the response body's usage object (no inference).
+ * Compatible with OpenAI Chat Completions, OpenAI Responses API, and Anthropic-style usage.
+ */
+export declare function extractTokenBreakdown(usage: any): {
+    promptTokens?: number;
+    completionTokens?: number;
+    totalTokens?: number;
+    cachedTokens?: number;
+    reasoningTokens?: number;
+    outputTextTokens?: number;
+};
+/**
+ * Splits an LLM SSE stream: one untouched for the client, one for metrics/capture.
+ */
+export declare function processStream(response: Response, requestStartTime: number, requestBody?: any, onChunk?: (body: any) => void): ProcessedStream;

package/server/dist/server/services/stream.js

@@ -0,0 +1,250 @@
+import { EventSourceParserStream } from 'eventsource-parser/stream';
+/**
+ * Extracts a normalized token breakdown from an upstream usage payload.
+ * Reads values directly from the response body's usage object (no inference).
+ * Compatible with OpenAI Chat Completions, OpenAI Responses API, and Anthropic-style usage.
+ */
+export function extractTokenBreakdown(usage) {
+    if (!usage || typeof usage !== 'object')
+        return {};
+    const promptDetails = usage.prompt_tokens_details || usage.input_tokens_details || {};
+    const completionDetails = usage.completion_tokens_details || usage.output_tokens_details || {};
+    const promptTokens = usage.prompt_tokens ?? usage.input_tokens;
+    const completionTokens = usage.completion_tokens ?? usage.output_tokens;
+    const totalTokens = usage.total_tokens
+        ?? (promptTokens != null && completionTokens != null ? promptTokens + completionTokens : undefined);
+    const cachedTokens = promptDetails.cached_tokens ?? usage.cache_read_input_tokens;
+    const reasoningTokens = completionDetails.reasoning_tokens;
+    const outputTextTokens = completionDetails.text_tokens;
+    return {
+        promptTokens: promptTokens != null ? Number(promptTokens) : undefined,
+        completionTokens: completionTokens != null ? Number(completionTokens) : undefined,
+        totalTokens: totalTokens != null ? Number(totalTokens) : undefined,
+        cachedTokens: cachedTokens != null ? Number(cachedTokens) : undefined,
+        reasoningTokens: reasoningTokens != null ? Number(reasoningTokens) : undefined,
+        outputTextTokens: outputTextTokens != null ? Number(outputTextTokens) : undefined,
+    };
+}
+/**
+ * Splits an LLM SSE stream: one untouched for the client, one for metrics/capture.
+ */
+export function processStream(response, requestStartTime, requestBody, onChunk) {
+    // Split the stream: one for the client, one for our internal processing
+    const [clientStream, internalStream] = response.body.tee();
+    const metricsPromise = (async () => {
+        let firstTokenTime = null;
+        let lastTokenTime = requestStartTime;
+        const interTokenLatencies = [];
+        let tokenCount = 0;
+        let fullResponseBody = null;
+        const parserStream = internalStream
+            .pipeThrough(new TextDecoderStream())
+            .pipeThrough(new EventSourceParserStream());
+        let usageTokenCount = null;
+        let promptTokenCount = null;
+        // Node 24 allows async iteration over Web Streams natively
+        for await (const event of parserStream) {
+            if (event.data === '[DONE]')
+                continue;
+            try {
+                const data = JSON.parse(event.data);
+                const now = performance.now();
+                const eventType = event.event;
+                // Check for usage info in the chunk (OpenAI spec and others)
+                // Chat Completions usage is often in the last chunk or every chunk with stream_options
+                // Responses API usage is in the response.completed event
+                const usage = data.usage || data.statistics || data.stats;
+                if (usage) {
+                    if (usage.completion_tokens)
+                        usageTokenCount = usage.completion_tokens;
+                    if (usage.prompt_tokens)
+                        promptTokenCount = usage.prompt_tokens;
+                    if (usage.input_tokens)
+                        promptTokenCount = usage.input_tokens; // Anthropic/others
+                    if (usage.output_tokens)
+                        usageTokenCount = usage.output_tokens;
+                }
+                // Extract content based on API type
+                let content = '';
+                let reasoningContent = '';
+                let deltaToolCalls = [];
+                if (eventType === 'response.output_text.delta') {
+                    content = data.delta || '';
+                }
+                else if (eventType === 'response.reasoning_summary_text.delta') {
+                    // Some models send summary or reasoning in separate delta events
+                    reasoningContent = data.delta || '';
+                }
+                else {
+                    // Fallback to Chat Completions format
+                    const delta = data.choices?.[0]?.delta || data.delta || {};
+                    // Keep reasoning_content separate from content to preserve the distinction
+                    reasoningContent = delta.reasoning_content || '';
+                    content = delta.content || '';
+                    deltaToolCalls = Array.isArray(delta.tool_calls) ? delta.tool_calls : [];
+                }
+                if (content || reasoningContent || deltaToolCalls.length > 0) {
+                    tokenCount++;
+                    if (firstTokenTime === null) {
+                        firstTokenTime = now;
+                    }
+                    else {
+                        interTokenLatencies.push(now - lastTokenTime);
+                    }
+                    lastTokenTime = now;
+                }
+                // Reconstruct the full response body
+                if (!fullResponseBody) {
+                    fullResponseBody = { ...data };
+                    // Initialize for Chat Completions
+                    const choice = fullResponseBody.choices?.[0];
+                    if (choice?.delta) {
+                        choice.message = {
+                            role: choice.delta.role || 'assistant',
+                            content: choice.delta.content || ''
+                        };
+                        if (choice.delta.reasoning_content) {
+                            choice.message.reasoning_content = choice.delta.reasoning_content;
+                        }
+                        if (deltaToolCalls.length > 0) {
+                            choice.message.tool_calls = deltaToolCalls.map((tc) => ({
+                                index: tc.index ?? 0,
+                                id: tc.id || '',
+                                type: tc.type || 'function',
+                                function: {
+                                    name: tc.function?.name || '',
+                                    arguments: tc.function?.arguments || ''
+                                }
+                            }));
+                        }
+                        delete choice.delta;
+                    }
+                    // Initialize for Responses API
+                    if (eventType?.startsWith('response.')) {
+                        fullResponseBody.output = fullResponseBody.output || [];
+                        if (eventType === 'response.output_text.delta' && content) {
+                            fullResponseBody.output.push({
+                                type: 'message',
+                                role: 'assistant',
+                                content: content
+                            });
+                        }
+                        else if (eventType === 'response.reasoning_summary_text.delta' && reasoningContent) {
+                            fullResponseBody.output.push({
+                                type: 'reasoning',
+                                role: 'assistant',
+                                summary: [{ type: 'summary_text', text: reasoningContent }]
+                            });
+                        }
+                    }
+                }
+                else {
+                    // Update existing response body
+                    if (eventType === 'response.output_text.delta') {
+                        const outputItem = fullResponseBody.output?.find((i) => i.type === 'message');
+                        if (outputItem) {
+                            outputItem.content += content;
+                        }
+                        else {
+                            fullResponseBody.output = fullResponseBody.output || [];
+                            fullResponseBody.output.push({
+                                type: 'message',
+                                role: 'assistant',
+                                content: content
+                            });
+                        }
+                    }
+                    else if (eventType === 'response.reasoning_summary_text.delta') {
+                        const reasoningItem = fullResponseBody.output?.find((i) => i.type === 'reasoning');
+                        if (reasoningItem) {
+                            reasoningItem.summary[0].text += reasoningContent;
+                        }
+                        else {
+                            fullResponseBody.output = fullResponseBody.output || [];
+                            fullResponseBody.output.push({
+                                type: 'reasoning',
+                                role: 'assistant',
+                                summary: [{ type: 'summary_text', text: reasoningContent }]
+                            });
+                        }
+                    }
+                    else if (eventType === 'response.completed') {
+                        // Take the final response object as the ground truth
+                        fullResponseBody = { ...data };
+                    }
+                    else {
+                        if (content && fullResponseBody.choices?.[0]?.message) {
+                            fullResponseBody.choices[0].message.content += content;
+                        }
+                        if (reasoningContent && fullResponseBody.choices?.[0]?.message) {
+                            const msg = fullResponseBody.choices[0].message;
+                            msg.reasoning_content = (msg.reasoning_content || '') + reasoningContent;
+                        }
+                        if (deltaToolCalls.length > 0 && fullResponseBody.choices?.[0]?.message) {
+                            const msg = fullResponseBody.choices[0].message;
+                            msg.tool_calls = msg.tool_calls || [];
+                            for (const tc of deltaToolCalls) {
+                                const existing = msg.tool_calls.find((t) => t.index === (tc.index ?? 0));
+                                if (existing) {
+                                    if (tc.id)
+                                        existing.id = tc.id;
+                                    if (tc.type)
+                                        existing.type = tc.type;
+                                    if (tc.function?.name)
+                                        existing.function.name = tc.function.name;
+                                    if (tc.function?.arguments)
+                                        existing.function.arguments += tc.function.arguments;
+                                }
+                                else {
+                                    msg.tool_calls.push({
+                                        index: tc.index ?? 0,
+                                        id: tc.id || '',
+                                        type: tc.type || 'function',
+                                        function: {
+                                            name: tc.function?.name || '',
+                                            arguments: tc.function?.arguments || ''
+                                        }
+                                    });
+                                }
+                            }
+                        }
+                    }
+                    // If we got more usage info later
+                    if (usage) {
+                        fullResponseBody.usage = { ...fullResponseBody.usage, ...usage };
+                    }
+                }
+                if (onChunk && fullResponseBody) {
+                    onChunk(fullResponseBody);
+                }
+            }
+            catch (e) {
+                // Ignore malformed chunks
+            }
+        }
+        const now = performance.now();
+        const totalLatency = now - requestStartTime;
+        const ttft = firstTokenTime ? firstTokenTime - requestStartTime : totalLatency;
+        const generationTimeMs = totalLatency - ttft;
+        const finalTokenCount = usageTokenCount ?? tokenCount;
+        // Estimate prompt token count if not provided by response
+        if (promptTokenCount === null && requestBody?.messages) {
+            const promptText = JSON.stringify(requestBody.messages);
+            promptTokenCount = Math.ceil(promptText.length / 4); // Basic estimation
+        }
+        const prefillSpeed = (promptTokenCount && ttft > 0) ? (promptTokenCount / (ttft / 1000)) : 0;
+        return {
+            metrics: {
+                ttft,
+                totalLatency,
+                tokensPerSecond: generationTimeMs > 0 ? (finalTokenCount / (generationTimeMs / 1000)) : 0,
+                promptPrefillSpeed: prefillSpeed,
+                tokenCount: finalTokenCount,
+                interTokenLatencies,
+                completedAt: new Date().toISOString(),
+            },
+            responseBody: fullResponseBody,
+        };
+    })();
+    return { clientStream, metricsPromise };
+}

package/server/dist/shared/types.d.ts

@@ -0,0 +1,33 @@
+export interface Metrics {
+    ttft: number;
+    totalLatency: number;
+    tokensPerSecond: number;
+    promptPrefillSpeed: number;
+    tokenCount: number;
+    interTokenLatencies: number[];
+    completedAt: string;
+}
+export type RequestMode = 'chat' | 'observe' | 'benchmark';
+export interface StoredRequest {
+    id: number;
+    mode: RequestMode;
+    createdAt: string;
+    contextHash: string | null;
+    requestBody: any;
+    responseBody?: any;
+    metrics?: Metrics;
+}
+export interface Settings {
+    targetUrl: string;
+    targetApiKey: string;
+}
+export interface ProxyProfile {
+    id: string;
+    name: string;
+    targetUrl: string;
+    targetApiKey: string;
+    model?: string;
+    inputTokenPrice?: number;
+    outputTokenPrice?: number;
+    cachedInputTokenPrice?: number;
+}

package/server/dist/shared/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/server/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@ngvuhuy/promptglass-server",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.ts",
+  "type": "module",
+  "scripts": {
+    "dev": "tsx watch index.ts",
+    "build": "tsc",
+    "start": "node dist/index.js"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "packageManager": "pnpm@10.28.2",
+  "dependencies": {
+    "axios": "^1.13.6",
+    "cors": "^2.8.6",
+    "dotenv": "^17.3.1",
+    "eventsource-parser": "^3.0.6",
+    "express": "^5.2.1",
+    "zod": "^4.3.6"
+  },
+  "devDependencies": {
+    "@types/cors": "^2.8.19",
+    "@types/express": "^5.0.6",
+    "@types/node": "^25.5.0",
+    "tsx": "^4.21.0",
+    "typescript": "^5.9.3"
+  }
+}

package/shared/types.ts

@@ -0,0 +1,37 @@
+export interface Metrics {
+  ttft: number; // Time to first token (ms)
+  totalLatency: number; // Total request-response time (ms)
+  tokensPerSecond: number;
+  promptPrefillSpeed: number; // Tokens per second reading context window
+  tokenCount: number;
+  interTokenLatencies: number[]; // ms between tokens
+  completedAt: string;
+}
+
+export type RequestMode = 'chat' | 'observe' | 'benchmark';
+
+export interface StoredRequest {
+  id: number;
+  mode: RequestMode;
+  createdAt: string;
+  contextHash: string | null;
+  requestBody: any;
+  responseBody?: any;
+  metrics?: Metrics;
+}
+
+export interface Settings {
+  targetUrl: string;
+  targetApiKey: string;
+}
+
+export interface ProxyProfile {
+  id: string;
+  name: string;
+  targetUrl: string;
+  targetApiKey: string;
+  model?: string;
+  inputTokenPrice?: number;       // USD per 1M input tokens
+  outputTokenPrice?: number;      // USD per 1M output tokens
+  cachedInputTokenPrice?: number; // USD per 1M cached input tokens
+}