autoval 0.1.0

package/README.md

@@ -0,0 +1,83 @@
+# autoval
+
+> Two lines. Any codebase. Your LLM calls become observable, evaluable, and PR-fixable.
+
+```bash
+npm install autoval
+```
+
+```ts
+import OpenAI from 'openai';
+import { autoval } from 'autoval';
+
+const client = new OpenAI();
+autoval.instrument(client);  // ← that's it
+
+const res = await client.chat.completions.create({
+  model: 'gpt-4o',
+  messages: [{ role: 'user', content: prompt }],
+});
+```
+
+Every call now writes a row to `autoval.llm_call_logs` in your ClickHouse. The Autoval agent reads from that table, scans for unsafe outputs, web-grounds judgments via Nimble, generates eval test cases, and opens PRs against your prompt file.
+
+## Supported SDKs
+
+| Provider | Detection | Method wrapped |
+|----------|-----------|----------------|
+| OpenAI (`openai`) | `client.chat.completions.create` | `chat.completions.create` |
+| Anthropic (`@anthropic-ai/sdk`) | `client.messages.create` | `messages.create` |
+| Google (`@google/generative-ai`) | `client.getGenerativeModel(...)` | `model.generateContent` |
+
+Unknown clients pass through with a console warning — no exceptions.
+
+## Config
+
+By default the package reads ClickHouse credentials from env:
+
+```
+CLICKHOUSE_URL=https://xxxx.clickhouse.cloud:8443
+CLICKHOUSE_USER=default
+CLICKHOUSE_PASSWORD=...
+CLICKHOUSE_DATABASE=autoval
+```
+
+Or call `autoval.configure({ ... })` once at startup:
+
+```ts
+autoval.configure({
+  clickhouseUrl: process.env.CH_URL,
+  clickhouseUser: 'default',
+  clickhousePassword: process.env.CH_PASSWORD,
+  clickhouseDatabase: 'autoval',
+});
+```
+
+## Schema
+
+Each LLM call writes a row with:
+
+```sql
+CREATE TABLE autoval.llm_call_logs (
+  id String,
+  input String,
+  output String,
+  model String,
+  latency_ms UInt32,
+  scored UInt8 DEFAULT 0,
+  timestamp DateTime64(3) DEFAULT now()
+) ENGINE = MergeTree()
+ORDER BY timestamp;
+```
+
+`scored = 0` rows are what the Autoval scanner picks up. After the agent reviews a row it sets `scored = 1`.
+
+## Notes
+
+- Insert is `await`-ed inside the wrapper. On serverless (Vercel, AWS Lambda) unawaited promises get cancelled, so we await to guarantee the row lands. Adds ~150ms per call.
+- Insert errors are caught and logged — never thrown — so a ClickHouse hiccup can't break your user-facing chat.
+- The wrapper mutates the client in-place. If you need an un-instrumented client, instantiate a fresh one.
+
+## License
+
+MIT

package/dist/anthropic.d.ts

@@ -0,0 +1,4 @@
+/**
+ * Wraps `client.messages.create` so every call logs input+output to ClickHouse.
+ */
+export declare function instrumentAnthropic<T>(client: T): T;

package/dist/anthropic.js

@@ -0,0 +1,59 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.instrumentAnthropic = instrumentAnthropic;
+const clickhouse_1 = require("./clickhouse");
+function isAnthropicClient(client) {
+    if (typeof client !== 'object' || client === null)
+        return false;
+    const messages = client.messages;
+    if (typeof messages !== 'object' || messages === null)
+        return false;
+    return typeof messages.create === 'function';
+}
+function summarizeInput(params) {
+    const parts = [];
+    if (params.system)
+        parts.push(`system: ${params.system}`);
+    if (Array.isArray(params.messages)) {
+        for (const m of params.messages) {
+            const c = typeof m.content === 'string' ? m.content : JSON.stringify(m.content);
+            parts.push(`${m.role}: ${c}`);
+        }
+    }
+    return parts.join('\n\n');
+}
+function extractOutput(res) {
+    return (res.content ?? [])
+        .filter((b) => b.type === 'text' && typeof b.text === 'string')
+        .map((b) => b.text ?? '')
+        .join('');
+}
+/**
+ * Wraps `client.messages.create` so every call logs input+output to ClickHouse.
+ */
+function instrumentAnthropic(client) {
+    if (!isAnthropicClient(client))
+        return client;
+    const messages = client.messages;
+    const original = messages.create.bind(messages);
+    messages.create = async (params, ...rest) => {
+        const start = Date.now();
+        const p = (params ?? {});
+        const result = await original(params, ...rest);
+        const latencyMs = Date.now() - start;
+        try {
+            const res = result;
+            await (0, clickhouse_1.logLlmCall)({
+                input: summarizeInput(p),
+                output: extractOutput(res),
+                model: p.model ?? res.model ?? 'unknown',
+                latency_ms: latencyMs,
+            });
+        }
+        catch (err) {
+            console.error('[autoval] Anthropic log wrapper failed:', err);
+        }
+        return result;
+    };
+    return client;
+}

package/dist/clickhouse.d.ts

@@ -0,0 +1,25 @@
+export interface AutovalConfig {
+    clickhouseUrl?: string;
+    clickhouseUser?: string;
+    clickhousePassword?: string;
+    clickhouseDatabase?: string;
+    /** Override the table name. Defaults to `autoval.llm_call_logs`. */
+    table?: string;
+    /** Disable logging entirely (useful in tests). */
+    disabled?: boolean;
+}
+export declare function configure(config: AutovalConfig): void;
+export declare function newCallId(prefix?: string): string;
+export interface LlmCallLog {
+    id?: string;
+    input: string;
+    output: string;
+    model: string;
+    latency_ms: number;
+}
+/**
+ * Insert a single LLM-call row into ClickHouse. Awaitable — callers
+ * decide whether to await (recommended on serverless) or fire-and-forget.
+ * Errors are caught and logged; never throws.
+ */
+export declare function logLlmCall(entry: LlmCallLog): Promise<string>;

package/dist/clickhouse.js

@@ -0,0 +1,67 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.configure = configure;
+exports.newCallId = newCallId;
+exports.logLlmCall = logLlmCall;
+const client_1 = require("@clickhouse/client");
+const crypto_1 = require("crypto");
+let _client = null;
+let _disabled = false;
+let _config = {
+    table: 'autoval.llm_call_logs',
+};
+function configure(config) {
+    _config = { ..._config, ...config };
+    _disabled = config.disabled ?? false;
+    _client = null; // force re-init on next call
+}
+function getClient() {
+    if (_disabled)
+        return null;
+    if (_client)
+        return _client;
+    const url = _config.clickhouseUrl ?? process.env.CLICKHOUSE_URL;
+    const username = _config.clickhouseUser ?? process.env.CLICKHOUSE_USER;
+    const password = _config.clickhousePassword ?? process.env.CLICKHOUSE_PASSWORD;
+    const database = _config.clickhouseDatabase ?? process.env.CLICKHOUSE_DATABASE;
+    if (!url || !username || !password || !database) {
+        console.warn('[autoval] ClickHouse env not configured — logging disabled. Set CLICKHOUSE_URL/USER/PASSWORD/DATABASE or call autoval.configure({...}).');
+        _disabled = true;
+        return null;
+    }
+    _client = (0, client_1.createClient)({ url, username, password, database });
+    return _client;
+}
+function newCallId(prefix = 'ev') {
+    return `${prefix}_${(0, crypto_1.randomUUID)().replace(/-/g, '').slice(0, 12)}`;
+}
+/**
+ * Insert a single LLM-call row into ClickHouse. Awaitable — callers
+ * decide whether to await (recommended on serverless) or fire-and-forget.
+ * Errors are caught and logged; never throws.
+ */
+async function logLlmCall(entry) {
+    const id = entry.id ?? newCallId('ev');
+    const client = getClient();
+    if (!client)
+        return id;
+    try {
+        await client.insert({
+            table: _config.table,
+            values: [
+                {
+                    id,
+                    input: entry.input,
+                    output: entry.output,
+                    model: entry.model,
+                    latency_ms: entry.latency_ms,
+                },
+            ],
+            format: 'JSONEachRow',
+        });
+    }
+    catch (err) {
+        console.error('[autoval] log insert failed:', err);
+    }
+    return id;
+}

package/dist/google.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Wraps `client.getGenerativeModel(...)` so every model returned has its
+ * `generateContent` method wrapped to log to ClickHouse.
+ */
+export declare function instrumentGoogle<T>(client: T): T;

package/dist/google.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.instrumentGoogle = instrumentGoogle;
+const clickhouse_1 = require("./clickhouse");
+function isGoogleClient(client) {
+    if (typeof client !== 'object' || client === null)
+        return false;
+    return typeof client.getGenerativeModel === 'function';
+}
+function summarizeInput(req) {
+    if (typeof req === 'string')
+        return req;
+    const r = (req ?? {});
+    if (typeof r.contents === 'string')
+        return r.contents;
+    if (Array.isArray(r.contents)) {
+        return r.contents
+            .map((c) => {
+            if (typeof c === 'string')
+                return c;
+            if (c && typeof c === 'object' && 'parts' in c) {
+                const parts = c.parts;
+                return parts.map((p) => (p && typeof p === 'object' && 'text' in p ? p.text : '')).join('');
+            }
+            return '';
+        })
+            .join('\n');
+    }
+    try {
+        return JSON.stringify(req).slice(0, 4000);
+    }
+    catch {
+        return '';
+    }
+}
+function extractOutput(res) {
+    const out = res.response;
+    if (!out)
+        return '';
+    if (typeof out.text === 'function') {
+        try {
+            return out.text();
+        }
+        catch {
+            // fall through
+        }
+    }
+    const parts = out.candidates?.[0]?.content?.parts ?? [];
+    return parts.map((p) => p.text ?? '').join('');
+}
+/**
+ * Wraps `client.getGenerativeModel(...)` so every model returned has its
+ * `generateContent` method wrapped to log to ClickHouse.
+ */
+function instrumentGoogle(client) {
+    if (!isGoogleClient(client))
+        return client;
+    const originalGetModel = client.getGenerativeModel.bind(client);
+    client.getGenerativeModel = (params) => {
+        const model = originalGetModel(params);
+        const originalGen = model.generateContent.bind(model);
+        model.generateContent = async (req, ...rest) => {
+            const start = Date.now();
+            const result = await originalGen(req, ...rest);
+            const latencyMs = Date.now() - start;
+            try {
+                await (0, clickhouse_1.logLlmCall)({
+                    input: summarizeInput(req),
+                    output: extractOutput(result),
+                    model: params.model,
+                    latency_ms: latencyMs,
+                });
+            }
+            catch (err) {
+                console.error('[autoval] Google log wrapper failed:', err);
+            }
+            return result;
+        };
+        return model;
+    };
+    return client;
+}

package/dist/index.d.ts

@@ -0,0 +1,38 @@
+import { configure, logLlmCall, newCallId } from './clickhouse';
+export { configure, logLlmCall, newCallId };
+export type { AutovalConfig, LlmCallLog } from './clickhouse';
+/**
+ * Instrument an LLM client so every call logs to ClickHouse for Autoval to
+ * scan, eval, and PR-fix.
+ *
+ *   import OpenAI from 'openai';
+ *   import { autoval } from 'autoval';
+ *
+ *   const client = new OpenAI();
+ *   autoval.instrument(client);  // ← that's it
+ *
+ *   // ... your existing calls log automatically:
+ *   await client.chat.completions.create({ model: 'gpt-4o', messages: [...] });
+ *
+ * Auto-detects OpenAI, Anthropic (@anthropic-ai/sdk), and Google
+ * (@google/generative-ai). Mutates the passed client in-place and returns
+ * it for chaining. Pass-through (no-op) for unknown clients with a warning.
+ *
+ * Configure once at startup if you don't use env vars:
+ *
+ *   autoval.configure({
+ *     clickhouseUrl: '...',
+ *     clickhouseUser: 'default',
+ *     clickhousePassword: '...',
+ *     clickhouseDatabase: 'autoval',
+ *   })
+ */
+declare function instrument<T>(client: T): T;
+/** Public surface. Use as `autoval.instrument(client)` or named import. */
+export declare const autoval: {
+    instrument: typeof instrument;
+    configure: typeof configure;
+    newCallId: typeof newCallId;
+};
+export { instrument };
+export default autoval;

package/dist/index.js

@@ -0,0 +1,69 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.autoval = exports.newCallId = exports.logLlmCall = exports.configure = void 0;
+exports.instrument = instrument;
+const openai_1 = require("./openai");
+const google_1 = require("./google");
+const anthropic_1 = require("./anthropic");
+const clickhouse_1 = require("./clickhouse");
+Object.defineProperty(exports, "configure", { enumerable: true, get: function () { return clickhouse_1.configure; } });
+Object.defineProperty(exports, "logLlmCall", { enumerable: true, get: function () { return clickhouse_1.logLlmCall; } });
+Object.defineProperty(exports, "newCallId", { enumerable: true, get: function () { return clickhouse_1.newCallId; } });
+/**
+ * Instrument an LLM client so every call logs to ClickHouse for Autoval to
+ * scan, eval, and PR-fix.
+ *
+ *   import OpenAI from 'openai';
+ *   import { autoval } from 'autoval';
+ *
+ *   const client = new OpenAI();
+ *   autoval.instrument(client);  // ← that's it
+ *
+ *   // ... your existing calls log automatically:
+ *   await client.chat.completions.create({ model: 'gpt-4o', messages: [...] });
+ *
+ * Auto-detects OpenAI, Anthropic (@anthropic-ai/sdk), and Google
+ * (@google/generative-ai). Mutates the passed client in-place and returns
+ * it for chaining. Pass-through (no-op) for unknown clients with a warning.
+ *
+ * Configure once at startup if you don't use env vars:
+ *
+ *   autoval.configure({
+ *     clickhouseUrl: '...',
+ *     clickhouseUser: 'default',
+ *     clickhousePassword: '...',
+ *     clickhouseDatabase: 'autoval',
+ *   })
+ */
+function instrument(client) {
+    if (!client || typeof client !== 'object')
+        return client;
+    const c = client;
+    // Sniff SDK shape — order matters: OpenAI's `messages` lives on a different
+    // path than Anthropic's, so we check signatures, not just key presence.
+    const hasOpenAIShape = 'chat' in c &&
+        typeof c.chat === 'object' &&
+        c.chat !== null &&
+        'completions' in c.chat;
+    if (hasOpenAIShape)
+        return (0, openai_1.instrumentOpenAI)(client);
+    const hasGoogleShape = 'getGenerativeModel' in c && typeof c.getGenerativeModel === 'function';
+    if (hasGoogleShape)
+        return (0, google_1.instrumentGoogle)(client);
+    const hasAnthropicShape = 'messages' in c &&
+        typeof c.messages === 'object' &&
+        c.messages !== null &&
+        'create' in c.messages;
+    if (hasAnthropicShape)
+        return (0, anthropic_1.instrumentAnthropic)(client);
+    console.warn('[autoval] Unrecognized client shape — pass-through (not instrumented). ' +
+        'Supported: OpenAI, Anthropic, Google GenerativeAI.');
+    return client;
+}
+/** Public surface. Use as `autoval.instrument(client)` or named import. */
+exports.autoval = {
+    instrument,
+    configure: clickhouse_1.configure,
+    newCallId: clickhouse_1.newCallId,
+};
+exports.default = exports.autoval;

package/dist/openai.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Wraps `client.chat.completions.create` so every call logs input+output
+ * to ClickHouse. Mutates the client in-place and returns it for chaining.
+ */
+export declare function instrumentOpenAI<T>(client: T): T;

package/dist/openai.js

@@ -0,0 +1,55 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.instrumentOpenAI = instrumentOpenAI;
+const clickhouse_1 = require("./clickhouse");
+function isOpenAIClient(client) {
+    if (typeof client !== 'object' || client === null)
+        return false;
+    const chat = client.chat;
+    if (typeof chat !== 'object' || chat === null)
+        return false;
+    const completions = chat.completions;
+    if (typeof completions !== 'object' || completions === null)
+        return false;
+    return typeof completions.create === 'function';
+}
+function summarizeInput(params) {
+    if (!params.messages || !Array.isArray(params.messages))
+        return '';
+    return params.messages
+        .map((m) => `${m.role}: ${typeof m.content === 'string' ? m.content : JSON.stringify(m.content)}`)
+        .join('\n\n');
+}
+function extractOutput(res) {
+    return res.choices?.[0]?.message?.content ?? '';
+}
+/**
+ * Wraps `client.chat.completions.create` so every call logs input+output
+ * to ClickHouse. Mutates the client in-place and returns it for chaining.
+ */
+function instrumentOpenAI(client) {
+    if (!isOpenAIClient(client))
+        return client;
+    const completions = client.chat.completions;
+    const original = completions.create.bind(completions);
+    completions.create = async (params, ...rest) => {
+        const start = Date.now();
+        const p = (params ?? {});
+        const result = await original(params, ...rest);
+        const latencyMs = Date.now() - start;
+        try {
+            const res = result;
+            await (0, clickhouse_1.logLlmCall)({
+                input: summarizeInput(p),
+                output: extractOutput(res),
+                model: p.model ?? res.model ?? 'unknown',
+                latency_ms: latencyMs,
+            });
+        }
+        catch (err) {
+            console.error('[autoval] OpenAI log wrapper failed:', err);
+        }
+        return result;
+    };
+    return client;
+}

package/package.json

@@ -0,0 +1,43 @@
+{
+  "name": "autoval",
+  "version": "0.1.0",
+  "description": "Two lines, any codebase. Instrument your LLM client so Autoval can scan, eval, and PR-fix your prompts.",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "scripts": {
+    "build": "tsc",
+    "prepublishOnly": "npm run build"
+  },
+  "keywords": [
+    "llm",
+    "eval",
+    "observability",
+    "openai",
+    "anthropic",
+    "gemini",
+    "clickhouse",
+    "autoval"
+  ],
+  "author": "Autoval team",
+  "license": "MIT",
+  "dependencies": {
+    "@clickhouse/client": "^1.18.0"
+  },
+  "peerDependencies": {
+    "@anthropic-ai/sdk": ">=0.20.0",
+    "@google/generative-ai": ">=0.20.0",
+    "openai": ">=4.0.0"
+  },
+  "peerDependenciesMeta": {
+    "@anthropic-ai/sdk": { "optional": true },
+    "@google/generative-ai": { "optional": true },
+    "openai": { "optional": true }
+  },
+  "devDependencies": {
+    "typescript": "^5.0.0"
+  }
+}