@rulemetric/cli 0.1.1

package/dist/lib/auth.js

@@ -0,0 +1,137 @@
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+function getConfigDir() {
+    return process.env.RULEMETRIC_CONFIG_DIR || path.join(os.homedir(), '.config', 'rulemetric');
+}
+export function createAuthStore(configDir = getConfigDir()) {
+    const authFile = path.join(configDir, 'auth.json');
+    function saveToken(token) {
+        fs.mkdirSync(configDir, { recursive: true, mode: 0o700 });
+        fs.writeFileSync(authFile, JSON.stringify(token, null, 2), {
+            encoding: 'utf-8',
+            mode: 0o600,
+        });
+    }
+    function loadToken() {
+        try {
+            const data = fs.readFileSync(authFile, 'utf-8');
+            const parsed = JSON.parse(data);
+            if (parsed &&
+                typeof parsed === 'object' &&
+                'accessToken' in parsed &&
+                'expiresAt' in parsed &&
+                'email' in parsed &&
+                typeof parsed.accessToken === 'string' &&
+                typeof parsed.expiresAt === 'string') {
+                return parsed;
+            }
+            return null;
+        }
+        catch {
+            return null;
+        }
+    }
+    function clearToken() {
+        try {
+            fs.unlinkSync(authFile);
+        }
+        catch {
+            // File doesn't exist, nothing to clear
+        }
+    }
+    function isAuthenticated() {
+        const token = loadToken();
+        if (!token)
+            return false;
+        const expiresAt = new Date(token.expiresAt).getTime();
+        if (Number.isNaN(expiresAt))
+            return false;
+        return Date.now() < expiresAt;
+    }
+    function saveEnvFile(accessToken, apiUrl) {
+        fs.mkdirSync(configDir, { recursive: true, mode: 0o700 });
+        const envPath = path.join(configDir, 'env');
+        // Preserve existing RULEMETRIC_API_KEY if present
+        let existingApiKey = '';
+        try {
+            const existing = fs.readFileSync(envPath, 'utf-8');
+            const match = existing.match(/^RULEMETRIC_API_KEY=(.+)$/m);
+            if (match)
+                existingApiKey = match[1];
+        }
+        catch { /* file doesn't exist */ }
+        const lines = [
+            `RULEMETRIC_ACCESS_TOKEN=${accessToken}`,
+        ];
+        if (existingApiKey) {
+            lines.push(`RULEMETRIC_API_KEY=${existingApiKey}`);
+        }
+        if (apiUrl) {
+            lines.push(`RULEMETRIC_API_URL=${apiUrl}`);
+        }
+        fs.writeFileSync(envPath, lines.join('\n') + '\n', {
+            encoding: 'utf-8',
+            mode: 0o600,
+        });
+    }
+    function saveApiKeyToEnv(apiKey, apiUrl) {
+        fs.mkdirSync(configDir, { recursive: true, mode: 0o700 });
+        const envPath = path.join(configDir, 'env');
+        // Preserve existing ACCESS_TOKEN if present
+        let existingToken = '';
+        let existingUrl = '';
+        try {
+            const existing = fs.readFileSync(envPath, 'utf-8');
+            const tokenMatch = existing.match(/^RULEMETRIC_ACCESS_TOKEN=(.+)$/m);
+            if (tokenMatch)
+                existingToken = tokenMatch[1];
+            const urlMatch = existing.match(/^RULEMETRIC_API_URL=(.+)$/m);
+            if (urlMatch)
+                existingUrl = urlMatch[1];
+        }
+        catch { /* file doesn't exist */ }
+        const lines = [
+            `RULEMETRIC_API_KEY=${apiKey}`,
+        ];
+        if (existingToken) {
+            lines.push(`RULEMETRIC_ACCESS_TOKEN=${existingToken}`);
+        }
+        const url = apiUrl ?? existingUrl;
+        if (url) {
+            lines.push(`RULEMETRIC_API_URL=${url}`);
+        }
+        fs.writeFileSync(envPath, lines.join('\n') + '\n', {
+            encoding: 'utf-8',
+            mode: 0o600,
+        });
+    }
+    function clearEnvFile() {
+        const envPath = path.join(configDir, 'env');
+        try {
+            fs.unlinkSync(envPath);
+        }
+        catch { /* noop */ }
+    }
+    return { saveToken, loadToken, clearToken, isAuthenticated, saveEnvFile, saveApiKeyToEnv, clearEnvFile, configDir, authFile };
+}
+// Lazy accessors — re-evaluate config dir from env on each call
+function getStore() {
+    return createAuthStore();
+}
+export function saveToken(token) { getStore().saveToken(token); }
+export function loadToken() { return getStore().loadToken(); }
+export function clearToken() { getStore().clearToken(); }
+export function isAuthenticated() {
+    // API key counts as authenticated
+    if (process.env.RULEMETRIC_API_KEY)
+        return true;
+    return getStore().isAuthenticated();
+}
+export function saveEnvFile(accessToken, apiUrl) { getStore().saveEnvFile(accessToken, apiUrl); }
+export function saveApiKey(apiKey, apiUrl) { getStore().saveApiKeyToEnv(apiKey, apiUrl); }
+export function clearEnvFile() { getStore().clearEnvFile(); }
+export function getDefaultConfigDir() { return getConfigDir(); }
+// Re-exported for backward compat (evaluated at import time; prefer getDefaultConfigDir() for dynamic usage)
+export const CONFIG_DIR = getConfigDir();
+//# sourceMappingURL=auth.js.map

package/dist/lib/auth.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.js","sourceRoot":"","sources":["../../src/lib/auth.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAS9B,SAAS,YAAY;IACnB,OAAO,OAAO,CAAC,GAAG,CAAC,qBAAqB,IAAI,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,YAAY,CAAC,CAAC;AAC/F,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,YAAoB,YAAY,EAAE;IAChE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,WAAW,CAAC,CAAC;IAEnD,SAAS,SAAS,CAAC,KAAgB;QACjC,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;YACzD,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,KAAK;SACZ,CAAC,CAAC;IACL,CAAC;IAED,SAAS,SAAS;QAChB,IAAI,CAAC;YACH,MAAM,IAAI,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;YAChD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;YACzC,IACE,MAAM;gBACN,OAAO,MAAM,KAAK,QAAQ;gBAC1B,aAAa,IAAI,MAAM;gBACvB,WAAW,IAAI,MAAM;gBACrB,OAAO,IAAI,MAAM;gBACjB,OAAQ,MAAoB,CAAC,WAAW,KAAK,QAAQ;gBACrD,OAAQ,MAAoB,CAAC,SAAS,KAAK,QAAQ,EACnD,CAAC;gBACD,OAAO,MAAmB,CAAC;YAC7B,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,SAAS,UAAU;QACjB,IAAI,CAAC;YACH,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QAC1B,CAAC;QAAC,MAAM,CAAC;YACP,uCAAuC;QACzC,CAAC;IACH,CAAC;IAED,SAAS,eAAe;QACtB,MAAM,KAAK,GAAG,SAAS,EAAE,CAAC;QAC1B,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QACzB,MAAM,SAAS,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAC;QACtD,IAAI,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC;YAAE,OAAO,KAAK,CAAC;QAC1C,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAChC,CAAC;IAED,SAAS,WAAW,CAAC,WAAmB,EAAE,MAAe;QACvD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAE5C,kDAAkD;QAClD,IAAI,cAAc,GAAG,EAAE,CAAC;QACxB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC3D,IAAI,KAAK;gBAAE,cAAc,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;QAEpC,MAAM,KAAK,GAAG;YACZ,2BAA2B,WAAW,EAAE;SACzC,CAAC;QACF,IAAI,cAAc,EAAE,CAAC;YACnB,KAAK,CAAC,IAAI,CAAC,sBAAsB,cAAc,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,IAAI,MAAM,EAAE,CAAC;YACX,KAAK,CAAC,IAAI,CAAC,sBAAsB,MAAM,EAAE,CAAC,CAAC;QAC7C,CAAC;QAED,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE;YACjD,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,KAAK;SACZ,CAAC,CAAC;IACL,CAAC;IAED,SAAS,eAAe,CAAC,MAAc,EAAE,MAAe;QACtD,EAAE,CAAC,SAAS,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAE5C,4CAA4C;QAC5C,IAAI,aAAa,GAAG,EAAE,CAAC;QACvB,IAAI,WAAW,GAAG,EAAE,CAAC;QACrB,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YACnD,MAAM,UAAU,GAAG,QAAQ,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACrE,IAAI,UAAU;gBAAE,aAAa,GAAG,UAAU,CAAC,CAAC,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;YAC9D,IAAI,QAAQ;gBAAE,WAAW,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;QAAC,MAAM,CAAC,CAAC,wBAAwB,CAAC,CAAC;QAEpC,MAAM,KAAK,GAAG;YACZ,sBAAsB,MAAM,EAAE;SAC/B,CAAC;QACF,IAAI,aAAa,EAAE,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,2BAA2B,aAAa,EAAE,CAAC,CAAC;QACzD,CAAC;QACD,MAAM,GAAG,GAAG,MAAM,IAAI,WAAW,CAAC;QAClC,IAAI,GAAG,EAAE,CAAC;YACR,KAAK,CAAC,IAAI,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAC;QAC1C,CAAC;QAED,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,EAAE;YACjD,QAAQ,EAAE,OAAO;YACjB,IAAI,EAAE,KAAK;SACZ,CAAC,CAAC;IACL,CAAC;IAED,SAAS,YAAY;QACnB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAC5C,IAAI,CAAC;YAAC,EAAE,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;QAAC,CAAC;QAAC,MAAM,CAAC,CAAC,UAAU,CAAC,CAAC;IACtD,CAAC;IAED,OAAO,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,eAAe,EAAE,WAAW,EAAE,eAAe,EAAE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,CAAC;AAChI,CAAC;AAED,gEAAgE;AAChE,SAAS,QAAQ;IACf,OAAO,eAAe,EAAE,CAAC;AAC3B,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,KAAgB,IAAU,QAAQ,EAAE,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;AAClF,MAAM,UAAU,SAAS,KAAuB,OAAO,QAAQ,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;AAChF,MAAM,UAAU,UAAU,KAAW,QAAQ,EAAE,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;AAC/D,MAAM,UAAU,eAAe;IAC7B,kCAAkC;IAClC,IAAI,OAAO,CAAC,GAAG,CAAC,kBAAkB;QAAE,OAAO,IAAI,CAAC;IAChD,OAAO,QAAQ,EAAE,CAAC,eAAe,EAAE,CAAC;AACtC,CAAC;AACD,MAAM,UAAU,WAAW,CAAC,WAAmB,EAAE,MAAe,IAAU,QAAQ,EAAE,CAAC,WAAW,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AACxH,MAAM,UAAU,UAAU,CAAC,MAAc,EAAE,MAAe,IAAU,QAAQ,EAAE,CAAC,eAAe,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC;AACjH,MAAM,UAAU,YAAY,KAAW,QAAQ,EAAE,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC;AACnE,MAAM,UAAU,mBAAmB,KAAa,OAAO,YAAY,EAAE,CAAC,CAAC,CAAC;AACxE,6GAA6G;AAC7G,MAAM,CAAC,MAAM,UAAU,GAAG,YAAY,EAAE,CAAC"}

package/dist/lib/auth.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=auth.test.d.ts.map

package/dist/lib/auth.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.test.d.ts","sourceRoot":"","sources":["../../src/lib/auth.test.ts"],"names":[],"mappings":""}

package/dist/lib/auth.test.js

@@ -0,0 +1,110 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import * as fs from 'node:fs';
+import * as path from 'node:path';
+import * as os from 'node:os';
+import { createAuthStore } from './auth.js';
+let tmpDir;
+let store;
+const validToken = {
+    accessToken: 'eyJhbGciOiJIUzI1NiJ9.test-access-token',
+    refreshToken: 'eyJhbGciOiJIUzI1NiJ9.test-refresh-token',
+    expiresAt: new Date(Date.now() + 3600_000).toISOString(),
+    email: 'test@example.com',
+};
+beforeEach(() => {
+    tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'rulemetric-test-'));
+    store = createAuthStore(tmpDir);
+});
+afterEach(() => {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+});
+describe('saveToken', () => {
+    it('creates the config directory and writes auth file', () => {
+        const nestedDir = path.join(tmpDir, 'nested', 'config');
+        const nestedStore = createAuthStore(nestedDir);
+        nestedStore.saveToken(validToken);
+        expect(fs.existsSync(nestedStore.authFile)).toBe(true);
+        const content = JSON.parse(fs.readFileSync(nestedStore.authFile, 'utf-8'));
+        expect(content.accessToken).toBe(validToken.accessToken);
+        expect(content.email).toBe(validToken.email);
+    });
+    it('writes file with restricted permissions', () => {
+        store.saveToken(validToken);
+        const stat = fs.statSync(store.authFile);
+        // 0o600 = owner read+write only (decimal 384)
+        expect(stat.mode & 0o777).toBe(0o600);
+    });
+    it('overwrites existing token', () => {
+        store.saveToken(validToken);
+        const newToken = { ...validToken, email: 'new@example.com' };
+        store.saveToken(newToken);
+        const loaded = store.loadToken();
+        expect(loaded?.email).toBe('new@example.com');
+    });
+});
+describe('loadToken', () => {
+    it('returns null when no auth file exists', () => {
+        expect(store.loadToken()).toBeNull();
+    });
+    it('returns the saved token', () => {
+        store.saveToken(validToken);
+        const loaded = store.loadToken();
+        expect(loaded).not.toBeNull();
+        expect(loaded.accessToken).toBe(validToken.accessToken);
+        expect(loaded.refreshToken).toBe(validToken.refreshToken);
+        expect(loaded.expiresAt).toBe(validToken.expiresAt);
+        expect(loaded.email).toBe(validToken.email);
+    });
+    it('returns null for corrupted JSON', () => {
+        fs.mkdirSync(tmpDir, { recursive: true });
+        fs.writeFileSync(store.authFile, 'not valid json', 'utf-8');
+        expect(store.loadToken()).toBeNull();
+    });
+    it('returns null for JSON missing required fields', () => {
+        fs.mkdirSync(tmpDir, { recursive: true });
+        fs.writeFileSync(store.authFile, JSON.stringify({ foo: 'bar' }), 'utf-8');
+        expect(store.loadToken()).toBeNull();
+    });
+    it('returns null for JSON with wrong field types', () => {
+        fs.mkdirSync(tmpDir, { recursive: true });
+        fs.writeFileSync(store.authFile, JSON.stringify({ accessToken: 123, expiresAt: true, email: 'a@b.com' }), 'utf-8');
+        expect(store.loadToken()).toBeNull();
+    });
+});
+describe('clearToken', () => {
+    it('removes the auth file', () => {
+        store.saveToken(validToken);
+        expect(fs.existsSync(store.authFile)).toBe(true);
+        store.clearToken();
+        expect(fs.existsSync(store.authFile)).toBe(false);
+    });
+    it('does not throw when no file exists', () => {
+        expect(() => store.clearToken()).not.toThrow();
+    });
+});
+describe('isAuthenticated', () => {
+    it('returns false when no token exists', () => {
+        expect(store.isAuthenticated()).toBe(false);
+    });
+    it('returns true when token is not expired', () => {
+        store.saveToken(validToken);
+        expect(store.isAuthenticated()).toBe(true);
+    });
+    it('returns false when token is expired', () => {
+        const expiredToken = {
+            ...validToken,
+            expiresAt: new Date(Date.now() - 3600_000).toISOString(),
+        };
+        store.saveToken(expiredToken);
+        expect(store.isAuthenticated()).toBe(false);
+    });
+    it('returns false when expiresAt is invalid date', () => {
+        const badToken = {
+            ...validToken,
+            expiresAt: 'not-a-date',
+        };
+        store.saveToken(badToken);
+        expect(store.isAuthenticated()).toBe(false);
+    });
+});
+//# sourceMappingURL=auth.test.js.map

package/dist/lib/auth.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"auth.test.js","sourceRoot":"","sources":["../../src/lib/auth.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACrE,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,KAAK,IAAI,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EAAE,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,eAAe,EAAkB,MAAM,WAAW,CAAC;AAE5D,IAAI,MAAc,CAAC;AACnB,IAAI,KAAyC,CAAC;AAE9C,MAAM,UAAU,GAAc;IAC5B,WAAW,EAAE,wCAAwC;IACrD,YAAY,EAAE,yCAAyC;IACvD,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,CAAC,WAAW,EAAE;IACxD,KAAK,EAAE,kBAAkB;CAC1B,CAAC;AAEF,UAAU,CAAC,GAAG,EAAE;IACd,MAAM,GAAG,EAAE,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,kBAAkB,CAAC,CAAC,CAAC;IACpE,KAAK,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC;AAClC,CAAC,CAAC,CAAC;AAEH,SAAS,CAAC,GAAG,EAAE;IACb,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AACtD,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,mDAAmD,EAAE,GAAG,EAAE;QAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACxD,MAAM,WAAW,GAAG,eAAe,CAAC,SAAS,CAAC,CAAC;QAC/C,WAAW,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAElC,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,YAAY,CAAC,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,CAAC;QAC3E,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yCAAyC,EAAE,GAAG,EAAE;QACjD,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC5B,MAAM,IAAI,GAAG,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACzC,8CAA8C;QAC9C,MAAM,CAAC,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2BAA2B,EAAE,GAAG,EAAE;QACnC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAc,EAAE,GAAG,UAAU,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC;QACxE,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAE1B,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QACjC,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,WAAW,EAAE,GAAG,EAAE;IACzB,EAAE,CAAC,uCAAuC,EAAE,GAAG,EAAE;QAC/C,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yBAAyB,EAAE,GAAG,EAAE;QACjC,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,EAAE,CAAC;QAEjC,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,MAAO,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QACzD,MAAM,CAAC,MAAO,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;QAC3D,MAAM,CAAC,MAAO,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC;QACrD,MAAM,CAAC,MAAO,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iCAAiC,EAAE,GAAG,EAAE;QACzC,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,gBAAgB,EAAE,OAAO,CAAC,CAAC;QAC5D,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,EAAE,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,EAAE,OAAO,CAAC,CAAC;QAC1E,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;IACvC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,EAAE,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1C,EAAE,CAAC,aAAa,CACd,KAAK,CAAC,QAAQ,EACd,IAAI,CAAC,SAAS,CAAC,EAAE,WAAW,EAAE,GAAG,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,EACvE,OAAO,CACR,CAAC;QACF,MAAM,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC;IACvC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,YAAY,EAAE,GAAG,EAAE;IAC1B,EAAE,CAAC,uBAAuB,EAAE,GAAG,EAAE;QAC/B,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC5B,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAEjD,KAAK,CAAC,UAAU,EAAE,CAAC;QACnB,MAAM,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACpD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC;IACjD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,iBAAiB,EAAE,GAAG,EAAE;IAC/B,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;QAC5B,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,YAAY,GAAc;YAC9B,GAAG,UAAU;YACb,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,QAAQ,CAAC,CAAC,WAAW,EAAE;SACzD,CAAC;QACF,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,QAAQ,GAAc;YAC1B,GAAG,UAAU;YACb,SAAS,EAAE,YAAY;SACxB,CAAC;QACF,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QAC1B,MAAM,CAAC,KAAK,CAAC,eAAe,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}

package/dist/lib/eval-analyzer.d.ts

@@ -0,0 +1,23 @@
+export interface AnalysisResult {
+    suggestions: Array<{
+        section: string;
+        current: string;
+        proposed: string;
+        reasoning: string;
+    }>;
+    overallAssessment: string;
+    proposedContent: string;
+}
+export declare function analyzeEvalResults(options: {
+    content: string;
+    evalResults: Array<{
+        evalName: string;
+        prompt: string;
+        expectations: string[];
+        grading: Record<string, unknown>;
+    }>;
+    model?: string;
+    timeout?: number;
+    userFeedback?: string;
+}): Promise<AnalysisResult>;
+//# sourceMappingURL=eval-analyzer.d.ts.map

package/dist/lib/eval-analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-analyzer.d.ts","sourceRoot":"","sources":["../../src/lib/eval-analyzer.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,cAAc;IAC7B,WAAW,EAAE,KAAK,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;QAChB,OAAO,EAAE,MAAM,CAAC;QAChB,QAAQ,EAAE,MAAM,CAAC;QACjB,SAAS,EAAE,MAAM,CAAC;KACnB,CAAC,CAAC;IACH,iBAAiB,EAAE,MAAM,CAAC;IAC1B,eAAe,EAAE,MAAM,CAAC;CACzB;AA+BD,wBAAsB,kBAAkB,CAAC,OAAO,EAAE;IAChD,OAAO,EAAE,MAAM,CAAC;IAChB,WAAW,EAAE,KAAK,CAAC;QACjB,QAAQ,EAAE,MAAM,CAAC;QACjB,MAAM,EAAE,MAAM,CAAC;QACf,YAAY,EAAE,MAAM,EAAE,CAAC;QACvB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KAClC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB,GAAG,OAAO,CAAC,cAAc,CAAC,CA+D1B"}

package/dist/lib/eval-analyzer.js

@@ -0,0 +1,89 @@
+import { executeEval } from './eval-executor.js';
+const ANALYZER_SYSTEM_PROMPT = `You are an expert at improving AI coding instructions, skills, and prompts. You receive eval results showing how well the current content performs, and your job is to propose specific improvements.
+
+You will receive:
+1. The current content of the eval target
+2. Eval results: for each eval, the prompt, expectations, pass/fail results, and grader feedback
+3. The pass rate and areas of weakness
+
+Analyze the failures and propose concrete edits to improve the content. Focus on:
+- Expectations that consistently fail — what's missing from the content?
+- Grader feedback suggesting new assertions — what outcomes aren't being tested?
+- Patterns across failures — is there a systemic gap?
+
+Your proposed content should be a complete replacement, not a diff. Keep what works, fix what doesn't.
+
+You MUST respond with ONLY a valid JSON object:
+
+{
+  "suggestions": [
+    {
+      "section": "which part of the content",
+      "current": "brief quote of current text",
+      "proposed": "what it should say instead",
+      "reasoning": "why this change helps"
+    }
+  ],
+  "overallAssessment": "brief summary of what's working and what's not",
+  "proposedContent": "the full improved content"
+}`;
+export async function analyzeEvalResults(options) {
+    const { content, evalResults, model, timeout = 120_000, userFeedback } = options;
+    const resultsText = evalResults.map((r) => {
+        const grading = r.grading;
+        const expectations = grading?.expectations ?? [];
+        const passRate = grading?.summary?.pass_rate ?? 'N/A';
+        const feedback = grading?.eval_feedback?.overall ?? '';
+        return `## Eval: ${r.evalName}
+Prompt: ${r.prompt}
+Pass rate: ${typeof passRate === 'number' ? (passRate * 100).toFixed(0) + '%' : passRate}
+
+Expectations:
+${expectations.map((e) => `  ${e.passed ? 'PASS' : 'FAIL'} — ${e.text}: ${e.evidence}`).join('\n')}
+
+Feedback: ${feedback}`;
+    }).join('\n\n---\n\n');
+    const feedbackSection = userFeedback
+        ? `\n\n## User Feedback\n\n${userFeedback}`
+        : '\n\n## User Feedback\n\nNo specific feedback provided.';
+    const analyzerPrompt = `## Current Content
+
+${content}
+
+## Eval Results
+
+${resultsText}${feedbackSection}
+
+Analyze the results and propose improvements. Respond with ONLY valid JSON.`;
+    const result = await executeEval({
+        prompt: analyzerPrompt,
+        targetContent: ANALYZER_SYSTEM_PROMPT,
+        timeout,
+        model,
+    });
+    if (result.error) {
+        return {
+            suggestions: [],
+            overallAssessment: `Analysis failed: ${result.error}`,
+            proposedContent: content,
+        };
+    }
+    try {
+        let jsonStr = result.output.trim();
+        const jsonMatch = jsonStr.match(/```(?:json)?\s*([\s\S]*?)```/);
+        if (jsonMatch)
+            jsonStr = jsonMatch[1].trim();
+        const objMatch = jsonStr.match(/\{[\s\S]*\}/);
+        if (objMatch)
+            jsonStr = objMatch[0];
+        return JSON.parse(jsonStr);
+    }
+    catch {
+        return {
+            suggestions: [],
+            overallAssessment: `Analysis returned unparseable output: ${result.output.slice(0, 200)}`,
+            proposedContent: content,
+        };
+    }
+}
+//# sourceMappingURL=eval-analyzer.js.map

package/dist/lib/eval-analyzer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-analyzer.js","sourceRoot":"","sources":["../../src/lib/eval-analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAajD,MAAM,sBAAsB,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;EA2B7B,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,OAWxC;IACC,MAAM,EAAE,OAAO,EAAE,WAAW,EAAE,KAAK,EAAE,OAAO,GAAG,OAAO,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC;IAEjF,MAAM,WAAW,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QACxC,MAAM,OAAO,GAAG,CAAC,CAAC,OAAc,CAAC;QACjC,MAAM,YAAY,GAAG,OAAO,EAAE,YAAY,IAAI,EAAE,CAAC;QACjD,MAAM,QAAQ,GAAG,OAAO,EAAE,OAAO,EAAE,SAAS,IAAI,KAAK,CAAC;QACtD,MAAM,QAAQ,GAAG,OAAO,EAAE,aAAa,EAAE,OAAO,IAAI,EAAE,CAAC;QAEvD,OAAO,YAAY,CAAC,CAAC,QAAQ;UACvB,CAAC,CAAC,MAAM;aACL,OAAO,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,QAAQ;;;EAGtF,YAAY,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC;;YAE3F,QAAQ,EAAE,CAAC;IACrB,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IAEvB,MAAM,eAAe,GAAG,YAAY;QAClC,CAAC,CAAC,2BAA2B,YAAY,EAAE;QAC3C,CAAC,CAAC,wDAAwD,CAAC;IAE7D,MAAM,cAAc,GAAG;;EAEvB,OAAO;;;;EAIP,WAAW,GAAG,eAAe;;4EAE6C,CAAC;IAE3E,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC;QAC/B,MAAM,EAAE,cAAc;QACtB,aAAa,EAAE,sBAAsB;QACrC,OAAO;QACP,KAAK;KACN,CAAC,CAAC;IAEH,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO;YACL,WAAW,EAAE,EAAE;YACf,iBAAiB,EAAE,oBAAoB,MAAM,CAAC,KAAK,EAAE;YACrD,eAAe,EAAE,OAAO;SACzB,CAAC;IACJ,CAAC;IAED,IAAI,CAAC;QACH,IAAI,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,CAAC,8BAA8B,CAAC,CAAC;QAChE,IAAI,SAAS;YAAE,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;QAC9C,IAAI,QAAQ;YAAE,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QAEpC,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAmB,CAAC;IAC/C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,WAAW,EAAE,EAAE;YACf,iBAAiB,EAAE,yCAAyC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE;YACzF,eAAe,EAAE,OAAO;SACzB,CAAC;IACJ,CAAC;AACH,CAAC"}

package/dist/lib/eval-benchmark.d.ts

@@ -0,0 +1,17 @@
+interface Stats {
+    mean: number;
+    stddev: number;
+    min: number;
+    max: number;
+}
+export interface BenchmarkResult {
+    runSummary: Record<string, Record<string, Stats>>;
+    notes: string[];
+    totalRuns: number;
+}
+export declare function aggregateBenchmark(targetId: string, iteration?: number): Promise<BenchmarkResult>;
+export declare function postBenchmark(targetId: string, iteration: number, result: BenchmarkResult): Promise<{
+    id: string;
+}>;
+export {};
+//# sourceMappingURL=eval-benchmark.d.ts.map

package/dist/lib/eval-benchmark.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-benchmark.d.ts","sourceRoot":"","sources":["../../src/lib/eval-benchmark.ts"],"names":[],"mappings":"AAiBA,UAAU,KAAK;IACb,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACb;AAcD,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC,CAAC;IAClD,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,MAAM,EAChB,SAAS,CAAC,EAAE,MAAM,GACjB,OAAO,CAAC,eAAe,CAAC,CA0E1B;AAED,wBAAsB,aAAa,CACjC,QAAQ,EAAE,MAAM,EAChB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,eAAe,GACtB,OAAO,CAAC;IAAE,EAAE,EAAE,MAAM,CAAA;CAAE,CAAC,CAMzB"}

package/dist/lib/eval-benchmark.js

@@ -0,0 +1,92 @@
+import { apiGet, apiPost } from './api-client.js';
+function computeStats(values) {
+    if (values.length === 0)
+        return { mean: 0, stddev: 0, min: 0, max: 0 };
+    const mean = values.reduce((a, b) => a + b, 0) / values.length;
+    const variance = values.reduce((sum, v) => sum + (v - mean) ** 2, 0) / values.length;
+    return {
+        mean: Math.round(mean * 1000) / 1000,
+        stddev: Math.round(Math.sqrt(variance) * 1000) / 1000,
+        min: Math.min(...values),
+        max: Math.max(...values),
+    };
+}
+export async function aggregateBenchmark(targetId, iteration) {
+    const query = iteration ? `?iteration=${iteration}` : '';
+    const runs = await apiGet(`/api/eval-targets/${targetId}/runs${query}`);
+    const completedRuns = runs.filter((r) => r.status === 'completed');
+    if (completedRuns.length === 0) {
+        return { runSummary: {}, notes: ['No completed runs found'], totalRuns: 0 };
+    }
+    // Group by configuration
+    const byConfig = {};
+    for (const run of completedRuns) {
+        if (!byConfig[run.configuration])
+            byConfig[run.configuration] = [];
+        byConfig[run.configuration].push(run);
+    }
+    const runSummary = {};
+    const notes = [];
+    for (const [config, configRuns] of Object.entries(byConfig)) {
+        const passRates = configRuns
+            .filter((r) => Object.keys(r.grading).length > 0)
+            .map((r) => r.grading.summary?.pass_rate ?? 0);
+        const durations = configRuns
+            .map((r) => r.timing.duration_ms / 1000)
+            .filter((v) => !isNaN(v));
+        const tokens = configRuns
+            .map((r) => r.timing.total_tokens ?? 0)
+            .filter((v) => v > 0);
+        runSummary[config] = {};
+        if (passRates.length > 0)
+            runSummary[config].pass_rate = computeStats(passRates);
+        if (durations.length > 0)
+            runSummary[config].time_seconds = computeStats(durations);
+        if (tokens.length > 0)
+            runSummary[config].tokens = computeStats(tokens);
+    }
+    // Compute delta if both configurations exist
+    if (runSummary.with_target && runSummary.without_target) {
+        const delta = {};
+        for (const metric of ['pass_rate', 'time_seconds', 'tokens']) {
+            const withVal = runSummary.with_target[metric]?.mean ?? 0;
+            const withoutVal = runSummary.without_target[metric]?.mean ?? 0;
+            const diff = withVal - withoutVal;
+            delta[metric] = `${diff >= 0 ? '+' : ''}${Math.round(diff * 1000) / 1000}`;
+        }
+        runSummary.delta = delta;
+        // Generate analysis notes
+        const passRateDelta = (runSummary.with_target.pass_rate?.mean ?? 0) - (runSummary.without_target.pass_rate?.mean ?? 0);
+        if (passRateDelta > 0.1) {
+            notes.push(`Target improves pass rate by ${(passRateDelta * 100).toFixed(0)}%`);
+        }
+        else if (passRateDelta < -0.1) {
+            notes.push(`Target decreases pass rate by ${(Math.abs(passRateDelta) * 100).toFixed(0)}%`);
+        }
+        else {
+            notes.push('Target has minimal impact on pass rate');
+        }
+        const timeDelta = (runSummary.with_target.time_seconds?.mean ?? 0) - (runSummary.without_target.time_seconds?.mean ?? 0);
+        if (Math.abs(timeDelta) > 5) {
+            notes.push(`Target ${timeDelta > 0 ? 'adds' : 'saves'} ${Math.abs(timeDelta).toFixed(1)}s average execution time`);
+        }
+    }
+    // Check for high variance
+    for (const [config, stats] of Object.entries(runSummary)) {
+        if (config === 'delta')
+            continue;
+        const passRate = stats.pass_rate;
+        if (passRate && passRate.stddev > 0.2) {
+            notes.push(`${config} shows high pass rate variance (stddev=${passRate.stddev.toFixed(2)}) — results may be unreliable`);
+        }
+    }
+    return { runSummary, notes, totalRuns: completedRuns.length };
+}
+export async function postBenchmark(targetId, iteration, result) {
+    return apiPost(`/api/eval-targets/${targetId}/benchmarks`, {
+        iteration,
+        runSummary: result.runSummary,
+        notes: result.notes,
+    });
+}
+//# sourceMappingURL=eval-benchmark.js.map

package/dist/lib/eval-benchmark.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-benchmark.js","sourceRoot":"","sources":["../../src/lib/eval-benchmark.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAC;AAwBlD,SAAS,YAAY,CAAC,MAAgB;IACpC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;IACvE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAC/D,MAAM,QAAQ,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IACrF,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI;QACpC,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI;QACrD,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;QACxB,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;KACzB,CAAC;AACJ,CAAC;AAQD,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACtC,QAAgB,EAChB,SAAkB;IAElB,MAAM,KAAK,GAAG,SAAS,CAAC,CAAC,CAAC,cAAc,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;IACzD,MAAM,IAAI,GAAG,MAAM,MAAM,CAAY,qBAAqB,QAAQ,QAAQ,KAAK,EAAE,CAAC,CAAC;IAEnF,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC;IAEnE,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,UAAU,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,yBAAyB,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC;IAC9E,CAAC;IAED,yBAAyB;IACzB,MAAM,QAAQ,GAA8B,EAAE,CAAC;IAC/C,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;QAChC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,EAAE,CAAC;QACnE,QAAQ,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,CAAC;IAED,MAAM,UAAU,GAA0C,EAAE,CAAC;IAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC5D,MAAM,SAAS,GAAG,UAAU;aACzB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;aAChD,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,OAAe,CAAC,OAAO,EAAE,SAAS,IAAI,CAAC,CAAC,CAAC;QAC1D,MAAM,SAAS,GAAG,UAAU;aACzB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,MAAc,CAAC,WAAW,GAAG,IAAI,CAAC;aAChD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,UAAU;aACtB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,MAAc,CAAC,YAAY,IAAI,CAAC,CAAC;aAC/C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAExB,UAAU,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACxB,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,UAAU,CAAC,MAAM,CAAC,CAAC,SAAS,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QACjF,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YAAE,UAAU,CAAC,MAAM,CAAC,CAAC,YAAY,GAAG,YAAY,CAAC,SAAS,CAAC,CAAC;QACpF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,UAAU,CAAC,MAAM,CAAC,CAAC,MAAM,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAC1E,CAAC;IAED,6CAA6C;IAC7C,IAAI,UAAU,CAAC,WAAW,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;QACxD,MAAM,KAAK,GAA2B,EAAE,CAAC;QACzC,KAAK,MAAM,MAAM,IAAI,CAAC,WAAW,EAAE,cAAc,EAAE,QAAQ,CAAC,EAAE,CAAC;YAC7D,MAAM,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;YAC1D,MAAM,UAAU,GAAG,UAAU,CAAC,cAAc,CAAC,MAAM,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;YAChE,MAAM,IAAI,GAAG,OAAO,GAAG,UAAU,CAAC;YAClC,KAAK,CAAC,MAAM,CAAC,GAAG,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC;QAC7E,CAAC;QACA,UAAkB,CAAC,KAAK,GAAG,KAAK,CAAC;QAElC,0BAA0B;QAC1B,MAAM,aAAa,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,SAAS,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;QACvH,IAAI,aAAa,GAAG,GAAG,EAAE,CAAC;YACxB,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC,aAAa,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAClF,CAAC;aAAM,IAAI,aAAa,GAAG,CAAC,GAAG,EAAE,CAAC;YAChC,KAAK,CAAC,IAAI,CAAC,iCAAiC,CAAC,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAC7F,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,wCAAwC,CAAC,CAAC;QACvD,CAAC;QAED,MAAM,SAAS,GAAG,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,UAAU,CAAC,cAAc,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC,CAAC,CAAC;QACzH,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;YAC5B,KAAK,CAAC,IAAI,CAAC,UAAU,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,IAAI,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,0BAA0B,CAAC,CAAC;QACrH,CAAC;IACH,CAAC;IAED,0BAA0B;IAC1B,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,CAAC;QACzD,IAAI,MAAM,KAAK,OAAO;YAAE,SAAS;QACjC,MAAM,QAAQ,GAAG,KAAK,CAAC,SAAS,CAAC;QACjC,IAAI,QAAQ,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,0CAA0C,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,+BAA+B,CAAC,CAAC;QAC3H,CAAC;IACH,CAAC;IAED,OAAO,EAAE,UAAU,EAAE,KAAK,EAAE,SAAS,EAAE,aAAa,CAAC,MAAM,EAAE,CAAC;AAChE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,QAAgB,EAChB,SAAiB,EACjB,MAAuB;IAEvB,OAAO,OAAO,CAAiB,qBAAqB,QAAQ,aAAa,EAAE;QACzE,SAAS;QACT,UAAU,EAAE,MAAM,CAAC,UAAU;QAC7B,KAAK,EAAE,MAAM,CAAC,KAAK;KACpB,CAAC,CAAC;AACL,CAAC"}

package/dist/lib/eval-conversation-handler.d.ts

@@ -0,0 +1,9 @@
+interface ConversationJobConfig {
+    type: 'conversation';
+    conversationId: string;
+    messageId: string;
+    messageContent: string;
+}
+export declare function processConversationJob(jobConfig: ConversationJobConfig, evalTargetId: string, logger: (msg: string) => void): Promise<void>;
+export {};
+//# sourceMappingURL=eval-conversation-handler.d.ts.map

package/dist/lib/eval-conversation-handler.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-conversation-handler.d.ts","sourceRoot":"","sources":["../../src/lib/eval-conversation-handler.ts"],"names":[],"mappings":"AA+BA,UAAU,qBAAqB;IAC7B,IAAI,EAAE,cAAc,CAAC;IACrB,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;CACxB;AAQD,wBAAsB,sBAAsB,CAC1C,SAAS,EAAE,qBAAqB,EAChC,YAAY,EAAE,MAAM,EACpB,MAAM,EAAE,CAAC,GAAG,EAAE,MAAM,KAAK,IAAI,GAC5B,OAAO,CAAC,IAAI,CAAC,CAsDf"}