impact-testing-sdk 0.1.0

package/dist/ImpactClient.d.ts

@@ -0,0 +1,20 @@
+import type { ImpactConfig } from './types.js';
+type OpenAILike = {
+    chat: {
+        completions: {
+            create(...args: unknown[]): Promise<unknown>;
+        };
+    };
+};
+type AnthropicLike = {
+    messages: {
+        create(...args: unknown[]): Promise<unknown>;
+    };
+};
+export declare class ImpactClient {
+    private readonly config;
+    constructor(config: ImpactConfig);
+    wrapOpenAI<T extends OpenAILike>(client: T): T;
+    wrapAnthropic<T extends AnthropicLike>(client: T): T;
+}
+export {};

package/dist/ImpactClient.js

@@ -0,0 +1,19 @@
+import { wrapOpenAI } from './interceptors/openai.js';
+import { wrapAnthropic } from './interceptors/anthropic.js';
+export class ImpactClient {
+    config;
+    constructor(config) {
+        if (!config.apiKey)
+            throw new Error('[ImpactClient] apiKey is required');
+        if (!config.shadowModel)
+            throw new Error('[ImpactClient] shadowModel is required');
+        this.config = config;
+    }
+    wrapOpenAI(client) {
+        return wrapOpenAI(client, this.config);
+    }
+    wrapAnthropic(client) {
+        return wrapAnthropic(client, this.config);
+    }
+}
+//# sourceMappingURL=ImpactClient.js.map

package/dist/ImpactClient.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ImpactClient.js","sourceRoot":"","sources":["../src/ImpactClient.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAA;AACrD,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAA;AAU3D,MAAM,OAAO,YAAY;IACN,MAAM,CAAc;IAErC,YAAY,MAAoB;QAC9B,IAAI,CAAC,MAAM,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,mCAAmC,CAAC,CAAA;QACxE,IAAI,CAAC,MAAM,CAAC,WAAW;YAAE,MAAM,IAAI,KAAK,CAAC,wCAAwC,CAAC,CAAA;QAClF,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;IACtB,CAAC;IAED,UAAU,CAAuB,MAAS;QACxC,OAAO,UAAU,CAAC,MAAe,EAAE,IAAI,CAAC,MAAM,CAAM,CAAA;IACtD,CAAC;IAED,aAAa,CAA0B,MAAS;QAC9C,OAAO,aAAa,CAAC,MAAe,EAAE,IAAI,CAAC,MAAM,CAAM,CAAA;IACzD,CAAC;CACF"}

package/dist/client.d.ts

@@ -0,0 +1,3 @@
+import type { ImpactConfig, RunPayload } from './types.js';
+export declare function estimateCost(model: string, totalTokens: number): number;
+export declare function sendRun(payload: RunPayload, config: ImpactConfig): void;

package/dist/client.js

@@ -0,0 +1,31 @@
+// Cost per 1K tokens (input + output combined) — approximate, update as needed
+const COST_PER_1K = {
+    'gpt-4o': 0.0075,
+    'gpt-4o-mini': 0.000225,
+    'gpt-4-turbo': 0.015,
+    'gpt-3.5-turbo': 0.001,
+    'claude-3-5-sonnet': 0.009,
+    'claude-3-haiku': 0.000375,
+    'claude-3-opus': 0.045,
+};
+export function estimateCost(model, totalTokens) {
+    const base = Object.entries(COST_PER_1K).find(([key]) => model.toLowerCase().includes(key));
+    if (!base)
+        return 0;
+    return (totalTokens / 1000) * base[1];
+}
+// Fire-and-forget — never throws, never blocks the caller
+export function sendRun(payload, config) {
+    const url = `${config.serverUrl ?? 'http://localhost:4000'}/runs`;
+    fetch(url, {
+        method: 'POST',
+        headers: {
+            'Content-Type': 'application/json',
+            'x-api-key': config.apiKey,
+        },
+        body: JSON.stringify(payload),
+    }).catch((err) => {
+        console.warn('[impact-sdk] Failed to send run (non-blocking):', err?.message);
+    });
+}
+//# sourceMappingURL=client.js.map

package/dist/client.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"client.js","sourceRoot":"","sources":["../src/client.ts"],"names":[],"mappings":"AAEA,+EAA+E;AAC/E,MAAM,WAAW,GAA2B;IAC1C,QAAQ,EAAgB,MAAM;IAC9B,aAAa,EAAW,QAAQ;IAChC,aAAa,EAAW,KAAK;IAC7B,eAAe,EAAS,KAAK;IAC7B,mBAAmB,EAAK,KAAK;IAC7B,gBAAgB,EAAQ,QAAQ;IAChC,eAAe,EAAS,KAAK;CAC9B,CAAA;AAED,MAAM,UAAU,YAAY,CAAC,KAAa,EAAE,WAAmB;IAC7D,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,EAAE,CACtD,KAAK,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAClC,CAAA;IACD,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAA;IACnB,OAAO,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAA;AACvC,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,OAAO,CAAC,OAAmB,EAAE,MAAoB;IAC/D,MAAM,GAAG,GAAG,GAAG,MAAM,CAAC,SAAS,IAAI,uBAAuB,OAAO,CAAA;IAEjE,KAAK,CAAC,GAAG,EAAE;QACT,MAAM,EAAE,MAAM;QACd,OAAO,EAAE;YACP,cAAc,EAAE,kBAAkB;YAClC,WAAW,EAAE,MAAM,CAAC,MAAM;SAC3B;QACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC;KAC9B,CAAC,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;QACf,OAAO,CAAC,IAAI,CAAC,iDAAiD,EAAE,GAAG,EAAE,OAAO,CAAC,CAAA;IAC/E,CAAC,CAAC,CAAA;AACJ,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,4 @@
+export { ImpactClient } from './ImpactClient.js';
+export { wrapOpenAI } from './interceptors/openai.js';
+export { wrapAnthropic } from './interceptors/anthropic.js';
+export type { ImpactConfig, RunPayload } from './types.js';

package/dist/index.js

@@ -0,0 +1,4 @@
+export { ImpactClient } from './ImpactClient.js';
+export { wrapOpenAI } from './interceptors/openai.js';
+export { wrapAnthropic } from './interceptors/anthropic.js';
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAmB,mBAAmB,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAoB,0BAA0B,CAAA;AACnE,OAAO,EAAE,aAAa,EAAE,MAAiB,6BAA6B,CAAA"}

package/dist/interceptors/anthropic.d.ts

@@ -0,0 +1,8 @@
+import type { ImpactConfig, AnthropicParams, AnthropicResponse } from '../types.js';
+type AnthropicLike = {
+    messages: {
+        create(params: AnthropicParams, options?: unknown): Promise<AnthropicResponse>;
+    };
+};
+export declare function wrapAnthropic<T extends AnthropicLike>(client: T, config: ImpactConfig): T;
+export {};

package/dist/interceptors/anthropic.js

@@ -0,0 +1,29 @@
+import { sendRun, estimateCost } from '../client.js';
+export function wrapAnthropic(client, config) {
+    const originalCreate = client.messages.create.bind(client.messages);
+    client.messages.create = async (params, options) => {
+        const start = Date.now();
+        const response = await originalCreate(params, options);
+        const latencyMs = Date.now() - start;
+        const output = response.content
+            .filter((b) => b.type === 'text')
+            .map((b) => b.text ?? '')
+            .join('');
+        const totalTokens = (response.usage?.input_tokens ?? 0) + (response.usage?.output_tokens ?? 0);
+        const prompt = params.messages.map((m) => m.content).join('\n');
+        sendRun({
+            prompt,
+            model: params.model,
+            shadowModel: config.shadowModel,
+            primaryResponse: {
+                output,
+                latencyMs,
+                tokens: totalTokens,
+                cost: estimateCost(params.model, totalTokens),
+            },
+        }, config);
+        return response;
+    };
+    return client;
+}
+//# sourceMappingURL=anthropic.js.map

package/dist/interceptors/anthropic.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"anthropic.js","sourceRoot":"","sources":["../../src/interceptors/anthropic.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAQpD,MAAM,UAAU,aAAa,CAA0B,MAAS,EAAE,MAAoB;IACpF,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;IAEnE,MAAM,CAAC,QAAQ,CAAC,MAAM,GAAG,KAAK,EAC5B,MAAuB,EACvB,OAAiB,EACW,EAAE;QAC9B,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;QAEpC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO;aAC5B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC;aAChC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,EAAE,CAAC;aACxB,IAAI,CAAC,EAAE,CAAC,CAAA;QAEX,MAAM,WAAW,GACf,CAAC,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,CAAC,CAAA;QAE5E,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAE/D,OAAO,CACL;YACE,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,eAAe,EAAE;gBACf,MAAM;gBACN,SAAS;gBACT,MAAM,EAAE,WAAW;gBACnB,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC;aAC9C;SACF,EACD,MAAM,CACP,CAAA;QAED,OAAO,QAAQ,CAAA;IACjB,CAAC,CAAA;IAED,OAAO,MAAM,CAAA;AACf,CAAC"}

package/dist/interceptors/openai.d.ts

@@ -0,0 +1,10 @@
+import type { ImpactConfig, ChatParams, ChatResponse } from '../types.js';
+type OpenAILike = {
+    chat: {
+        completions: {
+            create(params: ChatParams, options?: unknown): Promise<ChatResponse>;
+        };
+    };
+};
+export declare function wrapOpenAI<T extends OpenAILike>(client: T, config: ImpactConfig): T;
+export {};

package/dist/interceptors/openai.js

@@ -0,0 +1,26 @@
+import { sendRun, estimateCost } from '../client.js';
+export function wrapOpenAI(client, config) {
+    const originalCreate = client.chat.completions.create.bind(client.chat.completions);
+    client.chat.completions.create = async (params, options) => {
+        const start = Date.now();
+        const response = await originalCreate(params, options);
+        const latencyMs = Date.now() - start;
+        const output = response.choices[0]?.message?.content ?? '';
+        const totalTokens = response.usage?.total_tokens ?? 0;
+        const prompt = params.messages.map((m) => m.content).join('\n');
+        sendRun({
+            prompt,
+            model: params.model,
+            shadowModel: config.shadowModel,
+            primaryResponse: {
+                output,
+                latencyMs,
+                tokens: totalTokens,
+                cost: estimateCost(params.model, totalTokens),
+            },
+        }, config);
+        return response;
+    };
+    return client;
+}
+//# sourceMappingURL=openai.js.map

package/dist/interceptors/openai.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../src/interceptors/openai.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAA;AAUpD,MAAM,UAAU,UAAU,CAAuB,MAAS,EAAE,MAAoB;IAC9E,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;IAEnF,MAAM,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,KAAK,EACpC,MAAkB,EAClB,OAAiB,EACM,EAAE;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;QACxB,MAAM,QAAQ,GAAG,MAAM,cAAc,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;QACtD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;QAEpC,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,IAAI,EAAE,CAAA;QAC1D,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAA;QACrD,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAE/D,OAAO,CACL;YACE,MAAM;YACN,KAAK,EAAE,MAAM,CAAC,KAAK;YACnB,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,eAAe,EAAE;gBACf,MAAM;gBACN,SAAS;gBACT,MAAM,EAAE,WAAW;gBACnB,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,EAAE,WAAW,CAAC;aAC9C;SACF,EACD,MAAM,CACP,CAAA;QAED,OAAO,QAAQ,CAAA;IACjB,CAAC,CAAA;IAED,OAAO,MAAM,CAAA;AACf,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,57 @@
+export type ImpactConfig = {
+    apiKey: string;
+    shadowModel: string;
+    serverUrl?: string;
+};
+export type RunPayload = {
+    prompt: string;
+    model: string;
+    shadowModel: string;
+    primaryResponse: {
+        output: string;
+        latencyMs: number;
+        tokens: number;
+        cost: number;
+    };
+};
+export type ChatResponse = {
+    choices: Array<{
+        message?: {
+            content?: string | null;
+        };
+    }>;
+    usage?: {
+        total_tokens?: number;
+        prompt_tokens?: number;
+        completion_tokens?: number;
+    };
+    model?: string;
+};
+export type ChatParams = {
+    model: string;
+    messages: Array<{
+        role: string;
+        content: string;
+    }>;
+    [key: string]: unknown;
+};
+export type AnthropicResponse = {
+    content: Array<{
+        type: string;
+        text?: string;
+    }>;
+    usage?: {
+        input_tokens?: number;
+        output_tokens?: number;
+    };
+    model?: string;
+};
+export type AnthropicParams = {
+    model: string;
+    messages: Array<{
+        role: string;
+        content: string;
+    }>;
+    max_tokens: number;
+    [key: string]: unknown;
+};

package/dist/types.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":""}

package/package.json

@@ -0,0 +1,40 @@
+{
+  "name": "impact-testing-sdk",
+  "version": "0.1.0",
+  "description": "Shadow LLM testing SDK — intercept OpenAI and Anthropic calls and compare responses against a shadow model.",
+  "keywords": ["llm", "openai", "anthropic", "testing", "shadow", "ai"],
+  "license": "MIT",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "import": "./dist/index.js",
+      "types": "./dist/index.d.ts"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "prepublishOnly": "npm run build"
+  },
+  "peerDependencies": {
+    "@anthropic-ai/sdk": ">=0.20.0",
+    "openai": ">=4.0.0"
+  },
+  "peerDependenciesMeta": {
+    "openai": { "optional": true },
+    "@anthropic-ai/sdk": { "optional": true }
+  },
+  "devDependencies": {
+    "@types/node": "^20.14.0",
+    "typescript": "^5.4.5"
+  }
+}