@warpmetrics/warp 0.0.2

package/README.md

@@ -0,0 +1,142 @@
+# warp
+
+Measure your agents, not your LLM calls.
+
+Warp is a lightweight SDK that wraps your existing OpenAI or Anthropic client and gives you full observability over your AI agent's execution — runs, groups, costs, and outcomes — with zero config changes to your LLM calls.
+
+## Install
+
+```bash
+npm install warpmetrics
+```
+
+## Quick start
+
+```js
+import OpenAI from 'openai';
+import { warp, run, group, add, outcome } from 'warpmetrics';
+
+const openai = warp(new OpenAI(), { apiKey: 'wm_...' });
+
+const r = run('code-review', { name: 'Review PR #42' });
+const planning = group('planning');
+
+const response = await openai.chat.completions.create({
+  model: 'gpt-4o',
+  messages: [{ role: 'user', content: 'Review this PR...' }],
+});
+
+add(planning, response);
+add(r, planning);
+outcome(r, 'completed', { reason: 'Approved' });
+```
+
+Every LLM call is automatically tracked. You structure the execution with `run` and `group`, then record the result with `outcome`.
+
+## API
+
+### `warp(client, options?)`
+
+Wrap an OpenAI or Anthropic client. Every call to `.chat.completions.create()` or `.messages.create()` is automatically intercepted and tracked.
+
+```js
+const openai = warp(new OpenAI(), { apiKey: 'wm_...' });
+const anthropic = warp(new Anthropic(), { apiKey: 'wm_...' });
+```
+
+Options are only needed on the first call. After that, config is shared across all wrapped clients.
+
+| Option | Type | Default | Description |
+|---|---|---|---|
+| `apiKey` | `string` | `WARPMETRICS_API_KEY` env var | Your Warpmetrics API key |
+| `baseUrl` | `string` | `https://api.warpmetrics.com` | API endpoint |
+| `enabled` | `boolean` | `true` | Disable tracking entirely |
+| `debug` | `boolean` | `false` | Log events to console |
+| `flushInterval` | `number` | `1000` | Auto-flush interval in ms |
+| `maxBatchSize` | `number` | `100` | Max events per batch |
+
+### `run(label, options?)`
+
+Create a run — the top-level unit that tracks one agent execution.
+
+```js
+const r = run('code-review', { name: 'PR #42', link: 'https://github.com/org/repo/pull/42' });
+```
+
+### `group(label, options?)`
+
+Create a group — a logical phase or step inside a run.
+
+```js
+const planning = group('planning', { name: 'Planning phase' });
+const coding = group('coding');
+```
+
+### `add(target, ...items)`
+
+Link groups or LLM responses to a run or group.
+
+```js
+add(planning, response1, response2);  // LLM responses to a group
+add(r, planning, coding);             // groups to a run
+add(planning, subGroup);              // groups can nest
+```
+
+### `outcome(target, name, options?)`
+
+Record an outcome on any tracked target.
+
+```js
+outcome(r, 'completed', {
+  reason: 'All checks passed',
+  source: 'ci',
+  tags: ['approved'],
+  metadata: { reviewer: 'alice' },
+});
+```
+
+### `ref(target)`
+
+Resolve any target (run, group, or LLM response) to its string ID. Useful for passing IDs to your frontend or storing them.
+
+```js
+ref(r)         // 'wm_run_a1b2c3d4e5f6'
+ref(response)  // 'wm_call_x9y8z7w6v5u4'
+ref('wm_run_abc')  // pass-through
+```
+
+### `cost(target)`
+
+Get the estimated cost in USD for any target. Aggregates across all nested calls for runs and groups.
+
+```js
+cost(response)  // 0.0012
+cost(r)         // 0.0036 (sum of all calls in the run)
+```
+
+### `flush()`
+
+Manually flush pending events. Events are auto-flushed on an interval and on process exit, but you can force it.
+
+```js
+await flush();
+```
+
+## Supported providers
+
+- **OpenAI** — `client.chat.completions.create()` and `client.responses.create()`
+- **Anthropic** — `client.messages.create()`
+
+Need another provider? [Open an issue](https://github.com/warpmetrics/warp/issues).
+
+## Environment variables
+
+| Variable | Description |
+|---|---|
+| `WARPMETRICS_API_KEY` | API key (fallback if not passed to `warp()`) |
+| `WARPMETRICS_API_URL` | Custom API endpoint |
+| `WARPMETRICS_DEBUG` | Set to `"true"` to enable debug logging |
+
+## License
+
+MIT

package/package.json

@@ -0,0 +1,47 @@
+{
+  "name": "@warpmetrics/warp",
+  "version": "0.0.2",
+  "description": "Measure your agents, not your LLM calls.",
+  "type": "module",
+  "main": "src/index.js",
+  "exports": {
+    ".": {
+      "import": "./src/index.js",
+      "types": "./src/index.d.ts"
+    }
+  },
+  "files": [
+    "src"
+  ],
+  "scripts": {
+    "test": "vitest run",
+    "test:watch": "vitest",
+    "test:coverage": "vitest run --coverage",
+    "release:patch": "npm version patch && git push origin main --tags",
+    "release:minor": "npm version minor && git push origin main --tags"
+  },
+  "keywords": [
+    "ai",
+    "agents",
+    "llm",
+    "observability",
+    "openai",
+    "anthropic",
+    "claude",
+    "gpt",
+    "tracing",
+    "monitoring"
+  ],
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/nikolaionken/warpmetrics"
+  },
+  "dependencies": {
+    "ulid": "^3.0.2"
+  },
+  "devDependencies": {
+    "@vitest/coverage-v8": "^1.6.1",
+    "vitest": "^1.2.0"
+  }
+}

package/src/core/registry.js

@@ -0,0 +1,17 @@
+// Warpmetrics SDK — Registries
+// Module-level state that tracks runs, groups, and calls in memory.
+
+/** @type {Map<string, object>}  run id → run data */
+export const runRegistry = new Map();
+
+/** @type {Map<string, object>}  group id → group data */
+export const groupRegistry = new Map();
+
+/** @type {WeakMap<object, string>}  LLM response object → call id */
+export const responseRegistry = new WeakMap();
+
+/** @type {WeakMap<object, number>}  LLM response object → cost in USD */
+export const costRegistry = new WeakMap();
+
+/** @type {Map<string, number>}  call id → cost in USD (for aggregation) */
+export const costByCallId = new Map();

package/src/core/transport.js

@@ -0,0 +1,196 @@
+// Warpmetrics SDK — Transport
+// Batches events and flushes them to the API over HTTP.
+
+import { createRequire } from 'module';
+const { version: SDK_VERSION } = createRequire(import.meta.url)('../../package.json');
+
+const env = typeof process !== 'undefined' ? process.env : {};
+
+let config = {
+  apiKey: env.WARPMETRICS_API_KEY || null,
+  baseUrl: env.WARPMETRICS_API_URL || 'https://api.warpmetrics.com',
+  enabled: true,
+  flushInterval: 1000,
+  maxBatchSize: 100,
+  debug: env.WARPMETRICS_DEBUG === 'true',
+};
+
+const queue = {
+  runs: [],
+  groups: [],
+  calls: [],
+  links: [],
+  outcomes: [],
+};
+
+let flushTimeout = null;
+
+// ---------------------------------------------------------------------------
+// Config
+// ---------------------------------------------------------------------------
+
+export function setConfig(updates) {
+  config = { ...config, ...updates };
+}
+
+export function getConfig() {
+  return config;
+}
+
+/** Clear all pending events without sending. For testing only. */
+export function clearQueue() {
+  queue.runs.length = 0;
+  queue.groups.length = 0;
+  queue.calls.length = 0;
+  queue.links.length = 0;
+  queue.outcomes.length = 0;
+}
+
+// ---------------------------------------------------------------------------
+// Queue
+// ---------------------------------------------------------------------------
+
+function enqueue(type, event) {
+  if (!config.enabled) return;
+
+  queue[type].push(event);
+
+  const total = queue.runs.length + queue.groups.length + queue.calls.length
+    + queue.links.length + queue.outcomes.length;
+
+  if (total >= config.maxBatchSize) {
+    flush();
+  } else if (!flushTimeout) {
+    flushTimeout = setTimeout(flush, config.flushInterval);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Flush
+// ---------------------------------------------------------------------------
+
+export async function flush() {
+  if (flushTimeout) {
+    clearTimeout(flushTimeout);
+    flushTimeout = null;
+  }
+
+  const batch = {
+    runs: queue.runs.splice(0),
+    groups: queue.groups.splice(0),
+    calls: queue.calls.splice(0),
+    links: queue.links.splice(0),
+    outcomes: queue.outcomes.splice(0),
+  };
+
+  const total = batch.runs.length + batch.groups.length + batch.calls.length
+    + batch.links.length + batch.outcomes.length;
+
+  if (total === 0) return;
+
+  if (!config.apiKey) {
+    if (config.debug) {
+      console.log('[warpmetrics] No API key — events discarded.');
+    }
+    return;
+  }
+
+  if (config.debug) {
+    console.log(
+      `[warpmetrics] Flushing ${total} events`
+      + ` (runs=${batch.runs.length} groups=${batch.groups.length}`
+      + ` calls=${batch.calls.length} links=${batch.links.length}`
+      + ` outcomes=${batch.outcomes.length})`
+    );
+  }
+
+  try {
+    const res = await fetch(`${config.baseUrl}/v1/events`, {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json',
+        'Authorization': `Bearer ${config.apiKey}`,
+        'X-SDK-Version': SDK_VERSION,
+      },
+      body: JSON.stringify(batch),
+    });
+
+    if (!res.ok) {
+      const body = await res.text().catch(() => '');
+      throw new Error(`HTTP ${res.status}: ${body}`);
+    }
+
+    if (config.debug) {
+      const result = await res.json();
+      console.log(`[warpmetrics] Flush OK — received=${result.received} processed=${result.processed}`);
+    }
+  } catch (err) {
+    if (config.debug) {
+      console.error('[warpmetrics] Flush failed:', err.message);
+    }
+    // Re-queue so nothing is lost.
+    queue.runs.unshift(...batch.runs);
+    queue.groups.unshift(...batch.groups);
+    queue.calls.unshift(...batch.calls);
+    queue.links.unshift(...batch.links);
+    queue.outcomes.unshift(...batch.outcomes);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Log helpers — called by the public API modules
+// ---------------------------------------------------------------------------
+
+export function logRun(data) {
+  enqueue('runs', {
+    id: data.id,
+    label: data.label,
+    link: data.link,
+    name: data.name,
+    timestamp: new Date().toISOString(),
+  });
+}
+
+export function logGroup(data) {
+  enqueue('groups', {
+    id: data.id,
+    label: data.label,
+    name: data.name,
+    timestamp: new Date().toISOString(),
+  });
+}
+
+export function logCall(data) {
+  enqueue('calls', data);
+}
+
+export function logLink(data) {
+  enqueue('links', {
+    parentId: data.parentId,
+    childId: data.childId,
+    type: data.type,
+    timestamp: new Date().toISOString(),
+  });
+}
+
+export function logOutcome(data) {
+  enqueue('outcomes', {
+    targetId: data.targetId,
+    name: data.name,
+    reason: data.reason,
+    source: data.source,
+    tags: data.tags,
+    metadata: data.metadata,
+    timestamp: new Date().toISOString(),
+  });
+}
+
+// ---------------------------------------------------------------------------
+// Auto-flush on process exit (Node.js only)
+// ---------------------------------------------------------------------------
+
+if (typeof process !== 'undefined' && process.on) {
+  process.on('beforeExit', flush);
+  process.on('SIGTERM', () => flush().then(() => process.exit(0)));
+  process.on('SIGINT', () => flush().then(() => process.exit(0)));
+}

package/src/core/transport.test.js

@@ -0,0 +1,41 @@
+import { describe, it, expect } from 'vitest';
+import { createRequire } from 'module';
+import { run, flush } from '../index.js';
+import { setupBeforeEach } from '../../test/setup.js';
+
+const { version } = createRequire(import.meta.url)('../../package.json');
+
+setupBeforeEach();
+
+describe('transport', () => {
+  it('sends batched events with correct auth header', async () => {
+    run('test');
+    await flush();
+
+    expect(global.fetch).toHaveBeenCalledTimes(1);
+    const [url, opts] = global.fetch.mock.calls[0];
+    expect(url).toBe('https://api.warpmetrics.com/v1/events');
+    expect(opts.headers['Authorization']).toBe('Bearer wm_test_123');
+    expect(opts.headers['X-SDK-Version']).toBe(version);
+  });
+
+  it('re-queues events on failure and succeeds on retry', async () => {
+    global.fetch.mockRejectedValueOnce(new Error('Network error'));
+
+    run('test');
+    await flush();
+    expect(global.fetch).toHaveBeenCalledTimes(1);
+
+    global.fetch.mockResolvedValueOnce({ ok: true, json: () => Promise.resolve({}) });
+    await flush();
+    expect(global.fetch).toHaveBeenCalledTimes(2);
+
+    const body = JSON.parse(global.fetch.mock.calls[1][1].body);
+    expect(body.runs).toHaveLength(1);
+  });
+
+  it('skips flush when queue is empty', async () => {
+    await flush();
+    expect(global.fetch).not.toHaveBeenCalled();
+  });
+});

package/src/core/utils.js

@@ -0,0 +1,58 @@
+// Warpmetrics SDK — Utilities
+
+import { monotonicFactory } from 'ulid';
+
+const ulid = monotonicFactory();
+
+/**
+ * Generate a prefixed ULID-based unique ID (lowercase).
+ * @param {'run' | 'grp' | 'call'} prefix
+ * @returns {string} e.g. "wm_run_01jkx3ndek0gh4r5tmqp9a3bcv"
+ */
+export function generateId(prefix) {
+  return `wm_${prefix}_${ulid().toLowerCase()}`;
+}
+
+// ---------------------------------------------------------------------------
+// Pricing (per 1 M tokens, USD)
+// Best-effort — returns 0 for unknown models.
+// ---------------------------------------------------------------------------
+
+const PRICING = {
+  // OpenAI
+  'gpt-4o':                       { prompt: 2.50,  completion: 10.00 },
+  'gpt-4o-2024-11-20':            { prompt: 2.50,  completion: 10.00 },
+  'gpt-4o-mini':                  { prompt: 0.15,  completion: 0.60  },
+  'gpt-4o-mini-2024-07-18':       { prompt: 0.15,  completion: 0.60  },
+  'gpt-4-turbo':                  { prompt: 10.00, completion: 30.00 },
+  'gpt-4-turbo-preview':          { prompt: 10.00, completion: 30.00 },
+  'gpt-4':                        { prompt: 30.00, completion: 60.00 },
+  'gpt-3.5-turbo':                { prompt: 0.50,  completion: 1.50  },
+  'o1':                           { prompt: 15.00, completion: 60.00 },
+  'o1-mini':                      { prompt: 3.00,  completion: 12.00 },
+  'o3-mini':                      { prompt: 1.10,  completion: 4.40  },
+
+  // Anthropic
+  'claude-sonnet-4-5-20250514':   { prompt: 3.00,  completion: 15.00 },
+  'claude-opus-4-6':              { prompt: 15.00, completion: 75.00 },
+  'claude-3-5-sonnet-20241022':   { prompt: 3.00,  completion: 15.00 },
+  'claude-3-5-sonnet-latest':     { prompt: 3.00,  completion: 15.00 },
+  'claude-3-5-haiku-20241022':    { prompt: 0.80,  completion: 4.00  },
+  'claude-3-5-haiku-latest':      { prompt: 0.80,  completion: 4.00  },
+  'claude-3-opus-20240229':       { prompt: 15.00, completion: 75.00 },
+  'claude-3-haiku-20240307':      { prompt: 0.25,  completion: 1.25  },
+};
+
+/**
+ * Estimate cost from model name and token counts.
+ * Returns 0 for unknown models.
+ * @param {string} model
+ * @param {{ prompt: number, completion: number }} tokens
+ * @returns {number} cost in USD
+ */
+export function calculateCost(model, tokens) {
+  const p = PRICING[model];
+  if (!p) return 0;
+  return (tokens.prompt / 1_000_000) * p.prompt
+       + (tokens.completion / 1_000_000) * p.completion;
+}

package/src/core/warp.js

@@ -0,0 +1,155 @@
+// Warpmetrics SDK — warp()
+// Wraps an LLM client so every API call is automatically tracked.
+
+import { generateId, calculateCost } from './utils.js';
+import { responseRegistry, costRegistry, costByCallId } from './registry.js';
+import { logCall, setConfig, getConfig } from './transport.js';
+import * as openai from '../providers/openai.js';
+import * as anthropic from '../providers/anthropic.js';
+
+const providers = [openai, anthropic];
+
+// ---------------------------------------------------------------------------
+// Provider resolution
+// ---------------------------------------------------------------------------
+
+function findProvider(client) {
+  return providers.find(p => p.detect(client)) || null;
+}
+
+// ---------------------------------------------------------------------------
+// Interceptor
+// ---------------------------------------------------------------------------
+
+function createInterceptor(originalFn, context, provider) {
+  return async function (...args) {
+    const start    = Date.now();
+    const callId   = generateId('call');
+    const model    = args[0]?.model || 'unknown';
+    const messages = args[0]?.messages || args[0]?.input || [];
+    const tools    = args[0]?.tools || null;
+    const stream   = args[0]?.stream === true;
+
+    try {
+      const result  = await originalFn.apply(context, args);
+      const latency = Date.now() - start;
+
+      if (stream) {
+        return wrapStream(result, { callId, provider, model, messages, tools, start });
+      }
+
+      const ext  = provider.extract(result);
+      const cost = calculateCost(model, ext.tokens);
+
+      logCall({
+        id: callId, provider: provider.name, model, messages,
+        response: ext.response,
+        tools: tools ? tools.map(t => t.function?.name || t.name).filter(Boolean) : null,
+        toolCalls: ext.toolCalls,
+        tokens: ext.tokens, cost, latency,
+        timestamp: new Date().toISOString(),
+        status: 'success',
+      });
+
+      responseRegistry.set(result, callId);
+      costRegistry.set(result, cost);
+      costByCallId.set(callId, cost);
+
+      return result;
+    } catch (error) {
+      logCall({
+        id: callId, provider: provider.name, model, messages,
+        error: error.message,
+        latency: Date.now() - start,
+        timestamp: new Date().toISOString(),
+        status: 'error',
+      });
+      throw error;
+    }
+  };
+}
+
+// ---------------------------------------------------------------------------
+// Streaming wrapper
+// ---------------------------------------------------------------------------
+
+function wrapStream(stream, ctx) {
+  const wrapped = {
+    async *[Symbol.asyncIterator]() {
+      let content = '';
+      let usage = null;
+
+      for await (const chunk of stream) {
+        const delta = ctx.provider.extractStreamDelta(chunk);
+        if (delta.content) content += delta.content;
+        if (delta.usage) usage = delta.usage;
+        yield chunk;
+      }
+
+      const tokens = usage
+        ? ctx.provider.normalizeUsage(usage)
+        : { prompt: 0, completion: 0, total: 0 };
+
+      const cost = calculateCost(ctx.model, tokens);
+
+      logCall({
+        id: ctx.callId, provider: ctx.provider.name, model: ctx.model, messages: ctx.messages,
+        response: content,
+        tools: ctx.tools ? ctx.tools.map(t => t.function?.name || t.name).filter(Boolean) : null,
+        tokens, cost,
+        latency: Date.now() - ctx.start,
+        timestamp: new Date().toISOString(),
+        status: 'success',
+      });
+
+      responseRegistry.set(wrapped, ctx.callId);
+      costRegistry.set(wrapped, cost);
+      costByCallId.set(ctx.callId, cost);
+    },
+  };
+  return wrapped;
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Wrap an LLM client to automatically track every API call.
+ *
+ * @param {object} client  — OpenAI or Anthropic client instance
+ * @param {object} [options]
+ * @param {string} [options.apiKey]
+ * @param {string} [options.baseUrl]
+ * @param {boolean} [options.enabled]
+ * @param {number} [options.flushInterval]
+ * @param {number} [options.maxBatchSize]
+ * @param {boolean} [options.debug]
+ * @returns {object} — the same client, proxied
+ */
+export function warp(client, options) {
+  if (options) {
+    const cfg = getConfig();
+    setConfig({
+      apiKey:        options.apiKey        ?? cfg.apiKey,
+      baseUrl:       options.baseUrl       ?? cfg.baseUrl,
+      enabled:       options.enabled       ?? cfg.enabled,
+      flushInterval: options.flushInterval ?? cfg.flushInterval,
+      maxBatchSize:  options.maxBatchSize  ?? cfg.maxBatchSize,
+      debug:         options.debug         ?? cfg.debug,
+    });
+  }
+
+  const provider = findProvider(client);
+
+  if (!provider) {
+    if (getConfig().debug) {
+      console.warn('[warpmetrics] Unknown client type — supported: OpenAI, Anthropic.');
+    }
+    return client;
+  }
+
+  return provider.proxy(client, (originalFn, context) =>
+    createInterceptor(originalFn, context, provider),
+  );
+}