@warpmetrics/warp 0.0.2

package/src/core/warp.test.js

@@ -0,0 +1,101 @@
+import { describe, it, expect, vi } from 'vitest';
+import { warp, flush } from '../index.js';
+import { responseRegistry, costRegistry } from '../core/registry.js';
+import {
+  setupBeforeEach,
+  createMockOpenAI, createMockAnthropic,
+  OPENAI_RESPONSE, OPENAI_RESPONSES_API_RESPONSE, ANTHROPIC_RESPONSE,
+} from '../../test/setup.js';
+
+setupBeforeEach();
+
+describe('warp() — OpenAI', () => {
+  it('intercepts chat.completions.create and tracks the call', async () => {
+    const client = createMockOpenAI(OPENAI_RESPONSE);
+    const wrapped = warp(client);
+
+    const result = await wrapped.chat.completions.create({
+      model: 'gpt-4o-mini',
+      messages: [{ role: 'user', content: 'Hi' }],
+    });
+
+    expect(result.choices[0].message.content).toBe('Hello!');
+    expect(responseRegistry.has(result)).toBe(true);
+    expect(costRegistry.has(result)).toBe(true);
+
+    await flush();
+    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
+    expect(body.calls).toHaveLength(1);
+    expect(body.calls[0].model).toBe('gpt-4o-mini');
+    expect(body.calls[0].tokens.total).toBe(15);
+    expect(body.calls[0].status).toBe('success');
+  });
+
+  it('tracks errors without swallowing them', async () => {
+    const error = new Error('Rate limit exceeded');
+    const client = createMockOpenAI(null);
+    client.chat.completions.create = vi.fn().mockRejectedValue(error);
+    const wrapped = warp(client);
+
+    await expect(
+      wrapped.chat.completions.create({ model: 'gpt-4o', messages: [] })
+    ).rejects.toThrow('Rate limit exceeded');
+
+    await flush();
+    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
+    expect(body.calls).toHaveLength(1);
+    expect(body.calls[0].status).toBe('error');
+    expect(body.calls[0].error).toBe('Rate limit exceeded');
+  });
+
+  it('accepts config options on first call', () => {
+    const client = createMockOpenAI(OPENAI_RESPONSE);
+    warp(client, { apiKey: 'wm_live_xyz', debug: true });
+  });
+});
+
+describe('warp() — OpenAI Responses API', () => {
+  it('intercepts responses.create and tracks the call', async () => {
+    const client = createMockOpenAI(OPENAI_RESPONSE, OPENAI_RESPONSES_API_RESPONSE);
+    const wrapped = warp(client);
+
+    const result = await wrapped.responses.create({
+      model: 'gpt-4o',
+      input: 'Hi',
+    });
+
+    expect(result.output_text).toBe('Hello from Responses API!');
+    expect(responseRegistry.has(result)).toBe(true);
+    expect(costRegistry.has(result)).toBe(true);
+
+    await flush();
+    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
+    expect(body.calls).toHaveLength(1);
+    expect(body.calls[0].model).toBe('gpt-4o');
+    expect(body.calls[0].tokens.prompt).toBe(8);
+    expect(body.calls[0].tokens.completion).toBe(6);
+    expect(body.calls[0].status).toBe('success');
+  });
+});
+
+describe('warp() — Anthropic', () => {
+  it('intercepts messages.create and tracks the call', async () => {
+    const client = createMockAnthropic(ANTHROPIC_RESPONSE);
+    const wrapped = warp(client);
+
+    const result = await wrapped.messages.create({
+      model: 'claude-3-5-sonnet-latest',
+      messages: [{ role: 'user', content: 'Hi' }],
+    });
+
+    expect(result.content[0].text).toBe('Hello from Claude!');
+    expect(responseRegistry.has(result)).toBe(true);
+
+    await flush();
+    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
+    expect(body.calls).toHaveLength(1);
+    expect(body.calls[0].provider).toBe('anthropic');
+    expect(body.calls[0].tokens.prompt).toBe(12);
+    expect(body.calls[0].tokens.completion).toBe(8);
+  });
+});

package/src/e2e.test.js

@@ -0,0 +1,41 @@
+import { describe, it, expect } from 'vitest';
+import { warp, run, group, add, outcome, ref, cost, flush } from './index.js';
+import { setupBeforeEach, createMockOpenAI, OPENAI_RESPONSE } from '../test/setup.js';
+
+setupBeforeEach();
+
+describe('end-to-end', () => {
+  it('full agent flow: warp -> run -> group -> add -> outcome -> flush', async () => {
+    const client = createMockOpenAI(OPENAI_RESPONSE);
+    const openai = warp(client);
+
+    const r = run('code-review', { link: 'ticket:123', name: 'Review PR' });
+    const planning = group('planning', { name: 'Plan Phase' });
+
+    const response = await openai.chat.completions.create({
+      model: 'gpt-4o-mini',
+      messages: [{ role: 'user', content: 'Plan a review' }],
+    });
+
+    add(planning, response);
+    add(r, planning);
+    outcome(r, 'completed', { reason: 'Looks good', source: 'ci' });
+
+    expect(ref(r)).toBe(r.id);
+    expect(ref(planning)).toBe(planning.id);
+    expect(ref(response)).toMatch(/^wm_call_/);
+    expect(cost(r)).toBeGreaterThan(0);
+    expect(cost(response)).toBeGreaterThan(0);
+    expect(cost(r)).toEqual(cost(response));
+
+    await flush();
+
+    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
+    expect(body.runs).toHaveLength(1);
+    expect(body.groups).toHaveLength(1);
+    expect(body.calls).toHaveLength(1);
+    expect(body.links).toHaveLength(2);
+    expect(body.outcomes).toHaveLength(1);
+    expect(body.outcomes[0].name).toBe('completed');
+  });
+});

package/src/index.d.ts

@@ -0,0 +1,74 @@
+// Warpmetrics SDK — Type Definitions
+
+export interface Run {
+  readonly id: string;
+  readonly _type: 'run';
+}
+
+export interface Group {
+  readonly id: string;
+  readonly _type: 'group';
+}
+
+export interface WarpOptions {
+  apiKey?: string;
+  baseUrl?: string;
+  enabled?: boolean;
+  flushInterval?: number;
+  maxBatchSize?: number;
+  debug?: boolean;
+}
+
+export interface RunOptions {
+  /** External reference (e.g. "ticket:PROJ-101", a PR URL, etc.) */
+  link?: string;
+  /** Human-readable name */
+  name?: string;
+}
+
+export interface GroupOptions {
+  /** Human-readable name */
+  name?: string;
+}
+
+export interface OutcomeOptions {
+  /** Why this outcome occurred */
+  reason?: string;
+  /** Who / what recorded this outcome */
+  source?: string;
+  /** Categorisation tags */
+  tags?: string[];
+  /** Arbitrary extra data */
+  metadata?: Record<string, any>;
+}
+
+/**
+ * Wrap an LLM client to automatically track every API call.
+ * Pass options on the first call to configure the SDK; env vars are used as defaults.
+ */
+export function warp<T>(client: T, options?: WarpOptions): T;
+
+/** Create a run — the top-level unit that tracks one agent execution. */
+export function run(label: string, options?: RunOptions): Run;
+
+/** Create a group — a logical phase or step inside a run. */
+export function group(label: string, options?: GroupOptions): Group;
+
+/** Add items (groups or LLM responses) to a run or group. */
+export function add(target: Run | Group | string, ...items: any[]): void;
+
+/** Record an outcome on any tracked target. */
+export function outcome(
+  target: Run | Group | object | string,
+  name: string,
+  options?: OutcomeOptions,
+): void;
+
+/** Resolve any trackable target to its string ID. */
+export function ref(target: Run | Group | object | string): string | undefined;
+
+/** Get the cost in USD for any tracked target. */
+export function cost(target: Run | Group | object | string): number;
+
+/** Manually flush pending events to the API. */
+export function flush(): Promise<void>;

package/src/index.js

@@ -0,0 +1,19 @@
+// Warpmetrics SDK
+// Measure your agents, not your LLM calls.
+//
+//   warp(client, options?)           — wrap an LLM client
+//   run(label, options?)             — create a run
+//   group(label, options?)           — create a group
+//   add(target, ...items)            — add groups / calls to a run or group
+//   outcome(target, name, options?)  — record a result
+//   ref(target)                      — get tracking ID
+//   cost(target)                     — get cost in USD
+
+export { warp } from './core/warp.js';
+export { run } from './trace/run.js';
+export { group } from './trace/group.js';
+export { add } from './trace/add.js';
+export { outcome } from './trace/outcome.js';
+export { ref } from './trace/ref.js';
+export { cost } from './trace/cost.js';
+export { flush } from './core/transport.js';

package/src/providers/anthropic.js

@@ -0,0 +1,51 @@
+// Warpmetrics SDK — Anthropic Provider
+
+export const name = 'anthropic';
+
+export function detect(client) {
+  return client?.constructor?.name === 'Anthropic';
+}
+
+export function extract(result) {
+  const input  = result?.usage?.input_tokens || 0;
+  const output = result?.usage?.output_tokens || 0;
+  return {
+    response: Array.isArray(result?.content)
+      ? result.content.filter(c => c.type === 'text').map(c => c.text).join('')
+      : '',
+    tokens: { prompt: input, completion: output, total: input + output },
+    toolCalls: null,
+  };
+}
+
+export function extractStreamDelta(chunk) {
+  return {
+    content: chunk.type === 'content_block_delta' ? (chunk.delta?.text || null) : null,
+    usage:   chunk.type === 'message_delta' ? (chunk.usage || null) : null,
+  };
+}
+
+export function normalizeUsage(usage) {
+  const prompt     = usage?.input_tokens || 0;
+  const completion = usage?.output_tokens || 0;
+  return { prompt, completion, total: prompt + completion };
+}
+
+export function proxy(client, intercept) {
+  return new Proxy(client, {
+    get(target, prop, receiver) {
+      const value = Reflect.get(target, prop, receiver);
+      if (prop !== 'messages') return value;
+
+      return new Proxy(value, {
+        get(msgTarget, msgProp, msgReceiver) {
+          const msgValue = Reflect.get(msgTarget, msgProp, msgReceiver);
+          if (msgProp === 'create' && typeof msgValue === 'function') {
+            return intercept(msgValue, msgTarget);
+          }
+          return msgValue;
+        },
+      });
+    },
+  });
+}

package/src/providers/openai.js

@@ -0,0 +1,130 @@
+// Warpmetrics SDK — OpenAI Provider
+// Supports both Chat Completions and Responses API.
+
+export const name = 'openai';
+
+export function detect(client) {
+  return client?.constructor?.name === 'OpenAI';
+}
+
+// ---------------------------------------------------------------------------
+// Response extraction
+// ---------------------------------------------------------------------------
+
+function isResponsesAPI(result) {
+  return result?.object === 'response' || Array.isArray(result?.output);
+}
+
+function extractChatCompletions(result) {
+  return {
+    response: result?.choices?.[0]?.message?.content || '',
+    tokens: {
+      prompt:     result?.usage?.prompt_tokens || 0,
+      completion: result?.usage?.completion_tokens || 0,
+      total:      result?.usage?.total_tokens || 0,
+    },
+    toolCalls: result?.choices?.[0]?.message?.tool_calls?.map(tc => ({
+      id: tc.id,
+      name: tc.function?.name,
+      arguments: tc.function?.arguments,
+    })) || null,
+  };
+}
+
+function extractResponses(result) {
+  const textItems = (result?.output || [])
+    .filter(item => item.type === 'message')
+    .flatMap(item => (item.content || []).filter(c => c.type === 'output_text'));
+
+  const text = result?.output_text || textItems.map(c => c.text).join('') || '';
+
+  const input  = result?.usage?.input_tokens || 0;
+  const output = result?.usage?.output_tokens || 0;
+
+  const fnCalls = (result?.output || []).filter(item => item.type === 'function_call');
+
+  return {
+    response: text,
+    tokens: { prompt: input, completion: output, total: input + output },
+    toolCalls: fnCalls.length > 0
+      ? fnCalls.map(fc => ({ id: fc.id, name: fc.name, arguments: fc.arguments }))
+      : null,
+  };
+}
+
+export function extract(result) {
+  return isResponsesAPI(result) ? extractResponses(result) : extractChatCompletions(result);
+}
+
+// ---------------------------------------------------------------------------
+// Streaming extraction
+// ---------------------------------------------------------------------------
+
+export function extractStreamDelta(chunk) {
+  // Responses API streaming
+  if (chunk.type === 'response.output_text.delta') {
+    return { content: chunk.delta || null, usage: null };
+  }
+  if (chunk.type === 'response.completed') {
+    return { content: null, usage: chunk.response?.usage || null };
+  }
+
+  // Chat Completions streaming
+  return {
+    content: chunk.choices?.[0]?.delta?.content || null,
+    usage:   chunk.usage || null,
+  };
+}
+
+export function normalizeUsage(usage) {
+  const prompt     = usage?.prompt_tokens || usage?.input_tokens || 0;
+  const completion = usage?.completion_tokens || usage?.output_tokens || 0;
+  return { prompt, completion, total: prompt + completion };
+}
+
+// ---------------------------------------------------------------------------
+// Proxy
+// ---------------------------------------------------------------------------
+
+function interceptCreate(target, prop, intercept) {
+  const value = Reflect.get(target, prop);
+  if (prop === 'create' && typeof value === 'function') {
+    return intercept(value, target);
+  }
+  return value;
+}
+
+export function proxy(client, intercept) {
+  return new Proxy(client, {
+    get(target, prop, receiver) {
+      const value = Reflect.get(target, prop, receiver);
+
+      // client.responses.create()
+      if (prop === 'responses' && value) {
+        return new Proxy(value, {
+          get(rTarget, rProp) {
+            return interceptCreate(rTarget, rProp, intercept);
+          },
+        });
+      }
+
+      // client.chat.completions.create()
+      if (prop === 'chat' && value) {
+        return new Proxy(value, {
+          get(chatTarget, chatProp) {
+            const chatValue = Reflect.get(chatTarget, chatProp);
+            if (chatProp !== 'completions') return chatValue;
+
+            return new Proxy(chatValue, {
+              get(compTarget, compProp) {
+                return interceptCreate(compTarget, compProp, intercept);
+              },
+            });
+          },
+        });
+      }
+
+      return value;
+    },
+  });
+}

package/src/trace/add.js

@@ -0,0 +1,53 @@
+// Warpmetrics SDK — add()
+
+import { runRegistry, groupRegistry, responseRegistry } from '../core/registry.js';
+import { logLink, getConfig } from '../core/transport.js';
+import { ref as getRef } from './ref.js';
+
+/**
+ * Add items (groups or LLM responses) to a run or group.
+ *
+ * @param {object | string} target  — Run, Group, or ref string
+ * @param {...object} items         — Group objects or LLM response objects
+ */
+export function add(target, ...items) {
+  const targetId = getRef(target);
+  if (!targetId) {
+    if (getConfig().debug) console.warn('[warpmetrics] add() — target not recognised.');
+    return;
+  }
+
+  const data = runRegistry.get(targetId) || groupRegistry.get(targetId);
+  if (!data) {
+    if (getConfig().debug) console.warn(`[warpmetrics] add() — target not in registry: ${targetId}`);
+    return;
+  }
+
+  for (const item of items) {
+    // Group
+    if (item && item._type === 'group') {
+      const groupData = groupRegistry.get(item.id);
+      if (groupData) {
+        groupData.parentId = targetId;
+        data.groups.push(item.id);
+        logLink({ parentId: targetId, childId: item.id, type: 'group' });
+      }
+      continue;
+    }
+
+    // Run → cannot nest runs
+    if (item && item._type === 'run') {
+      if (getConfig().debug) console.warn('[warpmetrics] add() — cannot add a run to another target.');
+      continue;
+    }
+
+    // LLM response
+    const callId = responseRegistry.get(item);
+    if (callId) {
+      data.calls.push(callId);
+      logLink({ parentId: targetId, childId: callId, type: 'call' });
+    } else if (getConfig().debug) {
+      console.warn('[warpmetrics] add() — item not tracked. Was it from a warp()-ed client?');
+    }
+  }
+}

package/src/trace/add.test.js

@@ -0,0 +1,65 @@
+import { describe, it, expect } from 'vitest';
+import { warp, run, group, add, flush } from '../index.js';
+import { runRegistry, groupRegistry } from '../core/registry.js';
+import { setupBeforeEach, createMockOpenAI, OPENAI_RESPONSE } from '../../test/setup.js';
+
+setupBeforeEach();
+
+describe('add()', () => {
+  it('links a group to a run', async () => {
+    const r = run('test');
+    const g = group('step');
+    add(r, g);
+
+    const runData = runRegistry.get(r.id);
+    expect(runData.groups).toContain(g.id);
+
+    const groupData = groupRegistry.get(g.id);
+    expect(groupData.parentId).toBe(r.id);
+
+    await flush();
+    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
+    const link = body.links.find(l => l.childId === g.id);
+    expect(link).toBeDefined();
+    expect(link.parentId).toBe(r.id);
+    expect(link.type).toBe('group');
+  });
+
+  it('links an LLM response to a group', async () => {
+    const client = createMockOpenAI(OPENAI_RESPONSE);
+    const wrapped = warp(client);
+    const response = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
+
+    const g = group('step');
+    add(g, response);
+
+    const groupData = groupRegistry.get(g.id);
+    expect(groupData.calls).toHaveLength(1);
+    expect(groupData.calls[0]).toMatch(/^wm_call_/);
+  });
+
+  it('accepts multiple items at once', () => {
+    const r = run('test');
+    const g1 = group('a');
+    const g2 = group('b');
+    add(r, g1, g2);
+
+    const data = runRegistry.get(r.id);
+    expect(data.groups).toHaveLength(2);
+  });
+
+  it('nests groups inside groups', () => {
+    const r = run('test');
+    const parent = group('outer');
+    const child = group('inner');
+
+    add(parent, child);
+    add(r, parent);
+
+    const parentData = groupRegistry.get(parent.id);
+    expect(parentData.groups).toContain(child.id);
+
+    const childData = groupRegistry.get(child.id);
+    expect(childData.parentId).toBe(parent.id);
+  });
+});

package/src/trace/cost.js

@@ -0,0 +1,50 @@
+// Warpmetrics SDK — cost()
+
+import { ref as getRef } from './ref.js';
+import { runRegistry, groupRegistry, costRegistry, costByCallId } from '../core/registry.js';
+
+/**
+ * Get the cost in USD for any tracked target.
+ *
+ * - Response object → cost of that single call
+ * - Run / Group      → sum of all nested calls
+ * - Ref string       → lookup in registries
+ *
+ * @param {object | string} target
+ * @returns {number} cost in USD
+ */
+export function cost(target) {
+  // Response object — direct lookup
+  if (typeof target === 'object' && costRegistry.has(target)) {
+    return costRegistry.get(target);
+  }
+
+  const id = getRef(target);
+  if (!id) return 0;
+
+  // Single call by ref string
+  if (id.startsWith('wm_call_')) {
+    return costByCallId.get(id) || 0;
+  }
+
+  // Run or Group — aggregate
+  const container = runRegistry.get(id) || groupRegistry.get(id);
+  if (container) return sumCosts(container);
+
+  return 0;
+}
+
+function sumCosts(container) {
+  let total = 0;
+
+  for (const callId of container.calls) {
+    total += costByCallId.get(callId) || 0;
+  }
+
+  for (const groupId of container.groups) {
+    const g = groupRegistry.get(groupId);
+    if (g) total += sumCosts(g);
+  }
+
+  return total;
+}

package/src/trace/cost.test.js

@@ -0,0 +1,42 @@
+import { describe, it, expect } from 'vitest';
+import { warp, run, group, add, cost } from '../index.js';
+import { setupBeforeEach, createMockOpenAI, OPENAI_RESPONSE } from '../../test/setup.js';
+
+setupBeforeEach();
+
+describe('cost()', () => {
+  it('returns cost for an LLM response object', async () => {
+    const client = createMockOpenAI(OPENAI_RESPONSE);
+    const wrapped = warp(client);
+    const response = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
+
+    const c = cost(response);
+    expect(c).toBeGreaterThan(0);
+    // gpt-4o-mini: prompt=$0.15/1M, completion=$0.60/1M
+    // 10 prompt + 5 completion -> expected ~0.0000045
+    expect(c).toBeCloseTo(0.0000045, 7);
+  });
+
+  it('aggregates cost across a run with calls', async () => {
+    const client = createMockOpenAI(OPENAI_RESPONSE);
+    const wrapped = warp(client);
+
+    const r = run('test');
+    const g = group('step');
+
+    const r1 = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
+    const r2 = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
+
+    add(g, r1, r2);
+    add(r, g);
+
+    const singleCost = cost(r1);
+    const runCost = cost(r);
+    expect(runCost).toBeCloseTo(singleCost * 2, 10);
+  });
+
+  it('returns 0 for unknown targets', () => {
+    expect(cost({})).toBe(0);
+    expect(cost('wm_run_nonexistent')).toBe(0);
+  });
+});

package/src/trace/group.js

@@ -0,0 +1,31 @@
+// Warpmetrics SDK — group()
+
+import { generateId } from '../core/utils.js';
+import { groupRegistry } from '../core/registry.js';
+import { logGroup } from '../core/transport.js';
+
+/**
+ * Create a group — a logical phase or step inside a run.
+ *
+ * @param {string} label  — group type used for aggregation ("planner", "coder")
+ * @param {object} [options]
+ * @param {string} [options.name]  — human-readable name
+ * @returns {{ readonly id: string, readonly _type: 'group' }}
+ */
+export function group(label, options = {}) {
+  const id = generateId('grp');
+
+  const data = {
+    id,
+    label,
+    name: options.name || null,
+    parentId: null,
+    groups: [],
+    calls: [],
+  };
+
+  groupRegistry.set(id, data);
+  logGroup(data);
+
+  return Object.freeze({ id, _type: 'group' });
+}

package/src/trace/group.test.js

@@ -0,0 +1,22 @@
+import { describe, it, expect } from 'vitest';
+import { group } from '../index.js';
+import { groupRegistry } from '../core/registry.js';
+import { setupBeforeEach } from '../../test/setup.js';
+
+setupBeforeEach();
+
+describe('group()', () => {
+  it('returns a frozen object with id and _type', () => {
+    const g = group('planner');
+    expect(g.id).toMatch(/^wm_grp_/);
+    expect(g._type).toBe('group');
+    expect(Object.isFrozen(g)).toBe(true);
+  });
+
+  it('stores data in groupRegistry', () => {
+    const g = group('planner', { name: 'Planning Phase' });
+    const data = groupRegistry.get(g.id);
+    expect(data.label).toBe('planner');
+    expect(data.name).toBe('Planning Phase');
+  });
+});