@warpmetrics/warp 0.0.2 → 0.0.7

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Warpmetrics
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -7,14 +7,14 @@ Warp is a lightweight SDK that wraps your existing OpenAI or Anthropic client an
 ## Install
 
 ```bash
-npm install warpmetrics
+npm install @warpmetrics/warp
 ```
 
 ## Quick start
 
 ```js
 import OpenAI from 'openai';
-import { warp, run, group, add, outcome } from 'warpmetrics';
+import { warp, run, group, add, outcome } from '@warpmetrics/warp';
 
 const openai = warp(new OpenAI(), { apiKey: 'wm_...' });
 
@@ -100,9 +100,9 @@ outcome(r, 'completed', {
 Resolve any target (run, group, or LLM response) to its string ID. Useful for passing IDs to your frontend or storing them.
 
 ```js
-ref(r)         // 'wm_run_a1b2c3d4e5f6'
-ref(response)  // 'wm_call_x9y8z7w6v5u4'
-ref('wm_run_abc')  // pass-through
+ref(r)         // 'wm_run_01jkx3ndek0gh4r5tmqp9a3bcv'
+ref(response)  // 'wm_call_01jkx3ndef8mn2q7kpvhc4e9ws'
+ref('wm_run_01jkx3ndek0gh4r5tmqp9a3bcv')  // pass-through
 ```
 
 ### `cost(target)`

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@warpmetrics/warp",
-  "version": "0.0.2",
+  "version": "0.0.7",
   "description": "Measure your agents, not your LLM calls.",
   "type": "module",
   "main": "src/index.js",
@@ -11,7 +11,8 @@
     }
   },
   "files": [
-    "src"
+    "src",
+    "!src/**/*.test.js"
   ],
   "scripts": {
     "test": "vitest run",
@@ -35,7 +36,7 @@
   "license": "MIT",
   "repository": {
     "type": "git",
-    "url": "https://github.com/nikolaionken/warpmetrics"
+    "url": "https://github.com/warpmetrics/warp"
   },
   "dependencies": {
     "ulid": "^3.0.2"

package/src/core/transport.test.js

@@ -1,41 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { createRequire } from 'module';
-import { run, flush } from '../index.js';
-import { setupBeforeEach } from '../../test/setup.js';
-
-const { version } = createRequire(import.meta.url)('../../package.json');
-
-setupBeforeEach();
-
-describe('transport', () => {
-  it('sends batched events with correct auth header', async () => {
-    run('test');
-    await flush();
-
-    expect(global.fetch).toHaveBeenCalledTimes(1);
-    const [url, opts] = global.fetch.mock.calls[0];
-    expect(url).toBe('https://api.warpmetrics.com/v1/events');
-    expect(opts.headers['Authorization']).toBe('Bearer wm_test_123');
-    expect(opts.headers['X-SDK-Version']).toBe(version);
-  });
-
-  it('re-queues events on failure and succeeds on retry', async () => {
-    global.fetch.mockRejectedValueOnce(new Error('Network error'));
-
-    run('test');
-    await flush();
-    expect(global.fetch).toHaveBeenCalledTimes(1);
-
-    global.fetch.mockResolvedValueOnce({ ok: true, json: () => Promise.resolve({}) });
-    await flush();
-    expect(global.fetch).toHaveBeenCalledTimes(2);
-
-    const body = JSON.parse(global.fetch.mock.calls[1][1].body);
-    expect(body.runs).toHaveLength(1);
-  });
-
-  it('skips flush when queue is empty', async () => {
-    await flush();
-    expect(global.fetch).not.toHaveBeenCalled();
-  });
-});

package/src/core/warp.test.js

@@ -1,101 +0,0 @@
-import { describe, it, expect, vi } from 'vitest';
-import { warp, flush } from '../index.js';
-import { responseRegistry, costRegistry } from '../core/registry.js';
-import {
-  setupBeforeEach,
-  createMockOpenAI, createMockAnthropic,
-  OPENAI_RESPONSE, OPENAI_RESPONSES_API_RESPONSE, ANTHROPIC_RESPONSE,
-} from '../../test/setup.js';
-
-setupBeforeEach();
-
-describe('warp() — OpenAI', () => {
-  it('intercepts chat.completions.create and tracks the call', async () => {
-    const client = createMockOpenAI(OPENAI_RESPONSE);
-    const wrapped = warp(client);
-
-    const result = await wrapped.chat.completions.create({
-      model: 'gpt-4o-mini',
-      messages: [{ role: 'user', content: 'Hi' }],
-    });
-
-    expect(result.choices[0].message.content).toBe('Hello!');
-    expect(responseRegistry.has(result)).toBe(true);
-    expect(costRegistry.has(result)).toBe(true);
-
-    await flush();
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    expect(body.calls).toHaveLength(1);
-    expect(body.calls[0].model).toBe('gpt-4o-mini');
-    expect(body.calls[0].tokens.total).toBe(15);
-    expect(body.calls[0].status).toBe('success');
-  });
-
-  it('tracks errors without swallowing them', async () => {
-    const error = new Error('Rate limit exceeded');
-    const client = createMockOpenAI(null);
-    client.chat.completions.create = vi.fn().mockRejectedValue(error);
-    const wrapped = warp(client);
-
-    await expect(
-      wrapped.chat.completions.create({ model: 'gpt-4o', messages: [] })
-    ).rejects.toThrow('Rate limit exceeded');
-
-    await flush();
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    expect(body.calls).toHaveLength(1);
-    expect(body.calls[0].status).toBe('error');
-    expect(body.calls[0].error).toBe('Rate limit exceeded');
-  });
-
-  it('accepts config options on first call', () => {
-    const client = createMockOpenAI(OPENAI_RESPONSE);
-    warp(client, { apiKey: 'wm_live_xyz', debug: true });
-  });
-});
-
-describe('warp() — OpenAI Responses API', () => {
-  it('intercepts responses.create and tracks the call', async () => {
-    const client = createMockOpenAI(OPENAI_RESPONSE, OPENAI_RESPONSES_API_RESPONSE);
-    const wrapped = warp(client);
-
-    const result = await wrapped.responses.create({
-      model: 'gpt-4o',
-      input: 'Hi',
-    });
-
-    expect(result.output_text).toBe('Hello from Responses API!');
-    expect(responseRegistry.has(result)).toBe(true);
-    expect(costRegistry.has(result)).toBe(true);
-
-    await flush();
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    expect(body.calls).toHaveLength(1);
-    expect(body.calls[0].model).toBe('gpt-4o');
-    expect(body.calls[0].tokens.prompt).toBe(8);
-    expect(body.calls[0].tokens.completion).toBe(6);
-    expect(body.calls[0].status).toBe('success');
-  });
-});
-
-describe('warp() — Anthropic', () => {
-  it('intercepts messages.create and tracks the call', async () => {
-    const client = createMockAnthropic(ANTHROPIC_RESPONSE);
-    const wrapped = warp(client);
-
-    const result = await wrapped.messages.create({
-      model: 'claude-3-5-sonnet-latest',
-      messages: [{ role: 'user', content: 'Hi' }],
-    });
-
-    expect(result.content[0].text).toBe('Hello from Claude!');
-    expect(responseRegistry.has(result)).toBe(true);
-
-    await flush();
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    expect(body.calls).toHaveLength(1);
-    expect(body.calls[0].provider).toBe('anthropic');
-    expect(body.calls[0].tokens.prompt).toBe(12);
-    expect(body.calls[0].tokens.completion).toBe(8);
-  });
-});

package/src/e2e.test.js

@@ -1,41 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { warp, run, group, add, outcome, ref, cost, flush } from './index.js';
-import { setupBeforeEach, createMockOpenAI, OPENAI_RESPONSE } from '../test/setup.js';
-
-setupBeforeEach();
-
-describe('end-to-end', () => {
-  it('full agent flow: warp -> run -> group -> add -> outcome -> flush', async () => {
-    const client = createMockOpenAI(OPENAI_RESPONSE);
-    const openai = warp(client);
-
-    const r = run('code-review', { link: 'ticket:123', name: 'Review PR' });
-    const planning = group('planning', { name: 'Plan Phase' });
-
-    const response = await openai.chat.completions.create({
-      model: 'gpt-4o-mini',
-      messages: [{ role: 'user', content: 'Plan a review' }],
-    });
-
-    add(planning, response);
-    add(r, planning);
-    outcome(r, 'completed', { reason: 'Looks good', source: 'ci' });
-
-    expect(ref(r)).toBe(r.id);
-    expect(ref(planning)).toBe(planning.id);
-    expect(ref(response)).toMatch(/^wm_call_/);
-    expect(cost(r)).toBeGreaterThan(0);
-    expect(cost(response)).toBeGreaterThan(0);
-    expect(cost(r)).toEqual(cost(response));
-
-    await flush();
-
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    expect(body.runs).toHaveLength(1);
-    expect(body.groups).toHaveLength(1);
-    expect(body.calls).toHaveLength(1);
-    expect(body.links).toHaveLength(2);
-    expect(body.outcomes).toHaveLength(1);
-    expect(body.outcomes[0].name).toBe('completed');
-  });
-});

package/src/trace/add.test.js

@@ -1,65 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { warp, run, group, add, flush } from '../index.js';
-import { runRegistry, groupRegistry } from '../core/registry.js';
-import { setupBeforeEach, createMockOpenAI, OPENAI_RESPONSE } from '../../test/setup.js';
-
-setupBeforeEach();
-
-describe('add()', () => {
-  it('links a group to a run', async () => {
-    const r = run('test');
-    const g = group('step');
-    add(r, g);
-
-    const runData = runRegistry.get(r.id);
-    expect(runData.groups).toContain(g.id);
-
-    const groupData = groupRegistry.get(g.id);
-    expect(groupData.parentId).toBe(r.id);
-
-    await flush();
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    const link = body.links.find(l => l.childId === g.id);
-    expect(link).toBeDefined();
-    expect(link.parentId).toBe(r.id);
-    expect(link.type).toBe('group');
-  });
-
-  it('links an LLM response to a group', async () => {
-    const client = createMockOpenAI(OPENAI_RESPONSE);
-    const wrapped = warp(client);
-    const response = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
-
-    const g = group('step');
-    add(g, response);
-
-    const groupData = groupRegistry.get(g.id);
-    expect(groupData.calls).toHaveLength(1);
-    expect(groupData.calls[0]).toMatch(/^wm_call_/);
-  });
-
-  it('accepts multiple items at once', () => {
-    const r = run('test');
-    const g1 = group('a');
-    const g2 = group('b');
-    add(r, g1, g2);
-
-    const data = runRegistry.get(r.id);
-    expect(data.groups).toHaveLength(2);
-  });
-
-  it('nests groups inside groups', () => {
-    const r = run('test');
-    const parent = group('outer');
-    const child = group('inner');
-
-    add(parent, child);
-    add(r, parent);
-
-    const parentData = groupRegistry.get(parent.id);
-    expect(parentData.groups).toContain(child.id);
-
-    const childData = groupRegistry.get(child.id);
-    expect(childData.parentId).toBe(parent.id);
-  });
-});

package/src/trace/cost.test.js

@@ -1,42 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { warp, run, group, add, cost } from '../index.js';
-import { setupBeforeEach, createMockOpenAI, OPENAI_RESPONSE } from '../../test/setup.js';
-
-setupBeforeEach();
-
-describe('cost()', () => {
-  it('returns cost for an LLM response object', async () => {
-    const client = createMockOpenAI(OPENAI_RESPONSE);
-    const wrapped = warp(client);
-    const response = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
-
-    const c = cost(response);
-    expect(c).toBeGreaterThan(0);
-    // gpt-4o-mini: prompt=$0.15/1M, completion=$0.60/1M
-    // 10 prompt + 5 completion -> expected ~0.0000045
-    expect(c).toBeCloseTo(0.0000045, 7);
-  });
-
-  it('aggregates cost across a run with calls', async () => {
-    const client = createMockOpenAI(OPENAI_RESPONSE);
-    const wrapped = warp(client);
-
-    const r = run('test');
-    const g = group('step');
-
-    const r1 = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
-    const r2 = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
-
-    add(g, r1, r2);
-    add(r, g);
-
-    const singleCost = cost(r1);
-    const runCost = cost(r);
-    expect(runCost).toBeCloseTo(singleCost * 2, 10);
-  });
-
-  it('returns 0 for unknown targets', () => {
-    expect(cost({})).toBe(0);
-    expect(cost('wm_run_nonexistent')).toBe(0);
-  });
-});

package/src/trace/group.test.js

@@ -1,22 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { group } from '../index.js';
-import { groupRegistry } from '../core/registry.js';
-import { setupBeforeEach } from '../../test/setup.js';
-
-setupBeforeEach();
-
-describe('group()', () => {
-  it('returns a frozen object with id and _type', () => {
-    const g = group('planner');
-    expect(g.id).toMatch(/^wm_grp_/);
-    expect(g._type).toBe('group');
-    expect(Object.isFrozen(g)).toBe(true);
-  });
-
-  it('stores data in groupRegistry', () => {
-    const g = group('planner', { name: 'Planning Phase' });
-    const data = groupRegistry.get(g.id);
-    expect(data.label).toBe('planner');
-    expect(data.name).toBe('Planning Phase');
-  });
-});

package/src/trace/outcome.test.js

@@ -1,40 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { warp, run, outcome, flush } from '../index.js';
-import { setupBeforeEach, createMockOpenAI, OPENAI_RESPONSE } from '../../test/setup.js';
-
-setupBeforeEach();
-
-describe('outcome()', () => {
-  it('enqueues an outcome event for a run', async () => {
-    const r = run('test');
-    outcome(r, 'completed', { reason: 'All good' });
-    await flush();
-
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    const o = body.outcomes.find(e => e.name === 'completed');
-    expect(o).toBeDefined();
-    expect(o.targetId).toBe(r.id);
-    expect(o.reason).toBe('All good');
-  });
-
-  it('works with a ref string', async () => {
-    outcome('wm_run_abc123', 'shipped');
-    await flush();
-
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    expect(body.outcomes[0].targetId).toBe('wm_run_abc123');
-  });
-
-  it('works on an LLM response', async () => {
-    const client = createMockOpenAI(OPENAI_RESPONSE);
-    const wrapped = warp(client);
-    const response = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
-
-    outcome(response, 'helpful');
-    await flush();
-
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    const o = body.outcomes.find(e => e.name === 'helpful');
-    expect(o.targetId).toMatch(/^wm_call_/);
-  });
-});

package/src/trace/ref.test.js

@@ -1,35 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { warp, run, group, ref } from '../index.js';
-import { setupBeforeEach, createMockOpenAI, OPENAI_RESPONSE } from '../../test/setup.js';
-
-setupBeforeEach();
-
-describe('ref()', () => {
-  it('passes through strings', () => {
-    expect(ref('wm_run_abc')).toBe('wm_run_abc');
-  });
-
-  it('extracts id from Run', () => {
-    const r = run('test');
-    expect(ref(r)).toBe(r.id);
-  });
-
-  it('extracts id from Group', () => {
-    const g = group('test');
-    expect(ref(g)).toBe(g.id);
-  });
-
-  it('resolves LLM response to call id', async () => {
-    const client = createMockOpenAI(OPENAI_RESPONSE);
-    const wrapped = warp(client);
-    const response = await wrapped.chat.completions.create({ model: 'gpt-4o-mini', messages: [] });
-
-    const id = ref(response);
-    expect(id).toMatch(/^wm_call_/);
-  });
-
-  it('returns undefined for unknown objects', () => {
-    expect(ref({})).toBeUndefined();
-    expect(ref(null)).toBeUndefined();
-  });
-});

package/src/trace/run.test.js

@@ -1,34 +0,0 @@
-import { describe, it, expect } from 'vitest';
-import { run, flush } from '../index.js';
-import { runRegistry } from '../core/registry.js';
-import { setupBeforeEach } from '../../test/setup.js';
-
-setupBeforeEach();
-
-describe('run()', () => {
-  it('returns a frozen object with id and _type', () => {
-    const r = run('code-review');
-    expect(r.id).toMatch(/^wm_run_/);
-    expect(r._type).toBe('run');
-    expect(Object.isFrozen(r)).toBe(true);
-  });
-
-  it('stores data in runRegistry', () => {
-    const r = run('code-review', { name: 'PR #42', link: 'https://github.com/pr/42' });
-    const data = runRegistry.get(r.id);
-    expect(data.label).toBe('code-review');
-    expect(data.name).toBe('PR #42');
-    expect(data.link).toBe('https://github.com/pr/42');
-    expect(data.groups).toEqual([]);
-    expect(data.calls).toEqual([]);
-  });
-
-  it('enqueues a run event for transport', async () => {
-    run('test-label');
-    await flush();
-    expect(global.fetch).toHaveBeenCalledTimes(1);
-    const body = JSON.parse(global.fetch.mock.calls[0][1].body);
-    expect(body.runs).toHaveLength(1);
-    expect(body.runs[0].label).toBe('test-label');
-  });
-});