deadpipe 1.0.0 → 2.0.0

package/LICENSE

@@ -0,0 +1,29 @@
+Deadpipe SDK License
+
+Copyright 2024 Deadpipe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to use
+the Software solely for the purpose of integrating with the Deadpipe monitoring
+service (https://deadpipe.com), subject to the following conditions:
+
+1. The Software may only be used to send data to and receive data from the
+   official Deadpipe service.
+
+2. The Software may not be modified, distributed, sublicensed, or used in any
+   commercial product or service without explicit written permission from
+   Deadpipe.
+
+3. The Software may not be used to create competing monitoring services or
+   products.
+
+4. The above copyright notice and this permission notice shall be included in
+   all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,6 +1,6 @@
 # Deadpipe Node.js SDK
 
-Dead simple pipeline monitoring. Know when your pipelines die.
+LLM observability that answers one question: **"Is this prompt behaving the same as when it was last safe?"**
 
 ## Installation
 
@@ -14,160 +14,293 @@ pnpm add deadpipe
 
 ## Quick Start
 
-### Option 1: Wrapper (Recommended)
+### Option 1: Context Manager (Recommended)
 
 ```typescript
-import { Deadpipe } from 'deadpipe';
+import { track } from 'deadpipe';
+import OpenAI from 'openai';
 
-const dp = new Deadpipe('your-api-key');
+const client = new OpenAI();
 
-// Wrap any async function
-await dp.run('daily-sales-etl', async () => {
-  await processData();
-  return { recordsProcessed: 1500 }; // Optional: track records
+const response = await track('checkout_agent', async (t) => {
+  const response = await client.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: 'Process refund for order 1938' }]
+  });
+  t.record(response);
+  return response;
 });
-
-// Deadpipe automatically sends success/failed heartbeat when done
 ```
 
-### Option 2: Create a Wrapped Function
+### Option 2: Auto-Wrapping (Zero Code Changes)
 
 ```typescript
-import { Deadpipe } from 'deadpipe';
+import { wrapOpenAI } from 'deadpipe';
+import OpenAI from 'openai';
 
-const dp = new Deadpipe('your-api-key');
+const client = wrapOpenAI(new OpenAI(), { promptId: 'checkout_agent' });
 
-// Create a wrapped version of your function
-const myPipeline = dp.wrap('hourly-sync', async () => {
-  await syncData();
+// All calls automatically tracked
+const response = await client.chat.completions.create({
+  model: 'gpt-4',
+  messages: [{ role: 'user', content: 'Process refund for order 1938' }]
 });
-
-// Call it later
-await myPipeline();
 ```
 
-### Option 3: Manual Ping
+### Option 3: With Schema Validation (Zod)
 
 ```typescript
-import { Deadpipe } from 'deadpipe';
+import { track } from 'deadpipe';
+import { z } from 'zod';
+import OpenAI from 'openai';
+
+const RefundResponse = z.object({
+  order_id: z.string(),
+  amount: z.number(),
+  status: z.string(),
+});
 
-const dp = new Deadpipe('your-api-key');
+const client = new OpenAI();
 
-try {
-  await runMyJob();
-  await dp.ping('my-job', { status: 'success', recordsProcessed: 1000 });
-} catch (error) {
-  await dp.ping('my-job', { status: 'failed' });
-  throw error;
-}
+const result = await track('checkout_agent', async (t) => {
+  const response = await client.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: 'Process refund for order 1938' }],
+    response_format: { type: 'json_object' }
+  });
+  
+  return t.record(response);
+}, {
+  schema: {
+    validate: (data) => {
+      const result = RefundResponse.safeParse(data);
+      return {
+        success: result.success,
+        data: result.success ? result.data : undefined,
+        errors: result.success ? undefined : result.error.errors.map(e => e.message)
+      };
+    }
+  }
+});
+// result is typed as RefundResponse | null
 ```
 
-### Option 4: Environment Variable
+## What Gets Tracked
+
+Every prompt execution captures:
+
+| Category | Metrics |
+|----------|---------|
+| **Identity** | prompt_id, model, provider, app_id, environment, version |
+| **Timing** | request_start, first_token_time, total_latency |
+| **Volume** | input_tokens, output_tokens, estimated_cost_usd |
+| **Reliability** | http_status, timeout, retry_count, error_message |
+| **Output Integrity** | output_length, empty_output, truncated, json_parse_success, schema_validation_pass |
+| **Behavioral Fingerprint** | output_hash, refusal_flag, tool_calls_count |
+| **Safety Proxies** | enum_out_of_range, numeric_out_of_bounds |
+| **Change Context** | prompt_hash, tool_schema_hash, system_prompt_hash |
+
+## Advanced Usage
 
-Set `DEADPIPE_API_KEY` and use module-level functions:
+### Track Streaming Responses
 
 ```typescript
-import { run, ping } from 'deadpipe';
+const response = await track('streaming_agent', async (t) => {
+  const stream = await client.chat.completions.create({
+    model: 'gpt-4',
+    messages: [{ role: 'user', content: 'Tell me a story' }],
+    stream: true,
+  });
+
+  let fullContent = '';
+  for await (const chunk of stream) {
+    if (chunk.choices[0]?.delta?.content) {
+      t.markFirstToken(); // Call once on first token
+      fullContent += chunk.choices[0].delta.content;
+    }
+  }
+
+  // Record manually for streams
+  t.record({ 
+    model: 'gpt-4',
+    choices: [{ message: { content: fullContent } }],
+    usage: { prompt_tokens: 10, completion_tokens: 100, total_tokens: 110 }
+  });
 
-// Uses DEADPIPE_API_KEY from environment
-await run('my-pipeline', async () => {
-  await doWork();
+  return fullContent;
 });
+```
+
+### Track Retries
+
+```typescript
+const response = await track('checkout_agent', async (t) => {
+  for (let attempt = 0; attempt < 3; attempt++) {
+    try {
+      const response = await client.chat.completions.create({...});
+      t.record(response);
+      return response;
+    } catch (error) {
+      t.markRetry();
+      if (attempt === 2) throw error;
+    }
+  }
+});
+```
 
-// Or manual ping
-await ping('my-pipeline', { status: 'success' });
+### With Anthropic
+
+```typescript
+import { track } from 'deadpipe';
+import Anthropic from '@anthropic-ai/sdk';
+
+const client = new Anthropic();
+
+const response = await track('claude_agent', async (t) => {
+  const response = await client.messages.create({
+    model: 'claude-3-sonnet-20240229',
+    max_tokens: 1024,
+    messages: [{ role: 'user', content: 'Hello, Claude!' }]
+  });
+  t.record(response);
+  return response;
+}, { provider: 'anthropic' });
 ```
 
-## Express.js / API Routes
+### Environment-Based Configuration
 
 ```typescript
-import { Deadpipe } from 'deadpipe';
+// Uses these environment variables:
+// DEADPIPE_API_KEY - Your API key
+// DEADPIPE_APP_ID - Application identifier
+// DEADPIPE_ENVIRONMENT - e.g., 'production', 'staging'
+// DEADPIPE_VERSION or GIT_COMMIT - Version/commit hash
 
-const dp = new Deadpipe(process.env.DEADPIPE_API_KEY);
+import { track } from 'deadpipe';
 
-// Wrap a cron job handler
-export const handler = dp.wrap('daily-report', async () => {
-  const report = await generateReport();
-  await sendReport(report);
-  return { recordsProcessed: report.rows.length };
+// API key auto-loaded from DEADPIPE_API_KEY
+await track('my_prompt', async (t) => {
+  // ...
+});
+```
+
+### Full Options
+
+```typescript
+await track('checkout_agent', fn, {
+  // Authentication
+  apiKey: 'dp_...',
+  baseUrl: 'https://www.deadpipe.com/api/v1',
+  timeout: 10000,
+
+  // Identity
+  appId: 'my-app',
+  environment: 'production',
+  version: '1.2.3',
+  provider: 'openai', // or 'anthropic'
+
+  // Validation
+  schema: { validate: (data) => ({ success: true, data }) },
+  enumFields: { status: ['pending', 'approved', 'rejected'] },
+  numericBounds: { amount: [0, 10000] },
+
+  // Context (for change detection)
+  messages: [...],
+  tools: [...],
+  systemPrompt: 'You are a helpful assistant...',
 });
 ```
 
 ## Next.js API Routes
 
 ```typescript
-import { Deadpipe } from 'deadpipe';
+import { track } from 'deadpipe';
+import OpenAI from 'openai';
 
-const dp = new Deadpipe(process.env.DEADPIPE_API_KEY);
+const client = new OpenAI();
 
 export async function POST(request: Request) {
-  return dp.run('webhook-processor', async () => {
-    const data = await request.json();
-    await processWebhook(data);
-    return Response.json({ success: true });
+  const { prompt } = await request.json();
+
+  const response = await track('api_handler', async (t) => {
+    const completion = await client.chat.completions.create({
+      model: 'gpt-4',
+      messages: [{ role: 'user', content: prompt }]
+    });
+    t.record(completion);
+    return completion;
   });
+
+  return Response.json({ result: response.choices[0].message.content });
 }
 ```
 
-## GitHub Actions
-
-```yaml
-- name: Run ETL
-  env:
-    DEADPIPE_API_KEY: ${{ secrets.DEADPIPE_API_KEY }}
-  run: node scripts/etl.js
-```
+## Express.js
 
-```javascript
-// scripts/etl.js
-import { run } from 'deadpipe';
+```typescript
+import express from 'express';
+import { track } from 'deadpipe';
+import OpenAI from 'openai';
+
+const app = express();
+const client = new OpenAI();
+
+app.post('/generate', async (req, res) => {
+  const response = await track('express_endpoint', async (t) => {
+    const completion = await client.chat.completions.create({
+      model: 'gpt-4',
+      messages: req.body.messages
+    });
+    t.record(completion);
+    return completion;
+  });
 
-await run('github-actions-etl', async () => {
-  // Your ETL code
+  res.json(response);
 });
 ```
 
 ## API Reference
 
-### `new Deadpipe(apiKey, options)`
+### `track(promptId, fn, options?)`
 
-Create a client instance.
+Track a prompt execution with full telemetry.
 
-- `apiKey`: Your API key (or set `DEADPIPE_API_KEY` env var)
-- `options.baseUrl`: Override for self-hosted (default: `https://www.deadpipe.com/api/v1`)
-- `options.timeout`: Request timeout in ms (default: 10000)
+- `promptId`: Unique identifier for this prompt
+- `fn`: Async function that receives a `PromptTracker`
+- `options`: Configuration options (see above)
 
-### `dp.ping(pipelineId, options)`
+Returns: `Promise<T>` (result of fn)
 
-Send a heartbeat.
+### `wrapOpenAI(client, options)`
 
-- `pipelineId`: Unique identifier for this pipeline
-- `options.status`: `'success'` or `'failed'` (default: `'success'`)
-- `options.durationMs`: How long the run took
-- `options.recordsProcessed`: Number of records
-- `options.appName`: Group pipelines under an app
+Wrap an OpenAI client to auto-track all completions.
 
-Returns: `Promise<HeartbeatResponse | null>`
+- `client`: OpenAI client instance
+- `options.promptId`: Unique identifier for prompts
 
-### `dp.run(pipelineId, fn, options)`
+Returns: Wrapped client with identical API
 
-Run a function with automatic heartbeat.
+### `PromptTracker`
 
-- `pipelineId`: Unique identifier for this pipeline
-- `fn`: Async function to run
-- `options.appName`: Group pipelines under an app
+The tracker object passed to your function:
 
-Returns: `Promise<T>` (result of fn)
+- `record(response)` - Record the LLM response
+- `markFirstToken()` - Mark when first token received (streaming)
+- `markRetry()` - Mark a retry attempt
+- `recordError(error)` - Record an error
+- `getTelemetry()` - Get the telemetry object
 
-### `dp.wrap(pipelineId, fn, options)`
+### `estimateCost(model, inputTokens, outputTokens)`
 
-Create a wrapped version of a function.
+Estimate USD cost for a completion.
 
-Returns: A new function that auto-sends heartbeats.
+### `detectRefusal(text)`
+
+Detect if response is a refusal/decline.
 
 ## Zero Dependencies
 
-This SDK has zero runtime dependencies. It uses native `fetch` (Node 18+).
+This SDK has zero runtime dependencies. Uses native `fetch` (Node 18+).
 
 ## TypeScript
 
@@ -175,5 +308,4 @@ Full TypeScript support with type definitions included.
 
 ## License
 
-MIT
-
+Deadpipe SDK License - see [LICENSE](LICENSE) file.

package/dist/index.d.mts

@@ -1,105 +1,162 @@
 /**
- * Deadpipe - Dead simple pipeline monitoring.
+ * Deadpipe - LLM observability that answers one question:
+ * "Is this prompt behaving the same as when it was last safe?"
  *
  * @example
- * import { Deadpipe } from 'deadpipe';
+ * import { track } from 'deadpipe';
+ * import OpenAI from 'openai';
  *
- * const dp = new Deadpipe('your-api-key');
+ * const client = new OpenAI();
  *
- * // Option 1: Wrapper
- * await dp.run('my-pipeline', async () => {
- *   // your code here
+ * const { response, tracker } = await track('checkout_agent', async (t) => {
+ *   const response = await client.chat.completions.create({
+ *     model: 'gpt-4',
+ *     messages: [{ role: 'user', content: 'Process refund for order 1938' }]
+ *   });
+ *   t.record(response);
+ *   return response;
  * });
  *
- * // Option 2: Manual
- * await dp.ping('my-pipeline', { status: 'success' });
+ * @example Auto-wrapping (zero code changes):
+ * import { wrapOpenAI } from 'deadpipe';
+ * import OpenAI from 'openai';
+ *
+ * const client = wrapOpenAI(new OpenAI(), { promptId: 'checkout_agent' });
+ * // All calls automatically tracked
+ * const response = await client.chat.completions.create(...);
  */
-type Status = 'success' | 'failed';
-interface PingOptions {
-    status?: Status;
-    durationMs?: number;
-    recordsProcessed?: number;
-    appName?: string;
+declare const VERSION = "2.0.0";
+type StatusType = 'success' | 'error' | 'timeout' | 'empty' | 'schema_violation' | 'refusal';
+interface PromptTelemetry {
+    prompt_id: string;
+    model?: string;
+    provider?: string;
+    app_id?: string;
+    environment?: string;
+    version?: string;
+    request_start?: string;
+    first_token_time?: number;
+    end_time?: string;
+    total_latency?: number;
+    input_tokens?: number;
+    output_tokens?: number;
+    total_tokens?: number;
+    estimated_cost_usd?: number;
+    http_status?: number;
+    timeout?: boolean;
+    retry_count?: number;
+    provider_error_code?: string;
+    error_message?: string;
+    output_length?: number;
+    empty_output?: boolean;
+    truncated?: boolean;
+    json_parse_success?: boolean;
+    schema_validation_pass?: boolean;
+    missing_required_fields?: string;
+    output_hash?: string;
+    output_embedding?: string;
+    top_logprob_mean?: number;
+    refusal_flag?: boolean;
+    tool_call_flag?: boolean;
+    tool_calls_count?: number;
+    enum_out_of_range?: boolean;
+    numeric_out_of_bounds?: boolean;
+    hallucination_flags?: string;
+    prompt_hash?: string;
+    tool_schema_hash?: string;
+    system_prompt_hash?: string;
+    status?: StatusType;
 }
-interface DeadpipeOptions {
+interface TrackOptions {
     apiKey?: string;
     baseUrl?: string;
     timeout?: number;
+    appId?: string;
+    environment?: string;
+    version?: string;
+    provider?: 'openai' | 'anthropic' | string;
+    schema?: SchemaValidator;
+    enumFields?: Record<string, unknown[]>;
+    numericBounds?: Record<string, [number | null, number | null]>;
+    messages?: Array<{
+        role: string;
+        content: string;
+        [key: string]: unknown;
+    }>;
+    tools?: Array<Record<string, unknown>>;
+    systemPrompt?: string;
+}
+interface SchemaValidator {
+    validate: (data: unknown) => {
+        success: boolean;
+        data?: unknown;
+        errors?: string[];
+    };
+}
+interface WrapOpenAIOptions extends Omit<TrackOptions, 'messages' | 'tools' | 'systemPrompt'> {
+    promptId: string;
 }
-interface HeartbeatResponse {
-    received: boolean;
-    pipeline_id: string;
-    timestamp: string;
-    freshness: 'fresh' | 'stale' | 'critical';
-    next_expected: string;
-    created?: boolean;
-    message?: string;
+declare function estimateCost(model: string, inputTokens: number, outputTokens: number): number | null;
+declare function detectRefusal(text: string): boolean;
+declare function validateEnumBounds(data: Record<string, unknown>, enumFields?: Record<string, unknown[]>): boolean;
+declare function validateNumericBounds(data: Record<string, unknown>, numericBounds?: Record<string, [number | null, number | null]>): boolean;
+interface ExtractedResponse {
+    model: string;
+    content: string;
+    inputTokens: number | null;
+    outputTokens: number | null;
+    totalTokens: number | null;
+    finishReason: string | null;
+    toolCalls: Array<{
+        name: string;
+        arguments: string;
+    }>;
+    logprobs: unknown;
 }
-declare class Deadpipe {
+declare function extractOpenAIResponse(response: any): ExtractedResponse;
+declare function extractAnthropicResponse(response: any): ExtractedResponse;
+declare class PromptTracker {
+    private promptId;
     private apiKey;
     private baseUrl;
-    private timeout;
-    /**
-     * Create a Deadpipe client.
-     *
-     * @param apiKey - Your Deadpipe API key. Falls back to DEADPIPE_API_KEY env var.
-     * @param options - Configuration options.
-     */
-    constructor(apiKey?: string, options?: Omit<DeadpipeOptions, 'apiKey'>);
-    /**
-     * Send a heartbeat ping for a pipeline.
-     *
-     * @param pipelineId - Unique identifier for this pipeline.
-     * @param options - Ping options.
-     * @returns The heartbeat response, or null if the request failed.
-     */
-    ping(pipelineId: string, options?: PingOptions): Promise<HeartbeatResponse | null>;
-    /**
-     * Run a function with automatic heartbeat on completion.
-     *
-     * @param pipelineId - Unique identifier for this pipeline.
-     * @param fn - The function to run.
-     * @param options - Additional options.
-     * @returns The result of the function.
-     *
-     * @example
-     * const result = await dp.run('daily-etl', async () => {
-     *   const records = await processData();
-     *   return { recordsProcessed: records.length };
-     * });
-     */
-    run<T>(pipelineId: string, fn: () => T | Promise<T>, options?: {
-        appName?: string;
-    }): Promise<T>;
-    /**
-     * Create a wrapper function that auto-sends heartbeats.
-     *
-     * @param pipelineId - Unique identifier for this pipeline.
-     * @param fn - The function to wrap.
-     * @param options - Additional options.
-     * @returns A wrapped function.
-     *
-     * @example
-     * const myPipeline = dp.wrap('daily-etl', async () => {
-     *   await processData();
-     * });
-     *
-     * // Later...
-     * await myPipeline();
-     */
-    wrap<T extends (...args: unknown[]) => unknown>(pipelineId: string, fn: T, options?: {
-        appName?: string;
-    }): (...args: Parameters<T>) => Promise<Awaited<ReturnType<T>>>;
+    private timeoutMs;
+    private appId;
+    private environment;
+    private versionStr;
+    private provider;
+    private schema;
+    private enumFields;
+    private numericBounds;
+    private promptHash;
+    private toolSchemaHash;
+    private systemPromptHash;
+    private startTime;
+    private firstTokenTime;
+    private endTime;
+    private telemetry;
+    private recorded;
+    private retryCount;
+    constructor(promptId: string, options?: TrackOptions);
+    start(): void;
+    markFirstToken(): void;
+    markRetry(): void;
+    record(response: any, parsedOutput?: unknown): unknown;
+    recordError(error: Error): void;
+    private send;
+    isRecorded(): boolean;
+    getTelemetry(): PromptTelemetry;
 }
-/**
- * Send a heartbeat using DEADPIPE_API_KEY from environment.
- */
-declare function ping(pipelineId: string, options?: PingOptions): Promise<HeartbeatResponse | null>;
-/**
- * Run a function with automatic heartbeat using DEADPIPE_API_KEY from environment.
- */
-declare function run<T>(pipelineId: string, fn: () => T | Promise<T>, options?: {
-    appName?: string;
-}): Promise<T>;
+declare function track<T>(promptId: string, fn: (tracker: PromptTracker) => Promise<T>, options?: TrackOptions): Promise<T>;
+type OpenAIClient = any;
+interface TrackedCompletions {
+    create: (params: any) => Promise<any>;
+}
+interface TrackedChat {
+    completions: TrackedCompletions;
+}
+interface TrackedOpenAIClient extends OpenAIClient {
+    chat: TrackedChat;
+}
+declare function wrapOpenAI(client: OpenAIClient, options: WrapOpenAIOptions): TrackedOpenAIClient;
 
-export { Deadpipe, type DeadpipeOptions, type HeartbeatResponse, type PingOptions, type Status, Deadpipe as default, ping, run };
+export { type PromptTelemetry, PromptTracker, type SchemaValidator, type StatusType, type TrackOptions, VERSION, type WrapOpenAIOptions, detectRefusal, estimateCost, extractAnthropicResponse, extractOpenAIResponse, track, validateEnumBounds, validateNumericBounds, wrapOpenAI };