deadpipe 1.0.1 → 2.0.0

package/dist/index.mjs

@@ -1,44 +1,382 @@
 // src/index.ts
-var Deadpipe = class {
+var VERSION = "2.0.0";
+var MODEL_COSTS = {
+  // OpenAI
+  "gpt-4": { input: 0.03, output: 0.06 },
+  // legacy
+  "gpt-4-turbo": { input: 0.01, output: 0.03 },
+  // legacy
+  "gpt-4o": { input: 5e-3, output: 0.015 },
+  "gpt-4o-mini": { input: 15e-5, output: 6e-4 },
+  "gpt-4.1": { input: 2e-3, output: 8e-3 },
+  "gpt-3.5-turbo": { input: 5e-4, output: 15e-4 },
+  "gpt-5": { input: 175e-5, output: 0.014 },
+  "gpt-5-mini": { input: 25e-5, output: 2e-3 },
+  "gpt-5.2-pro": { input: 0.021, output: 0.168 },
+  // Anthropic
+  "claude-3-opus": { input: 0.015, output: 0.075 },
+  "claude-3-sonnet": { input: 3e-3, output: 0.015 },
+  "claude-3-haiku": { input: 25e-5, output: 125e-5 },
+  "claude-3.5-sonnet": { input: 3e-3, output: 0.015 },
+  "claude-opus-4": { input: 0.015, output: 0.075 },
+  "claude-sonnet-4": { input: 3e-3, output: 0.015 },
+  "claude-haiku-4": { input: 25e-5, output: 125e-5 }
+};
+function estimateCost(model, inputTokens, outputTokens) {
+  const modelLower = model.toLowerCase();
+  for (const [knownModel, costs] of Object.entries(MODEL_COSTS)) {
+    if (modelLower.includes(knownModel)) {
+      const inputCost = inputTokens / 1e3 * costs.input;
+      const outputCost = outputTokens / 1e3 * costs.output;
+      return Math.round((inputCost + outputCost) * 1e6) / 1e6;
+    }
+  }
+  return null;
+}
+function hashContentSync(content) {
+  let hash = 5381;
+  for (let i = 0; i < content.length; i++) {
+    hash = (hash << 5) + hash + content.charCodeAt(i);
+  }
+  return Math.abs(hash).toString(16).slice(0, 16).padStart(16, "0");
+}
+function hashMessages(messages) {
+  const serialized = JSON.stringify(messages);
+  return hashContentSync(serialized);
+}
+function hashTools(tools) {
+  if (!tools || tools.length === 0) return void 0;
+  const serialized = JSON.stringify(tools);
+  return hashContentSync(serialized);
+}
+var REFUSAL_PATTERNS = [
+  "i can't help with",
+  "i cannot help with",
+  "i'm not able to",
+  "i am not able to",
+  "i won't be able to",
+  "i'm unable to",
+  "i cannot provide",
+  "i can't provide",
+  "i must decline",
+  "i cannot assist with",
+  "i can't assist with",
+  "as an ai",
+  "as a language model",
+  "i don't have the ability",
+  "i cannot comply",
+  "i'm designed to",
+  "my purpose is to",
+  "violates my guidelines",
+  "against my guidelines",
+  "ethical guidelines",
+  "i apologize, but i cannot",
+  "i'm sorry, but i can't"
+];
+function detectRefusal(text) {
+  const textLower = text.toLowerCase();
+  return REFUSAL_PATTERNS.some((pattern) => textLower.includes(pattern));
+}
+function validateEnumBounds(data, enumFields) {
+  if (!enumFields) return true;
+  for (const [fieldName, validValues] of Object.entries(enumFields)) {
+    if (fieldName in data && !validValues.includes(data[fieldName])) {
+      return false;
+    }
+  }
+  return true;
+}
+function validateNumericBounds(data, numericBounds) {
+  if (!numericBounds) return true;
+  for (const [fieldName, [minVal, maxVal]] of Object.entries(numericBounds)) {
+    if (fieldName in data) {
+      const value = data[fieldName];
+      if (typeof value === "number") {
+        if (minVal !== null && value < minVal) return false;
+        if (maxVal !== null && value > maxVal) return false;
+      }
+    }
+  }
+  return true;
+}
+function extractOpenAIResponse(response) {
+  const result = {
+    model: "",
+    content: "",
+    inputTokens: null,
+    outputTokens: null,
+    totalTokens: null,
+    finishReason: null,
+    toolCalls: [],
+    logprobs: null
+  };
+  if (response?.model) {
+    result.model = response.model;
+  }
+  if (response?.choices?.[0]) {
+    const choice = response.choices[0];
+    if (choice.message) {
+      result.content = choice.message.content || "";
+      if (choice.message.tool_calls) {
+        result.toolCalls = choice.message.tool_calls.map((tc) => ({
+          name: tc.function.name,
+          arguments: tc.function.arguments
+        }));
+      }
+    }
+    if (choice.finish_reason) {
+      result.finishReason = choice.finish_reason;
+    }
+    if (choice.logprobs) {
+      result.logprobs = choice.logprobs;
+    }
+  }
+  if (response?.output) {
+    result.content = response.output || "";
+  }
+  if (response?.usage) {
+    result.inputTokens = response.usage.prompt_tokens ?? null;
+    result.outputTokens = response.usage.completion_tokens ?? null;
+    result.totalTokens = response.usage.total_tokens ?? null;
+  }
+  return result;
+}
+function extractAnthropicResponse(response) {
+  const result = {
+    model: "",
+    content: "",
+    inputTokens: null,
+    outputTokens: null,
+    totalTokens: null,
+    finishReason: null,
+    toolCalls: [],
+    logprobs: null
+  };
+  if (response?.model) {
+    result.model = response.model;
+  }
+  if (response?.content && Array.isArray(response.content)) {
+    const textBlocks = response.content.filter((block) => block.type === "text" || block.text).map((block) => block.text);
+    result.content = textBlocks.join("");
+    const toolBlocks = response.content.filter(
+      (block) => block.type === "tool_use"
+    );
+    result.toolCalls = toolBlocks.map((block) => ({
+      name: block.name,
+      arguments: JSON.stringify(block.input)
+    }));
+  }
+  if (response?.stop_reason) {
+    result.finishReason = response.stop_reason;
+  }
+  if (response?.usage) {
+    result.inputTokens = response.usage.input_tokens ?? null;
+    result.outputTokens = response.usage.output_tokens ?? null;
+    if (result.inputTokens !== null && result.outputTokens !== null) {
+      result.totalTokens = result.inputTokens + result.outputTokens;
+    }
+  }
+  return result;
+}
+function calculateLogprobMean(logprobs) {
+  if (!logprobs?.content) return null;
+  try {
+    const probs = logprobs.content.filter((token) => typeof token.logprob === "number").map((token) => token.logprob);
+    if (probs.length > 0) {
+      return probs.reduce((a, b) => a + b, 0) / probs.length;
+    }
+  } catch {
+  }
+  return null;
+}
+var PromptTracker = class {
+  promptId;
   apiKey;
   baseUrl;
-  timeout;
-  /**
-   * Create a Deadpipe client.
-   *
-   * @param apiKey - Your Deadpipe API key. Falls back to DEADPIPE_API_KEY env var.
-   * @param options - Configuration options.
-   */
-  constructor(apiKey, options = {}) {
-    this.apiKey = apiKey || process.env.DEADPIPE_API_KEY || "";
-    if (!this.apiKey) {
-      throw new Error(
-        "API key required. Pass apiKey or set DEADPIPE_API_KEY environment variable."
-      );
-    }
+  timeoutMs;
+  // Identity
+  appId;
+  environment;
+  versionStr;
+  provider;
+  // Validation
+  schema;
+  enumFields;
+  numericBounds;
+  // Context hashes
+  promptHash;
+  toolSchemaHash;
+  systemPromptHash;
+  // Timing
+  startTime = null;
+  firstTokenTime = null;
+  endTime = null;
+  // State
+  telemetry;
+  recorded = false;
+  retryCount = 0;
+  constructor(promptId, options = {}) {
+    this.promptId = promptId;
+    this.apiKey = options.apiKey || process.env.DEADPIPE_API_KEY;
     this.baseUrl = (options.baseUrl || "https://www.deadpipe.com/api/v1").replace(/\/$/, "");
-    this.timeout = options.timeout || 1e4;
-  }
-  /**
-   * Send a heartbeat ping for a pipeline.
-   *
-   * @param pipelineId - Unique identifier for this pipeline.
-   * @param options - Ping options.
-   * @returns The heartbeat response, or null if the request failed.
-   */
-  async ping(pipelineId, options = {}) {
-    const { status = "success", durationMs, recordsProcessed, appName } = options;
-    const payload = {
-      pipeline_id: pipelineId,
-      status
+    this.timeoutMs = options.timeout || 1e4;
+    this.appId = options.appId || process.env.DEADPIPE_APP_ID;
+    this.environment = options.environment || process.env.DEADPIPE_ENVIRONMENT;
+    this.versionStr = options.version || process.env.DEADPIPE_VERSION || process.env.GIT_COMMIT;
+    this.provider = options.provider || "openai";
+    this.schema = options.schema;
+    this.enumFields = options.enumFields;
+    this.numericBounds = options.numericBounds;
+    this.promptHash = options.messages ? hashMessages(options.messages) : void 0;
+    this.toolSchemaHash = hashTools(options.tools);
+    this.systemPromptHash = options.systemPrompt ? hashContentSync(options.systemPrompt) : void 0;
+    this.telemetry = {
+      prompt_id: this.promptId,
+      provider: this.provider,
+      app_id: this.appId,
+      environment: this.environment,
+      version: this.versionStr,
+      prompt_hash: this.promptHash,
+      tool_schema_hash: this.toolSchemaHash,
+      system_prompt_hash: this.systemPromptHash,
+      status: "success"
     };
-    if (durationMs !== void 0) payload.duration_ms = durationMs;
-    if (recordsProcessed !== void 0) payload.records_processed = recordsProcessed;
-    if (appName !== void 0) payload.app_name = appName;
+  }
+  start() {
+    this.startTime = Date.now();
+    this.telemetry.request_start = new Date(this.startTime).toISOString();
+  }
+  markFirstToken() {
+    if (this.firstTokenTime === null && this.startTime !== null) {
+      this.firstTokenTime = Date.now();
+      this.telemetry.first_token_time = this.firstTokenTime - this.startTime;
+    }
+  }
+  markRetry() {
+    this.retryCount++;
+    this.telemetry.retry_count = this.retryCount;
+  }
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  record(response, parsedOutput) {
+    this.endTime = Date.now();
+    this.telemetry.end_time = new Date(this.endTime).toISOString();
+    this.telemetry.total_latency = this.startTime ? this.endTime - this.startTime : 0;
+    const extracted = this.provider === "anthropic" ? extractAnthropicResponse(response) : extractOpenAIResponse(response);
+    this.telemetry.model = extracted.model;
+    this.telemetry.input_tokens = extracted.inputTokens ?? void 0;
+    this.telemetry.output_tokens = extracted.outputTokens ?? void 0;
+    this.telemetry.total_tokens = extracted.totalTokens ?? void 0;
+    this.telemetry.http_status = 200;
+    const content = extracted.content;
+    this.telemetry.output_length = content?.length ?? 0;
+    this.telemetry.empty_output = !content || content.trim().length === 0;
+    this.telemetry.truncated = extracted.finishReason === "length";
+    this.telemetry.tool_call_flag = extracted.toolCalls.length > 0;
+    this.telemetry.tool_calls_count = extracted.toolCalls.length;
+    if (content) {
+      this.telemetry.output_hash = hashContentSync(content);
+    }
+    if (extracted.logprobs) {
+      this.telemetry.top_logprob_mean = calculateLogprobMean(extracted.logprobs) ?? void 0;
+    }
+    if (this.telemetry.input_tokens && this.telemetry.output_tokens && this.telemetry.model) {
+      this.telemetry.estimated_cost_usd = estimateCost(
+        this.telemetry.model,
+        this.telemetry.input_tokens,
+        this.telemetry.output_tokens
+      ) ?? void 0;
+    }
+    if (content) {
+      this.telemetry.refusal_flag = detectRefusal(content);
+      if (this.telemetry.refusal_flag) {
+        this.telemetry.status = "refusal";
+      }
+    }
+    if (this.telemetry.empty_output) {
+      this.telemetry.status = "empty";
+    }
+    let parsedData = parsedOutput;
+    if (parsedData === void 0 && content) {
+      try {
+        const contentStripped = content.trim();
+        if (contentStripped.startsWith("{") || contentStripped.startsWith("[")) {
+          parsedData = JSON.parse(contentStripped);
+          this.telemetry.json_parse_success = true;
+        } else if (contentStripped.includes("```json")) {
+          const start = contentStripped.indexOf("```json") + 7;
+          const end = contentStripped.indexOf("```", start);
+          if (end > start) {
+            parsedData = JSON.parse(contentStripped.slice(start, end).trim());
+            this.telemetry.json_parse_success = true;
+          }
+        }
+      } catch {
+        this.telemetry.json_parse_success = false;
+      }
+    }
+    let validatedResult = null;
+    if (this.schema && parsedData !== null) {
+      const validation = this.schema.validate(parsedData);
+      this.telemetry.schema_validation_pass = validation.success;
+      if (!validation.success) {
+        this.telemetry.status = "schema_violation";
+        if (validation.errors) {
+          this.telemetry.missing_required_fields = JSON.stringify(validation.errors);
+        }
+      }
+      validatedResult = validation.data;
+    }
+    if (this.enumFields && parsedData !== null) {
+      if (!validateEnumBounds(parsedData, this.enumFields)) {
+        this.telemetry.enum_out_of_range = true;
+        this.telemetry.status = "schema_violation";
+      }
+    }
+    if (this.numericBounds && parsedData !== null) {
+      if (!validateNumericBounds(parsedData, this.numericBounds)) {
+        this.telemetry.numeric_out_of_bounds = true;
+        this.telemetry.status = "schema_violation";
+      }
+    }
+    this.send();
+    this.recorded = true;
+    if (this.schema) {
+      return validatedResult;
+    }
+    return response;
+  }
+  recordError(error) {
+    this.endTime = Date.now();
+    this.telemetry.end_time = new Date(this.endTime).toISOString();
+    this.telemetry.total_latency = this.startTime ? this.endTime - this.startTime : 0;
+    this.telemetry.status = "error";
+    this.telemetry.error_message = error.message;
+    const err = error;
+    if (err.status) {
+      this.telemetry.http_status = err.status;
+    }
+    if (err.code) {
+      this.telemetry.provider_error_code = String(err.code);
+    }
+    if (error.message.toLowerCase().includes("timeout")) {
+      this.telemetry.status = "timeout";
+      this.telemetry.timeout = true;
+    }
+    this.send();
+    this.recorded = true;
+  }
+  async send() {
+    if (!this.apiKey) return;
     try {
       const controller = new AbortController();
-      const timeoutId = setTimeout(() => controller.abort(), this.timeout);
-      const response = await fetch(`${this.baseUrl}/heartbeat`, {
+      const timeoutId = setTimeout(() => controller.abort(), this.timeoutMs);
+      const payload = {};
+      for (const [key, value] of Object.entries(this.telemetry)) {
+        if (value !== void 0) {
+          payload[key] = value;
+        }
+      }
+      await fetch(`${this.baseUrl}/prompt`, {
         method: "POST",
         headers: {
           "Content-Type": "application/json",
@@ -48,90 +386,102 @@ var Deadpipe = class {
         signal: controller.signal
       });
       clearTimeout(timeoutId);
-      if (!response.ok) {
-        return null;
-      }
-      return await response.json();
     } catch {
-      return null;
-    }
-  }
-  /**
-   * Run a function with automatic heartbeat on completion.
-   *
-   * @param pipelineId - Unique identifier for this pipeline.
-   * @param fn - The function to run.
-   * @param options - Additional options.
-   * @returns The result of the function.
-   *
-   * @example
-   * const result = await dp.run('daily-etl', async () => {
-   *   const records = await processData();
-   *   return { recordsProcessed: records.length };
-   * });
-   */
-  async run(pipelineId, fn, options = {}) {
-    const startTime = Date.now();
-    let status = "success";
-    let recordsProcessed;
-    try {
-      const result = await fn();
-      if (result && typeof result === "object" && "recordsProcessed" in result) {
-        recordsProcessed = result.recordsProcessed;
-      }
-      return result;
-    } catch (error) {
-      status = "failed";
-      throw error;
-    } finally {
-      const durationMs = Date.now() - startTime;
-      await this.ping(pipelineId, {
-        status,
-        durationMs,
-        recordsProcessed,
-        appName: options.appName
-      });
     }
   }
-  /**
-   * Create a wrapper function that auto-sends heartbeats.
-   *
-   * @param pipelineId - Unique identifier for this pipeline.
-   * @param fn - The function to wrap.
-   * @param options - Additional options.
-   * @returns A wrapped function.
-   *
-   * @example
-   * const myPipeline = dp.wrap('daily-etl', async () => {
-   *   await processData();
-   * });
-   *
-   * // Later...
-   * await myPipeline();
-   */
-  wrap(pipelineId, fn, options = {}) {
-    return async (...args) => {
-      return this.run(pipelineId, () => fn(...args), options);
-    };
+  isRecorded() {
+    return this.recorded;
+  }
+  getTelemetry() {
+    return { ...this.telemetry };
   }
 };
-var defaultClient = null;
-function getDefaultClient() {
-  if (!defaultClient) {
-    defaultClient = new Deadpipe();
+async function track(promptId, fn, options = {}) {
+  const tracker = new PromptTracker(promptId, options);
+  tracker.start();
+  try {
+    const result = await fn(tracker);
+    if (!tracker.isRecorded()) {
+      tracker.recordError(new Error("No response recorded"));
+    }
+    return result;
+  } catch (error) {
+    if (!tracker.isRecorded()) {
+      tracker.recordError(error instanceof Error ? error : new Error(String(error)));
+    }
+    throw error;
   }
-  return defaultClient;
-}
-async function ping(pipelineId, options = {}) {
-  return getDefaultClient().ping(pipelineId, options);
 }
-async function run(pipelineId, fn, options = {}) {
-  return getDefaultClient().run(pipelineId, fn, options);
+function wrapOpenAI(client, options) {
+  const { promptId, ...trackOptions } = options;
+  const wrappedClient = Object.create(client);
+  if (client.chat?.completions) {
+    wrappedClient.chat = {
+      ...client.chat,
+      completions: {
+        ...client.chat.completions,
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        create: async (params) => {
+          const messages = params.messages || [];
+          const tools = params.tools;
+          let systemPrompt;
+          for (const msg of messages) {
+            if (msg.role === "system") {
+              systemPrompt = msg.content || "";
+              break;
+            }
+          }
+          return track(
+            promptId,
+            async (t) => {
+              const response = await client.chat.completions.create(params);
+              t.record(response);
+              return response;
+            },
+            {
+              ...trackOptions,
+              messages,
+              tools,
+              systemPrompt
+            }
+          );
+        }
+      }
+    };
+  }
+  if (client.responses) {
+    wrappedClient.responses = {
+      ...client.responses,
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      create: async (params) => {
+        const inputContent = params.input || "";
+        const messages = typeof inputContent === "string" ? [{ role: "user", content: inputContent }] : inputContent;
+        return track(
+          promptId,
+          async (t) => {
+            const response = await client.responses.create(params);
+            t.record(response);
+            return response;
+          },
+          {
+            ...trackOptions,
+            messages
+          }
+        );
+      }
+    };
+  }
+  return wrappedClient;
 }
-var index_default = Deadpipe;
 export {
-  Deadpipe,
-  index_default as default,
-  ping,
-  run
+  PromptTracker,
+  VERSION,
+  detectRefusal,
+  estimateCost,
+  extractAnthropicResponse,
+  extractOpenAIResponse,
+  track,
+  validateEnumBounds,
+  validateNumericBounds,
+  wrapOpenAI
 };

package/package.json

@@ -1,15 +1,19 @@
 {
   "name": "deadpipe",
-  "version": "1.0.1",
-  "description": "Dead simple pipeline monitoring. Know when your pipelines die.",
+  "version": "2.0.0",
+  "description": "LLM observability that answers: Is this prompt behaving the same as when it was last safe?",
   "keywords": [
-    "monitoring",
-    "pipelines",
-    "etl",
+    "llm",
+    "openai",
+    "anthropic",
+    "gpt",
+    "claude",
     "observability",
-    "heartbeat",
-    "dead-mans-switch",
-    "alerting"
+    "monitoring",
+    "telemetry",
+    "prompt-tracking",
+    "ai",
+    "machine-learning"
   ],
   "homepage": "https://deadpipe.com",
   "bugs": {
@@ -23,9 +27,9 @@
   "type": "commonjs",
   "exports": {
     ".": {
+      "types": "./dist/index.d.ts",
       "import": "./dist/index.mjs",
-      "require": "./dist/index.js",
-      "types": "./dist/index.d.ts"
+      "require": "./dist/index.js"
     }
   },
   "main": "dist/index.js",