langwatch 0.1.0 → 0.1.3

package/dist/index.d.ts

@@ -1,8 +1,331 @@
 import EventEmitter from 'events';
-import { M as Metadata, C as CollectorRESTParams, S as Span, P as PendingBaseSpan, a as PendingLLMSpan, b as PendingRAGSpan, c as SpanTypes } from './utils-DDcm0z9v.js';
-export { B as BaseSpan, d as ChatMessage, e as ChatRichContent, L as LLMSpan, R as RAGSpan, f as SpanInputOutput, h as captureError, g as convertFromVercelAIMessages } from './utils-DDcm0z9v.js';
+import { AgentAction, AgentFinish } from '@langchain/core/agents';
+import { BaseCallbackHandler } from '@langchain/core/callbacks/base';
+import { DocumentInterface } from '@langchain/core/documents';
+import { Serialized } from '@langchain/core/load/serializable';
+import { BaseMessage } from '@langchain/core/messages';
+import { LLMResult } from '@langchain/core/outputs';
+import { ChainValues } from '@langchain/core/utils/types';
+import { R as RAGChunk, M as Metadata, C as CollectorRESTParams, S as Span, a as RESTEvaluation, P as PendingBaseSpan, b as PendingLLMSpan, c as PendingRAGSpan, d as SpanTypes } from './utils-CFtM8VVg.js';
+export { B as BaseSpan, e as ChatMessage, f as ChatRichContent, L as LLMSpan, g as RAGSpan, h as SpanInputOutput, i as autoconvertTypedValues, j as captureError, k as convertFromVercelAIMessages } from './utils-CFtM8VVg.js';
 import 'ai';
-import 'llm-cost/model_prices_and_context_window.json';
+
+type EvaluatorTypes = keyof Evaluators;
+type Evaluators = {
+    "huggingface/llama_guard": {
+        settings: {
+            policy: string;
+            evaluate: "input" | "output" | "both";
+            model: "cloudflare/thebloke/llamaguard-7b-awq";
+        };
+    };
+    "example/word_count": {
+        settings: Record<string, never>;
+    };
+    "openai/moderation": {
+        settings: {
+            model: "text-moderation-stable" | "text-moderation-latest";
+            categories: {
+                harassment: boolean;
+                harassment_threatening: boolean;
+                hate: boolean;
+                hate_threatening: boolean;
+                self_harm: boolean;
+                self_harm_instructions: boolean;
+                self_harm_intent: boolean;
+                sexual: boolean;
+                sexual_minors: boolean;
+                violence: boolean;
+                violence_graphic: boolean;
+            };
+        };
+    };
+    "haystack/faithfulness": {
+        settings: {
+            model: "openai/gpt-3.5-turbo" | "openai/gpt-3.5-turbo-0125" | "openai/gpt-3.5-turbo-1106" | "openai/gpt-4-turbo" | "openai/gpt-4-0125-preview" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "openai/gpt-4-1106-preview" | "azure/gpt-35-turbo-1106" | "azure/gpt-4o" | "azure/gpt-4-turbo-2024-04-09" | "azure/gpt-4-1106-preview" | "groq/llama3-70b-8192" | "anthropic/claude-3-haiku-20240307" | "anthropic/claude-3-sonnet-20240229" | "anthropic/claude-3-opus-20240229";
+            max_tokens: number;
+        };
+    };
+    "langevals/basic": {
+        settings: {
+            rules: {
+                field: "input" | "output";
+                rule: "contains" | "not_contains" | "matches_regex" | "not_matches_regex";
+                value: string;
+            }[];
+        };
+    };
+    "langevals/competitor_blocklist": {
+        settings: {
+            competitors: string[];
+        };
+    };
+    "langevals/competitor_llm": {
+        settings: {
+            model: "openai/gpt-3.5-turbo" | "openai/gpt-3.5-turbo-0125" | "openai/gpt-3.5-turbo-1106" | "openai/gpt-4-turbo" | "openai/gpt-4-0125-preview" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "openai/gpt-4-1106-preview" | "azure/gpt-35-turbo-1106" | "azure/gpt-4o" | "azure/gpt-4-turbo-2024-04-09" | "azure/gpt-4-1106-preview" | "groq/llama3-70b-8192" | "anthropic/claude-3-haiku-20240307" | "anthropic/claude-3-sonnet-20240229" | "anthropic/claude-3-opus-20240229";
+            max_tokens: number;
+            name: string;
+            description: string;
+        };
+    };
+    "langevals/competitor_llm_function_call": {
+        settings: {
+            model: "openai/gpt-3.5-turbo" | "openai/gpt-3.5-turbo-0125" | "openai/gpt-3.5-turbo-1106" | "openai/gpt-4-turbo" | "openai/gpt-4-0125-preview" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "openai/gpt-4-1106-preview" | "azure/gpt-35-turbo-1106" | "azure/gpt-4o" | "azure/gpt-4-turbo-2024-04-09" | "azure/gpt-4-1106-preview" | "groq/llama3-70b-8192" | "anthropic/claude-3-haiku-20240307" | "anthropic/claude-3-sonnet-20240229" | "anthropic/claude-3-opus-20240229";
+            max_tokens: number;
+            name: string;
+            description: string;
+            competitors: string[];
+        };
+    };
+    "langevals/llm_boolean": {
+        settings: {
+            model: "openai/gpt-3.5-turbo" | "openai/gpt-3.5-turbo-0125" | "openai/gpt-3.5-turbo-1106" | "openai/gpt-4-turbo" | "openai/gpt-4-0125-preview" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "openai/gpt-4-1106-preview" | "azure/gpt-35-turbo-1106" | "azure/gpt-4o" | "azure/gpt-4-turbo-2024-04-09" | "azure/gpt-4-1106-preview" | "groq/llama3-70b-8192" | "anthropic/claude-3-haiku-20240307" | "anthropic/claude-3-sonnet-20240229" | "anthropic/claude-3-opus-20240229";
+            max_tokens: number;
+            prompt: string;
+        };
+    };
+    "langevals/llm_score": {
+        settings: {
+            model: "openai/gpt-3.5-turbo" | "openai/gpt-3.5-turbo-0125" | "openai/gpt-3.5-turbo-1106" | "openai/gpt-4-turbo" | "openai/gpt-4-0125-preview" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "openai/gpt-4-1106-preview" | "azure/gpt-35-turbo-1106" | "azure/gpt-4o" | "azure/gpt-4-turbo-2024-04-09" | "azure/gpt-4-1106-preview" | "groq/llama3-70b-8192" | "anthropic/claude-3-haiku-20240307" | "anthropic/claude-3-sonnet-20240229" | "anthropic/claude-3-opus-20240229";
+            max_tokens: number;
+            prompt: string;
+        };
+    };
+    "langevals/off_topic": {
+        settings: {
+            model: "openai/gpt-3.5-turbo" | "openai/gpt-3.5-turbo-0125" | "openai/gpt-3.5-turbo-1106" | "openai/gpt-4-turbo" | "openai/gpt-4-0125-preview" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "openai/gpt-4-1106-preview" | "azure/gpt-35-turbo-1106" | "azure/gpt-4o" | "azure/gpt-4-turbo-2024-04-09" | "azure/gpt-4-1106-preview" | "groq/llama3-70b-8192" | "anthropic/claude-3-haiku-20240307" | "anthropic/claude-3-sonnet-20240229" | "anthropic/claude-3-opus-20240229";
+            max_tokens: number;
+            allowed_topics: {
+                topic: string;
+                description: string;
+            }[];
+        };
+    };
+    "langevals/product_sentiment_polarity": {
+        settings: Record<string, never>;
+    };
+    "langevals/query_resolution": {
+        settings: {
+            model: "openai/gpt-3.5-turbo" | "openai/gpt-3.5-turbo-0125" | "openai/gpt-3.5-turbo-1106" | "openai/gpt-4-turbo" | "openai/gpt-4-0125-preview" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "openai/gpt-4-1106-preview" | "azure/gpt-35-turbo-1106" | "azure/gpt-4o" | "azure/gpt-4-turbo-2024-04-09" | "azure/gpt-4-1106-preview" | "groq/llama3-70b-8192" | "anthropic/claude-3-haiku-20240307" | "anthropic/claude-3-sonnet-20240229" | "anthropic/claude-3-opus-20240229";
+            max_tokens: number;
+        };
+    };
+    "langevals/similarity": {
+        settings: {
+            field: "input" | "output";
+            rule: "is_not_similar_to" | "is_similar_to";
+            value: string;
+            threshold: number;
+            embeddings_model: "openai/text-embedding-3-small" | "azure/text-embedding-ada-002";
+        };
+    };
+    "lingua/language_detection": {
+        settings: {
+            check_for: "input_matches_output" | "output_matches_language";
+            expected_language?: "AF" | "AR" | "AZ" | "BE" | "BG" | "BN" | "BS" | "CA" | "CS" | "CY" | "DA" | "DE" | "EL" | "EN" | "EO" | "ES" | "ET" | "EU" | "FA" | "FI" | "FR" | "GA" | "GU" | "HE" | "HI" | "HR" | "HU" | "HY" | "ID" | "IS" | "IT" | "JA" | "KA" | "KK" | "KO" | "LA" | "LG" | "LT" | "LV" | "MI" | "MK" | "MN" | "MR" | "MS" | "NB" | "NL" | "NN" | "PA" | "PL" | "PT" | "RO" | "RU" | "SK" | "SL" | "SN" | "SO" | "SQ" | "SR" | "ST" | "SV" | "SW" | "TA" | "TE" | "TH" | "TL" | "TN" | "TR" | "TS" | "UK" | "UR" | "VI" | "XH" | "YO" | "ZH" | "ZU";
+            min_words: number;
+            threshold: number;
+        };
+    };
+    "aws/comprehend_pii_detection": {
+        settings: {
+            entity_types: {
+                BANK_ACCOUNT_NUMBER: boolean;
+                BANK_ROUTING: boolean;
+                CREDIT_DEBIT_NUMBER: boolean;
+                CREDIT_DEBIT_CVV: boolean;
+                CREDIT_DEBIT_EXPIRY: boolean;
+                PIN: boolean;
+                EMAIL: boolean;
+                ADDRESS: boolean;
+                NAME: boolean;
+                PHONE: boolean;
+                SSN: boolean;
+                DATE_TIME: boolean;
+                PASSPORT_NUMBER: boolean;
+                DRIVER_ID: boolean;
+                URL: boolean;
+                AGE: boolean;
+                USERNAME: boolean;
+                PASSWORD: boolean;
+                AWS_ACCESS_KEY: boolean;
+                AWS_SECRET_KEY: boolean;
+                IP_ADDRESS: boolean;
+                MAC_ADDRESS: boolean;
+                LICENSE_PLATE: boolean;
+                VEHICLE_IDENTIFICATION_NUMBER: boolean;
+                UK_NATIONAL_INSURANCE_NUMBER: boolean;
+                CA_SOCIAL_INSURANCE_NUMBER: boolean;
+                US_INDIVIDUAL_TAX_IDENTIFICATION_NUMBER: boolean;
+                UK_UNIQUE_TAXPAYER_REFERENCE_NUMBER: boolean;
+                IN_PERMANENT_ACCOUNT_NUMBER: boolean;
+                IN_NREGA: boolean;
+                INTERNATIONAL_BANK_ACCOUNT_NUMBER: boolean;
+                SWIFT_CODE: boolean;
+                UK_NATIONAL_HEALTH_SERVICE_NUMBER: boolean;
+                CA_HEALTH_NUMBER: boolean;
+                IN_AADHAAR: boolean;
+                IN_VOTER_NUMBER: boolean;
+            };
+            language_code: "en" | "es" | "fr" | "de" | "it" | "pt" | "ar" | "hi" | "ja" | "ko" | "zh" | "zh-TW";
+            min_confidence: number;
+            aws_region: "us-east-1" | "us-east-2" | "us-west-1" | "us-west-2" | "ap-east-1" | "ap-south-1" | "ap-northeast-3" | "ap-northeast-2" | "ap-southeast-1" | "ap-southeast-2" | "ap-northeast-1" | "ca-central-1" | "eu-central-1" | "eu-west-1" | "eu-west-2" | "eu-south-1" | "eu-west-3" | "eu-north-1" | "me-south-1" | "sa-east-1";
+        };
+    };
+    "google_cloud/dlp_pii_detection": {
+        settings: {
+            info_types: {
+                phone_number: boolean;
+                email_address: boolean;
+                credit_card_number: boolean;
+                iban_code: boolean;
+                ip_address: boolean;
+                passport: boolean;
+                vat_number: boolean;
+                medical_record_number: boolean;
+            };
+            min_likelihood: "VERY_UNLIKELY" | "UNLIKELY" | "POSSIBLE" | "LIKELY" | "VERY_LIKELY";
+        };
+    };
+    "azure/content_safety": {
+        settings: {
+            severity_threshold: 1 | 2 | 3 | 4 | 5 | 6 | 7;
+            categories: {
+                Hate: boolean;
+                SelfHarm: boolean;
+                Sexual: boolean;
+                Violence: boolean;
+            };
+            output_type: "FourSeverityLevels" | "EightSeverityLevels";
+        };
+    };
+    "azure/jailbreak": {
+        settings: Record<string, never>;
+    };
+    "azure/prompt_injection": {
+        settings: Record<string, never>;
+    };
+    "ragas/answer_correctness": {
+        settings: {
+            model: "openai/gpt-3.5-turbo-16k" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "azure/gpt-35-turbo-16k" | "azure/gpt-4o" | "anthropic/claude-3-5-sonnet-20240620";
+            embeddings_model: "openai/text-embedding-ada-002" | "azure/text-embedding-ada-002";
+            max_tokens: number;
+        };
+    };
+    "ragas/answer_relevancy": {
+        settings: {
+            model: "openai/gpt-3.5-turbo-16k" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "azure/gpt-35-turbo-16k" | "azure/gpt-4o" | "anthropic/claude-3-5-sonnet-20240620";
+            embeddings_model: "openai/text-embedding-ada-002" | "azure/text-embedding-ada-002";
+            max_tokens: number;
+        };
+    };
+    "ragas/context_precision": {
+        settings: {
+            model: "openai/gpt-3.5-turbo-16k" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "azure/gpt-35-turbo-16k" | "azure/gpt-4o" | "anthropic/claude-3-5-sonnet-20240620";
+            embeddings_model: "openai/text-embedding-ada-002" | "azure/text-embedding-ada-002";
+            max_tokens: number;
+        };
+    };
+    "ragas/context_recall": {
+        settings: {
+            model: "openai/gpt-3.5-turbo-16k" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "azure/gpt-35-turbo-16k" | "azure/gpt-4o" | "anthropic/claude-3-5-sonnet-20240620";
+            embeddings_model: "openai/text-embedding-ada-002" | "azure/text-embedding-ada-002";
+            max_tokens: number;
+        };
+    };
+    "ragas/context_relevancy": {
+        settings: {
+            model: "openai/gpt-3.5-turbo-16k" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "azure/gpt-35-turbo-16k" | "azure/gpt-4o" | "anthropic/claude-3-5-sonnet-20240620";
+            embeddings_model: "openai/text-embedding-ada-002" | "azure/text-embedding-ada-002";
+            max_tokens: number;
+        };
+    };
+    "ragas/context_utilization": {
+        settings: {
+            model: "openai/gpt-3.5-turbo-16k" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "azure/gpt-35-turbo-16k" | "azure/gpt-4o" | "anthropic/claude-3-5-sonnet-20240620";
+            embeddings_model: "openai/text-embedding-ada-002" | "azure/text-embedding-ada-002";
+            max_tokens: number;
+        };
+    };
+    "ragas/faithfulness": {
+        settings: {
+            model: "openai/gpt-3.5-turbo-16k" | "openai/gpt-4o" | "openai/gpt-4o-mini" | "azure/gpt-35-turbo-16k" | "azure/gpt-4o" | "anthropic/claude-3-5-sonnet-20240620";
+            embeddings_model: "openai/text-embedding-ada-002" | "azure/text-embedding-ada-002";
+            max_tokens: number;
+        };
+    };
+};
+
+type Conversation = {
+    input?: string;
+    output?: string;
+}[];
+
+type Money = {
+    currency: string;
+    amount: number;
+};
+type EvaluationResultModel = {
+    status: "processed" | "skipped" | "error";
+    passed?: boolean;
+    score?: number;
+    details?: string;
+    label?: string;
+    cost?: Money;
+};
+type CommonEvaluationParams = {
+    name?: string;
+    input?: string;
+    output?: string;
+    expectedOutput?: string;
+    contexts?: RAGChunk[] | string[];
+    conversation?: Conversation;
+    asGuardrail?: boolean;
+    trace?: LangWatchTrace;
+    span?: LangWatchSpan;
+};
+type SavedEvaluationParams = {
+    slug: string;
+    settings?: Record<string, unknown>;
+} & CommonEvaluationParams;
+type LangEvalsEvaluationParams<T extends EvaluatorTypes> = {
+    evaluator: T;
+    settings?: Evaluators[T]["settings"];
+} & CommonEvaluationParams;
+type EvaluationParams = SavedEvaluationParams | LangEvalsEvaluationParams<EvaluatorTypes>;
+
+declare class LangWatchCallbackHandler extends BaseCallbackHandler {
+    name: string;
+    trace: LangWatchTrace;
+    spans: Record<string, LangWatchSpan>;
+    constructor({ trace }: {
+        trace: LangWatchTrace;
+    });
+    handleLLMStart(llm: Serialized, prompts: string[], runId: string, parentRunId?: string | undefined, extraParams?: Record<string, unknown> | undefined, _tags?: string[] | undefined, metadata?: Record<string, unknown> | undefined, name?: string): Promise<void>;
+    private buildLLMSpan;
+    handleChatModelStart(llm: Serialized, messages: BaseMessage[][], runId: string, parentRunId?: string | undefined, extraParams?: Record<string, unknown> | undefined, tags?: string[] | undefined, metadata?: Record<string, unknown> | undefined, name?: string): Promise<void>;
+    handleNewToken(_token: string, runId: string): Promise<void>;
+    handleLLMEnd(response: LLMResult, runId: string, _parentRunId?: string | undefined): Promise<void>;
+    handleLLMError(err: Error, runId: string, _parentRunId?: string | undefined): Promise<void>;
+    handleChainStart(chain: Serialized, inputs: ChainValues, runId: string, parentRunId?: string | undefined, _tags?: string[] | undefined, _metadata?: Record<string, unknown> | undefined, _runType?: string, name?: string): Promise<void>;
+    handleChainEnd(output: ChainValues, runId: string, _parentRunId?: string | undefined): Promise<void>;
+    handleChainError(err: Error, runId: string, _parentRunId?: string | undefined, _tags?: string[] | undefined, _kwargs?: {
+        inputs?: Record<string, unknown> | undefined;
+    } | undefined): Promise<void>;
+    handleToolStart(tool: Serialized, input: string, runId: string, parentRunId?: string | undefined, _tags?: string[] | undefined, _metadata?: Record<string, unknown> | undefined, name?: string): Promise<void>;
+    handleToolEnd(output: string, runId: string, _parentRunId?: string | undefined): Promise<void>;
+    handleToolError(err: Error, runId: string, _parentRunId?: string | undefined, _tags?: string[] | undefined): Promise<void>;
+    handleRetrieverStart(retriever: Serialized, query: string, runId: string, parentRunId?: string | undefined, _tags?: string[] | undefined, _metadata?: Record<string, unknown> | undefined, name?: string | undefined): Promise<void>;
+    handleRetrieverEnd(documents: DocumentInterface<Record<string, any>>[], runId: string, _parentRunId?: string | undefined, _tags?: string[] | undefined): Promise<void>;
+    handleRetrieverError(err: Error, runId: string, _parentRunId?: string | undefined, _tags?: string[] | undefined): Promise<void>;
+    handleAgentAction(_action: AgentAction, runId: string, _parentRunId?: string | undefined, _tags?: string[] | undefined): Promise<void>;
+    handleAgentEnd(action: AgentFinish, runId: string, _parentRunId?: string | undefined, _tags?: string[] | undefined): Promise<void>;
+    private buildSpan;
+    private endSpan;
+    private errorSpan;
+    private autoconvertTypedValues;
+    private getParent;
+}
 
 declare class LangWatch extends EventEmitter {
     apiKey: string | undefined;
@@ -18,12 +341,29 @@ declare class LangWatch extends EventEmitter {
     sendTrace(params: CollectorRESTParams): Promise<void>;
     _sendTrace(params: CollectorRESTParams): Promise<void>;
 }
+type AddEvaluationParams = {
+    evaluationId?: string;
+    span?: LangWatchSpan;
+    name: string;
+    type?: string;
+    isGuardrail?: boolean;
+    status?: "processed" | "skipped" | "error";
+    passed?: boolean;
+    score?: number;
+    label?: string;
+    details?: string;
+    error?: Error;
+    timestamps?: RESTEvaluation["timestamps"];
+};
 declare class LangWatchTrace {
     client: LangWatch;
     traceId: string;
     metadata?: Metadata;
     finishedSpans: Record<string, Span>;
-    timeoutRef?: NodeJS.Timeout;
+    langchainCallback?: LangWatchCallbackHandler;
+    evaluations: RESTEvaluation[];
+    private currentSpan?;
+    private timeoutRef?;
     constructor({ client, traceId, metadata, }: {
         client: LangWatch;
         traceId: string;
@@ -32,9 +372,15 @@ declare class LangWatchTrace {
     update({ metadata }: {
         metadata: Metadata;
     }): void;
+    setCurrentSpan(span: LangWatchSpan): void;
+    getCurrentSpan(): LangWatchSpan | undefined;
+    resetCurrentSpan(): void;
     startSpan(params: Omit<Partial<PendingBaseSpan>, "parentId">): LangWatchSpan;
     startLLMSpan(params: Omit<Partial<PendingLLMSpan>, "parentId">): LangWatchLLMSpan;
     startRAGSpan(params: Omit<Partial<PendingRAGSpan>, "parentId">): LangWatchRAGSpan;
+    addEvaluation: ({ evaluationId, span, name, type, isGuardrail, status, passed, score, label, details, error, timestamps, }: AddEvaluationParams) => void;
+    evaluate(params: EvaluationParams): Promise<EvaluationResultModel>;
+    getLangChainCallback(): LangWatchCallbackHandler;
     onEnd(span: Span): void;
     delayedSendSpans(): void;
     sendSpans(): Promise<void>;
@@ -57,6 +403,8 @@ declare class LangWatchSpan implements PendingBaseSpan {
     startSpan(params: Omit<Partial<PendingBaseSpan>, "parentId">): LangWatchSpan;
     startLLMSpan(params: Omit<Partial<PendingLLMSpan>, "parentId">): LangWatchLLMSpan;
     startRAGSpan(params: Omit<Partial<PendingRAGSpan>, "parentId">): LangWatchRAGSpan;
+    addEvaluation(params: AddEvaluationParams): void;
+    evaluate(params: EvaluationParams): Promise<EvaluationResultModel>;
     end(params?: Partial<Omit<PendingBaseSpan, "spanId" | "parentId">>): void;
 }
 declare class LangWatchLLMSpan extends LangWatchSpan implements PendingLLMSpan {