@evalguard/langchain 1.0.0

package/dist/index.d.ts

@@ -0,0 +1,231 @@
+/**
+ * EvalGuard instrumentation for LangChain + LangGraph.
+ *
+ * LangChain's instrumentation surface is `@langchain/core/callbacks`. The
+ * recommended integration shape is a class that extends `BaseCallbackHandler`
+ * and overrides `handleLLMStart` / `handleLLMEnd` (and friends). The same
+ * handler works across LangChain.js + LangGraph because LangGraph reuses
+ * LangChain's callback dispatching for node-level events.
+ *
+ * Two usage patterns:
+ *
+ *   // 1. Pass directly to a chain/runnable
+ *   import { ChatOpenAI } from "@langchain/openai";
+ *   import { EvalGuardCallbackHandler } from "@evalguard/langchain";
+ *
+ *   const handler = new EvalGuardCallbackHandler({
+ *     apiKey: process.env.EVALGUARD_API_KEY!,
+ *     projectId: "proj-123",
+ *   });
+ *   const model = new ChatOpenAI({ model: "gpt-4o", callbacks: [handler] });
+ *
+ *   // 2. Or globally via env (auto-installed when present)
+ *   import { autoInstall } from "@evalguard/langchain";
+ *   autoInstall(); // reads EVALGUARD_API_KEY + EVALGUARD_PROJECT_ID
+ *
+ * Implementation notes:
+ *   - We do NOT depend on `@langchain/core` at install time. The handler
+ *     extends a duck-typed `BaseCallbackHandler` shape so the package
+ *     compiles + tests without LangChain present. Customers get full types
+ *     from their own `@langchain/core` install via TypeScript's module
+ *     resolution.
+ *   - Fail-CLOSED semantics match openai-wrapper / anthropic-wrapper /
+ *     llamaindex-wrapper (changed 2026-05-28): when `blockOnViolation` is
+ *     on (the default), an UNREACHABLE guardrail blocks the run rather
+ *     than waving it through. Set `blockOnViolation: false` for
+ *     monitor-only/availability-first deployments.
+ *   - `blockOnViolation` raises `EvalguardBlockedError` from `handleLLMStart`,
+ *     which aborts the run ONLY because this handler sets
+ *     `raiseError = true` and `awaitHandlers = true` (see the class body).
+ *     LangChain swallows handler throws by default. The customer catches
+ *     via `try { await chain.invoke(...) } catch {}`. Verified end-to-end
+ *     against @langchain/core@1.1.48 through `model.invoke`, `model.stream`
+ *     and a `RunnableSequence` in
+ *     `src/__tests__/blocking-enforcement.test.ts` — NOT by inspection.
+ *   - Because `raiseError = true` makes every throw from this class abort
+ *     the customer's run, the split is strict: the guardrail path is the
+ *     ONLY thing allowed to throw. Prompt-extraction failure fails CLOSED
+ *     (an unscannable prompt is rejected, never silently skipped); model-name
+ *     resolution degrades to "unknown"; every logging hook is wrapped in
+ *     `safeLog` so a trace-log failure can never bubble into the chain.
+ *   - Options that cannot be honoured are rejected in the constructor
+ *     (`EvalguardConfigError`) rather than accepted and ignored.
+ *   - **OpenInference compatibility**: every trace emits a `openinference`
+ *     sidecar with the standard `openinference.span.kind`, `llm.input_messages`,
+ *     `llm.output_messages`, `llm.model_name`, `llm.token_count.*` attributes.
+ *     This frees rendering in Phoenix / Arize / any OpenInference-aware viewer.
+ *   - LangGraph: the same handler emits `chain` + `tool` + `agent` events
+ *     so node-level activity in a graph shows up as nested spans, mirroring
+ *     LangSmith's per-node evaluator surface.
+ */
+import { type GuardrailCheckResult } from "./guardrail-client.js";
+/** Minimal LangChain message shape used in chat-model handler args. */
+interface LangChainMessage {
+    content: string | unknown;
+    type?: string;
+    _getType?: () => string;
+}
+interface LangChainSerialized {
+    name?: string;
+    id?: string[];
+    kwargs?: Record<string, unknown>;
+}
+interface LangChainLLMResult {
+    generations: Array<Array<{
+        text?: string;
+        message?: {
+            content?: string;
+        };
+        generationInfo?: Record<string, unknown>;
+    }>>;
+    llmOutput?: {
+        tokenUsage?: {
+            promptTokens?: number;
+            completionTokens?: number;
+            totalTokens?: number;
+        };
+        modelName?: string;
+        model_name?: string;
+        model?: string;
+        [k: string]: unknown;
+    };
+}
+export interface EvalGuardHandlerConfig {
+    apiKey: string;
+    projectId?: string;
+    baseUrl?: string;
+    blockOnViolation?: boolean;
+    disableLogging?: boolean;
+    disableGuardrails?: boolean;
+    metadata?: Record<string, unknown>;
+}
+export declare class EvalguardBlockedError extends Error {
+    readonly violations: GuardrailCheckResult["violations"];
+    constructor(message: string, violations: GuardrailCheckResult["violations"]);
+}
+/**
+ * Thrown from the constructor when the supplied options cannot all be
+ * honoured. A security option that the handler is structurally unable to
+ * enforce must fail loudly at wiring time — silently accepting it is how
+ * a customer ends up believing they are protected when they are not.
+ */
+export declare class EvalguardConfigError extends Error {
+    constructor(message: string);
+}
+/**
+ * The handler keeps state per runId so async overlapping calls are tracked
+ * correctly. LangChain assigns each LLM/chain/tool invocation a unique runId
+ * (UUID) which is passed to every handle* method.
+ */
+export declare class EvalGuardCallbackHandler {
+    readonly name = "evalguard-handler";
+    readonly awaitHandlers = true;
+    readonly raiseError = true;
+    private readonly client;
+    private readonly config;
+    private readonly inflight;
+    private readonly blockOnViolation;
+    private readonly enableLogging;
+    private readonly enableGuardrails;
+    constructor(config: EvalGuardHandlerConfig);
+    /**
+     * Called when an LLM (completion-style) starts. LangChain emits both
+     * `handleLLMStart` and `handleChatModelStart` — we cover both so the
+     * same handler works across `LLM` and `BaseChatModel` subclasses.
+     */
+    handleLLMStart(llm: LangChainSerialized, prompts: string[], runId: string, _parentRunId?: string, extraParams?: Record<string, unknown>): Promise<void>;
+    /**
+     * Called when a chat-style LLM (e.g. ChatOpenAI, ChatAnthropic) starts.
+     * `messages` is `BaseMessage[][]` — one inner array per parallel
+     * generation. We collapse EVERY role into a single prompt for the
+     * guardrail check (A287: user-only collapse hid tool results and
+     * retrieved documents from the firewall).
+     */
+    handleChatModelStart(llm: LangChainSerialized, messages: LangChainMessage[][], runId: string, _parentRunId?: string, extraParams?: Record<string, unknown>): Promise<void>;
+    /**
+     * Called when the LLM finishes (success). Emits a trace with latency,
+     * token usage, cost, and OpenInference-compliant span attributes.
+     */
+    handleLLMEnd(output: LangChainLLMResult, runId: string): Promise<void>;
+    /**
+     * Called when the LLM call errors. We still emit a trace so customers
+     * can see failed calls in the dashboard (latency + error type + model).
+     */
+    handleLLMError(err: unknown, runId: string): Promise<void>;
+    /**
+     * Chain start — covers LangChain Runnables + LangGraph node entry.
+     * We log these as nested spans via OpenInference `chain` kind so the
+     * trace tree mirrors the customer's actual graph topology.
+     */
+    handleChainStart(chain: LangChainSerialized, inputs: Record<string, unknown>, runId: string): Promise<void>;
+    /**
+     * Tool execution start (LangGraph node calling a tool). Treated as its
+     * own span so trajectory-style assertions can recover the tool-call
+     * sequence from the trace.
+     */
+    handleToolStart(tool: LangChainSerialized, input: string, runId: string): Promise<void>;
+    /**
+     * Agent action (LangChain agents + LangGraph). Captures the decided
+     * action so trajectory-grading metrics can score whether the action
+     * was the right next step.
+     */
+    handleAgentAction(action: {
+        tool: string;
+        toolInput: unknown;
+        log?: string;
+    }, runId: string): Promise<void>;
+    /** Shared startup path for both `handleLLMStart` and `handleChatModelStart`. */
+    private startRun;
+    /**
+     * Model name is trace METADATA, not a security input. A failure to
+     * resolve it must never abort the customer's run and must never skip the
+     * firewall — degrade to "unknown" and carry on to the scan.
+     */
+    private safeResolveModel;
+    /**
+     * The prompt could not be read, so the firewall cannot see the bytes that
+     * are about to reach the model. Silently returning here (what this handler
+     * did until 2026-07-30) skipped the scan entirely and let the call through
+     * unguarded — a total bypass for any customer using a message class whose
+     * `_getType()`/content accessor throws.
+     *
+     * Fail CLOSED instead, with an explicit `prompt_unreadable` marker so the
+     * trace records that nothing was scanned. Note this is a REJECTION, not a
+     * shortened/partial scan: never hand the scanner less than the real input.
+     */
+    private onUnreadablePrompt;
+    /**
+     * Fire-and-forget trace logging that can NEVER throw into the customer's
+     * chain. This class sets `raiseError = true` so LangChain propagates our
+     * throws — that is required for a guardrail block to actually block, but
+     * it also means an incidental failure in a logging-only hook would abort a
+     * perfectly good LLM call. The published contract (README, "Outage
+     * semantics") is that a trace-log failure never bubbles into your chain;
+     * this is where that contract is enforced.
+     *
+     * `GuardrailClient.logTrace` already resolves to null on transport failure
+     * (wrapper-core `withRetry`), so this guards the synchronous payload
+     * construction and any future rejection path.
+     */
+    private safeLog;
+    private resolveModel;
+}
+/**
+ * Auto-install path for the lazy customer. Reads EVALGUARD_API_KEY from env.
+ * Returns the handler instance if installed; null if EVALGUARD_API_KEY is
+ * not set (no-op safe to call at module load).
+ *
+ * Usage:
+ *
+ *   import { autoInstall } from "@evalguard/langchain";
+ *   autoInstall();
+ *
+ * Then pass the returned handler to a Runnable's `callbacks` array, OR set
+ * the global LangChain callbacks via `setGlobalCallbacks([handler])` from
+ * `@langchain/core/callbacks/manager`.
+ */
+export declare function autoInstall(): EvalGuardCallbackHandler | null;
+export type { GuardrailCheckResult, GuardrailViolation, TraceLogData } from "./guardrail-client.js";
+export { estimateCost, estimateCostDetailed, isModelPriced } from "./cost.js";
+export type { CostEstimate, PricingSource } from "./cost.js";
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2DG;AAEH,OAAO,EAAmB,KAAK,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AASnF,uEAAuE;AACvE,UAAU,gBAAgB;IACxB,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC;IAC1B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,MAAM,CAAC;CACzB;AAED,UAAU,mBAAmB;IAC3B,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAClC;AAED,UAAU,kBAAkB;IAC1B,WAAW,EAAE,KAAK,CAChB,KAAK,CAAC;QACJ,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,OAAO,CAAC,EAAE;YAAE,OAAO,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QAC/B,cAAc,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KAC1C,CAAC,CACH,CAAC;IACF,SAAS,CAAC,EAAE;QACV,UAAU,CAAC,EAAE;YACX,YAAY,CAAC,EAAE,MAAM,CAAC;YACtB,gBAAgB,CAAC,EAAE,MAAM,CAAC;YAC1B,WAAW,CAAC,EAAE,MAAM,CAAC;SACtB,CAAC;QACF,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,CAAC,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC;KACtB,CAAC;CACH;AAED,MAAM,WAAW,sBAAsB;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,cAAc,CAAC,EAAE,OAAO,CAAC;IACzB,iBAAiB,CAAC,EAAE,OAAO,CAAC;IAC5B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,qBAAa,qBAAsB,SAAQ,KAAK;IAC9C,QAAQ,CAAC,UAAU,EAAE,oBAAoB,CAAC,YAAY,CAAC,CAAC;gBAC5C,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,oBAAoB,CAAC,YAAY,CAAC;CAK5E;AAED;;;;;GAKG;AACH,qBAAa,oBAAqB,SAAQ,KAAK;gBACjC,OAAO,EAAE,MAAM;CAI5B;AASD;;;;GAIG;AACH,qBAAa,wBAAwB;IAInC,QAAQ,CAAC,IAAI,uBAAuB;IAyBpC,QAAQ,CAAC,aAAa,QAAQ;IAC9B,QAAQ,CAAC,UAAU,QAAQ;IAE3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAkB;IACzC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAyB;IAChD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAkC;IAC3D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;IAC3C,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAU;IACxC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAU;gBAE/B,MAAM,EAAE,sBAAsB;IAuB1C;;;;OAIG;IACG,cAAc,CAClB,GAAG,EAAE,mBAAmB,EACxB,OAAO,EAAE,MAAM,EAAE,EACjB,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACpC,OAAO,CAAC,IAAI,CAAC;IAWhB;;;;;;OAMG;IACG,oBAAoB,CACxB,GAAG,EAAE,mBAAmB,EACxB,QAAQ,EAAE,gBAAgB,EAAE,EAAE,EAC9B,KAAK,EAAE,MAAM,EACb,YAAY,CAAC,EAAE,MAAM,EACrB,WAAW,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACpC,OAAO,CAAC,IAAI,CAAC;IAWhB;;;OAGG;IACG,YAAY,CAAC,MAAM,EAAE,kBAAkB,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAiD5E;;;OAGG;IACG,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAyChE;;;;OAIG;IACG,gBAAgB,CACpB,KAAK,EAAE,mBAAmB,EAC1B,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;IAwBhB;;;;OAIG;IACG,eAAe,CACnB,IAAI,EAAE,mBAAmB,EACzB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;IAyBhB;;;;OAIG;IACG,iBAAiB,CACrB,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,OAAO,CAAC;QAAC,GAAG,CAAC,EAAE,MAAM,CAAA;KAAE,EAC1D,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,IAAI,CAAC;IA0BhB,gFAAgF;YAClE,QAAQ;IA6CtB;;;;OAIG;IACH,OAAO,CAAC,gBAAgB;IAWxB;;;;;;;;;;OAUG;YACW,kBAAkB;IA6BhC;;;;;;;;;;;;OAYG;IACH,OAAO,CAAC,OAAO;IAUf,OAAO,CAAC,YAAY;CAoBrB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,WAAW,IAAI,wBAAwB,GAAG,IAAI,CAQ7D;AAoFD,YAAY,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AASpG,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAC9E,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC"}