@juspay/neurolink 12.17.3 → 12.18.0

package/dist/proxy/vertexAnthropicFallback.js

@@ -0,0 +1,339 @@
+/**
+ * Native Claude-to-Vertex passthrough.
+ *
+ * Claude on Vertex speaks the Anthropic Messages API, so a fallback to it needs
+ * no translation at all — the client's body goes upstream essentially verbatim
+ * and the SSE comes back in the shape the client already expects.
+ *
+ * This exists because the SDK/translation path cannot carry an agentic turn:
+ * it runs its own tool loop, and a request whose tools have no executor (every
+ * proxied request, since the caller executes them) ends with the pending tool
+ * call discarded and a step-cap apology returned as assistant text. Tools,
+ * `tool_use`, `tool_result`, `cache_control` and thinking blocks all survive
+ * here because nothing reshapes them.
+ */
+import { GoogleAuth } from "google-auth-library";
+/** Vertex pins the Anthropic API version in the body, not a header. */
+const VERTEX_ANTHROPIC_VERSION = "vertex-2023-10-16";
+const CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform";
+/**
+ * Fields the Anthropic first-party API accepts that Vertex's publisher endpoint
+ * does not.
+ *
+ * Deliberately short. `model` moves into the URL, and `context_management` is
+ * refused outright ("context_management: Extra inputs are not permitted").
+ * Everything else the client sends is carried, including `metadata` and
+ * `thinking`, both of which Vertex accepts — dropping them would silently
+ * discard caller intent.
+ */
+const UNSUPPORTED_FIELDS = ["model", "context_management"];
+/**
+ * Reasoning effort levels Vertex's Claude models accept, and how a level they
+ * do not accept maps onto one they do.
+ *
+ * Claude Code sends `xhigh` for the Claude 5 family; `claude-opus-4-6` refuses
+ * it by name ("Supported levels: high, low, max, medium"). Mapping it to the
+ * top supported level preserves the caller's intent — run this turn as hard as
+ * the model allows — where dropping the field would quietly downgrade it.
+ */
+const VERTEX_EFFORT_BY_LEVEL = {
+    low: "low",
+    medium: "medium",
+    high: "high",
+    max: "max",
+    xhigh: "max",
+};
+function record(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+/**
+ * Translate `output_config` for Vertex.
+ *
+ * An effort level Vertex knows passes through; one it does not is mapped onto
+ * the nearest level it does. A level with no mapping at all loses only the
+ * `effort` key, never the sibling settings beside it.
+ */
+export function mapVertexOutputConfig(value) {
+    if (!record(value)) {
+        return undefined;
+    }
+    const effort = value.effort;
+    if (typeof effort !== "string") {
+        return value;
+    }
+    const mapped = VERTEX_EFFORT_BY_LEVEL[effort.toLowerCase()];
+    if (mapped !== undefined) {
+        return { ...value, effort: mapped };
+    }
+    const { effort: _unsupported, ...rest } = value;
+    return Object.keys(rest).length > 0 ? rest : undefined;
+}
+/** Strip what Vertex rejects and pin the API version it requires. */
+export function buildVertexAnthropicPayload(body) {
+    const dropped = new Set(UNSUPPORTED_FIELDS);
+    const payload = Object.fromEntries(Object.entries(body).filter(([key]) => !dropped.has(key)));
+    if ("output_config" in payload) {
+        const mapped = mapVertexOutputConfig(payload.output_config);
+        if (mapped === undefined) {
+            delete payload.output_config;
+        }
+        else {
+            payload.output_config = mapped;
+        }
+    }
+    return { ...payload, anthropic_version: VERTEX_ANTHROPIC_VERSION };
+}
+/**
+ * Publisher endpoint for a Claude model on Vertex.
+ *
+ * Streaming and non-streaming are different verbs on the same resource, so the
+ * caller's `stream` flag selects the verb rather than changing the body.
+ */
+export function buildVertexAnthropicUrl(target) {
+    const verb = target.stream ? "streamRawPredict" : "rawPredict";
+    // The global endpoint is not a region: it drops the host prefix entirely,
+    // while the path still carries `locations/global`. Prefixing it yields
+    // `global-aiplatform.googleapis.com`, which does not resolve to an API.
+    const host = target.location === "global"
+        ? "aiplatform.googleapis.com"
+        : `${target.location}-aiplatform.googleapis.com`;
+    return (`https://${host}/v1/projects/${target.projectId}/locations/` +
+        `${target.location}/publishers/anthropic/models/${target.model}:${verb}`);
+}
+let cachedAuth;
+function auth() {
+    cachedAuth ??= new GoogleAuth({ scopes: [CLOUD_PLATFORM_SCOPE] });
+    return cachedAuth;
+}
+/**
+ * Send one Anthropic-shaped request to Vertex and return the upstream response
+ * untouched, so the caller can stream its bytes straight to the client.
+ */
+export async function dispatchVertexAnthropicPassthrough(request) {
+    const token = await auth().getAccessToken();
+    if (!token) {
+        throw new Error("Vertex passthrough could not obtain a Google access token; " +
+            "check Application Default Credentials for this project");
+    }
+    const stream = request.body.stream === true;
+    const url = buildVertexAnthropicUrl({
+        projectId: request.projectId,
+        location: request.location,
+        model: request.model,
+        stream,
+    });
+    return fetch(url, {
+        method: "POST",
+        headers: {
+            authorization: `Bearer ${token}`,
+            "content-type": "application/json",
+            accept: stream ? "text/event-stream" : "application/json",
+        },
+        body: JSON.stringify(buildVertexAnthropicPayload(request.body)),
+        ...(request.signal ? { signal: request.signal } : {}),
+    });
+}
+/** Claude models are the only publisher models this passthrough addresses. */
+export function isVertexAnthropicModel(model) {
+    return model.toLowerCase().startsWith("claude");
+}
+/**
+ * Resolve where Vertex traffic is addressed, using the same environment the
+ * Vertex provider reads so a fallback lands on the project and region the rest
+ * of the process already uses.
+ */
+export function resolveVertexTarget() {
+    const projectId = process.env.GOOGLE_CLOUD_PROJECT ??
+        process.env.VERTEX_PROJECT_ID ??
+        process.env.GOOGLE_VERTEX_PROJECT ??
+        process.env.GOOGLE_CLOUD_PROJECT_ID;
+    if (!projectId?.trim()) {
+        return undefined;
+    }
+    const location = process.env.GOOGLE_CLOUD_LOCATION ??
+        process.env.VERTEX_LOCATION ??
+        process.env.GOOGLE_VERTEX_LOCATION ??
+        "us-central1";
+    return { projectId: projectId.trim(), location };
+}
+/**
+ * Run one Claude-on-Vertex fallback attempt and hand back a response the route
+ * can serve directly.
+ *
+ * A non-2xx upstream throws so the caller's attempt loop records the failure
+ * and moves to the next hop, exactly as the other providers behave.
+ */
+export async function executeVertexAnthropicFallback(args) {
+    const target = resolveVertexTarget();
+    if (!target) {
+        throw new Error("Vertex fallback needs a Google Cloud project; set GOOGLE_CLOUD_PROJECT");
+    }
+    const upstream = await dispatchVertexAnthropicPassthrough({
+        body: args.body,
+        projectId: target.projectId,
+        location: target.location,
+        model: args.model,
+        ...(args.signal ? { signal: args.signal } : {}),
+    });
+    if (!upstream.ok) {
+        const detail = await upstream.text().catch(() => "");
+        throw new Error(`Vertex ${args.model} responded ${upstream.status}: ${detail.slice(0, 300)}`);
+    }
+    // A non-streaming reply is one JSON object with no `data:` lines, so the SSE
+    // observer would read nothing from it and report all-zero usage as if it were
+    // real. Parse that shape directly instead, and still hand the client the exact
+    // bytes Vertex sent.
+    if (args.onTerminal && args.body.stream !== true) {
+        const text = await upstream.text();
+        args.onTerminal({ usage: readJsonUsage(text), status: upstream.status });
+        return new Response(text, {
+            status: upstream.status,
+            headers: {
+                "content-type": upstream.headers.get("content-type") ?? "application/json",
+            },
+        });
+    }
+    // The body is already Anthropic-shaped, so it is forwarded untouched rather
+    // than parsed and re-serialized. Usage is read from a copy of each chunk so
+    // this hop is metered like every other one instead of going dark.
+    const body = upstream.body && args.onTerminal
+        ? observeVertexUsage(upstream.body, args.onTerminal)
+        : upstream.body;
+    return new Response(body, {
+        status: upstream.status,
+        headers: {
+            "content-type": upstream.headers.get("content-type") ??
+                (args.body.stream === true ? "text/event-stream" : "application/json"),
+        },
+    });
+}
+/**
+ * Read token usage out of an Anthropic SSE stream.
+ *
+ * `message_start` carries the input side and the cache breakdown;
+ * `message_delta` carries the running output count, the last of which is the
+ * turn's total. Both are accumulated so a passthrough hop reports the same
+ * usage as any other, rather than going unmetered.
+ */
+function readUsageEvent(event, into) {
+    const message = event.message;
+    const usage = (event.type === "message_start" && record(message)
+        ? message.usage
+        : event.usage);
+    if (!record(usage)) {
+        return;
+    }
+    const num = (value) => typeof value === "number" && Number.isFinite(value) ? value : 0;
+    if (event.type === "message_start") {
+        into.inputTokens = num(usage.input_tokens);
+        into.cacheCreationTokens = num(usage.cache_creation_input_tokens);
+        into.cacheReadTokens = num(usage.cache_read_input_tokens);
+    }
+    // Every message_delta restates the running total, so the last one wins.
+    const output = num(usage.output_tokens);
+    if (output > 0) {
+        into.outputTokens = output;
+    }
+}
+/**
+ * Read usage out of a non-streaming Anthropic reply.
+ *
+ * `rawPredict` returns a single `{"type":"message","usage":{...}}` object, which
+ * carries the whole turn's counts at once rather than splitting them across
+ * `message_start` and `message_delta`.
+ */
+export function readJsonUsage(text) {
+    const usage = {
+        inputTokens: 0,
+        outputTokens: 0,
+        cacheCreationTokens: 0,
+        cacheReadTokens: 0,
+    };
+    try {
+        const parsed = JSON.parse(text);
+        readUsageEvent({ ...parsed, type: "message_start", message: parsed }, usage);
+        // A single object states the final output count directly.
+        const raw = record(parsed.usage) ? parsed.usage.output_tokens : undefined;
+        if (typeof raw === "number" && Number.isFinite(raw)) {
+            usage.outputTokens = raw;
+        }
+    }
+    catch {
+        // An unparseable body is reported as zero rather than guessed at.
+    }
+    return usage;
+}
+/**
+ * Forward a stream unchanged while observing its usage events.
+ *
+ * The bytes are passed through untouched — parsing happens on a copy of each
+ * chunk — so the client sees exactly what Vertex sent.
+ *
+ * A transform's `flush` only runs on a clean end, which would leave a stream
+ * error or a client disconnect with no terminal record at all. An explicit
+ * reader loop is used instead, so `onTerminal` fires exactly once on every
+ * exit: normal end, upstream failure, and cancellation.
+ */
+export function observeVertexUsage(body, onTerminal) {
+    const usage = {
+        inputTokens: 0,
+        outputTokens: 0,
+        cacheCreationTokens: 0,
+        cacheReadTokens: 0,
+    };
+    const decoder = new TextDecoder();
+    const reader = body.getReader();
+    let pending = "";
+    let reported = false;
+    const report = (status, errorMessage) => {
+        if (reported) {
+            return;
+        }
+        reported = true;
+        onTerminal({ usage, status, ...(errorMessage ? { errorMessage } : {}) });
+    };
+    const consume = (chunk) => {
+        pending += decoder.decode(chunk, { stream: true });
+        const lines = pending.split("\n");
+        // Keep the trailing fragment; it completes on a later chunk.
+        pending = lines.pop() ?? "";
+        for (const line of lines) {
+            if (!line.startsWith("data:")) {
+                continue;
+            }
+            const raw = line.slice(5).trim();
+            if (!raw || raw === "[DONE]") {
+                continue;
+            }
+            try {
+                readUsageEvent(JSON.parse(raw), usage);
+            }
+            catch {
+                // A partial or non-JSON payload is never fatal to the passthrough.
+            }
+        }
+    };
+    return new ReadableStream({
+        async pull(controller) {
+            try {
+                const { done, value } = await reader.read();
+                if (done) {
+                    report(200);
+                    controller.close();
+                    return;
+                }
+                consume(value);
+                controller.enqueue(value);
+            }
+            catch (error) {
+                report(502, error instanceof Error ? error.message : String(error));
+                controller.error(error);
+            }
+        },
+        cancel(reason) {
+            // The client went away mid-stream; tokens already spent still count.
+            report(499, reason === undefined ? undefined : String(reason));
+            return reader.cancel(reason);
+        },
+    });
+}

package/dist/server/routes/claudeProxyRoutes.js

@@ -28,6 +28,7 @@ import { buildProxyLimitHeaders, summarizePoolHeadroom, } from "../../proxy/quot
 import { buildClaudeError, ClaudeStreamSerializer, parseClaudeRequest, serializeClaudeResponse, } from "../../proxy/claudeFormat.js";
 import { getProxyUpstreamFailure } from "../../proxy/proxyFailureDetails.js";
 import { CodexFallbackResponseError, CodexFallbackStreamError, consumeCodexFallbackResponse, createCodexFallbackStream, convertClaudeRequestToCodex, } from "../../proxy/codexFallback.js";
+import { executeVertexAnthropicFallback, isVertexAnthropicModel, } from "../../proxy/vertexAnthropicFallback.js";
 import { isProxyRequestFinalized, getProxyRequestAccounting, registerProxyResponseObserver, takeProxyResponseObservers, trackProxyResponse, } from "../../proxy/proxyActivity.js";
 import { buildAnthropicModelsListResponse, buildTranslationOptions, handleTranslatedJsonRequest, handleTranslatedStreamRequest, } from "../../proxy/proxyTranslationEngine.js";
 import { tracers } from "../../telemetry/tracers.js";
@@ -4046,6 +4047,44 @@ async function tryConfiguredClaudeFallbackChain(args) {
                     logFinalRequest,
                 });
             }
+            else if (fallback.provider === "vertex" &&
+                isVertexAnthropicModel(fallback.model)) {
+                // Claude on Vertex speaks the Anthropic Messages API, so the request
+                // goes up as-is. The SDK path below runs its own tool loop and drops
+                // the caller's pending tool call, which strands every agentic turn.
+                const vertexModel = fallback.model;
+                const vertexAccount = `vertex/${vertexModel}`;
+                response = await executeVertexAnthropicFallback({
+                    body,
+                    model: vertexModel,
+                    ...(ctx.abortSignal ? { signal: ctx.abortSignal } : {}),
+                    onTerminal: ({ usage, status, errorMessage }) => {
+                        // The hop finalizes itself, like the Codex path does. Nothing else
+                        // ends this tracer: the route returns before the stream drains, so
+                        // without this the root span stays open and cost, request and
+                        // duration metrics are never emitted.
+                        //
+                        // Billing has to follow the model that served the turn. Left on the
+                        // requested alias, the pricing lookup misses and the charge is
+                        // silently dropped, which is why the Vertex hop had token counters
+                        // but no `proxy_cost_usd_total` series at all.
+                        tracer?.setModelSubstitution(body.model, vertexModel, "vertex");
+                        // Without this the spend lands under the Anthropic account that
+                        // never served the turn, hiding Vertex cost inside a subscription
+                        // account's figures.
+                        tracer?.setServedAccount(vertexAccount, "vertex");
+                        // setUsage must precede end(), which is what turns usage into cost.
+                        tracer?.setUsage(usage);
+                        tracer?.end(status, Date.now() - requestStartTime);
+                        logFinalRequest(status, vertexAccount, "vertex", status === 200 ? undefined : "vertex_fallback_stream", errorMessage, {
+                            inputTokens: usage.inputTokens,
+                            outputTokens: usage.outputTokens,
+                            cacheCreationTokens: usage.cacheCreationTokens,
+                            cacheReadTokens: usage.cacheReadTokens,
+                        });
+                    },
+                });
+            }
             else {
                 const availability = await ProviderHealthChecker.checkFallbackProviderAvailability(fallback.provider, fallback.model);
                 if (!availability.available) {

package/dist/types/proxy.d.ts

@@ -4174,4 +4174,26 @@ export type ProxyServiceInstallSettings = {
     port?: number;
     environment: Record<string, string>;
 };
+/** Where a native Claude-to-Vertex passthrough is addressed. */
+export type VertexAnthropicPassthroughTarget = {
+    projectId: string;
+    location: string;
+    model: string;
+    stream: boolean;
+};
+/** One native Claude-to-Vertex passthrough dispatch. */
+export type VertexAnthropicPassthroughRequest = {
+    body: Readonly<Record<string, unknown>>;
+    projectId: string;
+    location: string;
+    model: string;
+    signal?: AbortSignal;
+};
+/** How a Claude-on-Vertex passthrough ended, so the hop can be finalized once. */
+export type VertexPassthroughTerminal = {
+    usage: UsageContext;
+    /** 200 served, 499 client cancelled, 502 upstream stream failure. */
+    status: number;
+    errorMessage?: string;
+};
 export {};

package/dist/types/proxyContext.d.ts

@@ -7,6 +7,10 @@ export type ProxyContextPolicy = {
     models?: Record<string, {
         contextWindow: number;
         maxOutputTokens?: number;
+        /** Input estimate above which history is reduced. Bounds per-turn cost. */
+        compactAtTokens?: number;
+        /** Input estimate to reduce to. Must be below compactAtTokens. */
+        compactToTokens?: number;
     }>;
 };
 export type ProxyContextEvidence = {
@@ -24,7 +28,11 @@ export type ProxyContextEvidence = {
     multimodalEstimate: boolean;
     originalToolCount: number;
     retainedToolCount: number;
-    historyModified: false;
+    historyModified: boolean;
+    /** Complete history units dropped by pre-dispatch truncation. */
+    historyUnitsRemoved?: number;
+    /** Input estimate before truncation, when truncation ran. */
+    inputTokensBeforeTruncation?: number;
 };
 export type ProxyPreparedContext<T> = {
     body: T;
@@ -33,3 +41,13 @@ export type ProxyPreparedContext<T> = {
     totalTokensReservation: number;
     evidence: ProxyContextEvidence;
 };
+/** Estimator injected into history truncation so it matches preflight accounting. */
+export type ProxyHistoryEstimate = (value: unknown) => number;
+/** The history container a proxy body carries. */
+export type ProxyHistoryField = "messages" | "input" | "conversationMessages";
+export type ProxyHistoryTruncationResult<T> = {
+    body: T;
+    historyModified: boolean;
+    unitsRemoved: number;
+    itemsRemoved: number;
+};