@juspay/neurolink 12.17.4 → 12.18.0

package/dist/proxy/proxyContextPreflight.js

@@ -1,5 +1,6 @@
 import { getRuntimeContextWindow, getRuntimeOutputCeiling, } from "../constants/contextWindows.js";
 import { estimateTokens, IMAGE_TOKEN_ESTIMATE, } from "../utils/tokenEstimation.js";
+import { truncateHistoryForBudget } from "./proxyHistoryTruncation.js";
 /** Local refusal before spending a provider request; never silently truncates. */
 export class ProxyContextPreflightError extends Error {
     code;
@@ -65,13 +66,32 @@ export function parseProxyContextPolicy(raw) {
             throw new ProxyContextPreflightError("invalid_context_policy", "models must be a bounded map of provider/model limits");
         }
         for (const [key, limit] of Object.entries(value.models)) {
+            // Read each budget into a local before comparing: narrowing from the
+            // `positive` predicate does not survive several `||` operands when it is
+            // applied to a property access, and the looser compiler flags used by the
+            // package build then see `unknown` on one side of `>=`.
+            const contextWindow = record(limit) ? limit.contextWindow : undefined;
+            const maxOutputTokens = record(limit) ? limit.maxOutputTokens : undefined;
+            const compactAt = record(limit) ? limit.compactAtTokens : undefined;
+            const compactTo = record(limit) ? limit.compactToTokens : undefined;
             if (!key.includes("/") ||
                 !record(limit) ||
-                !positive(limit.contextWindow) ||
-                (limit.maxOutputTokens !== undefined &&
-                    !positive(limit.maxOutputTokens)) ||
-                Object.keys(limit).some((k) => !["contextWindow", "maxOutputTokens"].includes(k))) {
-                throw new ProxyContextPreflightError("invalid_context_policy", "Each model needs an explicit provider/model key and positive contextWindow/output limit");
+                !positive(contextWindow) ||
+                (maxOutputTokens !== undefined && !positive(maxOutputTokens)) ||
+                (compactAt !== undefined &&
+                    (!positive(compactAt) || compactAt >= contextWindow)) ||
+                (compactTo !== undefined &&
+                    (!positive(compactTo) ||
+                        !positive(compactAt) ||
+                        compactTo >= compactAt)) ||
+                (compactAt !== undefined && compactTo === undefined) ||
+                Object.keys(limit).some((k) => ![
+                    "contextWindow",
+                    "maxOutputTokens",
+                    "compactAtTokens",
+                    "compactToTokens",
+                ].includes(k))) {
+                throw new ProxyContextPreflightError("invalid_context_policy", "Each model needs an explicit provider/model key, a positive contextWindow/output limit, and a compaction target below its trigger");
             }
         }
     }
@@ -234,16 +254,21 @@ export function prepareProxyRequestContext(args) {
                 : {}),
         };
     }
-    const wire = body;
-    const state = { multimodal: false };
+    let wire = body;
+    // Fixed cost and history are tracked separately: truncation can remove the
+    // only image in a request, and a sticky flag would then keep the ceiling
+    // check disabled for a request that no longer carries any media.
+    const fixedState = { multimodal: false };
+    let historyState = { multimodal: false };
+    const isMultimodal = () => fixedState.multimodal || historyState.multimodal;
     const provider = args.provider === "codex" ? "openai" : args.provider;
-    const toolsTokensEstimate = estimateValue(wire.tools, provider, state);
+    const toolsTokensEstimate = estimateValue(wire.tools, provider, fixedState);
     const instructionsTokensEstimate = [
         wire.instructions,
         wire.system,
         wire.systemPrompt,
         wire.systemInstruction,
-    ].reduce((sum, value) => sum + estimateValue(value, provider, state), 0);
+    ].reduce((sum, value) => sum + estimateValue(value, provider, fixedState), 0);
     const schemaTokensEstimate = [
         wire.response_format,
         wire.output_config,
@@ -256,19 +281,53 @@ export function prepareProxyRequestContext(args) {
         record(wire.generationConfig)
             ? wire.generationConfig.responseJsonSchema
             : undefined,
-    ].reduce((sum, value) => sum + estimateValue(value, provider, state), 0);
-    const inputTokensEstimate = 24 +
+    ].reduce((sum, value) => sum + estimateValue(value, provider, fixedState), 0);
+    const estimate = (value) => estimateValue(value, provider, historyState);
+    const fixedTokensEstimate = 24 +
         schemaTokensEstimate +
         toolsTokensEstimate +
-        instructionsTokensEstimate +
-        [
-            wire.messages,
-            wire.input,
-            wire.contents,
-            wire.conversationMessages,
-            wire.prompt,
-        ].reduce((sum, value) => sum + estimateValue(value, provider, state), 0);
+        instructionsTokensEstimate;
+    const historyTokensEstimate = () => [
+        wire.messages,
+        wire.input,
+        wire.contents,
+        wire.conversationMessages,
+        wire.prompt,
+    ].reduce((sum, value) => sum + estimate(value), 0);
+    let inputTokensEstimate = fixedTokensEstimate + historyTokensEstimate();
     const configured = policy.models?.[`${args.provider}/${args.model}`];
+    // Cost control: reduce history before the refusal checks below, so those see
+    // the reduced estimate and a bounded request is dispatched instead of a
+    // rejected one. Runs on every dispatch path because every path lands here.
+    let historyModified = false;
+    let historyUnitsRemoved = 0;
+    const inputTokensBeforeTruncation = inputTokensEstimate;
+    if (configured?.compactAtTokens !== undefined &&
+        configured.compactToTokens !== undefined &&
+        inputTokensEstimate > configured.compactAtTokens) {
+        const truncated = truncateHistoryForBudget({
+            body,
+            inputTokensEstimate,
+            targetTokens: configured.compactToTokens,
+            estimate,
+        });
+        if (truncated.historyModified) {
+            body = truncated.body;
+            wire = body;
+            historyModified = true;
+            historyUnitsRemoved = truncated.unitsRemoved;
+            historyState = { multimodal: false };
+            inputTokensEstimate = fixedTokensEstimate + historyTokensEstimate();
+        }
+        // The design requires a typed local failure rather than a silent dispatch
+        // above the configured bound: fixed context alone, or the one unit that
+        // must be retained, can exceed the target with nothing left to remove.
+        if (inputTokensEstimate > configured.compactToTokens) {
+            throw new ProxyContextPreflightError("proxy_context_compaction_failed", `Compaction could not bring ${args.provider}/${args.model} within ` +
+                `${configured.compactToTokens} tokens; ${inputTokensEstimate} remain ` +
+                `after removing every removable unit`);
+        }
+    }
     const discovered = getRuntimeContextWindow(args.provider, args.model);
     const contextWindow = configured?.contextWindow ?? discovered;
     const outputCeiling = configured?.maxOutputTokens ??
@@ -306,10 +365,13 @@ export function prepareProxyRequestContext(args) {
                 ? "discovered"
                 : "unknown",
         tokenCountSource: "estimated",
-        multimodalEstimate: state.multimodal,
+        multimodalEstimate: isMultimodal(),
         originalToolCount: toolCount(tools),
         retainedToolCount: toolCount(wire.tools),
-        historyModified: false,
+        historyModified,
+        ...(historyModified
+            ? { historyUnitsRemoved, inputTokensBeforeTruncation }
+            : {}),
     };
     if (policy.maxInputTokens !== undefined &&
         inputTokensEstimate > policy.maxInputTokens) {
@@ -323,9 +385,9 @@ export function prepareProxyRequestContext(args) {
     }
     if (contextWindow !== undefined &&
         policy.enforceDiscoveredLimits !== false &&
-        !state.multimodal &&
+        !isMultimodal() &&
         inputTokensEstimate + outputTokensReserve > contextWindow) {
-        throw new ProxyContextPreflightError("proxy_context_window_exceeded", "Estimated input plus output/reasoning reserve exceeds the serving model context window; no history was truncated", evidence);
+        throw new ProxyContextPreflightError("proxy_context_window_exceeded", `Estimated input plus output/reasoning reserve exceeds the serving model context window; ${historyModified ? "history was truncated and still does not fit" : "no history was truncated"}`, evidence);
     }
     return {
         body,

package/dist/proxy/proxyHistoryTruncation.d.ts

@@ -0,0 +1,24 @@
+/**
+ * Deterministic pre-dispatch history truncation.
+ *
+ * Runs inside the shared context preflight, so one implementation bounds input
+ * cost on every dispatch path: native Anthropic, the translated/SDK fallback
+ * that serves Vertex, and native or bridged Codex.
+ *
+ * Deliberately provider-free. Summarizing history would add an outbound model
+ * call to the very requests this exists to make cheaper, so reduction is pure
+ * removal of the oldest complete units.
+ */
+import type { ProxyHistoryEstimate, ProxyHistoryTruncationResult } from "../types/index.js";
+/**
+ * Remove the oldest complete units until the estimate reaches `targetTokens`.
+ *
+ * Returns the body unchanged when there is nothing safe to remove. The newest
+ * unit is always retained: it carries the turn being answered.
+ */
+export declare function truncateHistoryForBudget<T extends object>(args: {
+    body: T;
+    inputTokensEstimate: number;
+    targetTokens: number;
+    estimate: ProxyHistoryEstimate;
+}): ProxyHistoryTruncationResult<T>;

package/dist/proxy/proxyHistoryTruncation.js

@@ -0,0 +1,171 @@
+/**
+ * Deterministic pre-dispatch history truncation.
+ *
+ * Runs inside the shared context preflight, so one implementation bounds input
+ * cost on every dispatch path: native Anthropic, the translated/SDK fallback
+ * that serves Vertex, and native or bridged Codex.
+ *
+ * Deliberately provider-free. Summarizing history would add an outbound model
+ * call to the very requests this exists to make cheaper, so reduction is pure
+ * removal of the oldest complete units.
+ */
+function record(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+/**
+ * A turn that cannot legally open a history.
+ *
+ * The Messages API rejects a conversation whose first message is not from the
+ * user, so truncation that stops on an assistant turn would trade a local
+ * refusal for an upstream 400. Items with no role (Codex `function_call` and
+ * `function_call_output`) are not turns and are left alone.
+ */
+function isNonUserTurn(item) {
+    return record(item) && typeof item.role === "string" && item.role !== "user";
+}
+function blocksOf(item) {
+    return Array.isArray(item.content) ? item.content : [item];
+}
+/** Tool-call ids this item opens. */
+function definedCallIds(item) {
+    if (!record(item)) {
+        return [];
+    }
+    const ids = [];
+    if (item.type === "function_call" && typeof item.call_id === "string") {
+        ids.push(item.call_id);
+    }
+    for (const block of blocksOf(item)) {
+        if (record(block) &&
+            block.type === "tool_use" &&
+            typeof block.id === "string") {
+            ids.push(block.id);
+        }
+    }
+    return ids;
+}
+/** Tool-call ids this item answers. */
+function referencedCallIds(item) {
+    if (!record(item)) {
+        return [];
+    }
+    const ids = [];
+    if (item.type === "function_call_output" &&
+        typeof item.call_id === "string") {
+        ids.push(item.call_id);
+    }
+    for (const block of blocksOf(item)) {
+        if (record(block) &&
+            block.type === "tool_result" &&
+            typeof block.tool_use_id === "string") {
+            ids.push(block.tool_use_id);
+        }
+    }
+    return ids;
+}
+/**
+ * Group items so a tool call and every answer to it stay in one unit.
+ *
+ * A unit stays open while any call it opened is still unanswered, so removing
+ * whole units can never strand a `tool_result` or a `function_call_output`.
+ */
+function groupIntoUnits(items) {
+    const units = [];
+    let current = [];
+    const awaiting = new Set();
+    for (let index = 0; index < items.length; index += 1) {
+        const item = items[index];
+        current.push(index);
+        for (const id of referencedCallIds(item)) {
+            awaiting.delete(id);
+        }
+        for (const id of definedCallIds(item)) {
+            awaiting.add(id);
+        }
+        if (awaiting.size === 0) {
+            units.push(current);
+            current = [];
+        }
+    }
+    if (current.length > 0) {
+        // An unanswered trailing call still belongs with the turn that opened it.
+        units.push(current);
+    }
+    return units;
+}
+/**
+ * Pick the history container.
+ *
+ * `input` is overloaded across paths: the translated/SDK shape carries the
+ * current turn there as an object, while Codex carries the item array. Only an
+ * array is history, so the object form is never touched.
+ */
+function historyFieldOf(wire) {
+    if (Array.isArray(wire.messages)) {
+        return "messages";
+    }
+    if (Array.isArray(wire.conversationMessages)) {
+        return "conversationMessages";
+    }
+    if (Array.isArray(wire.input)) {
+        return "input";
+    }
+    return undefined;
+}
+/**
+ * Remove the oldest complete units until the estimate reaches `targetTokens`.
+ *
+ * Returns the body unchanged when there is nothing safe to remove. The newest
+ * unit is always retained: it carries the turn being answered.
+ */
+export function truncateHistoryForBudget(args) {
+    const unchanged = {
+        body: args.body,
+        historyModified: false,
+        unitsRemoved: 0,
+        itemsRemoved: 0,
+    };
+    const wire = args.body;
+    const field = historyFieldOf(wire);
+    if (field === undefined) {
+        return unchanged;
+    }
+    const items = wire[field];
+    const units = groupIntoUnits(items);
+    if (units.length <= 1) {
+        return unchanged;
+    }
+    // Everything outside the history container — tools, instructions, schema —
+    // is fixed cost that truncation must never touch, so only unit costs are
+    // subtracted from the running projection.
+    const unitTokens = units.map((unit) => args.estimate(unit.map((index) => items[index])));
+    // The translated shape keeps the current turn in object-valued `input`, so
+    // every `conversationMessages` unit is prior history and all of them may go.
+    // The other shapes carry the current turn in the array itself, so the newest
+    // unit there is the turn being answered and must survive.
+    const removable = field === "conversationMessages" ? units.length : units.length - 1;
+    let removedUnits = 0;
+    let projected = args.inputTokensEstimate;
+    while (removedUnits < removable && projected > args.targetTokens) {
+        projected -= unitTokens[removedUnits];
+        removedUnits += 1;
+    }
+    if (removedUnits === 0) {
+        return unchanged;
+    }
+    // Advance by whole units so the kept history opens on a user turn without
+    // stranding a tool call from its answer.
+    while (removedUnits < removable &&
+        isNonUserTurn(items[units[removedUnits][0]])) {
+        removedUnits += 1;
+    }
+    const keptIndices = units.slice(removedUnits).flat();
+    const kept = keptIndices.map((index) => items[index]);
+    const itemsRemoved = items.length - kept.length;
+    return {
+        body: { ...args.body, [field]: kept },
+        historyModified: true,
+        unitsRemoved: removedUnits,
+        itemsRemoved,
+    };
+}

package/dist/proxy/vertexAnthropicFallback.d.ts

@@ -0,0 +1,82 @@
+/**
+ * Native Claude-to-Vertex passthrough.
+ *
+ * Claude on Vertex speaks the Anthropic Messages API, so a fallback to it needs
+ * no translation at all — the client's body goes upstream essentially verbatim
+ * and the SSE comes back in the shape the client already expects.
+ *
+ * This exists because the SDK/translation path cannot carry an agentic turn:
+ * it runs its own tool loop, and a request whose tools have no executor (every
+ * proxied request, since the caller executes them) ends with the pending tool
+ * call discarded and a step-cap apology returned as assistant text. Tools,
+ * `tool_use`, `tool_result`, `cache_control` and thinking blocks all survive
+ * here because nothing reshapes them.
+ */
+import type { UsageContext, VertexPassthroughTerminal, VertexAnthropicPassthroughRequest, VertexAnthropicPassthroughTarget } from "../types/index.js";
+/**
+ * Translate `output_config` for Vertex.
+ *
+ * An effort level Vertex knows passes through; one it does not is mapped onto
+ * the nearest level it does. A level with no mapping at all loses only the
+ * `effort` key, never the sibling settings beside it.
+ */
+export declare function mapVertexOutputConfig(value: unknown): Record<string, unknown> | undefined;
+/** Strip what Vertex rejects and pin the API version it requires. */
+export declare function buildVertexAnthropicPayload(body: Readonly<Record<string, unknown>>): Record<string, unknown>;
+/**
+ * Publisher endpoint for a Claude model on Vertex.
+ *
+ * Streaming and non-streaming are different verbs on the same resource, so the
+ * caller's `stream` flag selects the verb rather than changing the body.
+ */
+export declare function buildVertexAnthropicUrl(target: VertexAnthropicPassthroughTarget): string;
+/**
+ * Send one Anthropic-shaped request to Vertex and return the upstream response
+ * untouched, so the caller can stream its bytes straight to the client.
+ */
+export declare function dispatchVertexAnthropicPassthrough(request: VertexAnthropicPassthroughRequest): Promise<Response>;
+/** Claude models are the only publisher models this passthrough addresses. */
+export declare function isVertexAnthropicModel(model: string): boolean;
+/**
+ * Resolve where Vertex traffic is addressed, using the same environment the
+ * Vertex provider reads so a fallback lands on the project and region the rest
+ * of the process already uses.
+ */
+export declare function resolveVertexTarget(): {
+    projectId: string;
+    location: string;
+} | undefined;
+/**
+ * Run one Claude-on-Vertex fallback attempt and hand back a response the route
+ * can serve directly.
+ *
+ * A non-2xx upstream throws so the caller's attempt loop records the failure
+ * and moves to the next hop, exactly as the other providers behave.
+ */
+export declare function executeVertexAnthropicFallback(args: {
+    body: Readonly<Record<string, unknown>>;
+    model: string;
+    signal?: AbortSignal;
+    /** Fires exactly once on every exit, so the hop is finalized like any other. */
+    onTerminal?: (terminal: VertexPassthroughTerminal) => void;
+}): Promise<Response>;
+/**
+ * Read usage out of a non-streaming Anthropic reply.
+ *
+ * `rawPredict` returns a single `{"type":"message","usage":{...}}` object, which
+ * carries the whole turn's counts at once rather than splitting them across
+ * `message_start` and `message_delta`.
+ */
+export declare function readJsonUsage(text: string): UsageContext;
+/**
+ * Forward a stream unchanged while observing its usage events.
+ *
+ * The bytes are passed through untouched — parsing happens on a copy of each
+ * chunk — so the client sees exactly what Vertex sent.
+ *
+ * A transform's `flush` only runs on a clean end, which would leave a stream
+ * error or a client disconnect with no terminal record at all. An explicit
+ * reader loop is used instead, so `onTerminal` fires exactly once on every
+ * exit: normal end, upstream failure, and cancellation.
+ */
+export declare function observeVertexUsage(body: ReadableStream<Uint8Array>, onTerminal: (terminal: VertexPassthroughTerminal) => void): ReadableStream<Uint8Array>;