@cldmv/llm-translator 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -7
- package/dist/api/convert/anthropic/messages/to/openai/chat/request.mjs +10 -10
- package/dist/api/convert/openai/chat/to/anthropic/messages/response.mjs +1 -1
- package/dist/api/convert/openai/chat/to/anthropic/messages/stream.mjs +1 -1
- package/dist/api/detect.mjs +1 -1
- package/dist/api/driver/azure/detect.mjs +1 -1
- package/dist/api/driver/azure/info.mjs +1 -1
- package/dist/api/driver/deepseek/info.mjs +1 -1
- package/dist/api/driver/llamacpp/info.mjs +1 -1
- package/dist/api/driver/mistral/info.mjs +1 -1
- package/dist/api/driver/openai/info.mjs +1 -1
- package/dist/api/driver/openrouter/info.mjs +1 -1
- package/dist/api/request.mjs +2 -2
- package/dist/api/send.mjs +1 -1
- package/dist/lib/cache.mjs +2 -0
- package/dist/lib/constraint.mjs +2 -0
- package/dist/lib/context.mjs +2 -0
- package/dist/lib/passthrough.mjs +1 -1
- package/dist/lib/presets.mjs +1 -1
- package/dist/lib/routes.mjs +1 -1
- package/dist/lib/settings.mjs +1 -1
- package/package.json +1 -1
- package/types/src/api/detect.d.mts +1 -1
- package/types/src/api/driver/azure/info.d.mts +2 -1
- package/types/src/api/driver/deepseek/info.d.mts +2 -1
- package/types/src/api/driver/mistral/info.d.mts +2 -1
- package/types/src/api/driver/openai/info.d.mts +3 -1
- package/types/src/api/driver/openrouter/info.d.mts +2 -1
- package/types/src/api/request.d.mts +1 -1
- package/types/src/api/send.d.mts +1 -1
- package/types/src/lib/cache.d.mts +30 -0
- package/types/src/lib/constraint.d.mts +36 -0
- package/types/src/lib/context.d.mts +34 -0
- package/types/src/lib/presets.d.mts +2 -1
- package/types/src/lib/settings.d.mts +20 -0
package/README.md
CHANGED
|
@@ -17,20 +17,19 @@ Use it as a library when you want the conversion without any I/O (`request`, `re
|
|
|
17
17
|
|
|
18
18
|
## ✨ What's New
|
|
19
19
|
|
|
20
|
-
### Latest: v0.
|
|
20
|
+
### Latest: v0.11.0 (November 2026)
|
|
21
21
|
|
|
22
|
-
- **
|
|
23
|
-
-
|
|
24
|
-
- **
|
|
25
|
-
-
|
|
26
|
-
- [View full v0.10.0 Changelog](https://github.com/CLDMV/llm-translator/blob/master/docs/changelog/v0/v0.10.0.md)
|
|
22
|
+
- **Constrained decoding** — a neutral `constraint` (JSON Schema, regex, choice list or grammar) is turned into each backend's native field: vLLM `structured_outputs` (or `guided_*` before 0.11.0), SGLang `regex` and `ebnf`, llama.cpp `grammar` and `json_schema`, and TGI's grammar. A kind a backend can't enforce is dropped and recorded, or rejected.
|
|
23
|
+
- **`clear_tool_uses` context edits** — `context_management` edits are applied locally before an OpenAI-protocol upstream sees the conversation, and the reply reports them in `applied_edits`.
|
|
24
|
+
- **Prompt-cache hints** — `cache_salt` (set by the server, vLLM only) and `prompt_cache_key` (OpenAI only) are mapped, with docs on what is and isn't cached. Hosted APIs such as `api.openai.com` now bind by host name before any driver probes them.
|
|
25
|
+
- [View full v0.11.0 Changelog](https://github.com/CLDMV/llm-translator/blob/master/docs/changelog/v0/v0.11.0.md)
|
|
27
26
|
|
|
28
27
|
### Recent Releases
|
|
29
28
|
|
|
29
|
+
- **v0.10.0** (November 2026) — native `/v1/messages` for vLLM, readiness probes and LoRA adapters, Groq, Together, Fireworks, xAI and Gemini drivers, and Ollama's native `/api/chat` ([Changelog](https://github.com/CLDMV/llm-translator/blob/master/docs/changelog/v0/v0.10.0.md))
|
|
30
30
|
- **v0.9.0** (November 2026) — Azure OpenAI, OpenRouter, Hugging Face TGI and LocalAI drivers, with Azure's `apiVersion` and `deployment` upstream fields and more startup probes ([Changelog](https://github.com/CLDMV/llm-translator/blob/master/docs/changelog/v0/v0.9.0.md))
|
|
31
31
|
- **v0.8.0** (November 2026) — Mistral and LM Studio drivers, an OpenAI client's `n` above 1 answered by parallel requests to Anthropic-protocol upstreams, and `stop_details.category` from content-filter results ([Changelog](https://github.com/CLDMV/llm-translator/blob/master/docs/changelog/v0/v0.8.0.md))
|
|
32
32
|
- **v0.7.0** (November 2026) — Anthropic-protocol servers detected by probe on any host, tool-call ids rewritten for strict upstream formats, embeddings usage through `onUsage`, and capture records for every failed-over attempt ([Changelog](https://github.com/CLDMV/llm-translator/blob/master/docs/changelog/v0/v0.7.0.md))
|
|
33
|
-
- **v0.6.0** (November 2026) — `POST /v1/embeddings` pass-through, sampling extensions per upstream and per request, `strict` tool use end to end with a schema check, and the `capture` option for debugging ([Changelog](https://github.com/CLDMV/llm-translator/blob/master/docs/changelog/v0/v0.6.0.md))
|
|
34
33
|
|
|
35
34
|
📚 **For complete version history and detailed release notes, see the [docs/changelog/](https://github.com/CLDMV/llm-translator/tree/master/docs/changelog/) folder.**
|
|
36
35
|
|
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
var
|
|
1
|
+
var O=Object.defineProperty;var d=(e,n)=>O(e,"name",{value:n,configurable:!0});import{resolve as q}from"../../../../../../../lib/settings.mjs";import{TranslatorError as j}from"../../../../../../../lib/errors.mjs";import{createIdFitter as B,createNameFitter as C}from"../../../../../../../lib/ids.mjs";import{sanitize as A}from"../../../../../../../lib/schema.mjs";import{NOTE as L}from"../../../../../../../lib/images.mjs";import{expand as U}from"../../../../../../../lib/documents.mjs";import{createBudget as W,unsign as z}from"../../../../../../../lib/thinking.mjs";import{clearToolUses as D,valid as P}from"../../../../../../../lib/context.mjs";import{plan as H,scan as K}from"../../../../../../../lib/toolsearch.mjs";const R=new Set(["model","messages","system","max_tokens","stream","temperature","top_p","top_k","stop_sequences","tools","tool_choice","thinking","output_config","output_format","context_management"]),G=new Set(["metadata","cache_control","service_tier"]),J=new Set(["mcp_servers","container","inference_geo"]),Q="x-anthropic-billing-header:";async function he(e,n,o){const t=q(n);if(e===null||typeof e!="object"||!Array.isArray(e.messages))throw new j("messages: Field required",{code:"invalid_request",param:"messages"});const s={front:"anthropic",model:e.model,stream:e.stream===!0,thinking:e.thinking?.type??null,reasoningField:t.reasoningField,toolNames:{},toolIds:{},dropped:[]};e.thinking?.type==="enabled"&&Number.isInteger(e.thinking.budget_tokens)&&e.output_config?.effort===void 0&&(s.thinkingBudget=e.thinking.budget_tokens),o?.producer!==void 0&&(s.producer=o.producer);const i=C({max:64,map:s.toolNames}),m=o?.toolCallIds?B({policy:o.toolCallIds,ids:V(e.messages),map:s.toolIds}):c=>c;for(const c of e.tools??[])(c.type===void 0||c.type==="custom")&&i(c.name);const g=K(e.messages,new Set((e.tools??[]).map(c=>c.name))),a=H(e.tools??[],g,t,e.tool_choice);s.toolSearch={mode:t.toolSearch,loaded:a.loaded,hidden:a.hidden,stubbed:a.stubbed,capped:g.capped};for(const c of Object.keys(e))if(!(R.has(c)||G.has(c)))if(J.has(c))h(s,c);else{if(t.unknownFields==="error")throw new j(`${c}: Extra inputs are not permitted`,{code:"unsupported_field",param:c});h(s,c)}const l={model:e.model},$=I(e.system,t),u=await X(e.messages,t,s,i,m,$,Y(e.context_management,s),o?.budget??W());l.messages=u.messages,u.prefill&&(l.continue_final_message=!0,l.add_generation_prompt=!1),e.max_tokens!==void 0&&(l[t.maxTokensField]=e.max_tokens),e.temperature!==void 0&&(l.temperature=e.temperature),e.top_p!==void 0&&(l.top_p=e.top_p),e.top_k!==void 0&&(t.topK?l.top_k=e.top_k:h(s,"top_k")),e.stop_sequences?.length>0&&(l.stop=e.stop_sequences),s.stream&&(l.stream=!0,t.streamUsage&&(l.stream_options={include_usage:!0},t.continuousUsage&&(l.stream_options.continuous_usage_stats=!0)));const x=oe(a.offer,t,s,i);if(x.length>0){l.tools=x;const c=se(e.tool_choice,t,s,i);c!==void 0&&(l.tool_choice=c),e.tool_choice?.disable_parallel_tool_use===!0&&t.parallelToolCalls&&(l.parallel_tool_calls=!1)}return Z(e.context_management,l,s),ie(e,t,l,o?.listed===!0),re(e,t,s,l),{body:l,state:s}}d(he,"request");function h(e,n){e.dropped.includes(n)||e.dropped.push(n)}d(h,"drop");function v(e,n,o){if(n.unknownFields==="error")throw new j(`${o}: is not supported by this translator`,{code:"unsupported_content",param:o});h(e,o)}d(v,"unsupported");function I(e,n){return e==null?"":(typeof e=="string"?[{type:"text",text:e}]:e).filter(t=>t.type==="text"&&!(n.stripBillingHeader&&t.text.trimStart().startsWith(Q))).map(t=>t.text).join(`
|
|
2
2
|
|
|
3
|
-
`)}
|
|
3
|
+
`)}d(I,"textOf");function S(e){return typeof e=="string"?[{type:"text",text:e}]:e??[]}d(S,"asBlocks");function V(e){const n=[];for(const o of e)for(const t of S(o.content))t.type==="tool_use"?n.push(t.id):t.type==="tool_result"&&n.push(t.tool_use_id);return n}d(V,"idsOf");async function X(e,n,o,t,s,i,m,g){const a=[],l=[],$=new Map,u=n.inlineSystem==="auto"?"user":n.inlineSystem,x=new Map;if(n.reasoningRoundTrip){for(let r=e.length-1;r>=0;r--)if(e[r].role==="assistant")for(const f of[...S(e[r].content)].reverse())f.type==="thinking"&&x.set(f,await z(f.signature,{producer:o.producer,budget:g}))}const c=d(r=>{const f=a.at(-1);f?.role==="user"?f.parts.push(...r):a.push({role:"user",parts:r})},"pushUser"),E=d(async(r,f)=>{switch(r.type){case"text":return r.text===""?[]:[{type:"text",text:r.text}];case"image":return ne(r,n,o);case"document":{const _=[];for(const p of await U(r,{settings:n,toolInputs:$,toolUseId:f}))_.push(...await E(p,f));return _}case"search_result":{const _=(r.content??[]).map(p=>p.text).join(`
|
|
4
4
|
`);return[{type:"text",text:`[Search result: ${r.title??""} (${r.source??""})]
|
|
5
|
-
${_}`}]}case"tool_reference":return
|
|
6
|
-
${
|
|
7
|
-
</system-reminder>`;c([{type:"text",text:_}])}}else if(r.role==="user"){const
|
|
8
|
-
`);
|
|
5
|
+
${_}`}]}case"tool_reference":return n.toolSearch==="native"&&n.deferredTools!=="drop"?[{type:"tool_reference",tool_name:r.tool_name}]:[{type:"text",text:o.toolSearch.loaded.includes(r.tool_name)?`[Tool loaded: ${r.tool_name}]`:`[Tool reference: ${r.tool_name}]`}];default:return v(o,n,`block:${r.type}`),[]}},"toParts");for(const r of e)if(r.role==="system"){const f=I(r.content,n);if(f===""||u==="drop")continue;if(u==="keep")a.push({role:"system",content:f});else if(u==="hoist")l.push(f);else{const _=f.trimStart().startsWith("<system-reminder>")?f:`<system-reminder>
|
|
6
|
+
${f}
|
|
7
|
+
</system-reminder>`;c([{type:"text",text:_}])}}else if(r.role==="user"){const f=[],_=[];for(const y of S(r.content)){if(y.type!=="tool_result"){f.push(...await E(y,null));continue}const T=[];for(const k of S(y.content))T.push(...await E(k,y.tool_use_id));const M=T.filter(k=>k.type==="text").map(k=>k.text),F=T.filter(k=>k.type==="image_url");let w=T.some(k=>k.type==="tool_reference")?T.filter(k=>k.type!=="image_url"):M.join(`
|
|
8
|
+
`);F.length>0&&(_.push({type:"text",text:`Image output of tool call ${s(y.tool_use_id)}:`},...F),w===""&&(w="(image output attached in the next message)")),y.is_error===!0&&n.toolErrors&&(w=typeof w=="string"?`${n.toolErrorMarker}${w}`:[{type:"text",text:n.toolErrorMarker.trimEnd()},...w]),a.push({role:"tool",tool_call_id:s(y.tool_use_id),content:w})}const p=[..._,...f];p.length>0&&c(p)}else if(r.role==="assistant"){const f={role:"assistant",texts:[],parts:[],tool_calls:[]};for(const p of S(r.content))if(p.type==="text")p.text!==""&&f.texts.push(p.text);else if(p.type==="thinking"){const y=x.get(p)??null;y?f.parts.push({restored:y}):p.thinking&&f.parts.push({text:p.thinking})}else p.type==="redacted_thinking"?h(o,"redacted_thinking"):p.type==="tool_use"?($.set(p.id,p.input),f.tool_calls.push({id:s(p.id),type:"function",function:{name:t(p.name),arguments:JSON.stringify(p.input??{})}})):v(o,n,`block:${p.type}`);const _=a.at(-1);_?.role==="assistant"?(_.texts.push(...f.texts),_.parts.push(...f.parts),_.tool_calls.push(...f.tool_calls)):a.push(f)}else throw new j(`messages: unsupported role "${r.role}"`,{code:"invalid_request",param:"messages"});return te(a,n,i,l,m,o)}d(X,"convertMessages");function Y(e,n){let o=1/0;for(const t of e?.edits??[])typeof t.type=="string"&&t.type.startsWith("clear_thinking")?t.keep==="all"?o=1/0:o=t.keep?.value??1:N(t)||h(n,`context_management.${t.type}`);return o}d(Y,"thinkingTurnsToKeep");function N(e){return typeof e.type=="string"&&e.type.startsWith("clear_tool_uses")}d(N,"isClearToolUses");function Z(e,n,o){for(const t of e?.edits??[]){if(!N(t))continue;if(!P(t)){h(o,`context_management.${t.type}`);continue}o.appliedEdits??=[];const s=D(t,n,o.toolNames);s&&o.appliedEdits.push(s)}}d(Z,"applyContextEdits");function b(e,n){const o=e.find(a=>a.restored&&(a.restored.reasoning_content!==void 0||a.restored.reasoning!==void 0)),t=o?o.restored.reasoning_content!==void 0?"reasoning_content":"reasoning":n;let s="",i,m=null;for(const a of e){const l=a.restored?a.restored.reasoning_content??a.restored.reasoning:a.text;a.restored?.reasoning_details&&(i=[...i??[],...a.restored.reasoning_details]),l!==void 0&&(s+=(m!==null&&!(m.restored&&a.restored)?`
|
|
9
9
|
|
|
10
|
-
`:"")+
|
|
10
|
+
`:"")+l,m=a)}const g={};return s!==""&&(g[t]=s),i&&(g.reasoning_details=i),g}d(b,"reasoningFields");function ee(e){return!!(e.reasoning_content||e.reasoning||e.reasoning_details)}d(ee,"reasoningSent");function te(e,n,o,t,s,i){let m=s;for(let u=e.length-1;u>=0;u--)e[u].role!=="assistant"||e[u].parts.length===0||(m>0?m--:e[u].parts=[]);const g=[],a=[o,...t].filter(u=>u!=="").join(`
|
|
11
11
|
|
|
12
|
-
`);a!==""&&
|
|
12
|
+
`);a!==""&&g.push({role:"system",content:a});for(const u of e)if(u.role==="user"){const x=u.parts.every(c=>c.type==="text");g.push({role:"user",content:x?u.parts.map(c=>c.text).join(`
|
|
13
13
|
|
|
14
|
-
`):
|
|
14
|
+
`):u.parts})}else if(u.role==="assistant"){const x={role:"assistant",content:u.texts.join(`
|
|
15
15
|
|
|
16
|
-
`)};
|
|
16
|
+
`)};u.parts.length>0&&Object.assign(x,b(u.parts,n.reasoningField)),n.reasoningRoundTrip&&u.tool_calls.length>0&&!ee(x)&&(i.reasoningMissing=(i.reasoningMissing??0)+1),u.tool_calls.length>0&&(x.tool_calls=u.tool_calls),g.push(x)}else g.push(u);const l=g.at(-1),$=n.prefill==="continue_final_message"&&l?.role==="assistant"&&!l.tool_calls&&l.content!=="";return{messages:g,prefill:$}}d(te,"finalize");function ne(e,n,o){if(!n.images)return h(o,"image"),[{type:"text",text:L}];const t=e.source;return t.type==="base64"?[{type:"image_url",image_url:{url:`data:${t.media_type};base64,${t.data}`}}]:t.type==="url"?[{type:"image_url",image_url:{url:t.url}}]:(h(o,"image.file"),[{type:"text",text:"[An image supplied as an uploaded file could not be forwarded.]"}])}d(ne,"imageParts");function oe(e,n,o,t){const s=[];for(const i of e){if(i.type!==void 0&&i.type!=="custom"){h(o,`tool:${i.name}`);continue}if(i.defer_loading===!0&&n.deferredTools==="drop"){h(o,"deferred_tools");continue}const m=i.input_schema??{type:"object",properties:{}},g={name:t(i.name),parameters:n.toolSchema==="sanitize"?A(m):m};i.description!==void 0&&(g.description=i.description),i.strict!==void 0&&(g.strict=i.strict),s.push(i.defer_loading===!0&&n.toolSearch==="native"?{type:"function",function:g,defer_loading:!0}:{type:"function",function:g})}return s}d(oe,"convertTools");function se(e,n,o,t){switch(e?.type){case void 0:return;case"auto":return"auto";case"any":return"required";case"none":return"none";case"tool":return{type:"function",function:{name:t(e.name)}};default:v(o,n,`tool_choice:${e.type}`);return}}d(se,"convertToolChoice");function ie(e,n,o,t){const s=e.thinking,i=s?.type;let m;i==="disabled"?(n.thinking==="reasoning_effort"&&(m="none"),n.thinking==="chat_template_kwargs"&&(o.chat_template_kwargs={enable_thinking:!1})):(n.effort&&e.output_config?.effort!==void 0&&(m=e.output_config.effort),i==="enabled"&&(n.thinking==="chat_template_kwargs"&&(o.chat_template_kwargs={enable_thinking:!0},n.thinkingBudget&&s.budget_tokens!==void 0&&e.max_tokens!==void 0&&(o.thinking_token_budget=Math.max(0,Math.min(s.budget_tokens,e.max_tokens-1)))),n.thinking==="reasoning_effort"&&m===void 0&&!t&&(m="medium"))),m!==void 0&&(o.reasoning_effort=m)}d(ie,"applyThinking");function re(e,n,o,t){const s=e.output_config?.format??e.output_format;s!==void 0&&(n.responseFormat&&s.type==="json_schema"?t.response_format={type:"json_schema",json_schema:{name:"output",schema:s.schema}}:h(o,"output_config.format"));for(const i of Object.keys(e.output_config??{}))i!=="effort"&&i!=="format"&&h(o,`output_config.${i}`)}d(re,"applyOutputConfig");export{he as request};
|
|
17
17
|
//# sourceMappingURL=request.mjs.map
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var
|
|
1
|
+
var y=Object.defineProperty;var f=(n,t)=>y(n,"name",{value:t,configurable:!0});import{resolve as k}from"../../../../../../../lib/settings.mjs";import{TranslatorError as d}from"../../../../../../../lib/errors.mjs";import{messageId as x,anthropicToolId as T,restoreName as A}from"../../../../../../../lib/ids.mjs";import{sign as w,reasoningOf as C,reasoningKey as E,detailsText as I,splitThink as N}from"../../../../../../../lib/thinking.mjs";import{isComplete as O,parseArguments as S}from"../../../../../../../lib/arguments.mjs";import{filterCategory as U,finishError as v,toAnthropicStop as b,toAnthropicUsage as j}from"../../../../../../../lib/finish.mjs";function G(n,t,m){const c=k(m),r=n?.choices?.[0];if(!r)throw new d("Upstream response contained no choices",{code:"invalid_upstream_response",status:502});const p=v(r.finish_reason);if(p)throw new d(p.message,{code:p.code,status:p.status,type:p.type});const e=r.message??{},a=[];let l=C(e),s=typeof e.content=="string"?e.content:"";const o={};if(c.reasoningRoundTrip){const i=E(e);i&&(o[i]=e[i]),Array.isArray(e.reasoning_details)&&e.reasoning_details.length>0&&(o.reasoning_details=e.reasoning_details)}l===""&&c.thinkTags&&({reasoning:l,text:s}=N(s,c.thinkOpen)),l===""&&o.reasoning_details&&(l=I(o.reasoning_details)),(l!==""||o.reasoning_details)&&(a.push({type:"thinking",thinking:l,signature:w(Object.keys(o).length>0?o:void 0,t?.producer)}),s=s.trimStart()),s===""&&typeof e.refusal=="string"&&(s=e.refusal),s!==""&&a.push({type:"text",text:s});const u=t?.toolNames??{},h=t?.toolIds??{},g=e.tool_calls??[];for(const i of g)a.push({type:"tool_use",id:T(i.id,h),name:A(u,i.function.name),input:S(i.function.arguments)});a.length===0&&a.push({type:"text",text:""});const _=b(r.finish_reason,{hasToolCalls:g.length>0,matchedStop:r.stop_reason,refusal:e.refusal,category:U(r.content_filter_results),argumentsComplete:g.length===0||O(g.at(-1).function.arguments)});return t&&n.usage&&(t.upstreamUsage=n.usage),{id:x(),type:"message",role:"assistant",model:t?.model??n.model,content:a,..._,usage:j(n.usage),...t?.appliedEdits?{context_management:{applied_edits:t.appliedEdits}}:{}}}f(G,"response");export{G as response};
|
|
2
2
|
//# sourceMappingURL=response.mjs.map
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var
|
|
1
|
+
var V=Object.defineProperty;var s=(b,r)=>V(b,"name",{value:r,configurable:!0});import{resolve as W}from"../../../../../../../lib/settings.mjs";import{TranslatorError as X,openaiToAnthropicType as Y,rewriteContextOverflow as Z}from"../../../../../../../lib/errors.mjs";import{isComplete as $}from"../../../../../../../lib/arguments.mjs";import{messageId as ee,anthropicToolId as te,restoreName as ne}from"../../../../../../../lib/ids.mjs";import{sign as ie,reasoningOf as oe,reasoningKey as le,detailsText as se,mergeDetail as re,createThinkSplitter as ae}from"../../../../../../../lib/thinking.mjs";import{filterCategory as de,finishError as fe,toAnthropicStop as ce,toAnthropicUsage as B}from"../../../../../../../lib/finish.mjs";import{messages as ue}from"../../../../../../../lib/sse.mjs";async function*be(b,r,F){const u=W(F),J=r?.toolNames??{},K=r?.toolIds??{},d=u.thinkTags?ae(u.thinkOpen):null,c=new Map;let m=!1,j=0,i=null,N=null,k=null,O=null,h="",T=null,v=!1,A=!1,E=0,w=!1,I=!1,U=!1;const S=[],l=s((e,t)=>({event:e,data:{type:e,...t}}),"event"),p=s((e,t)=>l("error",{error:{type:e,message:t}}),"fail"),M=s(e=>{const t={};return u.reasoningRoundTrip&&(e.key!==null&&(t[e.key]=e.text),e.details.length>0&&(t.reasoning_details=e.details)),Object.keys(t).length>0?t:void 0},"carried");function*y(){if(i!==null){i.kind==="thinking"?yield l("content_block_delta",{index:i.index,delta:{type:"signature_delta",signature:ie(M(i.carry),r?.producer)}}):i.kind==="tool"&&(i.entry.state="closed",i.entry.sent||(yield l("content_block_delta",{index:i.index,delta:{type:"input_json_delta",partial_json:"{}"}}))),yield l("content_block_stop",{index:i.index}),i=null;for(const e of c.values()){if(e.state!=="pending")continue;e.state="closed";const t=j++,n=e.args.join("");yield l("content_block_start",{index:t,content_block:e.block}),yield l("content_block_delta",{index:t,delta:{type:"input_json_delta",partial_json:n===""?"{}":n}}),yield l("content_block_stop",{index:t})}}}s(y,"close");function*G(){for(const e of R()){if(e.state!=="buffered")continue;yield*g("tool",e.block,e);const t=e.args.join("");t!==""&&(e.sent=!0,yield l("content_block_delta",{index:i.index,delta:{type:"input_json_delta",partial_json:t}})),yield*y()}}s(G,"flushBuffered");function R(){return[...c.entries()].sort((e,t)=>e[0]-t[0]).map(e=>e[1])}s(R,"ordered");function*g(e,t,n){yield*y(),i={kind:e,index:j++,entry:n,carry:{key:null,text:"",details:[]}},yield l("content_block_start",{index:i.index,content_block:t})}s(g,"begin");function*a(e,t,n=null){if(e==="text"&&U&&(t=t.trimStart()),t==="")return;if(U=e==="thinking",e==="text"&&(v=!0),i?.kind==="tool"){S.push({kind:e,text:t,key:n});return}i?.kind!==e&&(yield*g(e,e==="text"?{type:"text",text:""}:{type:"thinking",thinking:"",signature:""})),n!==null&&(i.carry.key??=n,i.carry.text+=t);const o=e==="text"?{type:"text_delta",text:t}:{type:"thinking_delta",thinking:t};yield l("content_block_delta",{index:i.index,delta:o})}s(a,"piece");function*z(e){if(i?.kind==="tool"){S.push({kind:"details",items:e});return}i?.kind!=="thinking"&&(yield*g("thinking",{type:"thinking",thinking:"",signature:""}));for(const t of e)re(i.carry.details,t);i.carry.key===null&&(yield*a("thinking",se(e)))}s(z,"details");function*D(e){yield*a("thinking",e.reasoning),yield*a("text",e.text)}s(D,"pieces");function*H(e){let t=e.index;t===void 0&&(t=e.id===void 0||e.id===null?E:c.size),E=t;let n=c.get(t);if(n===void 0){const x=e.function?.name??"",_={type:"tool_use",id:te(e.id,K),name:ne(J,x),input:{}};d?.pending()&&(I=!0);const f=u.toolStreaming==="buffer"||I;n={state:f?"buffered":"pending",sent:!1,args:[],block:_},c.set(t,n),!f&&i?.kind!=="tool"&&(n.state="open",yield*g("tool",_,n))}let o=e.function?.arguments;o!==null&&typeof o=="object"&&(o=JSON.stringify(o)),!(o==null||o==="")&&(n.args.push(o),n.state==="open"&&(n.sent=!0,yield l("content_block_delta",{index:i.index,delta:{type:"input_json_delta",partial_json:o}})))}s(H,"toolCall");try{for await(const e of ue(b)){if(e.data==="[DONE]"){A=!0;break}const t=e.data;if(t.error){const f=t.error.message??"Upstream error";yield p(Y(t.error),Z(f)??f);return}m||(m=!0,yield l("message_start",{message:{id:ee(),type:"message",role:"assistant",model:r?.model??t.model,content:[],stop_reason:null,stop_sequence:null,usage:B(t.usage)}})),t.usage&&(N=t.usage,r&&(r.upstreamUsage=t.usage));const n=t.choices?.[0];if(!n)continue;const o=n.delta??{},x=oe(o);let _="";x!==""&&!w&&(w=!0,d&&(_=d.release().text)),yield*a("thinking",x,le(o)),yield*a("text",_),u.reasoningRoundTrip&&Array.isArray(o.reasoning_details)&&o.reasoning_details.length>0&&(yield*z(o.reasoning_details)),typeof o.content=="string"&&o.content!==""&&(d&&!w?yield*D(d.push(o.content)):yield*a("text",o.content)),typeof o.refusal=="string"&&(h+=o.refusal),T=de(n.content_filter_results)??T;for(const f of o.tool_calls??[])yield*H(f);n.finish_reason&&(k=n.finish_reason),typeof n.stop_reason=="string"&&(O=n.stop_reason)}}catch(e){yield e instanceof X?p(e.type,e.message):p("api_error",e.message);return}if(!m||k===null&&!A){yield p("api_error",m?"Upstream stream ended before completion":"Upstream stream ended without data");return}const C=fe(k);if(C){yield p(C.type,C.message);return}d&&(yield*D(d.end())),yield*G();const q=R().at(-1),L=q===void 0||$(q.args.join(""));yield*y();for(const e of S)yield*e.kind==="details"?z(e.items):a(e.kind,e.text,e.key);!v&&h!==""&&(yield*a("text",h)),yield*y(),j===0&&(yield*g("text",{type:"text",text:""}),yield*y());const P=ce(k,{hasToolCalls:c.size>0,matchedStop:O,argumentsComplete:L,refusal:h,category:T}),Q=r?.appliedEdits?{context_management:{applied_edits:r.appliedEdits}}:{};yield l("message_delta",{delta:P,usage:B(N),...Q}),yield l("message_stop",{})}s(be,"stream");export{be as stream};
|
|
2
2
|
//# sourceMappingURL=stream.mjs.map
|
package/dist/api/detect.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var
|
|
1
|
+
var w=Object.defineProperty;var f=(t,e)=>w(t,"name",{value:e,configurable:!0});import{context as p,self as b}from"../lib/runtime.mjs";import{capture as y}from"../lib/capture.mjs";import{TranslatorError as d}from"../lib/errors.mjs";import{warn as k}from"../lib/warn.mjs";import{emit as $}from"../lib/events.mjs";import*as x from"../lib/http.mjs";import{validToolIdPolicy as U}from"../lib/ids.mjs";async function D(t={}){const e=y(b,p),{status:n}=b,l=t.network!==!1;for(const a of e.drivers){const s=a.info.fields;if(s!==void 0&&!(Array.isArray(s)&&s.every(i=>typeof i=="string")))throw new d(`Driver ${a.name}: info().fields must be an array of field names`,{code:"invalid_driver",status:500})}for(const a of e.drivers){const s=a.info.toolCallIds;if(s!==void 0&&!U(s))throw new d(`Driver ${a.name}: info().toolCallIds must be { alphabet, min?, max? } with a character-class alphabet and positive integer bounds`,{code:"invalid_driver",status:500})}const r={transport:e.state.transport,signal:t.signal,timeout:e.config.detectTimeout};return await Promise.all(e.state.endpoints.map(a=>A(e,a,l,r))),n()}f(D,"detect");async function A(t,e,n,l){let r,a=null;if(e.backend!=="auto"){if(r=t.drivers.find(s=>s.name===e.backend),!r)throw new d(`Unknown backend "${e.backend}" for upstream ${e.name}`,{code:"unknown_backend",status:500});a={version:e.version}}else if(n){const s=URL.canParse(e.url)?new URL(e.url).hostname:"",i={endpoint:e,...l,probes:new Map},c=t.drivers.filter(o=>I(o.info,s));for(const o of c)if(a=await h(o,i),a){r=o;break}let u=!1;for(const[o,m]of r?[]:t.drivers.entries()){if(!u&&m.info.probes?.length>0&&!m.info.hosts?.includes(s)){u=!0;for(const v of t.drivers.slice(o))for(const g of v.info.probes??[])x.probe(i,g,v.info)}if(a=await h(m,i),a){r=m;break}}}if(r||(r=t.drivers.find(s=>s.name==="compatible"),a={reachable:null,models:[]}),t.state.noMessages.delete(e.name),Object.assign(e.health,{ready:null,readyReason:null,checkedAt:null,unsupported:!1}),e.driver=r.name,e.protocol=r.info.protocol,e.api==="native"){if(!r.info.native){const s=t.drivers.filter(i=>i.info.native).map(i=>i.name);throw new d(`Upstream ${e.name}: api "native" needs a driver with a native API (${s.join(", ")}), but the upstream is bound to ${r.name}; set backend to one of them`,{code:"unsupported_api",status:500})}e.protocol=r.info.native.protocol}if(e.facts=a,e.driverSettings=await r.settings?.(a)??{},e.probed={},n&&e.backend==="auto"&&r.probe){const s=f(c=>e.settings[c]!==void 0||t.config.settings[c]!==void 0,"given"),i={inlineSystem:t.settings(e).inlineSystem==="auto",images:!s("images")};(i.inlineSystem||i.images)&&(e.probed=await r.probe({endpoint:e,facts:a,want:i,...l}))}a.reachable===!1?k(t.config,t.state,`Upstream ${e.name} (${e.url}) did not answer detection; using the generic OpenAI-compatible driver`,{type:"detect",kind:"unreachable",endpoint:e.name,url:e.url,driver:r.name}):$({config:t.config,state:t.state},{type:"detect",kind:"driver",level:"info",message:`Upstream ${e.name} (${e.url}): driver ${r.name}`,endpoint:e.name,url:e.url,driver:r.name})}f(A,"bind");function I(t,e){return e!==""&&(t.hosts?.includes(e)===!0||t.hostSuffixes?.some(n=>e.endsWith(n))===!0)}f(I,"matchesHost");async function h(t,e){try{return await t.detect?.(e)??null}catch{return null}}f(h,"detectWith");export{D as detect};
|
|
2
2
|
//# sourceMappingURL=detect.mjs.map
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var
|
|
1
|
+
var r=Object.defineProperty;var o=(t,e)=>r(t,"name",{value:e,configurable:!0});import{info as s}from"./info.mjs";function c({endpoint:t}){const e=URL.canParse(t.url)?new URL(t.url).hostname:"";return s().hostSuffixes.some(n=>e.endsWith(n))?{host:e}:null}o(c,"detect");export{c as detect};
|
|
2
2
|
//# sourceMappingURL=detect.mjs.map
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var a=Object.defineProperty;var e=(o,
|
|
1
|
+
var a=Object.defineProperty;var e=(o,i)=>a(o,"name",{value:i,configurable:!0});function n(){return{name:"azure",protocol:"openai.chat",order:32,auth:"api-key",strict:!0,hostSuffixes:[".openai.azure.com",".cognitiveservices.azure.com",".services.ai.azure.com"],paths:{chat:"/openai/v1/chat/completions",embeddings:"/openai/v1/embeddings",models:"/openai/v1/models"}}}e(n,"info");export{n as info};
|
|
2
2
|
//# sourceMappingURL=info.mjs.map
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var a=Object.defineProperty;var e=(o,t)=>a(o,"name",{value:t,configurable:!0});function
|
|
1
|
+
var a=Object.defineProperty;var e=(o,t)=>a(o,"name",{value:t,configurable:!0});function r(){return{name:"deepseek",protocol:"openai.chat",order:35,auth:"bearer",strict:!1,hosts:["api.deepseek.com"],paths:{chat:"/v1/chat/completions",models:"/v1/models"}}}e(r,"info");export{r as info};
|
|
2
2
|
//# sourceMappingURL=info.mjs.map
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var a=Object.defineProperty;var e=(p,t)=>a(p,"name",{value:t,configurable:!0});function r(){return{name:"llamacpp",protocol:"openai.chat",order:40,auth:"bearer",strict:!1,probes:["/props"],fields:["top_k","min_p","chat_template_kwargs","repetition_penalty","repeat_penalty","grammar"],paths:{chat:"/v1/chat/completions",models:"/v1/models",props:"/props"}}}e(r,"info");export{r as info};
|
|
1
|
+
var a=Object.defineProperty;var e=(p,t)=>a(p,"name",{value:t,configurable:!0});function r(){return{name:"llamacpp",protocol:"openai.chat",order:40,auth:"bearer",strict:!1,probes:["/props"],fields:["top_k","min_p","chat_template_kwargs","repetition_penalty","repeat_penalty","grammar","json_schema"],paths:{chat:"/v1/chat/completions",models:"/v1/models",props:"/props"}}}e(r,"info");export{r as info};
|
|
2
2
|
//# sourceMappingURL=info.mjs.map
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var e=Object.defineProperty;var a=(t,o)=>e(t,"name",{value:o,configurable:!0});function r(){return{name:"mistral",protocol:"openai.chat",order:36,auth:"bearer",strict:!1,toolCallIds:{alphabet:"a-zA-Z0-9",min:9,max:9},paths:{chat:"/v1/chat/completions",models:"/v1/models"}}}a(r,"info");export{r as info};
|
|
1
|
+
var e=Object.defineProperty;var a=(t,o)=>e(t,"name",{value:o,configurable:!0});function r(){return{name:"mistral",protocol:"openai.chat",order:36,auth:"bearer",strict:!1,hosts:["api.mistral.ai"],toolCallIds:{alphabet:"a-zA-Z0-9",min:9,max:9},paths:{chat:"/v1/chat/completions",models:"/v1/models"}}}a(r,"info");export{r as info};
|
|
2
2
|
//# sourceMappingURL=info.mjs.map
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var
|
|
1
|
+
var a=Object.defineProperty;var o=(e,t)=>a(e,"name",{value:t,configurable:!0});function c(){return{name:"openai",protocol:"openai.chat",order:30,auth:"bearer",strict:!0,hosts:["api.openai.com"],fields:["prompt_cache_key"],paths:{chat:"/v1/chat/completions",models:"/v1/models"}}}o(c,"info");export{c as info};
|
|
2
2
|
//# sourceMappingURL=info.mjs.map
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var r=Object.defineProperty;var o=(e,t)=>r(e,"name",{value:t,configurable:!0});function n(){return{name:"openrouter",protocol:"openai.chat",order:37,auth:"bearer",strict:!1,paths:{chat:"/v1/chat/completions",models:"/v1/models"}}}o(n,"info");export{n as info};
|
|
1
|
+
var r=Object.defineProperty;var o=(e,t)=>r(e,"name",{value:t,configurable:!0});function n(){return{name:"openrouter",protocol:"openai.chat",order:37,auth:"bearer",strict:!1,hosts:["openrouter.ai"],paths:{chat:"/v1/chat/completions",models:"/v1/models"}}}o(n,"info");export{n as info};
|
|
2
2
|
//# sourceMappingURL=info.mjs.map
|
package/dist/api/request.mjs
CHANGED
|
@@ -1,3 +1,3 @@
|
|
|
1
|
-
var
|
|
2
|
-
${
|
|
1
|
+
var M=Object.defineProperty;var h=(e,a)=>M(e,"name",{value:a,configurable:!0});import{context as B,self as E}from"../lib/runtime.mjs";import{capture as K}from"../lib/capture.mjs";import{resolve as P}from"../lib/route.mjs";import*as S from"../lib/http.mjs";import{filterChatBody as R}from"../lib/passthrough.mjs";import{addSampling as H}from"../lib/sampling.mjs";import{addConstraint as z}from"../lib/constraint.mjs";import{addCache as A}from"../lib/cache.mjs";import*as C from"../lib/images.mjs";import*as T from"../lib/tools.mjs";import{LIMITS as D}from"../lib/toolsearch.mjs";import{warn as v}from"../lib/warn.mjs";import{emit as G}from"../lib/events.mjs";import{fit as J,pinBody as N}from"../lib/effort.mjs";import{relax as Q,strictTools as V}from"../lib/strict.mjs";import{TranslatorError as W}from"../lib/errors.mjs";import*as X from"../lib/vllm.mjs";function Y(e){return`${e.driver}:${e.protocol}:${URL.canParse(e.url)?new URL(e.url).host:e.url}`}h(Y,"producerOf");async function ue(e,a={}){const r=K(E,B,a.front),f=P(r,e,{endpoint:a.endpoint,advance:a.hold===!0&&a.endpoint===void 0});if(a.hold!==!0)return q(r,e,a,f);f.endpoint.health.inflight++;try{return await q(r,e,a,f)}catch(n){throw f.endpoint.health.inflight--,n}}h(ue,"request");function Z(e,a,r,f,n,p){const k=V(a);if(k.length===0)return a;const l=k.filter(s=>s.problems.length>0);if(l.length>0&&n.unknownFields==="error")throw new W(`tools: ${l[0].name}: schema is outside the strict subset (${l[0].problems.join("; ")})`,{code:"unsupported_content",param:"tools"});const _=Q(a);if(p.retry===!0)return _;const g=h((s,j)=>v(e.config,e.state,s,{kind:j,endpoint:r.name,log:p.log}),"tell");for(const s of l)g(`Upstream ${r.name}: strict tool ${s.name} has a schema outside the strict subset: ${s.problems.join("; ")} (sent without strict)`,"strict");const u=k.filter(s=>s.problems.length===0);return f.info.strict!==!0&&u.length>0&&g(`Upstream ${r.name}: the ${f.name} backend does not enforce strict tool use (${u.map(s=>s.name).join(", ")})`,"strict"),_}h(Z,"checkStrict");async function q(e,a,r,f){const{endpoint:n,model:p,unserved:k,convert:l,pool:_,pin:g}=f,u=Object.hasOwn(e.config.efforts,p)?e.config.efforts[p]:void 0;g&&(a=N(a,g,e.front));const s=e.driver(n),j=e.front==="anthropic"?a.metadata?.prompt_cache_key:void 0;let w=e.front==="anthropic"?a.metadata?.constraint:void 0;e.front==="openai"&&a.constraint!==void 0&&({constraint:w,...a}=a);const $=`${n.name}
|
|
2
|
+
${p}`,U=r.images===!1||e.state.textOnly.has($)&&e.settings(n).imageFallback,b=e.settings(n),I=r.jsonObject==="instruction"||e.state.jsonInstruction.has($)&&b.jsonObjectFallback,c={...U?{...b,images:!1}:b,...I?{jsonObject:"instruction"}:{},...g?.effort===void 0?{}:{effort:!0}},y=a.stream===!0,m=l===null?e.front==="anthropic"?"anthropic.messages":"openai.chat":n.protocol,O=m==="anthropic.messages"&&n.protocol==="openai.chat";let t,o;l?({body:t,state:o}=await l.request(a,c,{producer:Y(n),budget:r.budget,listed:u!==void 0,toolCallIds:s.info.toolCallIds}),o.toolSearch?.capped&&v(e.config,e.state,`Tool search: more than ${D.definitions} tool definitions came back in ToolSearch results for ${n.name}; the rest are not offered.`,{kind:"toolsearch",endpoint:n.name,log:r.log})):O?t=X.request(a,c):t=e.front==="openai"&&(r.filter??c.passthroughFilter===!0)?R(a,s.info.fields):{...a};const F=e.front==="anthropic"?a.metadata?.sampling:void 0;if(m==="openai.chat"){const i=[];t=H(t,F,c,s.info.fields,i),o&&o.dropped.push(...i.filter(d=>!o.dropped.includes(d)))}else if(!l&&F!==void 0&&t.metadata?.sampling!==void 0){const{sampling:i,...d}=t.metadata;t={...t,metadata:d}}if(m==="openai.chat"){const i=[],d={name:s.info.name,version:n.facts?.version??n.version};t=z(t,w,c,d,i),t=A(t,{salt:r.cacheSalt,key:j},c,d.name,i),o&&o.dropped.push(...i.filter(L=>!o.dropped.includes(L)))}else{if(o&&!o.dropped.includes("cache_salt")&&(r.cacheSalt??c.cacheSalt)!=null&&o.dropped.push("cache_salt"),o&&!o.dropped.includes("prompt_cache_key")&&(j??c.promptCacheKey)!=null&&o.dropped.push("prompt_cache_key"),!l&&t.metadata?.prompt_cache_key!==void 0){const{prompt_cache_key:i,...d}=t.metadata;t={...t,metadata:d}}if(o&&(w??c.constraint)!=null&&!o.dropped.includes("constraint")&&o.dropped.push("constraint"),!l&&t.metadata?.constraint!==void 0){const{constraint:i,...d}=t.metadata;t={...t,metadata:d}}}if(r.usage===!0&&y&&e.front==="openai"){if(l)o.includeUsage=!0;else if(c.streamUsage){const i=t.stream_options&&typeof t.stream_options=="object"?t.stream_options:{};t.stream_options={...i,include_usage:!0}}}U&&!l&&e.front==="openai"&&(t=C.strip(t)),r.tools===!1&&(t=T.strip(t)),t.model=p,t=Z(e,t,n,s,c,r),u&&m==="openai.chat"&&J(t,u,p,(i,d)=>v(e.config,e.state,i,{type:"effort",endpoint:n.name,model:p,...d,log:r.log})),o?.reasoningMissing&&v(e.config,e.state,`Upstream ${n.name}: assistant turns with tool calls have no reasoning to send back (the client dropped its thinking blocks or their signatures), which servers that need it with tools will refuse.`,{kind:"reasoning",endpoint:n.name,log:r.log}),s.request&&!O&&(t=await s.request(t,{endpoint:n,settings:c,levels:u,state:o,warn:h(i=>v(e.config,e.state,i,{kind:"driver",endpoint:n.name,log:r.log}),"warn")})),o?.dropped?.length>0&&r.retry!==!0&&G(e,{type:"dropped",level:"debug",message:`Upstream ${n.name}: dropped ${o.dropped.join(", ")}`,endpoint:n.name,model:p,fields:[...o.dropped],log:r.log});const x=await S.pathOf(s,m==="anthropic.messages"?"messages":"chat",n,p);return{upstream:n.name,pool:_.map(i=>i.name),method:"POST",url:S.join(n.url,x),path:x,headers:{"content-type":"application/json",accept:y?"text/event-stream":"application/json",...s.info.headers,...l===null&&n.protocol==="anthropic.messages"?r.clientHeaders:void 0,...S.headers(n,s.info.auth,r.headers)},body:t,state:{v:1,upstream:n.name,front:e.front,native:l===null,messages:O,model:a.model,stream:y,unserved:k,convert:o},images:{sent:C.carries(t),retry:c.imageFallback&&m==="openai.chat",key:$},tools:{sent:T.carries(t),retry:c.toolFallback&&m==="openai.chat"},json:{sent:o?.jsonSchemaEmulated===!0,retry:c.jsonObjectFallback&&m==="anthropic.messages",key:$}}}h(q,"prepare");export{ue as request};
|
|
3
3
|
//# sourceMappingURL=request.mjs.map
|
package/dist/api/send.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var le=Object.defineProperty;var u=(c,
|
|
1
|
+
var le=Object.defineProperty;var u=(c,s)=>le(c,"name",{value:s,configurable:!0});import{context as K,self as ee}from"../lib/runtime.mjs";import{TranslatorError as z}from"../lib/errors.mjs";import*as te from"../lib/http.mjs";import*as ue from"../lib/images.mjs";import*as ce from"../lib/tools.mjs";import{parse as ae}from"../lib/upstream-error.mjs";import{meter as me,normalize as de}from"../lib/usage.mjs";import{capture as fe}from"../lib/capture.mjs";import{estimate as pe,bodyLength as ge,output as he,patch as ye,stream as we}from"../lib/estimate.mjs";import{warn as re}from"../lib/warn.mjs";import{start as be}from"../lib/record.mjs";import{emit as se}from"../lib/events.mjs";import*as ne from"../lib/pool.mjs";import{createBudget as ke}from"../lib/thinking.mjs";import*as oe from"../lib/fanout.mjs";import{refresh as je}from"../lib/health.mjs";const ve={"content-type":"text/event-stream; charset=utf-8","cache-control":"no-cache","x-accel-buffering":"no"};async function _e(c,s={}){const{transport:w,textOnly:f,jsonInstruction:v}=K.state,{request:_,response:$,stream:o,error:p}=ee,d=p.build,{config:i,state:b}=K,g=fe(ee,K,s.front),U=c&&typeof c=="object"&&!Array.isArray(c),T={model:null,requested:U&&typeof c.model=="string"?c.model:null,upstream:null,front:s.front??i.front,stream:U&&c.stream===!0},k=me(T,s.onUsage,(m,A)=>re(i,b,m,{type:"usage",...A,log:s.log})),q=u(m=>Object.defineProperty(m,"usage",{value:k.usage,enumerable:!1,configurable:!0}),"attach");let J=!1;try{const m=oe.wants(g,c);if(m>0){J=!0;const A=u((n,S)=>re(i,b,n,{type:"usage",...S,log:s.log}),"warning");return await oe.run({body:c,count:m,options:s,send:_e,warn:A})}return q(await $e({transport:w,textOnly:f,jsonInstruction:v,request:_,response:$,stream:o,error:p,captured:g},c,s,k))}catch(m){if(J)throw m;if(!(m instanceof z))throw s.signal?.aborted||se(g,{type:"error",kind:"exception",level:"error",message:String(m?.message??m),log:s.log}),k.finish(s.signal?.aborted?"aborted":"error"),m;return k.finish("error"),q(d(m.status,m.message,{type:m.type,code:m.code,param:m.param,front:s.front}))}}u(_e,"send");async function $e({transport:c,textOnly:s,jsonInstruction:w,request:f,response:v,stream:_,error:$,captured:o},p,d,i){const b={front:d.front,headers:d.headers,clientHeaders:d.clientHeaders,filter:d.filter,cacheSalt:d.cacheSalt,usage:!0,budget:ke(),log:d.log},g=u(t=>{se(o,{...t,log:d.log})},"announce"),U=i.meta.front==="openai"&&p?.stream_options?.include_usage!==!0,T=u(t=>{i.meta.model=t.body.model??null,i.meta.upstream=t.upstream,t.state.native?i.source(null):i.source(()=>t.state.convert?.upstreamUsage,t.state.front==="anthropic"?"openai":"folded")},"note"),k=u(t=>{if(t.state.native)return{input:!1,output:!1};const e=de(t.state.front==="anthropic"?"openai":"folded",t.state.convert?.upstreamUsage);return e===null?{input:!0,output:!0}:{input:e.input_tokens===null&&e.cache_read_input_tokens===null&&e.cache_creation_input_tokens===null,output:e.output_tokens===null}},"gaps"),q=u(async t=>{const e=o.endpoint(t.upstream),a=o.driver(e);return(a.count&&!(U&&t.state.stream)?await a.count({endpoint:e,request:t.body,transport:c,signal:d.signal,timeout:o.config.detectTimeout}):null)??pe(t.body)},"input"),J=u(async(t,e)=>{const a=k(t);return!a.input&&!a.output?null:{input_tokens:a.input?await q(t):null,output_tokens:a.output?he(e):null}},"counts"),m=u((t,e,a,r,R)=>(i.finish(R),$(t,e,a,r)),"fail"),A=u(async t=>{let e=t,a=await Q(c,e,d.signal,I());const r=a.status;if(r===404&&e.state.messages){const y=await C(a);return y?.type==="error"?{upstream:e,result:a,failure:y}:(o.state.noMessages.add(e.upstream),g({type:"fallback",kind:"messages",level:"warn",message:`Upstream ${e.upstream}: /v1/messages answered 404; sent the request converted to Chat Completions, and will do so for this upstream until the next detection. Set nativeMessages: false on this upstream to skip the first attempt.`,endpoint:e.upstream,model:e.body.model,status:r,reason:String(ae(y,r).message).trim().slice(0,500),action:"conversion"}),e=await f(p,{...b,endpoint:e.upstream,retry:!0}),T(e),{upstream:e,result:await Q(c,e,d.signal,I())})}const R=r===400&&[e.images,e.tools].some(y=>y.sent&&y.retry),F=(r===400||r===422)&&e.json.sent&&e.json.retry;if(!a.ok&&(R||F)){const y=await C(a),O=ae(y,r).message,h={};if(r===400&&e.images.sent&&e.images.retry&&ue.refused(O)?(s.add(e.images.key),h.images=!1):r===400&&e.tools.sent&&e.tools.retry&&ce.refused(O)?h.tools=!1:F&&(h.jsonObject="instruction"),!("images"in h||"tools"in h||"jsonObject"in h))return{upstream:e,result:a,failure:y};const j={upstream:e,result:a,failure:y};if("images"in h||"tools"in h){const G="images"in h?"images":"tools";g({type:"fallback",kind:G,level:"warn",message:`Upstream ${e.upstream}: ${e.state.model} refused ${G==="images"?"images":"tools"} (${r}: "${String(O).trim().slice(0,500)}"); sent the request again without them.`,endpoint:e.upstream,model:e.body.model,status:r,reason:String(O).trim().slice(0,500),action:G==="images"?"text-only":"without-tools"})}if(e=await f(p,{...b,...h,endpoint:e.upstream,retry:!0}),T(e),a=await Q(c,e,d.signal,I()),F&&"jsonObject"in h){if(!a.ok){try{await a.body?.cancel?.()}catch{}return T(j.upstream),j}w.add(j.upstream.json.key),g({type:"fallback",kind:"json_object",level:"warn",message:`Upstream ${j.upstream.upstream}: ${j.upstream.state.model} rejected the json_object schema emulation (${r}: "${String(O).trim().slice(0,500)}"); sent it as a system instruction instead. Set jsonObject: "instruction" on this upstream to skip the first attempt.`,endpoint:j.upstream.upstream,model:j.upstream.body.model,status:r,reason:String(O).trim().slice(0,500),action:"instruction"})}}return{upstream:e,result:a}},"attempt");await je(o,p,d.signal);let n=await f(p,{...b,hold:!0});const S=n.pool;let B=0;const D=S.length>1?o.config.pool:null,I=u(()=>D&&{connectTimeout:D.connectTimeout,headersTimeout:B+1<S.length?D.headersTimeout:0},"limitsNow");let E=o.endpoint(n.upstream);const V=u(()=>E.health.inflight--,"release");let l,x,W=null,X=0;const Y=u(t=>{const e=be(o,{body:p,sent:n,front:i.meta.front,requested:i.meta.requested,headers:t,log:d.log,exchange:W,attempt:X});return W??=e.id,e},"begin"),Z=u(async t=>{const e=Y();if(!e.on)return;if(l===void 0)return e.attempt({error:x.message},t);let a=null;try{a=x??await C(l)}catch{}e.attempt({status:l.status,headers:te.plain(l.headers),body:a},t)},"keep");try{for(;;){const t=E;t.health.requests++,X++,T(n);let e=null;try{({upstream:n,result:l,failure:x}=await A(n)),!l.ok&&l.status>=500&&(e=`HTTP ${l.status}`,g({type:"error",kind:"upstream",level:"error",message:`Upstream ${t.name} answered HTTP ${l.status}`,endpoint:t.name,model:n.body.model,status:l.status}))}catch(r){if(!(r instanceof z)||r.code!=="upstream_unreachable")throw r;e=r.message,l=void 0,x=r,g({type:"error",kind:"transport",level:"error",message:r.message,endpoint:t.name,model:n.body.model})}if(e===null){ne.succeeded(t);break}ne.failed(o.config,t,e);let a=null;for(;a===null&&!d.signal?.aborted&&B+1<S.length;)try{a=await f(p,{...b,endpoint:S[++B],hold:!0})}catch(r){if(!(r instanceof z)||r.code!=="endpoint_not_in_pool")throw r}if(a===null){if(l===void 0)throw await Z(),x;break}await Z(e),g({type:"failover",level:"warn",message:`Upstream ${t.name} failed (${e}); trying ${a.upstream}`,from:t.name,to:a.upstream,model:a.body.model,reason:e});try{await l?.body?.cancel?.()}catch{}V(),E=o.endpoint(a.upstream),n=a}}catch(t){throw V(),t}const ie=E;E=null,i.usage.then(()=>ie.health.inflight--);const H=te.plain(l.headers),M=Y(H);if(!l.ok){const t=x??await C(l);return M.reply(l.status,H,t),m(l.status,t,H,n.state,"upstream_error")}if(n.state.stream){const t=_(M.raw(l.body),n.state),e=n.state.native?t:we(i.meta.front,t,{gaps:u(()=>k(n),"gaps"),input:u(()=>q(n),"input"),report:u(a=>i.estimate(a),"report")});return{status:200,headers:ve,stream:M.events(i.track(e,d.signal,U))}}const P=await C(l);if(M.reply(l.status,H,P),P===null||typeof P!="object")return g({type:"error",kind:"upstream",level:"error",message:`Upstream ${n.upstream} returned a body that is not JSON`,endpoint:n.upstream,model:n.body.model,status:502}),m(502,"The upstream returned a body that is not JSON",H,n.state,"error");const N=await v(P,n.state),L=await J(n,ge(i.meta.front,N));return L&&(N.usage=ye(i.meta.front,N.usage,L),i.estimate(L)),i.json(N),i.finish("done"),{status:200,headers:{"content-type":"application/json"},body:N}}u($e,"round");async function Q(c,s,w,f=null){let v,_=!1,$=w;if(f&&f.headersTimeout>0){const o=new AbortController;$=w?AbortSignal.any([w,o.signal]):o.signal,v=setTimeout(()=>{_=!0,o.abort()},f.headersTimeout)}try{return await c(s.url,{method:s.method,headers:s.headers,body:JSON.stringify(s.body),signal:$,redirect:"error",...f&&f.connectTimeout>0?{connectTimeout:f.connectTimeout}:{}})}catch(o){if(w?.aborted)throw o;const p=_?`no response headers within ${f.headersTimeout} ms`:o.message;throw new z(`Could not reach the upstream (${s.upstream}): ${p}`,{code:"upstream_unreachable",status:502,details:{cause:o}})}finally{clearTimeout(v)}}u(Q,"call");async function C(c){const s=await c.text();try{return JSON.parse(s)}catch{return s===""?null:s}}u(C,"parse");export{_e as send};
|
|
2
2
|
//# sourceMappingURL=send.mjs.map
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
var h=Object.defineProperty;var a=(e,t)=>h(e,"name",{value:t,configurable:!0});import{TranslatorError as i}from"./errors.mjs";const f=["vllm"],_=["openai"];function d(e){return e==null||typeof e=="string"&&e!==""}a(d,"valid");function w(e,t,r,l,s=[]){if(!d(t.salt))throw new i("cacheSalt: expected a non-empty string",{code:"invalid_request",param:"cacheSalt"});if(!d(t.key))throw new i("metadata.prompt_cache_key: expected a non-empty string",{code:"invalid_request",param:"metadata.prompt_cache_key"});let c=e;const u=a(n=>{if(r.unknownFields==="error")throw new i(`${n}: is not accepted by this upstream`,{code:"unsupported_field",param:n});s.includes(n)||s.push(n)},"refuse"),o=t.salt??r.cacheSalt;o!=null&&(f.includes(l)?c={...c,cache_salt:o}:u("cache_salt"));const p=t.key??r.promptCacheKey;return p!=null&&!Object.hasOwn(e,"prompt_cache_key")&&(_.includes(l)?c={...c,prompt_cache_key:p}:u("prompt_cache_key")),c}a(w,"addCache");export{_ as KEY_DRIVERS,f as SALT_DRIVERS,w as addCache,d as valid};
|
|
2
|
+
//# sourceMappingURL=cache.mjs.map
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
var g=Object.defineProperty;var i=(r,o)=>g(r,"name",{value:o,configurable:!0});import{atLeast as h}from"./normalize.mjs";import{TranslatorError as l}from"./errors.mjs";const u=["json","regex","choice","grammar"],j="0.11.0",x=i(r=>r.replace(/[.*+?^${}()|[\]\\]/g,"\\$&"),"escape");function y(r){if(r==null)return null;if(typeof r!="object"||Array.isArray(r))return`expected an object with one of ${u.join(", ")}`;const o=Object.keys(r).filter(t=>r[t]!==void 0&&r[t]!==null);if(o.length!==1||!u.includes(o[0]))return`expected exactly one of ${u.join(", ")}`;const[n]=o,e=r[n];return n==="json"&&(typeof e!="object"||Array.isArray(e))?"json must be a JSON Schema object":(n==="regex"||n==="grammar")&&(typeof e!="string"||e==="")?`${n} must be a non-empty string`:n==="choice"&&!(Array.isArray(e)&&e.length>0&&e.every(t=>typeof t=="string"&&t!==""))?"choice must be a non-empty list of non-empty strings":null}i(y,"check");function _(r,o,n,e){const t=i(()=>`(?:${e.map(x).join("|")})`,"alternation"),f={type:"json_schema",json_schema:{name:"constraint",schema:e,strict:!0}};switch(r){case"vllm":case"compatible":return h(o,j)?{structured_outputs:{[n]:e}}:{[`guided_${n}`]:e};case"sglang":return n==="json"?{response_format:f}:n==="regex"?{regex:e}:n==="choice"?{regex:t()}:{ebnf:e};case"llamacpp":return n==="json"?{json_schema:e}:n==="grammar"?{grammar:e}:n==="choice"?{grammar:`root ::= ${e.map(s=>JSON.stringify(s)).join(" | ")}`}:null;case"tgi":return n==="json"?{response_format:{type:"json",value:e}}:n==="regex"?{response_format:{type:"regex",value:e}}:n==="choice"?{response_format:{type:"regex",value:t()}}:null;default:return n==="json"?{response_format:f}:null}}i(_,"fields");function A(r,o,n,e,t=[]){const f=y(o);if(f!==null)throw new l(`constraint: ${f}`,{code:"invalid_request",param:"constraint"});const s=o??n.constraint;if(s==null)return r;const a=u.find(c=>s[c]!==void 0&&s[c]!==null),p=i(c=>{t.includes(c)||t.push(c)},"note"),m=_(e.name,e.version,a,s[a]);if(m===null){if(n.unknownFields==="error")throw new l(`constraint.${a}: is not accepted by this upstream`,{code:"unsupported_field",param:`constraint.${a}`});return p(`constraint.${a}`),r}return["vllm","compatible"].includes(e.name)&&Array.isArray(r.tools)&&r.tools.length>0&&r.tool_choice!=="none"?(p(`constraint.${a}`),r):Object.keys(m).some(c=>Object.hasOwn(r,c))?r:{...r,...m}}i(A,"addConstraint");export{u as KINDS,j as STRUCTURED_OUTPUTS_VERSION,A as addConstraint,y as check,_ as fields};
|
|
2
|
+
//# sourceMappingURL=constraint.mjs.map
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
var m=Object.defineProperty;var u=(e,o)=>m(e,"name",{value:o,configurable:!0});import{estimate as x,tokens as d}from"./estimate.mjs";import{restoreName as y}from"./ids.mjs";const v=1e5,k=3,a="[Tool result cleared to save context]",c=u((e,o)=>e!==null&&typeof e=="object"&&o.includes(e.type)&&Number.isInteger(e.value)&&e.value>=0,"threshold");function N(e){return e.trigger!==void 0&&!c(e.trigger,["input_tokens","tool_uses"])||e.keep!==void 0&&!c(e.keep,["tool_uses"])||e.clear_at_least!==void 0&&!c(e.clear_at_least,["input_tokens"])||e.exclude_tools!==void 0&&!(Array.isArray(e.exclude_tools)&&e.exclude_tools.every(o=>typeof o=="string"))?!1:e.clear_tool_inputs===void 0||typeof e.clear_tool_inputs=="boolean"}u(N,"valid");function D(e,o,p){const l=[];for(const t of o.messages)if(t.role==="assistant")for(const n of t.tool_calls??[])l.push({call:n,result:o.messages.find(r=>r.role==="tool"&&r.tool_call_id===n.id)});const f=e.trigger??{type:"input_tokens",value:v};if((f.type==="tool_uses"?l.length:x(o))<=f.value)return null;const g=e.keep?.value??k,h=new Set(e.exclude_tools??[]),s=l.slice(0,Math.max(0,l.length-g)).filter(t=>t.result!==void 0&&t.result.content!==a&&!h.has(y(p,t.call.function.name))),i=[];for(const t of s){const n=JSON.stringify([t.result.content,e.clear_tool_inputs===!0?t.call.function.arguments:""]).length,r=JSON.stringify([a,e.clear_tool_inputs===!0?"{}":""]).length;i.push(Math.max(0,n-r))}const _=d(i.reduce((t,n)=>t+n,0));if(s.length===0||_<(e.clear_at_least?.value??0))return null;for(const t of s)t.result.content=a,e.clear_tool_inputs===!0&&(t.call.function.arguments="{}");return{type:e.type,cleared_tool_uses:s.length,cleared_input_tokens:_}}u(D,"clearToolUses");export{k as DEFAULT_KEEP,v as DEFAULT_TRIGGER,a as PLACEHOLDER,D as clearToolUses,N as valid};
|
|
2
|
+
//# sourceMappingURL=context.mjs.map
|
package/dist/lib/passthrough.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var
|
|
1
|
+
var a=Object.defineProperty;var i=(t,o)=>a(t,"name",{value:o,configurable:!0});const p=["model","messages","stream","stream_options","max_tokens","max_completion_tokens","temperature","top_p","n","stop","presence_penalty","frequency_penalty","logit_bias","logprobs","top_logprobs","seed","response_format","tools","tool_choice","parallel_tool_calls","reasoning_effort","user","metadata","store","service_tier"],d=["top_k","min_p","repetition_penalty","length_penalty","min_tokens","stop_token_ids","include_stop_str_in_output","ignore_eos","skip_special_tokens","spaces_between_special_tokens","allowed_token_ids","bad_words","prompt_logprobs","truncate_prompt_tokens","add_generation_prompt","continue_final_message","add_special_tokens","chat_template_kwargs","thinking_token_budget","include_reasoning","structured_outputs","guided_json","guided_regex","guided_choice","guided_grammar"],c=new Set(p),l=new Set(["include_usage","continuous_usage_stats"]);function g(t,o=[]){const s={};for(const[e,n]of Object.entries(t))if(!(!c.has(e)&&!o.includes(e)))if(e==="stream_options"){const _=Object.fromEntries(Object.entries(n??{}).filter(([r])=>l.has(r)));Object.keys(_).length>0&&(s.stream_options=_)}else s[e]=n;return s}i(g,"filterChatBody");export{d as VLLM_FIELDS,g as filterChatBody};
|
|
2
2
|
//# sourceMappingURL=passthrough.mjs.map
|
package/dist/lib/presets.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var f=Object.defineProperty;var s=(e,t)=>f(e,"name",{value:t,configurable:!0});import{noEmptyTools as d}from"./normalize.mjs";const p={chat:"/v1/chat/completions",embeddings:"/v1/embeddings",models:"/v1/models"},c={chat:"/chat/completions",embeddings:"/embeddings",models:"/models"};function m(e){return/^grok-(3-mini|4|code)/.test(String(e))&&!/non-reasoning/.test(String(e))}s(m,"grokReasons");const l=Object.freeze({groq:{order:21,hosts:["api.groq.com"],root:"/openai",paths:p,settings:{reasoningField:"reasoning",maxTokensField:"max_completion_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!1,prefill:"none",streamUsage:!0,continuousUsage:!1},drop:["logprobs","top_logprobs","logit_bias"],dropName:!0},together:{order:22,hosts:["api.together.ai","api.together.xyz","us-api.together.ai"],paths:p,fields:["top_k","min_p","repetition_penalty"],settings:{reasoningField:"reasoning",reasoningRoundTrip:!0,maxTokensField:"max_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!0,prefill:"none",parallelToolCalls:!1,streamUsage:!1,continuousUsage:!1}},fireworks:{order:23,hosts:["api.fireworks.ai"],root:"/inference",paths:p,fields:["top_k","min_p","repetition_penalty"],settings:{reasoningField:"reasoning_content",reasoningRoundTrip:!0,maxTokensField:"max_tokens",thinking:"reasoning_effort",thinkingBudget:!1,effort:!0,topK:!0,prefill:"none",streamUsage:!0,continuousUsage:!1}},xai:{order:24,hosts:["api.x.ai"],paths:p,settings:{reasoningField:"reasoning_content",maxTokensField:"max_completion_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!1,prefill:"none",streamUsage:!0,continuousUsage:!1},dropWhen:s(e=>m(e.model)?["stop","presence_penalty","frequency_penalty"]:[],"dropWhen")},gemini:{order:25,hosts:["generativelanguage.googleapis.com"],root:"/v1beta/openai",paths:c,settings:{maxTokensField:"max_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!1,prefill:"none",streamUsage:!0,continuousUsage:!1}}});function _(e){const t=l[e],o={name:e,protocol:"openai.chat",order:t.order,auth:"bearer",strict:!1,paths:{...t.paths}};return t.fields?{...o,fields:[...t.fields]}:o}s(_,"describe");function x(e,t){const o=URL.canParse(t.url)?new URL(t.url).hostname:"";return l[e].hosts.includes(o)?{host:o}:null}s(x,"match");function U(e){return{...l[e].settings}}s(U,"settingsOf");function b(e,t,o){const i=l[e],a=i.root??"";return(o.url.endsWith(a)?"":a)+i.paths[t]}s(b,"route");function y(e,t,o){const i=l[e],a=o?.state?.dropped,g=s(n=>{Array.isArray(a)&&!a.includes(n)&&a.push(n)},"note"),r={...d(t)};for(const n of[...i.drop??[],...i.dropWhen?.(t)??[]])r[n]!==void 0&&(delete r[n],g(n));return i.dropName&&Array.isArray(r.messages)&&r.messages.some(n=>n&&"name"in n)&&(r.messages=r.messages.map(n=>{const u={...n};return delete u.name,u}),g("messages[].name")),r}s(y,"trim");export{l as PRESETS,_ as describe,x as match,b as route,U as settingsOf,y as trim};
|
|
1
|
+
var f=Object.defineProperty;var s=(e,t)=>f(e,"name",{value:t,configurable:!0});import{noEmptyTools as d}from"./normalize.mjs";const p={chat:"/v1/chat/completions",embeddings:"/v1/embeddings",models:"/v1/models"},c={chat:"/chat/completions",embeddings:"/embeddings",models:"/models"};function m(e){return/^grok-(3-mini|4|code)/.test(String(e))&&!/non-reasoning/.test(String(e))}s(m,"grokReasons");const l=Object.freeze({groq:{order:21,hosts:["api.groq.com"],root:"/openai",paths:p,settings:{reasoningField:"reasoning",maxTokensField:"max_completion_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!1,prefill:"none",streamUsage:!0,continuousUsage:!1},drop:["logprobs","top_logprobs","logit_bias"],dropName:!0},together:{order:22,hosts:["api.together.ai","api.together.xyz","us-api.together.ai"],paths:p,fields:["top_k","min_p","repetition_penalty"],settings:{reasoningField:"reasoning",reasoningRoundTrip:!0,maxTokensField:"max_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!0,prefill:"none",parallelToolCalls:!1,streamUsage:!1,continuousUsage:!1}},fireworks:{order:23,hosts:["api.fireworks.ai"],root:"/inference",paths:p,fields:["top_k","min_p","repetition_penalty"],settings:{reasoningField:"reasoning_content",reasoningRoundTrip:!0,maxTokensField:"max_tokens",thinking:"reasoning_effort",thinkingBudget:!1,effort:!0,topK:!0,prefill:"none",streamUsage:!0,continuousUsage:!1}},xai:{order:24,hosts:["api.x.ai"],paths:p,settings:{reasoningField:"reasoning_content",maxTokensField:"max_completion_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!1,prefill:"none",streamUsage:!0,continuousUsage:!1},dropWhen:s(e=>m(e.model)?["stop","presence_penalty","frequency_penalty"]:[],"dropWhen")},gemini:{order:25,hosts:["generativelanguage.googleapis.com"],root:"/v1beta/openai",paths:c,settings:{maxTokensField:"max_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!1,prefill:"none",streamUsage:!0,continuousUsage:!1}}});function _(e){const t=l[e],o={name:e,protocol:"openai.chat",order:t.order,auth:"bearer",strict:!1,hosts:[...t.hosts],paths:{...t.paths}};return t.fields?{...o,fields:[...t.fields]}:o}s(_,"describe");function x(e,t){const o=URL.canParse(t.url)?new URL(t.url).hostname:"";return l[e].hosts.includes(o)?{host:o}:null}s(x,"match");function U(e){return{...l[e].settings}}s(U,"settingsOf");function b(e,t,o){const i=l[e],a=i.root??"";return(o.url.endsWith(a)?"":a)+i.paths[t]}s(b,"route");function y(e,t,o){const i=l[e],a=o?.state?.dropped,g=s(n=>{Array.isArray(a)&&!a.includes(n)&&a.push(n)},"note"),r={...d(t)};for(const n of[...i.drop??[],...i.dropWhen?.(t)??[]])r[n]!==void 0&&(delete r[n],g(n));return i.dropName&&Array.isArray(r.messages)&&r.messages.some(n=>n&&"name"in n)&&(r.messages=r.messages.map(n=>{const u={...n};return delete u.name,u}),g("messages[].name")),r}s(y,"trim");export{l as PRESETS,_ as describe,x as match,b as route,U as settingsOf,y as trim};
|
|
2
2
|
//# sourceMappingURL=presets.mjs.map
|
package/dist/lib/routes.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var
|
|
1
|
+
var H=Object.defineProperty;var c=(e,o)=>H(e,"name",{value:o,configurable:!0});import{timingSafeEqual as O}from"node:crypto";import{TranslatorError as p}from"./errors.mjs";import{messageId as T}from"./ids.mjs";import{sign as j}from"./thinking.mjs";const C=64*1024*1024,D={"content-type":"text/event-stream; charset=utf-8","cache-control":"no-cache","x-accel-buffering":"no"},v=Symbol("stalled"),x=Symbol("gone");async function I(e,o){if(!(o>0))return e;let s;const d=new Promise(f=>{s=setTimeout(f,o,v)});try{return await Promise.race([e,d])}finally{clearTimeout(s)}}c(I,"settle");async function*L(e,o,s){let d=!1;try{const f=o.front==="anthropic";f&&(yield{event:"message_start",data:{type:"message_start",message:{id:T(),type:"message",role:"assistant",model:s,content:[],stop_reason:null,stop_sequence:null,usage:{input_tokens:0,output_tokens:0,cache_creation_input_tokens:0,cache_read_input_tokens:0}}}},yield{event:"content_block_start",data:{type:"content_block_start",index:0,content_block:{type:"thinking",thinking:"",signature:""}}});const a=await e,b=c(function*(){yield{event:"content_block_delta",data:{type:"content_block_delta",index:0,delta:{type:"signature_delta",signature:j()}}},yield{event:"content_block_stop",data:{type:"content_block_stop",index:0}}},"closeEarly");if(a.stream){d=!0;let u=null;for await(const i of a.stream){if(!f){yield i;continue}if(i.event!=="message_start"){if(u===null&&i.event==="content_block_start"&&i.data.content_block.type==="thinking"){u=0;continue}u===null&&(yield*b(),u=1),yield u>0&&typeof i.data?.index=="number"?{...i,data:{...i.data,index:i.data.index+u}}:i}}return}f&&(yield*b()),yield*S(o,a.body)}finally{d||e.then(f=>f.stream?.return?.(),()=>{})}}c(L,"late");function*S(e,o){if(e.front==="anthropic"){yield{event:"error",data:o};return}yield{data:o},yield{data:"[DONE]"}}c(S,"failure");async function E(e,o,s,d,f,a=200,b=D){e.hijack();const u=e.raw;u.writeHead(a,b);try{await o.write(u,d,{front:o.front,interval:f})}catch(i){if(!s.signal.aborted){const y=i instanceof p;y||(e.log.error(i),o.emit({type:"error",kind:"route",level:"error",message:String(i?.message??i),log:!1}));const{body:g}=o.build(y?i.status:502,y?i.message:"Internal server error",y?{type:i.type,code:i.code}:{});for(const w of S(o,g))u.write(o.encode(w))}}return u.end(),e}c(E,"stream");function U(e,o){const s=Buffer.from(e),d=Buffer.from(o);return s.length===d.length&&O(s,d)}c(U,"same");function z(e){const o=/^Bearer\s+(.+)$/i.exec(e.authorization??"");return e["x-api-key"]||o?.[1]||""}c(z,"keyOf");function B(e){const o={};for(const[s,d]of Object.entries(e))(s==="anthropic-beta"||s==="anthropic-version"||s.startsWith("x-claude-code-"))&&(o[s]=String(d));return o}c(B,"forwardable");function P(e){if(e===!0)return{ok:!0,context:void 0};if(e&&typeof e=="object"&&e.ok===!0)return{ok:!0,context:e.context,cacheSalt:e.cacheSalt};const o=e&&typeof e=="object"&&e.ok===!1?e:{};return{ok:!1,status:Number.isInteger(o.status)&&o.status>=400&&o.status<=599?o.status:401,message:typeof o.message=="string"&&o.message!==""?o.message:"unauthorized"}}c(P,"verdictOf");function R(e,o,s){const{front:d,filter:f}=s,a={...s,build:c((t,n,r={})=>s.build(t,n,{...r,front:d}),"build"),send:c((t,n)=>s.send(t,{...n,front:d,filter:f}),"send"),route:c(t=>s.route(t,{front:d}),"route"),count:c((t,n)=>s.count(t,{...n,front:d}),"count"),embed:c((t,n)=>s.embed(t,n),"embed"),list:c(t=>s.list({...t,front:d}),"list"),get:c((t,n)=>s.get(t,{...n,front:d}),"get")},b=[o.apiKey??[]].flat().filter(Boolean),u=o.bodyLimit??C,i=new WeakMap,y=new WeakMap,g=c((t,n)=>t.code(n.status).headers(n.headers).send(n.body),"reject");e.setErrorHandler((t,n,r)=>{if(t instanceof p)return g(r,a.build(t.status,t.message,{type:t.type,code:t.code,param:t.param}));const l=t.statusCode??500;return l<500?g(r,a.build(l,t.message)):(n.log.error(t),a.emit({type:"error",kind:"route",level:"error",message:String(t.message),status:l,log:!1}),g(r,a.build(l,"Internal server error")))}),e.addHook("onRequest",async(t,n)=>{if(b.length===0||t.routeOptions.config?.public)return;const r=z(t.headers);if(!(r&&b.some(l=>U(r,l))))return g(n,a.build(401,"invalid x-api-key"))}),typeof o.authorize=="function"&&e.addHook("preHandler",async(t,n)=>{if(t.routeOptions.config?.public)return;let r=null;if(t.body&&typeof t.body=="object"&&!Array.isArray(t.body))try{r=a.route(t.body)}catch{}let l;try{l=P(await o.authorize({method:t.method,url:t.url,headers:t.headers,body:t.body,front:a.front,route:r}))}catch(m){t.log.error(m),a.emit({type:"error",kind:"route",level:"error",message:String(m?.message??m),log:!1}),l={ok:!1,status:401,message:"unauthorized"}}if(!l.ok)return g(n,a.build(l.status,l.message));i.set(t,l.context),typeof l.cacheSalt=="string"&&l.cacheSalt!==""&&y.set(t,l.cacheSalt)}),e.route({method:["GET","HEAD"],url:"/",config:{public:!0},handler:c(async(t,n)=>n.send({ok:!0}),"handler")}),e.route({method:["GET","HEAD"],url:"/api/hello",config:{public:!0},handler:c(async(t,n)=>n.send({ok:!0}),"handler")}),e.get("/v1/models",async(t,n)=>{const r=new AbortController;return n.raw.on("close",()=>r.abort()),a.list({signal:r.signal})}),e.get("/v1/models/*",async(t,n)=>{const r=new AbortController;return n.raw.on("close",()=>r.abort()),a.get(t.params["*"],{signal:r.signal})});const w=a.front==="anthropic"?"/v1/messages":"/v1/chat/completions";e.post(w,{bodyLimit:u},async(t,n)=>{if(n.raw.destroyed)return n;const r=new AbortController;n.raw.on("close",()=>{n.raw.writableFinished||r.abort()});const l=typeof o.onUsage=="function"?k=>o.onUsage(k,i.get(t)):void 0,m=a.interval(t.body),_=a.send(t.body,{signal:r.signal,clientHeaders:B(t.headers),cacheSalt:y.get(t),onUsage:l,log:t.log});_.catch(()=>{});const A=new Promise(k=>r.signal.addEventListener("abort",()=>k(x),{once:!0})),h=await I(Promise.race([_,A]),t.body?.stream===!0?m:0);return h===x?(_.then(k=>k.stream?.return?.(),()=>{}),n):h===v?E(n,a,r,L(_,a,t.body?.model),m):h.stream?E(n,a,r,h.stream,m,h.status,h.headers):g(n,h)}),a.front==="openai"&&e.post("/v1/embeddings",{bodyLimit:u},async(t,n)=>{const r=new AbortController;n.raw.on("close",()=>{n.raw.writableFinished||r.abort()});const l=typeof o.onUsage=="function"?m=>o.onUsage(m,i.get(t)):void 0;return g(n,await a.embed(t.body,{signal:r.signal,onUsage:l,log:t.log}))}),a.front==="anthropic"&&e.post("/v1/messages/count_tokens",{bodyLimit:u},async(t,n)=>{const r=new AbortController;return n.raw.on("close",()=>r.abort()),a.count(t.body,{signal:r.signal})})}c(R,"register");export{L as late,R as register};
|
|
2
2
|
//# sourceMappingURL=routes.mjs.map
|
package/dist/lib/settings.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var
|
|
1
|
+
var f=Object.defineProperty;var l=(o,e)=>f(o,"name",{value:e,configurable:!0});import{TranslatorError as n}from"./errors.mjs";import{check as c}from"./constraint.mjs";import{valid as d}from"./cache.mjs";const r=Object.freeze({inlineSystem:"auto",stripBillingHeader:!0,reasoningField:"reasoning",thinkTags:!0,thinkOpen:!1,reasoningRoundTrip:!1,thinking:"chat_template_kwargs",thinkingBudget:!1,effort:!1,images:!0,imageFallback:!0,toolFallback:!1,documents:"extract",documentConverter:null,pdf:Object.freeze({pages:!0,maxPages:100,maxBytes:32*1024*1024,maxImages:20,scale:1.5}),toolSchema:"pass",deferredTools:"forward",toolSearch:"expand",jsonObject:"schema",jsonObjectFallback:!0,toolErrors:!1,toolErrorMarker:"[error] ",toolErrorInference:!1,maxTokensField:"max_completion_tokens",streamUsage:!0,continuousUsage:!1,prefill:"continue_final_message",topK:!0,responseFormat:!0,parallelToolCalls:!0,unknownFields:"drop",sampling:Object.freeze({}),constraint:null,cacheSalt:null,promptCacheKey:null,pingInterval:1e4,toolStreaming:"buffer",effortPassthrough:!1,passthroughFilter:!1,nativeMessages:!1,contextWindow:0,defaultMaxTokens:8192}),g={inlineSystem:["auto","keep","hoist","user","drop"],reasoningField:["reasoning","reasoning_content"],thinking:["chat_template_kwargs","reasoning_effort","none"],documents:["extract","note","error"],toolSchema:["pass","sanitize"],deferredTools:["forward","drop"],toolSearch:["expand","inline","native"],jsonObject:["schema","instruction","drop"],maxTokensField:["max_completion_tokens","max_tokens"],prefill:["continue_final_message","none"],unknownFields:["drop","error"],toolStreaming:["buffer","stream"]},p=["effortPassthrough","passthroughFilter","imageFallback","toolFallback","jsonObjectFallback","thinkOpen","reasoningRoundTrip","toolErrorInference"];function y(...o){const e={...r,pdf:{...r.pdf},sampling:{...r.sampling}};for(const t of o)if(t)for(const[s,a]of Object.entries(t))a!==void 0&&(s==="pdf"?Object.assign(e.pdf,a):s==="sampling"&&a!==null&&typeof a=="object"&&!Array.isArray(a)?Object.assign(e.sampling,a):e[s]=a);for(const[t,s]of Object.entries(g))if(!s.includes(e[t]))throw new n(`Invalid setting ${t}: ${JSON.stringify(e[t])} (expected one of ${s.join(", ")})`,{code:"invalid_setting",status:500,details:{key:t,allowed:s}});for(const t of p)if(typeof e[t]!="boolean")throw new n(`Invalid setting ${t}: ${JSON.stringify(e[t])} (expected true or false)`,{code:"invalid_setting",status:500,details:{key:t}});if(e.nativeMessages!==!0&&e.nativeMessages!==!1&&e.nativeMessages!=="auto")throw new n(`Invalid setting nativeMessages: ${JSON.stringify(e.nativeMessages)} (expected true, false or "auto")`,{code:"invalid_setting",status:500,details:{key:"nativeMessages"}});if(typeof e.toolErrorMarker!="string"||e.toolErrorMarker.trim()==="")throw new n(`Invalid setting toolErrorMarker: ${JSON.stringify(e.toolErrorMarker)} (expected a non-empty string)`,{code:"invalid_setting",status:500,details:{key:"toolErrorMarker"}});if(e.sampling===null||typeof e.sampling!="object"||Array.isArray(e.sampling))throw new n(`Invalid setting sampling: ${JSON.stringify(e.sampling)} (expected an object of field names and values)`,{code:"invalid_setting",status:500,details:{key:"sampling"}});const i=c(e.constraint);if(i!==null)throw new n(`Invalid setting constraint: ${JSON.stringify(e.constraint)} (${i})`,{code:"invalid_setting",status:500,details:{key:"constraint"}});for(const t of["cacheSalt","promptCacheKey"])if(!d(e[t]))throw new n(`Invalid setting ${t}: ${JSON.stringify(e[t])} (expected a non-empty string or null)`,{code:"invalid_setting",status:500,details:{key:t}});if(!(Number.isFinite(e.pdf.scale)&&e.pdf.scale>0))throw new n(`Invalid setting pdf.scale: ${JSON.stringify(e.pdf.scale)} (expected a positive number)`,{code:"invalid_setting",status:500,details:{key:"pdf.scale"}});return e}l(y,"resolve");export{r as DEFAULTS,y as resolve};
|
|
2
2
|
//# sourceMappingURL=settings.mjs.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cldmv/llm-translator",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "Bidirectional Anthropic Messages ⇄ OpenAI Chat Completions translator with backend auto-detection (vLLM and other OpenAI-compatible servers), built on slothlet, with an optional standalone server for running Claude Code against local models.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Bind every endpoint to a driver. `backend: "auto"` endpoints are probed (drivers tried in `info().order
|
|
2
|
+
* Bind every endpoint to a driver. `backend: "auto"` endpoints are probed (drivers tried in `info().order` after any driver whose `info().hosts` names the endpoint host, the generic
|
|
3
3
|
* driver matches last); endpoints with an explicit `backend` skip detection. Call it again to re-probe.
|
|
4
4
|
* @param {any} [options] - `{ network, signal }`. `network: false` skips all probing (auto endpoints fall back to the generic driver).
|
|
5
5
|
* @returns {Promise<any>} The `status()` result.
|
|
@@ -19,7 +19,7 @@
|
|
|
19
19
|
/**
|
|
20
20
|
* Describe the Azure OpenAI driver. The key goes in the `api-key` header. The real paths come from the `path` leaf, because they depend on the
|
|
21
21
|
* upstream's `apiVersion` and `deployment`; `paths` holds the v1 API paths for a base URL without `/openai`.
|
|
22
|
-
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, paths: Record<string,string>}} Driver facts.
|
|
22
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, hostSuffixes: string[], paths: Record<string,string>}} Driver facts.
|
|
23
23
|
*/
|
|
24
24
|
export function info(): {
|
|
25
25
|
name: string;
|
|
@@ -27,5 +27,6 @@ export function info(): {
|
|
|
27
27
|
order: number;
|
|
28
28
|
auth: string;
|
|
29
29
|
strict: boolean;
|
|
30
|
+
hostSuffixes: string[];
|
|
30
31
|
paths: Record<string, string>;
|
|
31
32
|
};
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
/**
|
|
20
20
|
* Describe the DeepSeek driver.
|
|
21
|
-
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, paths: Record<string,string>}} Driver facts.
|
|
21
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, hosts: string[], paths: Record<string,string>}} Driver facts.
|
|
22
22
|
*/
|
|
23
23
|
export function info(): {
|
|
24
24
|
name: string;
|
|
@@ -26,5 +26,6 @@ export function info(): {
|
|
|
26
26
|
order: number;
|
|
27
27
|
auth: string;
|
|
28
28
|
strict: boolean;
|
|
29
|
+
hosts: string[];
|
|
29
30
|
paths: Record<string, string>;
|
|
30
31
|
};
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
/**
|
|
20
20
|
* Describe the Mistral driver. Mistral accepts only 9-character alphanumeric tool-call ids, so the translator rewrites the client's ids.
|
|
21
|
-
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, toolCallIds: {alphabet: string, min: number, max: number}, paths: Record<string,string>}} Driver facts.
|
|
21
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, hosts: string[], toolCallIds: {alphabet: string, min: number, max: number}, paths: Record<string,string>}} Driver facts.
|
|
22
22
|
*/
|
|
23
23
|
export function info(): {
|
|
24
24
|
name: string;
|
|
@@ -26,6 +26,7 @@ export function info(): {
|
|
|
26
26
|
order: number;
|
|
27
27
|
auth: string;
|
|
28
28
|
strict: boolean;
|
|
29
|
+
hosts: string[];
|
|
29
30
|
toolCallIds: {
|
|
30
31
|
alphabet: string;
|
|
31
32
|
min: number;
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
/**
|
|
20
20
|
* Describe the OpenAI driver.
|
|
21
|
-
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, paths: Record<string,string>}} Driver facts.
|
|
21
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, hosts: string[], fields: string[], paths: Record<string,string>}} Driver facts.
|
|
22
22
|
*/
|
|
23
23
|
export function info(): {
|
|
24
24
|
name: string;
|
|
@@ -26,5 +26,7 @@ export function info(): {
|
|
|
26
26
|
order: number;
|
|
27
27
|
auth: string;
|
|
28
28
|
strict: boolean;
|
|
29
|
+
hosts: string[];
|
|
30
|
+
fields: string[];
|
|
29
31
|
paths: Record<string, string>;
|
|
30
32
|
};
|
|
@@ -18,7 +18,7 @@
|
|
|
18
18
|
*/
|
|
19
19
|
/**
|
|
20
20
|
* Describe the OpenRouter driver. The API lives under `/api`; the `path` leaf adds it, so `paths` holds the usual `/v1/...` ones.
|
|
21
|
-
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, paths: Record<string,string>}} Driver facts.
|
|
21
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, hosts: string[], paths: Record<string,string>}} Driver facts.
|
|
22
22
|
*/
|
|
23
23
|
export function info(): {
|
|
24
24
|
name: string;
|
|
@@ -26,5 +26,6 @@ export function info(): {
|
|
|
26
26
|
order: number;
|
|
27
27
|
auth: string;
|
|
28
28
|
strict: boolean;
|
|
29
|
+
hosts: string[];
|
|
29
30
|
paths: Record<string, string>;
|
|
30
31
|
};
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
* Route a client request body (in the configured `front` protocol) to an upstream and convert it to that upstream's
|
|
3
3
|
* protocol. Does no I/O.
|
|
4
4
|
* @param {any} body - Client request body (Anthropic Messages or OpenAI Chat Completions, per `front`).
|
|
5
|
-
* @param {any} [options] - `{ front, headers, clientHeaders, filter, images, usage }`. `front` overrides the configured front for this call, `headers` are extra upstream headers, `clientHeaders` are client headers forwarded only to a native Anthropic upstream, `filter` restricts a forwarded OpenAI body to known fields (default: the `passthroughFilter` setting, off), `images: false` sends the request without images (used by the image fallback), `endpoint` sends to that member of the model's pool (used for failover; default: the first in the pool's order), `hold: true` (used by `send`) advances a round robin and takes the chosen endpoint's in-flight slot, which the caller then releases, `jsonObject: "instruction"` sends a `json_object` response format as a system instruction (used by the JSON fallback, and applied by itself once a refusal is remembered for the endpoint and model), `tools: false` sends the request without tools (used by the tool fallback), `usage: true` (used by `send`) makes a streaming OpenAI-front request always ask the upstream for its final usage chunk, `retry: true` (used by `send` for the same request sent again) keeps the `dropped` event from repeating, `log` is the logger events of this request also go to (used by the server), `budget` (from `createBudget`, one per `send` for all its attempts) limits the cost of decoding reasoning signatures.
|
|
5
|
+
* @param {any} [options] - `{ front, headers, clientHeaders, filter, images, usage }`. `front` overrides the configured front for this call, `headers` are extra upstream headers, `clientHeaders` are client headers forwarded only to a native Anthropic upstream, `filter` restricts a forwarded OpenAI body to known fields (default: the `passthroughFilter` setting, off), `images: false` sends the request without images (used by the image fallback), `endpoint` sends to that member of the model's pool (used for failover; default: the first in the pool's order), `hold: true` (used by `send`) advances a round robin and takes the chosen endpoint's in-flight slot, which the caller then releases, `jsonObject: "instruction"` sends a `json_object` response format as a system instruction (used by the JSON fallback, and applied by itself once a refusal is remembered for the endpoint and model), `tools: false` sends the request without tools (used by the tool fallback), `usage: true` (used by `send`) makes a streaming OpenAI-front request always ask the upstream for its final usage chunk, `cacheSalt` (a trusted per-call vLLM `cache_salt`, which replaces the upstream's `cacheSalt` setting), `retry: true` (used by `send` for the same request sent again) keeps the `dropped` event from repeating, `log` is the logger events of this request also go to (used by the server), `budget` (from `createBudget`, one per `send` for all its attempts) limits the cost of decoding reasoning signatures.
|
|
6
6
|
* @returns {Promise<{upstream: string, method: string, url: string, path: string, headers: Record<string,string>, body: any, state: any, pool: string[], images: {sent: boolean, retry: boolean, key: string}, tools: {sent: boolean, retry: boolean}, json: {sent: boolean, retry: boolean, key: string}}>} The upstream request and the state `response`/`stream`/`error` need. `pool` lists the names of the endpoints serving the model in the order they are tried (`upstream` is the first); `images` tells `send` whether the body carries images, whether a refusal may be retried without them and the key under which a refusal is remembered; `tools` tells it whether the body carries tools and whether a tool-parser refusal may be retried without them; `json` tells it whether the body carries the schema emulation of `json_object`, whether a refusal may be retried with the instruction and the key under which a refusal is remembered.
|
|
7
7
|
*/
|
|
8
8
|
export function request(body: any, options?: any): Promise<{
|
package/types/src/api/send.d.mts
CHANGED
|
@@ -25,7 +25,7 @@
|
|
|
25
25
|
* a stream has started, an error chunk and `[DONE]`. Aborting the caller's signal aborts every branch. An OpenAI-protocol upstream
|
|
26
26
|
* receives `n` unchanged and answers it itself.
|
|
27
27
|
* @param {any} body - Client request body (per `front`).
|
|
28
|
-
* @param {any} [options] - `{ signal, front, headers, clientHeaders, filter, onUsage, log }`: abort signal for the upstream call, per-call front, extra upstream headers, client headers forwarded to a native Anthropic upstream, the pass-through body filter (see `request`) and the usage callback (an exception in it is reported as a `usage` event and never breaks the reply) and the logger this request's events also go to (used by the server).
|
|
28
|
+
* @param {any} [options] - `{ signal, front, headers, clientHeaders, filter, cacheSalt, onUsage, log }`: abort signal for the upstream call, per-call front, extra upstream headers, client headers forwarded to a native Anthropic upstream, the pass-through body filter (see `request`) and the usage callback (an exception in it is reported as a `usage` event and never breaks the reply) and the logger this request's events also go to (used by the server).
|
|
29
29
|
* @returns {Promise<{status: number, headers: Record<string,string>, body: any, usage: Promise<any>} | {status: number, headers: Record<string,string>, stream: AsyncIterable<{event?: string, data: *}>, usage: Promise<any>}>} A JSON reply, or a stream reply when the client asked for streaming and the upstream accepted.
|
|
30
30
|
*/
|
|
31
31
|
export function send(body: any, options?: any): Promise<{
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Is a salt or key value usable?
|
|
3
|
+
* @param {any} value - Setting or request value.
|
|
4
|
+
* @returns {boolean} True for a non-empty string, or an absent value.
|
|
5
|
+
*/
|
|
6
|
+
export function valid(value: any): boolean;
|
|
7
|
+
/**
|
|
8
|
+
* Add the cache hints to an OpenAI-protocol body.
|
|
9
|
+
*
|
|
10
|
+
* - `cache_salt` comes only from trusted places: the `cacheSalt` call option (the server passes the `authorize` hook's `cacheSalt`) or the upstream's `cacheSalt` setting,
|
|
11
|
+
* in that order. It always replaces a `cache_salt` the body carries, so a client cannot choose its own tenant. A backend without the feature
|
|
12
|
+
* follows `unknownFields`: dropped (listed in `dropped`) or rejected.
|
|
13
|
+
* - `prompt_cache_key` comes from the client (`metadata.prompt_cache_key` on the Anthropic front; the OpenAI front's own field is kept) or the upstream's `promptCacheKey` setting. A
|
|
14
|
+
* key the body already carries wins over the setting.
|
|
15
|
+
* @param {any} body - Converted or forwarded Chat Completions body.
|
|
16
|
+
* @param {{ salt?: any, key?: any }} asked - The call's `cacheSalt` option and the request's prompt cache key.
|
|
17
|
+
* @param {any} settings - Resolved settings (`cacheSalt`, `promptCacheKey`, `unknownFields`).
|
|
18
|
+
* @param {string} driver - Driver name of the upstream.
|
|
19
|
+
* @param {string[]} [dropped] - Receives the names that were left out.
|
|
20
|
+
* @returns {any} The body, with the fields added (a copy when anything was added).
|
|
21
|
+
* @throws {TranslatorError} `invalid_request` for a malformed salt or key; `unsupported_field` when the backend lacks the feature and `unknownFields` is `"error"`.
|
|
22
|
+
*/
|
|
23
|
+
export function addCache(body: any, asked: {
|
|
24
|
+
salt?: any;
|
|
25
|
+
key?: any;
|
|
26
|
+
}, settings: any, driver: string, dropped?: string[]): any;
|
|
27
|
+
/** Drivers whose servers take `cache_salt` (vLLM's per-tenant prefix-cache isolation). */
|
|
28
|
+
export const SALT_DRIVERS: string[];
|
|
29
|
+
/** Drivers whose servers take `prompt_cache_key` (OpenAI's cache routing hint). */
|
|
30
|
+
export const KEY_DRIVERS: string[];
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What is wrong with a constraint value, if anything.
|
|
3
|
+
* @param {any} value - A `constraint` setting or request value.
|
|
4
|
+
* @returns {string | null} A message, or null when the value is a valid constraint (or absent).
|
|
5
|
+
*/
|
|
6
|
+
export function check(value: any): string | null;
|
|
7
|
+
/**
|
|
8
|
+
* The fields to add to a Chat Completions body for one constraint, or null when the backend cannot enforce that kind.
|
|
9
|
+
* @param {string} driver - Driver name.
|
|
10
|
+
* @param {string | undefined} version - Server version (vLLM), when known.
|
|
11
|
+
* @param {string} kind - One of {@link KINDS}.
|
|
12
|
+
* @param {any} value - The constraint's value.
|
|
13
|
+
* @returns {Record<string, any> | null} Body fields, or null when unsupported.
|
|
14
|
+
*/
|
|
15
|
+
export function fields(driver: string, version: string | undefined, kind: string, value: any): Record<string, any> | null;
|
|
16
|
+
/**
|
|
17
|
+
* Add a constraint to an OpenAI-protocol body in the backend's own fields. The request's constraint (`constraint` on the OpenAI front,
|
|
18
|
+
* `metadata.constraint` on the Anthropic front) replaces the upstream's `constraint` setting. A body that already carries the native field
|
|
19
|
+
* keeps it. vLLM applies a constraint to the whole generation, so with tools on offer the constraint is dropped and recorded (the tools win).
|
|
20
|
+
* A kind the backend cannot enforce follows `unknownFields`: dropped (listed in `dropped`) or rejected.
|
|
21
|
+
* @param {any} body - Converted or forwarded Chat Completions body.
|
|
22
|
+
* @param {any} asked - The request's constraint, if any.
|
|
23
|
+
* @param {any} settings - Resolved settings (`constraint`, `unknownFields`).
|
|
24
|
+
* @param {{ name: string, version?: string }} target - The upstream's driver name and server version.
|
|
25
|
+
* @param {string[]} [dropped] - Receives `constraint` names that were left out.
|
|
26
|
+
* @returns {any} The body, with the fields added (a copy when anything was added).
|
|
27
|
+
* @throws {TranslatorError} `invalid_request` for a malformed request constraint; `unsupported_field` when the backend cannot enforce the kind and `unknownFields` is `"error"`.
|
|
28
|
+
*/
|
|
29
|
+
export function addConstraint(body: any, asked: any, settings: any, target: {
|
|
30
|
+
name: string;
|
|
31
|
+
version?: string;
|
|
32
|
+
}, dropped?: string[]): any;
|
|
33
|
+
/** The constraint kinds a client can ask for. */
|
|
34
|
+
export const KINDS: string[];
|
|
35
|
+
/** First vLLM release whose Chat Completions endpoint takes `structured_outputs`; older releases take `guided_json`, `guided_regex`, `guided_choice` and `guided_grammar`. */
|
|
36
|
+
export const STRUCTURED_OUTPUTS_VERSION: "0.11.0";
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether an edit's parameters are all usable. Anything else cannot be honoured and the edit is dropped.
|
|
3
|
+
* @param {any} edit - A `clear_tool_uses` edit.
|
|
4
|
+
* @returns {boolean} True when every present parameter has its documented shape.
|
|
5
|
+
*/
|
|
6
|
+
export function valid(edit: any): boolean;
|
|
7
|
+
/**
|
|
8
|
+
* Clear the results (and optionally the inputs) of the oldest tool uses of a converted conversation, as Anthropic's
|
|
9
|
+
* `clear_tool_uses_20250919` edit does. Messages are changed in place.
|
|
10
|
+
*
|
|
11
|
+
* Tool uses are counted in conversation order across every assistant turn. The edit applies when the trigger is exceeded (default: more
|
|
12
|
+
* than 100000 estimated input tokens). The most recent `keep` tool uses (default 3, whatever tool they are) are never cleared, nor are
|
|
13
|
+
* uses of a tool named in `exclude_tools`. The rest, oldest first, have their result replaced by a placeholder (and, with
|
|
14
|
+
* `clear_tool_inputs`, their arguments by `{}`). When fewer than `clear_at_least` input tokens would be freed nothing changes. Input
|
|
15
|
+
* tokens are estimated at one per four characters (`lib/estimate`), for the trigger and for the amount cleared.
|
|
16
|
+
* @param {any} edit - A valid `clear_tool_uses` edit (see `valid`).
|
|
17
|
+
* @param {{ messages: any[], tools?: any[] }} body - Converted OpenAI body; its messages are edited.
|
|
18
|
+
* @param {Record<string, string>} toolNames - `fitted -> original` tool names, to match `exclude_tools`.
|
|
19
|
+
* @returns {{ type: string, cleared_tool_uses: number, cleared_input_tokens: number } | null} The applied edit, or null when nothing was cleared.
|
|
20
|
+
*/
|
|
21
|
+
export function clearToolUses(edit: any, body: {
|
|
22
|
+
messages: any[];
|
|
23
|
+
tools?: any[];
|
|
24
|
+
}, toolNames: Record<string, string>): {
|
|
25
|
+
type: string;
|
|
26
|
+
cleared_tool_uses: number;
|
|
27
|
+
cleared_input_tokens: number;
|
|
28
|
+
} | null;
|
|
29
|
+
/** Input tokens above which an edit with no `trigger` applies. */
|
|
30
|
+
export const DEFAULT_TRIGGER: 100000;
|
|
31
|
+
/** Recent tool uses an edit with no `keep` leaves alone. */
|
|
32
|
+
export const DEFAULT_KEEP: 3;
|
|
33
|
+
/** What a cleared tool result says. */
|
|
34
|
+
export const PLACEHOLDER: "[Tool result cleared to save context]";
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* The driver facts of a preset.
|
|
3
3
|
* @param {string} name - Preset name.
|
|
4
|
-
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, fields?: string[], paths: Record<string,string>}} Driver facts.
|
|
4
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, hosts: string[], fields?: string[], paths: Record<string,string>}} Driver facts.
|
|
5
5
|
*/
|
|
6
6
|
export function describe(name: string): {
|
|
7
7
|
name: string;
|
|
@@ -9,6 +9,7 @@ export function describe(name: string): {
|
|
|
9
9
|
order: number;
|
|
10
10
|
auth: string;
|
|
11
11
|
strict: boolean;
|
|
12
|
+
hosts: string[];
|
|
12
13
|
fields?: string[];
|
|
13
14
|
paths: Record<string, string>;
|
|
14
15
|
};
|
|
@@ -40,6 +40,9 @@ export function resolve(...layers: (Partial<Settings> | null | undefined)[]): Se
|
|
|
40
40
|
* @property {boolean} parallelToolCalls - Map `disable_parallel_tool_use` to `parallel_tool_calls: false`.
|
|
41
41
|
* @property {"drop"|"error"} unknownFields - Unknown request fields are dropped (and listed in `state.dropped`) or rejected.
|
|
42
42
|
* @property {Record<string, any>} sampling - Sampling extension fields (`min_p`, `repetition_penalty`, `seed`, ...) added to requests for an OpenAI-protocol upstream when the request does not carry them, merged key by key. A request may set them itself (OpenAI front, or `metadata.sampling` on the Anthropic front). A field the upstream's driver does not accept follows `unknownFields`.
|
|
43
|
+
* @property {{ json?: object, regex?: string, choice?: string[], grammar?: string } | null} constraint - Neutral constrained-decoding description sent with every request to an OpenAI-protocol upstream that does not carry its own: exactly one of `json` (a JSON Schema), `regex`, `choice` (a list of strings) or `grammar` (GBNF for llama.cpp, EBNF for SGLang, a vLLM grammar). Each driver turns it into its backend's native field. Replaced whole, not merged. A request overrides it (`constraint` on the OpenAI front, `metadata.constraint` on the Anthropic front).
|
|
44
|
+
* @property {string | null} cacheSalt - vLLM `cache_salt` sent with every request to this upstream: requests with different salts never share cached prefixes. A non-empty string, or null for none. Backends without the feature follow `unknownFields`. A trusted caller overrides it per call (`cacheSalt` option, or the `cacheSalt` an `authorize` hook returns); it is never read from the client.
|
|
45
|
+
* @property {string | null} promptCacheKey - OpenAI `prompt_cache_key` sent with requests to this upstream that do not carry their own. A non-empty string, or null for none.
|
|
43
46
|
* @property {number} pingInterval - Milliseconds of upstream silence before the SSE writer emits a `ping` (0 disables).
|
|
44
47
|
* @property {"buffer"|"stream"} toolStreaming - OpenAI to Anthropic streams: `buffer` holds each tool call until its arguments are complete and emits one whole `tool_use` block (text and thinking keep streaming and appear before the block); `stream` forwards argument fragments incrementally and holds text that arrives meanwhile until the tool blocks are done.
|
|
45
48
|
* @property {boolean} effortPassthrough - vLLM driver: send `xhigh` and `max` reasoning effort unchanged instead of clamping them to `high`.
|
|
@@ -188,6 +191,23 @@ export type Settings = {
|
|
|
188
191
|
* - Sampling extension fields (`min_p`, `repetition_penalty`, `seed`, ...) added to requests for an OpenAI-protocol upstream when the request does not carry them, merged key by key. A request may set them itself (OpenAI front, or `metadata.sampling` on the Anthropic front). A field the upstream's driver does not accept follows `unknownFields`.
|
|
189
192
|
*/
|
|
190
193
|
sampling: Record<string, any>;
|
|
194
|
+
/**
|
|
195
|
+
* - Neutral constrained-decoding description sent with every request to an OpenAI-protocol upstream that does not carry its own: exactly one of `json` (a JSON Schema), `regex`, `choice` (a list of strings) or `grammar` (GBNF for llama.cpp, EBNF for SGLang, a vLLM grammar). Each driver turns it into its backend's native field. Replaced whole, not merged. A request overrides it (`constraint` on the OpenAI front, `metadata.constraint` on the Anthropic front).
|
|
196
|
+
*/
|
|
197
|
+
constraint: {
|
|
198
|
+
json?: object;
|
|
199
|
+
regex?: string;
|
|
200
|
+
choice?: string[];
|
|
201
|
+
grammar?: string;
|
|
202
|
+
} | null;
|
|
203
|
+
/**
|
|
204
|
+
* - vLLM `cache_salt` sent with every request to this upstream: requests with different salts never share cached prefixes. A non-empty string, or null for none. Backends without the feature follow `unknownFields`. A trusted caller overrides it per call (`cacheSalt` option, or the `cacheSalt` an `authorize` hook returns); it is never read from the client.
|
|
205
|
+
*/
|
|
206
|
+
cacheSalt: string | null;
|
|
207
|
+
/**
|
|
208
|
+
* - OpenAI `prompt_cache_key` sent with requests to this upstream that do not carry their own. A non-empty string, or null for none.
|
|
209
|
+
*/
|
|
210
|
+
promptCacheKey: string | null;
|
|
191
211
|
/**
|
|
192
212
|
* - Milliseconds of upstream silence before the SSE writer emits a `ping` (0 disables).
|
|
193
213
|
*/
|