@cldmv/llm-translator 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -6
- package/dist/api/convert/anthropic/messages/to/ollama/chat/request.mjs +2 -0
- package/dist/api/convert/ollama/chat/to/anthropic/messages/response.mjs +2 -0
- package/dist/api/convert/ollama/chat/to/anthropic/messages/stream.mjs +2 -0
- package/dist/api/convert/ollama/chat/to/openai/chat/response.mjs +2 -0
- package/dist/api/convert/ollama/chat/to/openai/chat/stream.mjs +2 -0
- package/dist/api/convert/openai/chat/to/ollama/chat/request.mjs +2 -0
- package/dist/api/detect.mjs +1 -1
- package/dist/api/driver/fireworks/detect.mjs +2 -0
- package/dist/api/driver/fireworks/info.mjs +2 -0
- package/dist/api/driver/fireworks/path.mjs +2 -0
- package/dist/api/driver/fireworks/request.mjs +2 -0
- package/dist/api/driver/fireworks/settings.mjs +2 -0
- package/dist/api/driver/gemini/detect.mjs +2 -0
- package/dist/api/driver/gemini/info.mjs +2 -0
- package/dist/api/driver/gemini/path.mjs +2 -0
- package/dist/api/driver/gemini/request.mjs +2 -0
- package/dist/api/driver/gemini/settings.mjs +2 -0
- package/dist/api/driver/groq/detect.mjs +2 -0
- package/dist/api/driver/groq/info.mjs +2 -0
- package/dist/api/driver/groq/path.mjs +2 -0
- package/dist/api/driver/groq/request.mjs +2 -0
- package/dist/api/driver/groq/settings.mjs +2 -0
- package/dist/api/driver/ollama/info.mjs +1 -1
- package/dist/api/driver/ollama/request.mjs +1 -1
- package/dist/api/driver/together/detect.mjs +2 -0
- package/dist/api/driver/together/info.mjs +2 -0
- package/dist/api/driver/together/request.mjs +2 -0
- package/dist/api/driver/together/settings.mjs +2 -0
- package/dist/api/driver/vllm/detect.mjs +1 -1
- package/dist/api/driver/vllm/info.mjs +1 -1
- package/dist/api/driver/vllm/model/list.mjs +1 -1
- package/dist/api/driver/xai/detect.mjs +2 -0
- package/dist/api/driver/xai/info.mjs +2 -0
- package/dist/api/driver/xai/request.mjs +2 -0
- package/dist/api/driver/xai/settings.mjs +2 -0
- package/dist/api/embeddings/send.mjs +1 -1
- package/dist/api/health/check.mjs +2 -0
- package/dist/api/model/list.mjs +1 -1
- package/dist/api/request.mjs +2 -2
- package/dist/api/response.mjs +1 -1
- package/dist/api/send.mjs +1 -1
- package/dist/api/status.mjs +1 -1
- package/dist/api/stream.mjs +1 -1
- package/dist/index.mjs +1 -1
- package/dist/lib/capture.mjs +1 -1
- package/dist/lib/config.mjs +1 -1
- package/dist/lib/health.mjs +2 -0
- package/dist/lib/http.mjs +1 -1
- package/dist/lib/models.mjs +1 -1
- package/dist/lib/ndjson.mjs +4 -0
- package/dist/lib/ollama.mjs +3 -0
- package/dist/lib/pool.mjs +1 -1
- package/dist/lib/presets.mjs +2 -0
- package/dist/lib/route.mjs +1 -1
- package/dist/lib/settings.mjs +1 -1
- package/dist/lib/vllm.mjs +4 -0
- package/package.json +1 -1
- package/types/src/api/convert/anthropic/messages/to/ollama/chat/request.d.mts +14 -0
- package/types/src/api/convert/ollama/chat/to/anthropic/messages/response.d.mts +10 -0
- package/types/src/api/convert/ollama/chat/to/anthropic/messages/stream.d.mts +14 -0
- package/types/src/api/convert/ollama/chat/to/openai/chat/response.d.mts +10 -0
- package/types/src/api/convert/ollama/chat/to/openai/chat/stream.d.mts +13 -0
- package/types/src/api/convert/openai/chat/to/ollama/chat/request.d.mts +12 -0
- package/types/src/api/driver/fireworks/detect.d.mts +8 -0
- package/types/src/api/driver/fireworks/info.d.mts +13 -0
- package/types/src/api/driver/fireworks/path.d.mts +7 -0
- package/types/src/api/driver/fireworks/request.d.mts +7 -0
- package/types/src/api/driver/fireworks/settings.d.mts +5 -0
- package/types/src/api/driver/gemini/detect.d.mts +8 -0
- package/types/src/api/driver/gemini/info.d.mts +13 -0
- package/types/src/api/driver/gemini/path.d.mts +7 -0
- package/types/src/api/driver/gemini/request.d.mts +7 -0
- package/types/src/api/driver/gemini/settings.d.mts +5 -0
- package/types/src/api/driver/groq/detect.d.mts +8 -0
- package/types/src/api/driver/groq/info.d.mts +13 -0
- package/types/src/api/driver/groq/path.d.mts +7 -0
- package/types/src/api/driver/groq/request.d.mts +7 -0
- package/types/src/api/driver/groq/settings.d.mts +5 -0
- package/types/src/api/driver/ollama/info.d.mts +7 -2
- package/types/src/api/driver/together/detect.d.mts +8 -0
- package/types/src/api/driver/together/info.d.mts +13 -0
- package/types/src/api/driver/together/request.d.mts +7 -0
- package/types/src/api/driver/together/settings.d.mts +5 -0
- package/types/src/api/driver/vllm/detect.d.mts +1 -1
- package/types/src/api/driver/vllm/info.d.mts +1 -1
- package/types/src/api/driver/vllm/model/list.d.mts +3 -1
- package/types/src/api/driver/xai/detect.d.mts +8 -0
- package/types/src/api/driver/xai/info.d.mts +13 -0
- package/types/src/api/driver/xai/request.d.mts +7 -0
- package/types/src/api/driver/xai/settings.d.mts +5 -0
- package/types/src/api/endpoint/status.d.mts +5 -2
- package/types/src/api/health/check.d.mts +8 -0
- package/types/src/api/status.d.mts +1 -1
- package/types/src/lib/config.d.mts +4 -0
- package/types/src/lib/events.d.mts +2 -2
- package/types/src/lib/health.d.mts +22 -0
- package/types/src/lib/models.d.mts +25 -0
- package/types/src/lib/ndjson.d.mts +16 -0
- package/types/src/lib/ollama.d.mts +86 -0
- package/types/src/lib/pool.d.mts +30 -4
- package/types/src/lib/presets.d.mts +98 -0
- package/types/src/lib/route.d.mts +1 -1
- package/types/src/lib/settings.d.mts +5 -0
- package/types/src/lib/vllm.d.mts +47 -0
package/dist/lib/pool.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var
|
|
1
|
+
var k=Object.defineProperty;var i=(t,r)=>k(t,"name",{value:r,configurable:!0});import{TranslatorError as S}from"./errors.mjs";const d=Object.freeze(["failover","round_robin","least_busy"]),m=Object.freeze({base:1e3,max:6e4}),y=Object.freeze({connect:1e4,headers:0}),p=Object.freeze({interval:1e4,timeout:2e3});function F(){return{failures:0,until:0,inflight:0,requests:0,errors:0,lastError:null,lastFailure:null,lastSuccess:null,ready:null,checkedAt:null,readyReason:null,unsupported:!1,checking:null}}i(F,"health");function j(t={}){const r=i(a=>new S(`pool: ${a}`,{code:"INVALID_CONFIG",status:500}),"bad");if(!t||typeof t!="object"||Array.isArray(t))throw r("must be an object");const e=t.strategy??"failover",s={...t.models};for(const[a,f]of[["strategy",e],...Object.entries(s).map(([x,O])=>[`models.${x}`,O])])if(!d.includes(f))throw r(`${a} must be one of ${d.join(", ")} (got ${JSON.stringify(f)})`);const o=t.backoff?.base??m.base,l=t.backoff?.max??m.max;if(!(Number.isFinite(o)&&o>0)||!(Number.isFinite(l)&&l>=o))throw r("backoff.base must be a positive number of ms and backoff.max at least backoff.base");const u=t.connectTimeout??y.connect,c=t.headersTimeout??y.headers,b=t.healthInterval??p.interval,h=t.healthTimeout??p.timeout,n={connectTimeout:u,headersTimeout:c,healthInterval:b,healthTimeout:h};for(const a of Object.keys(n)){const f=n[a];if(typeof f!="number"||!(Number.isFinite(f)&&f>=0))throw r(`${a} must be a number of ms, 0 or more (got ${JSON.stringify(f)})`)}return{strategy:e,backoff:{base:o,max:l},connectTimeout:u,headersTimeout:c,healthInterval:b,healthTimeout:h,models:s}}i(j,"normalize");function g(t,r){return t.health.until<=r&&t.health.ready!==!1}i(g,"available");function A(t,r,e,s,o,l=Date.now()){const u=t.pool.models[s]??t.pool.strategy,c=e.filter(n=>g(n,l)),b=e.filter(n=>!g(n,l)).sort((n,a)=>n.health.until-a.health.until);let h=c;if(u==="least_busy")h=[...c].sort((n,a)=>n.health.inflight-a.health.inflight);else if(u==="round_robin"&&c.length>0){const n=r.cursors.get(s)??0;o&&r.cursors.set(s,n+1);const a=n%c.length;h=[...c.slice(a),...c.slice(0,a)]}return[...h,...b]}i(A,"order");function E(t,r,e,s=Date.now()){const o=r.health;o.failures++,o.errors++,o.lastError=e,o.lastFailure=s;const{base:l,max:u}=t.pool.backoff;o.until=s+Math.min(u,l*2**Math.min(o.failures-1,30))}i(E,"failed");function I(t,r=Date.now()){const e=t.health;e.failures=0,e.until=0,e.lastSuccess=r,e.ready===!1&&(e.ready=!0)}i(I,"succeeded");function N(t,r=Date.now()){const e=t.health,s=i(l=>l===null?null:new Date(l).toISOString(),"iso"),o=e.until>r;return{healthy:!o&&e.ready!==!1,ready:e.ready,readyReason:e.ready===!1?e.readyReason:null,checkedAt:s(e.checkedAt),failures:e.failures,retryAt:o?s(e.until):null,inflight:e.inflight,requests:e.requests,errors:e.errors,lastError:e.lastError,lastFailure:s(e.lastFailure),lastSuccess:s(e.lastSuccess)}}i(N,"describe");export{m as BACKOFF,p as PROBES,d as STRATEGIES,y as TIMEOUTS,N as describe,E as failed,F as health,j as normalize,A as order,I as succeeded};
|
|
2
2
|
//# sourceMappingURL=pool.mjs.map
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
var f=Object.defineProperty;var s=(e,t)=>f(e,"name",{value:t,configurable:!0});import{noEmptyTools as d}from"./normalize.mjs";const p={chat:"/v1/chat/completions",embeddings:"/v1/embeddings",models:"/v1/models"},c={chat:"/chat/completions",embeddings:"/embeddings",models:"/models"};function m(e){return/^grok-(3-mini|4|code)/.test(String(e))&&!/non-reasoning/.test(String(e))}s(m,"grokReasons");const l=Object.freeze({groq:{order:21,hosts:["api.groq.com"],root:"/openai",paths:p,settings:{reasoningField:"reasoning",maxTokensField:"max_completion_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!1,prefill:"none",streamUsage:!0,continuousUsage:!1},drop:["logprobs","top_logprobs","logit_bias"],dropName:!0},together:{order:22,hosts:["api.together.ai","api.together.xyz","us-api.together.ai"],paths:p,fields:["top_k","min_p","repetition_penalty"],settings:{reasoningField:"reasoning",reasoningRoundTrip:!0,maxTokensField:"max_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!0,prefill:"none",parallelToolCalls:!1,streamUsage:!1,continuousUsage:!1}},fireworks:{order:23,hosts:["api.fireworks.ai"],root:"/inference",paths:p,fields:["top_k","min_p","repetition_penalty"],settings:{reasoningField:"reasoning_content",reasoningRoundTrip:!0,maxTokensField:"max_tokens",thinking:"reasoning_effort",thinkingBudget:!1,effort:!0,topK:!0,prefill:"none",streamUsage:!0,continuousUsage:!1}},xai:{order:24,hosts:["api.x.ai"],paths:p,settings:{reasoningField:"reasoning_content",maxTokensField:"max_completion_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!1,prefill:"none",streamUsage:!0,continuousUsage:!1},dropWhen:s(e=>m(e.model)?["stop","presence_penalty","frequency_penalty"]:[],"dropWhen")},gemini:{order:25,hosts:["generativelanguage.googleapis.com"],root:"/v1beta/openai",paths:c,settings:{maxTokensField:"max_tokens",thinking:"none",thinkingBudget:!1,effort:!0,topK:!1,prefill:"none",streamUsage:!0,continuousUsage:!1}}});function _(e){const t=l[e],o={name:e,protocol:"openai.chat",order:t.order,auth:"bearer",strict:!1,paths:{...t.paths}};return t.fields?{...o,fields:[...t.fields]}:o}s(_,"describe");function x(e,t){const o=URL.canParse(t.url)?new URL(t.url).hostname:"";return l[e].hosts.includes(o)?{host:o}:null}s(x,"match");function U(e){return{...l[e].settings}}s(U,"settingsOf");function b(e,t,o){const i=l[e],a=i.root??"";return(o.url.endsWith(a)?"":a)+i.paths[t]}s(b,"route");function y(e,t,o){const i=l[e],a=o?.state?.dropped,g=s(n=>{Array.isArray(a)&&!a.includes(n)&&a.push(n)},"note"),r={...d(t)};for(const n of[...i.drop??[],...i.dropWhen?.(t)??[]])r[n]!==void 0&&(delete r[n],g(n));return i.dropName&&Array.isArray(r.messages)&&r.messages.some(n=>n&&"name"in n)&&(r.messages=r.messages.map(n=>{const u={...n};return delete u.name,u}),g("messages[].name")),r}s(y,"trim");export{l as PRESETS,_ as describe,x as match,b as route,U as settingsOf,y as trim};
|
|
2
|
+
//# sourceMappingURL=presets.mjs.map
|
package/dist/lib/route.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var m=Object.defineProperty;var u=(e,n)=>m(e,"name",{value:n,configurable:!0});import{TranslatorError as f}from"./errors.mjs";import{order as v}from"./pool.mjs";function
|
|
1
|
+
var m=Object.defineProperty;var u=(e,n)=>m(e,"name",{value:n,configurable:!0});import{TranslatorError as f}from"./errors.mjs";import{order as v}from"./pool.mjs";import{baseModels as h}from"./models.mjs";function w(e,n){return(e.models??e.facts?.models??[]).includes(n)?!0:e.protocol==="anthropic.messages"&&e.models===void 0&&/^claude-/.test(n)}u(w,"serves");function O(e,n,t,r,l=!1){const a=Object.hasOwn(e.aliases,t),o=a?e.aliases[t]:t,d=a&&Object.hasOwn(e.pins,t)?e.pins[t]:void 0,c=n.filter(s=>w(s,o)),i=n.find(s=>s.default)??n[0];if(c.length>0){const s=v(e,r,c,o,l);return{endpoint:s[0],model:o,unserved:!1,pool:s,pin:d}}const p=h(i);return p.length===1?{endpoint:i,model:p[0],unserved:!1,pool:[i],pin:d}:{endpoint:i,model:o,unserved:!0,pool:[i],pin:d}}u(O,"pick");function x(e,n,t={}){if(!n||typeof n!="object"||Array.isArray(n))throw new f("The request body must be a JSON object",{code:"invalid_body",status:400});const r=O(e.config,e.state.endpoints,n.model,e.state,t.advance===!0);if(t.endpoint!==void 0){const o=r.pool.find(d=>d.name===t.endpoint);if(!o)throw new f(`Endpoint ${t.endpoint} no longer serves ${r.model}`,{code:"endpoint_not_in_pool",status:500});return{endpoint:o,model:r.model,requested:n.model,unserved:r.unserved,pin:r.pin,convert:e.converters(o,n),pool:[o]}}const{endpoint:l,pool:a}=r;return{endpoint:l,model:r.model,requested:n.model,unserved:r.unserved,pin:r.pin,convert:e.converters(l,n),pool:a}}u(x,"resolve");export{O as pick,x as resolve,w as serves};
|
|
2
2
|
//# sourceMappingURL=route.mjs.map
|
package/dist/lib/settings.mjs
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
var l=Object.defineProperty;var i=(
|
|
1
|
+
var l=Object.defineProperty;var i=(o,e)=>l(o,"name",{value:e,configurable:!0});import{TranslatorError as a}from"./errors.mjs";const r=Object.freeze({inlineSystem:"auto",stripBillingHeader:!0,reasoningField:"reasoning",thinkTags:!0,thinkOpen:!1,reasoningRoundTrip:!1,thinking:"chat_template_kwargs",thinkingBudget:!1,effort:!1,images:!0,imageFallback:!0,toolFallback:!1,documents:"extract",documentConverter:null,pdf:Object.freeze({pages:!0,maxPages:100,maxBytes:32*1024*1024,maxImages:20,scale:1.5}),toolSchema:"pass",deferredTools:"forward",toolSearch:"expand",jsonObject:"schema",jsonObjectFallback:!0,toolErrors:!1,toolErrorMarker:"[error] ",toolErrorInference:!1,maxTokensField:"max_completion_tokens",streamUsage:!0,continuousUsage:!1,prefill:"continue_final_message",topK:!0,responseFormat:!0,parallelToolCalls:!0,unknownFields:"drop",sampling:Object.freeze({}),pingInterval:1e4,toolStreaming:"buffer",effortPassthrough:!1,passthroughFilter:!1,nativeMessages:!1,contextWindow:0,defaultMaxTokens:8192}),f={inlineSystem:["auto","keep","hoist","user","drop"],reasoningField:["reasoning","reasoning_content"],thinking:["chat_template_kwargs","reasoning_effort","none"],documents:["extract","note","error"],toolSchema:["pass","sanitize"],deferredTools:["forward","drop"],toolSearch:["expand","inline","native"],jsonObject:["schema","instruction","drop"],maxTokensField:["max_completion_tokens","max_tokens"],prefill:["continue_final_message","none"],unknownFields:["drop","error"],toolStreaming:["buffer","stream"]},d=["effortPassthrough","passthroughFilter","imageFallback","toolFallback","jsonObjectFallback","thinkOpen","reasoningRoundTrip","toolErrorInference"];function p(...o){const e={...r,pdf:{...r.pdf},sampling:{...r.sampling}};for(const t of o)if(t)for(const[n,s]of Object.entries(t))s!==void 0&&(n==="pdf"?Object.assign(e.pdf,s):n==="sampling"&&s!==null&&typeof s=="object"&&!Array.isArray(s)?Object.assign(e.sampling,s):e[n]=s);for(const[t,n]of Object.entries(f))if(!n.includes(e[t]))throw new a(`Invalid setting ${t}: ${JSON.stringify(e[t])} (expected one of ${n.join(", ")})`,{code:"invalid_setting",status:500,details:{key:t,allowed:n}});for(const t of d)if(typeof e[t]!="boolean")throw new a(`Invalid setting ${t}: ${JSON.stringify(e[t])} (expected true or false)`,{code:"invalid_setting",status:500,details:{key:t}});if(e.nativeMessages!==!0&&e.nativeMessages!==!1&&e.nativeMessages!=="auto")throw new a(`Invalid setting nativeMessages: ${JSON.stringify(e.nativeMessages)} (expected true, false or "auto")`,{code:"invalid_setting",status:500,details:{key:"nativeMessages"}});if(typeof e.toolErrorMarker!="string"||e.toolErrorMarker.trim()==="")throw new a(`Invalid setting toolErrorMarker: ${JSON.stringify(e.toolErrorMarker)} (expected a non-empty string)`,{code:"invalid_setting",status:500,details:{key:"toolErrorMarker"}});if(e.sampling===null||typeof e.sampling!="object"||Array.isArray(e.sampling))throw new a(`Invalid setting sampling: ${JSON.stringify(e.sampling)} (expected an object of field names and values)`,{code:"invalid_setting",status:500,details:{key:"sampling"}});if(!(Number.isFinite(e.pdf.scale)&&e.pdf.scale>0))throw new a(`Invalid setting pdf.scale: ${JSON.stringify(e.pdf.scale)} (expected a positive number)`,{code:"invalid_setting",status:500,details:{key:"pdf.scale"}});return e}i(p,"resolve");export{r as DEFAULTS,p as resolve};
|
|
2
2
|
//# sourceMappingURL=settings.mjs.map
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
var c=Object.defineProperty;var s=(t,r)=>c(t,"name",{value:r,configurable:!0});import{atLeast as f,semver as l}from"./normalize.mjs";import{sign as a}from"./thinking.mjs";const u="0.11.1",y="x-anthropic-billing-header:";function A(t,r){if(t.driver!=="vllm"||r.nativeMessages===!1)return!1;if(r.nativeMessages===!0)return!0;const n=t.facts?.version??t.version;return l(n)!==null&&f(n,u)}s(A,"wants");function*p(t){for(const r of t)if(Array.isArray(r?.content))for(const n of r.content)yield n,n?.type==="tool_result"&&Array.isArray(n.content)&&(yield*n.content)}s(p,"blocks");function v(t,r){const n=Array.isArray(t.messages)?t.messages:[];if(n.at(-1)?.role==="assistant")return"prefill";const e=[...p(n)];return e.some(o=>o?.type==="document")?"document":r.toolSearch!=="native"&&(Array.isArray(t.tools)&&t.tools.some(i=>i?.defer_loading!==void 0)||e.some(i=>["tool_reference","tool_addition","tool_removal"].includes(i?.type)))?"toolsearch":null}s(v,"unsuited");function d(t,r){return(typeof t=="string"?[{type:"text",text:t}]:t).filter(e=>e?.type==="text"&&typeof e.text=="string").filter(e=>!(r.stripBillingHeader&&e.text.trimStart().startsWith(y))).map(e=>e.text).join(`
|
|
2
|
+
|
|
3
|
+
`)}s(d,"systemText");function m(t,r){const n=r.toolErrorMarker;return typeof t.content=="string"?{...t,content:`${n}${t.content}`}:Array.isArray(t.content)?{...t,content:[{type:"text",text:n.trimEnd()},...t.content]}:{...t,content:n.trimEnd()}}s(m,"marked");function h(t,r){const n={...t};if(n.system!==void 0&&n.system!==null){const e=d(n.system,r);e===""?delete n.system:n.system=e}return r.toolErrors&&Array.isArray(n.messages)&&(n.messages=n.messages.map(e=>Array.isArray(e?.content)?{...e,content:e.content.map(o=>o?.type==="tool_result"&&o.is_error===!0?m(o,r):o)}:e)),n}s(h,"request");function E(t){return Array.isArray(t?.content)?{...t,content:t.content.map(r=>r?.type==="thinking"&&typeof r.signature!="string"?{...r,signature:a()}:r)}:t}s(E,"response");async function*M(t){const r=new Map;for await(const n of t){const e=n.data;e?.type==="content_block_start"&&e.content_block?.type==="thinking"?r.set(e.index,typeof e.content_block.signature=="string"):e?.type==="content_block_delta"&&e.delta?.type==="signature_delta"?r.set(e.index,!0):e?.type==="content_block_stop"&&r.get(e.index)===!1&&(yield{event:"content_block_delta",data:{type:"content_block_delta",index:e.index,delta:{type:"signature_delta",signature:a()}}}),e?.type==="content_block_stop"&&r.delete(e.index),yield n,e?.type==="message_start"&&(yield{event:"ping",data:{type:"ping"}})}}s(M,"stream");export{u as MIN_VERSION,h as request,E as response,M as stream,v as unsuited,A as wants};
|
|
4
|
+
//# sourceMappingURL=vllm.mjs.map
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@cldmv/llm-translator",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.10.0",
|
|
4
4
|
"description": "Bidirectional Anthropic Messages ⇄ OpenAI Chat Completions translator with backend auto-detection (vLLM and other OpenAI-compatible servers), built on slothlet, with an optional standalone server for running Claude Code against local models.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert an Anthropic Messages request to an Ollama `/api/chat` body. The Anthropic conversion to Chat Completions does the work (system, tool
|
|
3
|
+
* blocks, images, tool names, dropped fields); `chatBody` then puts the result in Ollama's shape: tool arguments as objects, tool results by tool
|
|
4
|
+
* name, `images` as base64, the sampling fields and `max_tokens` in `options`, `thinking` and `output_config.effort` as `think`, and
|
|
5
|
+
* `output_config.format` as `format`. The upstream's `sampling` setting and `metadata.sampling` become further `options`.
|
|
6
|
+
* @param {any} body - Anthropic request body.
|
|
7
|
+
* @param {Partial<import("../../../../../../../lib/settings.mjs").Settings>} [settings] - Translator settings.
|
|
8
|
+
* @param {any} [context] - `{ producer, budget, listed, toolCallIds }`, as for the Chat Completions conversion.
|
|
9
|
+
* @returns {Promise<{body: any, state: any}>} The Ollama body and the conversion state the response converters read.
|
|
10
|
+
*/
|
|
11
|
+
export function request(body: any, settings?: Partial<import("../../../../../../../lib/settings.mjs").Settings>, context?: any): Promise<{
|
|
12
|
+
body: any;
|
|
13
|
+
state: any;
|
|
14
|
+
}>;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert an Ollama reply to an Anthropic message: the reply is read as a Chat Completions `chat.completion` (thinking, tool calls with generated
|
|
3
|
+
* ids, `done_reason` as `finish_reason`, the evaluation counts as `usage`) and the Chat Completions conversion builds the message.
|
|
4
|
+
* @param {any} reply - Parsed Ollama reply.
|
|
5
|
+
* @param {any} [state] - State from the request converter.
|
|
6
|
+
* @param {Partial<import("../../../../../../../lib/settings.mjs").Settings>} [settings] - Translator settings.
|
|
7
|
+
* @returns {any} Anthropic message.
|
|
8
|
+
* @throws {import("../../../../../../../lib/errors.mjs").TranslatorError} When the reply is an error.
|
|
9
|
+
*/
|
|
10
|
+
export function response(reply: any, state?: any, settings?: Partial<import("../../../../../../../lib/settings.mjs").Settings>): any;
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert an Ollama stream (one JSON object per line, possibly split anywhere across chunks) to Anthropic events. The lines become Chat
|
|
3
|
+
* Completions chunks (one id for the reply, tool calls whole with generated ids, the final line as the finish and usage chunks) and the Chat
|
|
4
|
+
* Completions conversion writes the events. A line that is not JSON, an `{ error }` line and a stream that ends before its final line each
|
|
5
|
+
* end the stream with an `error` event.
|
|
6
|
+
* @param {ReadableStream|AsyncIterable<unknown>|Iterable<unknown>|string|Uint8Array} source - Ollama NDJSON bytes or text.
|
|
7
|
+
* @param {any} [state] - State from the request converter.
|
|
8
|
+
* @param {Partial<import("../../../../../../../lib/settings.mjs").Settings>} [settings] - Translator settings.
|
|
9
|
+
* @returns {AsyncGenerator<{ event: string, data: any }, void, undefined>} Anthropic events.
|
|
10
|
+
*/
|
|
11
|
+
export function stream(source: ReadableStream | AsyncIterable<unknown> | Iterable<unknown> | string | Uint8Array, state?: any, settings?: Partial<import("../../../../../../../lib/settings.mjs").Settings>): AsyncGenerator<{
|
|
12
|
+
event: string;
|
|
13
|
+
data: any;
|
|
14
|
+
}, void, undefined>;
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert an Ollama reply to a Chat Completions `chat.completion` (see `completion`). The model name is the one the client asked for, and the
|
|
3
|
+
* counts are kept on the state for the usage report.
|
|
4
|
+
* @param {any} reply - Parsed Ollama reply.
|
|
5
|
+
* @param {any} [state] - State from the request converter.
|
|
6
|
+
* @param {Partial<import("../../../../../../../lib/settings.mjs").Settings>} [settings] - Translator settings.
|
|
7
|
+
* @returns {any} `chat.completion` object.
|
|
8
|
+
* @throws {import("../../../../../../../lib/errors.mjs").TranslatorError} When the reply is an error.
|
|
9
|
+
*/
|
|
10
|
+
export function response(reply: any, state?: any, settings?: Partial<import("../../../../../../../lib/settings.mjs").Settings>): any;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert an Ollama stream (one JSON object per line, possibly split anywhere across chunks) to Chat Completions chunks followed by `[DONE]`.
|
|
3
|
+
* Every chunk carries one completion id, tool calls come whole with generated ids, and the final line gives the finish chunk and, unless
|
|
4
|
+
* `state.includeUsage` is false, a usage chunk. A line that is not JSON, an `{ error }` line and a stream that ends before its final line
|
|
5
|
+
* each become an `{ error }` chunk followed by `[DONE]`.
|
|
6
|
+
* @param {ReadableStream|AsyncIterable<unknown>|Iterable<unknown>|string|Uint8Array} source - Ollama NDJSON bytes or text.
|
|
7
|
+
* @param {any} [state] - State from the request converter (model, `includeUsage`).
|
|
8
|
+
* @param {Partial<import("../../../../../../../lib/settings.mjs").Settings>} [settings] - Translator settings.
|
|
9
|
+
* @returns {AsyncGenerator<{ data: any|string }, void, undefined>} Chat Completions chunks.
|
|
10
|
+
*/
|
|
11
|
+
export function stream(source: ReadableStream | AsyncIterable<unknown> | Iterable<unknown> | string | Uint8Array, state?: any, settings?: Partial<import("../../../../../../../lib/settings.mjs").Settings>): AsyncGenerator<{
|
|
12
|
+
data: any | string;
|
|
13
|
+
}, void, undefined>;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert a Chat Completions request to an Ollama `/api/chat` body (see `chatBody`). A client may also send Ollama's own `think`, `options`,
|
|
3
|
+
* `format` and `keep_alive`; they are used as sent. The upstream's `sampling` setting fills in `options` fields the request does not name.
|
|
4
|
+
* @param {any} body - Chat Completions request body.
|
|
5
|
+
* @param {Partial<import("../../../../../../../lib/settings.mjs").Settings>} [settings] - Translator settings.
|
|
6
|
+
* @returns {{body: any, state: any}} The Ollama body and the conversion state the response converters read.
|
|
7
|
+
* @throws {TranslatorError} `invalid_request` when `messages` is missing.
|
|
8
|
+
*/
|
|
9
|
+
export function request(body: any, settings?: Partial<import("../../../../../../../lib/settings.mjs").Settings>): {
|
|
10
|
+
body: any;
|
|
11
|
+
state: any;
|
|
12
|
+
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Describe the Fireworks AI driver (see `PRESETS.fireworks` in `lib/presets.mjs`).
|
|
3
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, fields?: string[], paths: Record<string,string>}} Driver facts.
|
|
4
|
+
*/
|
|
5
|
+
export function info(): {
|
|
6
|
+
name: string;
|
|
7
|
+
protocol: string;
|
|
8
|
+
order: number;
|
|
9
|
+
auth: string;
|
|
10
|
+
strict: boolean;
|
|
11
|
+
fields?: string[];
|
|
12
|
+
paths: Record<string, string>;
|
|
13
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Put the calls under the provider's path root, unless the upstream URL already ends in it.
|
|
3
|
+
* @param {"chat" | "embeddings" | "models"} kind - Which call.
|
|
4
|
+
* @param {any} context - `{ endpoint }`.
|
|
5
|
+
* @returns {string} Path starting with `/`.
|
|
6
|
+
*/
|
|
7
|
+
export function path(kind: "chat" | "embeddings" | "models", { endpoint }: any): string;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drop tool fields when there are no tools, and the fields Fireworks AI rejects (each recorded in `state.dropped`).
|
|
3
|
+
* @param {any} body - Chat Completions request body.
|
|
4
|
+
* @param {any} [context] - `{ state }`.
|
|
5
|
+
* @returns {any} Normalized body.
|
|
6
|
+
*/
|
|
7
|
+
export function request(body: any, context?: any): any;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Match by host name, no network access.
|
|
3
|
+
* @param {any} context - `{ endpoint }`.
|
|
4
|
+
* @returns {{host: string} | null} Facts, or null when the endpoint is not Gemini (OpenAI-compatible endpoint)'s.
|
|
5
|
+
*/
|
|
6
|
+
export function detect({ endpoint }: any): {
|
|
7
|
+
host: string;
|
|
8
|
+
} | null;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Describe the Gemini (OpenAI-compatible endpoint) driver (see `PRESETS.gemini` in `lib/presets.mjs`).
|
|
3
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, fields?: string[], paths: Record<string,string>}} Driver facts.
|
|
4
|
+
*/
|
|
5
|
+
export function info(): {
|
|
6
|
+
name: string;
|
|
7
|
+
protocol: string;
|
|
8
|
+
order: number;
|
|
9
|
+
auth: string;
|
|
10
|
+
strict: boolean;
|
|
11
|
+
fields?: string[];
|
|
12
|
+
paths: Record<string, string>;
|
|
13
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Put the calls under the provider's path root, unless the upstream URL already ends in it.
|
|
3
|
+
* @param {"chat" | "embeddings" | "models"} kind - Which call.
|
|
4
|
+
* @param {any} context - `{ endpoint }`.
|
|
5
|
+
* @returns {string} Path starting with `/`.
|
|
6
|
+
*/
|
|
7
|
+
export function path(kind: "chat" | "embeddings" | "models", { endpoint }: any): string;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drop tool fields when there are no tools, and the fields Gemini (OpenAI-compatible endpoint) rejects (each recorded in `state.dropped`).
|
|
3
|
+
* @param {any} body - Chat Completions request body.
|
|
4
|
+
* @param {any} [context] - `{ state }`.
|
|
5
|
+
* @returns {any} Normalized body.
|
|
6
|
+
*/
|
|
7
|
+
export function request(body: any, context?: any): any;
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Settings for Gemini (OpenAI-compatible endpoint): token field, reasoning field and controls, sampling fields and streaming usage as the provider documents them (see `PRESETS.gemini` in `lib/presets.mjs`).
|
|
3
|
+
* @returns {any} Partial settings.
|
|
4
|
+
*/
|
|
5
|
+
export function settings(): any;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Describe the Groq driver (see `PRESETS.groq` in `lib/presets.mjs`).
|
|
3
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, fields?: string[], paths: Record<string,string>}} Driver facts.
|
|
4
|
+
*/
|
|
5
|
+
export function info(): {
|
|
6
|
+
name: string;
|
|
7
|
+
protocol: string;
|
|
8
|
+
order: number;
|
|
9
|
+
auth: string;
|
|
10
|
+
strict: boolean;
|
|
11
|
+
fields?: string[];
|
|
12
|
+
paths: Record<string, string>;
|
|
13
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Put the calls under the provider's path root, unless the upstream URL already ends in it.
|
|
3
|
+
* @param {"chat" | "embeddings" | "models"} kind - Which call.
|
|
4
|
+
* @param {any} context - `{ endpoint }`.
|
|
5
|
+
* @returns {string} Path starting with `/`.
|
|
6
|
+
*/
|
|
7
|
+
export function path(kind: "chat" | "embeddings" | "models", { endpoint }: any): string;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drop tool fields when there are no tools, and the fields Groq rejects (each recorded in `state.dropped`).
|
|
3
|
+
* @param {any} body - Chat Completions request body.
|
|
4
|
+
* @param {any} [context] - `{ state }`.
|
|
5
|
+
* @returns {any} Normalized body.
|
|
6
|
+
*/
|
|
7
|
+
export function request(body: any, context?: any): any;
|
|
@@ -17,8 +17,9 @@
|
|
|
17
17
|
* @module @cldmv/llm-translator/driver/ollama
|
|
18
18
|
*/
|
|
19
19
|
/**
|
|
20
|
-
* Describe the Ollama driver.
|
|
21
|
-
*
|
|
20
|
+
* Describe the Ollama driver. `protocol` is the OpenAI-compatible `/v1` API; `native` is the mode an upstream selects with `api: "native"`, Ollama's own
|
|
21
|
+
* `/api/chat` (the `ollama.chat` protocol; the model list and embeddings stay on `/v1`).
|
|
22
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, probes: string[], paths: Record<string,string>, native: {protocol: string, paths: Record<string,string>}}} Driver facts.
|
|
22
23
|
*/
|
|
23
24
|
export function info(): {
|
|
24
25
|
name: string;
|
|
@@ -28,4 +29,8 @@ export function info(): {
|
|
|
28
29
|
strict: boolean;
|
|
29
30
|
probes: string[];
|
|
30
31
|
paths: Record<string, string>;
|
|
32
|
+
native: {
|
|
33
|
+
protocol: string;
|
|
34
|
+
paths: Record<string, string>;
|
|
35
|
+
};
|
|
31
36
|
};
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Describe the Together AI driver (see `PRESETS.together` in `lib/presets.mjs`).
|
|
3
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, fields?: string[], paths: Record<string,string>}} Driver facts.
|
|
4
|
+
*/
|
|
5
|
+
export function info(): {
|
|
6
|
+
name: string;
|
|
7
|
+
protocol: string;
|
|
8
|
+
order: number;
|
|
9
|
+
auth: string;
|
|
10
|
+
strict: boolean;
|
|
11
|
+
fields?: string[];
|
|
12
|
+
paths: Record<string, string>;
|
|
13
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drop tool fields when there are no tools, and the fields Together AI rejects (each recorded in `state.dropped`).
|
|
3
|
+
* @param {any} body - Chat Completions request body.
|
|
4
|
+
* @param {any} [context] - `{ state }`.
|
|
5
|
+
* @returns {any} Normalized body.
|
|
6
|
+
*/
|
|
7
|
+
export function request(body: any, context?: any): any;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Probe an endpoint for vLLM.
|
|
3
3
|
* @param {any} context - `{ endpoint, transport, signal, timeout, probes }`.
|
|
4
|
-
* @returns {Promise<any | null>} Facts `{ version, models, maxModelLen }`, or null when the endpoint is not vLLM (or is LocalAI, SGLang, llama.cpp or TGI, which can also answer `/version`).
|
|
4
|
+
* @returns {Promise<any | null>} Facts `{ version, models, adapters, maxModelLen }` (`models` lists the base models and the LoRA adapters, `adapters` the adapters alone as `{ id, parent }`, taken from the entries whose `parent` names the base model), or null when the endpoint is not vLLM (or is LocalAI, SGLang, llama.cpp or TGI, which can also answer `/version`).
|
|
5
5
|
*/
|
|
6
6
|
export function detect({ endpoint, transport, signal, timeout, probes }: any): Promise<any | null>;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Describe the vLLM driver.
|
|
2
|
+
* Describe the vLLM driver. `paths.health` (`GET /health`: 200 when the engine is up, 503 once it is dead; outside the API-key guard) is what `lib/health` probes before routing a request to a pool.
|
|
3
3
|
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, fields: string[], paths: Record<string,string>}} Driver facts.
|
|
4
4
|
*/
|
|
5
5
|
export function info(): {
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* List the models the vLLM server serves (including `max_model_len`).
|
|
2
|
+
* List the models the vLLM server serves (including `max_model_len`, and `parent` on a LoRA adapter). What the server answers is also what the
|
|
3
|
+
* upstream serves from now on: adapters loaded at runtime (`/v1/load_lora_adapter`) become routable, and unloaded ones stop being. An upstream
|
|
4
|
+
* with a configured `models` list keeps that list.
|
|
3
5
|
* @param {any} context - `{ endpoint, transport, signal, timeout }`.
|
|
4
6
|
* @returns {Promise<any[] | null>} Models, or null when unreachable.
|
|
5
7
|
*/
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Describe the xAI driver (see `PRESETS.xai` in `lib/presets.mjs`).
|
|
3
|
+
* @returns {{name: string, protocol: string, order: number, auth: string, strict: boolean, fields?: string[], paths: Record<string,string>}} Driver facts.
|
|
4
|
+
*/
|
|
5
|
+
export function info(): {
|
|
6
|
+
name: string;
|
|
7
|
+
protocol: string;
|
|
8
|
+
order: number;
|
|
9
|
+
auth: string;
|
|
10
|
+
strict: boolean;
|
|
11
|
+
fields?: string[];
|
|
12
|
+
paths: Record<string, string>;
|
|
13
|
+
};
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drop tool fields when there are no tools, and the fields xAI rejects (each recorded in `state.dropped`).
|
|
3
|
+
* @param {any} body - Chat Completions request body.
|
|
4
|
+
* @param {any} [context] - `{ state }`.
|
|
5
|
+
* @returns {any} Normalized body.
|
|
6
|
+
*/
|
|
7
|
+
export function request(body: any, context?: any): any;
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Health of every endpoint, for status output. An endpoint is unhealthy while it is backing off after a connection error or a 5xx; it is still
|
|
2
|
+
* Health of every endpoint, for status output. An endpoint is unhealthy while it is backing off after a connection error or a 5xx, or while its last readiness probe (`GET /health` for vLLM, see `health.check`) said it is not ready; it is still
|
|
3
3
|
* tried when nothing healthy serves the model. Secrets are never included.
|
|
4
|
-
* @returns {{strategy: string, backoff: {base: number, max: number}, models: Record<string, string>, endpoints: {name: string, url: string, models: string[], healthy: boolean, failures: number, retryAt: string | null, inflight: number, requests: number, errors: number, lastError: string | null, lastFailure: string | null, lastSuccess: string | null}[]}} The pool config and one record per endpoint (`inflight` counts requests whose reply has not finished, `requests` and `errors` count attempts since start).
|
|
4
|
+
* @returns {{strategy: string, backoff: {base: number, max: number}, models: Record<string, string>, endpoints: {name: string, url: string, models: string[], healthy: boolean, ready: boolean | null, readyReason: string | null, checkedAt: string | null, failures: number, retryAt: string | null, inflight: number, requests: number, errors: number, lastError: string | null, lastFailure: string | null, lastSuccess: string | null}[]}} The pool config and one record per endpoint (`inflight` counts requests whose reply has not finished, `requests` and `errors` count attempts since start).
|
|
5
5
|
*/
|
|
6
6
|
export function status(): {
|
|
7
7
|
strategy: string;
|
|
@@ -15,6 +15,9 @@ export function status(): {
|
|
|
15
15
|
url: string;
|
|
16
16
|
models: string[];
|
|
17
17
|
healthy: boolean;
|
|
18
|
+
ready: boolean | null;
|
|
19
|
+
readyReason: string | null;
|
|
20
|
+
checkedAt: string | null;
|
|
18
21
|
failures: number;
|
|
19
22
|
retryAt: string | null;
|
|
20
23
|
inflight: number;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Probe every upstream whose driver names a health path (vLLM: `GET /health`) right away, whatever `pool.healthInterval` says, and return the
|
|
3
|
+
* pool status. `send` already does this before it routes a request to a model that has more than one endpoint; call this to refresh the
|
|
4
|
+
* `ready` field of `endpoint.status()` and `status()` for a lone upstream, or on a schedule of your own.
|
|
5
|
+
* @param {any} [options] - `{ signal }`.
|
|
6
|
+
* @returns {Promise<any>} The `endpoint.status()` result.
|
|
7
|
+
*/
|
|
8
|
+
export function check(options?: any): Promise<any>;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Describe the translator's upstreams. Secrets (API keys, headers) are never included.
|
|
3
|
-
* @returns {{front: string, upstreams: any[]}} Front protocol and per-upstream detection results and health (see `endpoint.status`).
|
|
3
|
+
* @returns {{front: string, upstreams: any[]}} Front protocol and per-upstream detection results and health (see `endpoint.status`). `messages` is true while an Anthropic-front request to a vLLM upstream goes to its native `/v1/messages` (the `nativeMessages` setting; a request it cannot take is still converted), false when it is converted, and null for any other driver.
|
|
4
4
|
*/
|
|
5
5
|
export function status(): {
|
|
6
6
|
front: string;
|
|
@@ -36,6 +36,10 @@ export type UpstreamConfig = {
|
|
|
36
36
|
* Azure OpenAI deployment name for the classic URLs. Defaults to the request's model.
|
|
37
37
|
*/
|
|
38
38
|
deployment?: string;
|
|
39
|
+
/**
|
|
40
|
+
* Which API of the server to use: `"openai"` (default) is the OpenAI-compatible one, `"native"` the server's own chat API, for a driver that has one (Ollama's `/api/chat`).
|
|
41
|
+
*/
|
|
42
|
+
api?: "openai" | "native";
|
|
39
43
|
/**
|
|
40
44
|
* Driver name, or `"auto"` (default) to detect.
|
|
41
45
|
*/
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* @typedef {object} TranslatorEvent
|
|
3
|
-
* @property {string} type Specific event name (`fallback`, `failover`, `effort`, `dropped`, `detect`, `usage`, `warning`, `error` or `log`).
|
|
3
|
+
* @property {string} type Specific event name (`fallback`, `failover`, `health`, `effort`, `dropped`, `detect`, `usage`, `warning`, `error` or `log`).
|
|
4
4
|
* @property {string} [kind] Sub-kind within the type (for example `json_object` for a `fallback`).
|
|
5
5
|
* @property {"debug" | "info" | "warn" | "error"} level Severity.
|
|
6
6
|
* @property {string} message Human readable text.
|
|
@@ -64,7 +64,7 @@ export function emit({ config, state }: {
|
|
|
64
64
|
}): TranslatorEvent;
|
|
65
65
|
export type TranslatorEvent = {
|
|
66
66
|
/**
|
|
67
|
-
* Specific event name (`fallback`, `failover`, `effort`, `dropped`, `detect`, `usage`, `warning`, `error` or `log`).
|
|
67
|
+
* Specific event name (`fallback`, `failover`, `health`, `effort`, `dropped`, `detect`, `usage`, `warning`, `error` or `log`).
|
|
68
68
|
*/
|
|
69
69
|
type: string;
|
|
70
70
|
/**
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Probe the endpoints that name a health path and whose last verdict is older than `pool.healthInterval` (all of them with `force`). A request
|
|
3
|
+
* that was answered counts as a verdict too, so an endpoint that is serving is not probed. Probes run in parallel, and requests that arrive
|
|
4
|
+
* while one is running share it.
|
|
5
|
+
* @param {any} c - Captured translator parts (`lib/capture`).
|
|
6
|
+
* @param {any[]} endpoints - Runtime endpoints.
|
|
7
|
+
* @param {{ signal?: AbortSignal, force?: boolean }} [options] - `force` probes regardless of the interval and of an earlier "no such path" answer.
|
|
8
|
+
* @returns {Promise<void>} Resolves when every probe has finished.
|
|
9
|
+
*/
|
|
10
|
+
export function probe(c: any, endpoints: any[], options?: {
|
|
11
|
+
signal?: AbortSignal;
|
|
12
|
+
force?: boolean;
|
|
13
|
+
}): Promise<void>;
|
|
14
|
+
/**
|
|
15
|
+
* Refresh the readiness of the endpoints a request could go to, before it is routed. Only a model with more than one endpoint is probed: a
|
|
16
|
+
* lone endpoint is tried whatever it reports, so asking would only add a round trip.
|
|
17
|
+
* @param {any} c - Captured translator parts (`lib/capture`).
|
|
18
|
+
* @param {any} body - Client request body (only its `model` is read; anything that is not an object is left for `request` to reject).
|
|
19
|
+
* @param {AbortSignal | undefined} signal - The caller's abort signal.
|
|
20
|
+
* @returns {Promise<void>} Resolves when the pool's probes have finished.
|
|
21
|
+
*/
|
|
22
|
+
export function refresh(c: any, body: any, signal: AbortSignal | undefined): Promise<void>;
|
|
@@ -12,6 +12,31 @@ export function modelsOf(result: {
|
|
|
12
12
|
owned_by?: string;
|
|
13
13
|
max_model_len?: number;
|
|
14
14
|
}[] | null;
|
|
15
|
+
/**
|
|
16
|
+
* The LoRA adapters in a `/v1/models` list: vLLM lists each one as a model whose `parent` is the base model it adapts (`--lora-modules`, or
|
|
17
|
+
* `/v1/load_lora_adapter` when `VLLM_ALLOW_RUNTIME_LORA_UPDATING` is on); base models have no `parent`.
|
|
18
|
+
* @param {any[]} models - Entries of the model list.
|
|
19
|
+
* @returns {{id: string, parent: string}[]} The adapters.
|
|
20
|
+
*/
|
|
21
|
+
export function adaptersOf(models: any[]): {
|
|
22
|
+
id: string;
|
|
23
|
+
parent: string;
|
|
24
|
+
}[];
|
|
25
|
+
/**
|
|
26
|
+
* The model ids an endpoint serves that are not LoRA adapters (all of them for a driver that detects none).
|
|
27
|
+
* @param {any} endpoint - Runtime endpoint.
|
|
28
|
+
* @returns {string[]} Configured or detected model ids, adapters left out.
|
|
29
|
+
*/
|
|
30
|
+
export function baseModels(endpoint: any): string[];
|
|
31
|
+
/**
|
|
32
|
+
* The entries a model list shows for an endpoint when the server was not asked: its configured or detected ids, an adapter with its `parent`.
|
|
33
|
+
* @param {any} endpoint - Runtime endpoint.
|
|
34
|
+
* @returns {{id: string, parent?: string}[]} Entries.
|
|
35
|
+
*/
|
|
36
|
+
export function knownModels(endpoint: any): {
|
|
37
|
+
id: string;
|
|
38
|
+
parent?: string;
|
|
39
|
+
}[];
|
|
15
40
|
/**
|
|
16
41
|
* Fetch and normalize the models an upstream reports.
|
|
17
42
|
* @param {any} context - `{ endpoint, transport, signal, timeout, auth, extraHeaders, path }`. `path` replaces `/v1/models`.
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Create an incremental NDJSON parser. A line may be split across chunks at any byte; the part after the last newline is held until the next
|
|
3
|
+
* chunk (or `flush`) completes it. Handles `\n` and `\r\n`, and skips blank lines.
|
|
4
|
+
* @returns {{ push: (chunk: string) => any[], flush: () => any[] }} Parser.
|
|
5
|
+
*/
|
|
6
|
+
export function createParser(): {
|
|
7
|
+
push: (chunk: string) => any[];
|
|
8
|
+
flush: () => any[];
|
|
9
|
+
};
|
|
10
|
+
/**
|
|
11
|
+
* Parse NDJSON text (any `lib/source.mjs` input) into objects. Parsed objects in the source pass through, so a stream that was already
|
|
12
|
+
* decoded can be fed in again.
|
|
13
|
+
* @param {ReadableStream|AsyncIterable<string|Uint8Array|object>|Iterable<string|Uint8Array|object>|string|Uint8Array} source - NDJSON bytes or text.
|
|
14
|
+
* @returns {AsyncGenerator<any, void, undefined>} One object per line.
|
|
15
|
+
*/
|
|
16
|
+
export function parse(source: ReadableStream | AsyncIterable<string | Uint8Array | object> | Iterable<string | Uint8Array | object> | string | Uint8Array): AsyncGenerator<any, void, undefined>;
|