@north-light/cli 0.3.493 → 0.3.494

package/dist/core/llm/run.js

@@ -1,4 +1,3 @@
-var de=Object.defineProperty;var p=(t,e)=>de(t,"name",{value:e,configurable:!0});import{randomUUID as ce}from"node:crypto";import{streamSimple as fe}from"@earendil-works/pi-ai/compat";import{expandModelCandidates as Y,parseModelSpec as ue,probeRouteAvailability as me}from"../model-routes.js";import{readMergedLaunchConfig as ee}from"../config.js";import{openModelRegistry as pe}from"../runtime/model-registry.js";import{coolingUntilFor as te,isManagedProvider as E,markSubscriptionRateLimited as he,markSubscriptionSuccess as ye,readSubscriptionPool as _e,retryAfterMsFromHeaders as Z,retryAfterMsFromError as ge,refreshSubscriptionCredential as ve,subscriptionNeedsSuccessWrite as be}from"../subscription-state.js";import{classifyRateLimit as we,describeRateLimitScope as oe,rateLimitHeaders as xe}from"../rate-limit-scope.js";import{RunError as l}from"../runs/operations.js";import{recognizeProviderError as ne}from"../provider-error-recognition.js";import{isSafetyRefusal as ke}from"./refusal.js";import{assertUnderCap as je,recordModelUsage as qe}from"../usage/ledger.js";function Ae(t){return typeof t=="string"||t===null||Array.isArray(t)&&t.every(e=>e&&typeof e=="object"&&(e.type==="text"&&typeof e.text=="string"||e.type==="image_url"&&typeof e.image_url?.url=="string"&&/^data:image\/(png|jpeg|gif|webp);base64,[A-Za-z0-9+/]+={0,2}$/.test(e.image_url.url)))}p(Ae,"validContent");function re(t){return Array.isArray(t)?t.map(e=>{if(e.type==="text")return e;const[,n,r]=/^data:(image\/(?:png|jpeg|gif|webp));base64,(.*)$/.exec(e.image_url.url);return{type:"image",mimeType:n,data:r}}):t??""}p(re,"imageContent");function We(t){if(!t||typeof t!="object"||Array.isArray(t))throw new l(400,"invalid_request","body must be an object");const e=t;if(!Array.isArray(e.messages)||!e.messages.every(n=>n&&typeof n=="object"&&["system","user","assistant","tool"].includes(n.role)&&Ae(n.content)))throw new l(400,"invalid_request","messages must be chat messages","messages");if(e.messages.some(n=>["system","assistant"].includes(n.role)&&typeof n.content!="string"&&n.content!==null))throw new l(400,"invalid_request","system and assistant content must be text","messages");if(e.model!==void 0&&typeof e.model!="string")throw new l(400,"invalid_request","model must be a string","model");if(e.model_policy!==void 0&&!["exact","substitute"].includes(e.model_policy))throw new l(400,"invalid_request","invalid model_policy","model_policy");if(e.payer!==void 0&&!["user","app"].includes(e.payer))throw new l(400,"invalid_request","invalid payer","payer");if(e.reasoning_effort!==void 0&&!["minimal","low","medium","high","xhigh","max"].includes(e.reasoning_effort))throw new l(400,"invalid_request","invalid reasoning_effort","reasoning_effort");if(e.stream!==void 0&&typeof e.stream!="boolean")throw new l(400,"invalid_request","stream must be boolean","stream");if(e.tools!==void 0&&(!Array.isArray(e.tools)||!e.tools.every(n=>n?.type==="function"&&typeof n.function?.name=="string"&&n.function.name.length>0&&(n.function.description===void 0||typeof n.function.description=="string")&&(n.function.parameters===void 0||n.function.parameters!==null&&typeof n.function.parameters=="object"&&!Array.isArray(n.function.parameters)&&n.function.parameters.type==="object"))))throw new l(400,"invalid_request","invalid tools","tools");if(e.tool_choice!==void 0&&e.tool_choice!=="auto"&&e.tool_choice!=="none"&&e.tool_choice!=="required"&&(typeof e.tool_choice!="object"||e.tool_choice===null||e.tool_choice.type!=="function"||typeof e.tool_choice.function?.name!="string"))throw new l(400,"invalid_request","invalid tool_choice","tool_choice");if(e.messages.some(n=>n.role==="assistant"&&n.tool_calls!==void 0&&(!Array.isArray(n.tool_calls)||!n.tool_calls.every(r=>r&&typeof r.id=="string"&&typeof r.function?.name=="string"&&typeof r.function.arguments=="string"&&Me(r.function.arguments)))))throw new l(400,"invalid_request","invalid assistant tool_calls","messages");if(e.stop!==void 0&&!(typeof e.stop=="string"||Array.isArray(e.stop)&&e.stop.every(n=>typeof n=="string")))throw new l(400,"invalid_request","invalid stop","stop");if(e.seed!==void 0&&!Number.isSafeInteger(e.seed))throw new l(400,"invalid_request","invalid seed","seed");for(const[n,r]of[["temperature",e.temperature],["top_p",e.top_p],["max_tokens",e.max_tokens]])if(r!==void 0&&(typeof r!="number"||!Number.isFinite(r)||n==="temperature"&&(r<0||r>2)||n==="top_p"&&(r<0||r>1)||n==="max_tokens"&&(!Number.isSafeInteger(r)||r<1)))throw new l(400,"invalid_request",`invalid ${n}`,n);if(e.response_format!==void 0){const n=e.response_format;if(!n||typeof n!="object"||Array.isArray(n)||!["text","json_object","json_schema"].includes(n.type)||n.type==="json_schema"&&(typeof n.json_schema?.name!="string"||!n.json_schema.name||!n.json_schema.schema||typeof n.json_schema.schema!="object"||Array.isArray(n.json_schema.schema)))throw new l(400,"invalid_request","invalid response_format","response_format")}if(e.metadata!==void 0&&Buffer.byteLength(JSON.stringify(e.metadata)??"")>4096)throw new l(400,"invalid_request","metadata exceeds 4 KB","metadata");return e}p(We,"parseLlmRequest");function Me(t){try{const e=JSON.parse(t);return e!==null&&typeof e=="object"&&!Array.isArray(e)}catch{return!1}}p(Me,"isJsonObject");function $e(t,e){const n=Date.now(),r=t.messages.map(s=>s.role==="system"?{role:"system",content:s.content??"",timestamp:n}:s.role==="tool"?{role:"toolResult",toolCallId:s.tool_call_id??"",toolName:"",content:typeof s.content=="string"||s.content===null?[{type:"text",text:s.content??""}]:re(s.content),isError:!1,timestamp:n}:s.role==="assistant"?{role:"assistant",content:[...s.content?[{type:"text",text:s.content}]:[],...(s.tool_calls??[]).map(c=>({type:"toolCall",id:c.id,name:c.function.name,arguments:JSON.parse(c.function.arguments)}))],api:"openai-completions",provider:"openai",model:t.model??"",usage:{input:0,output:0,cacheRead:0,cacheWrite:0,totalTokens:0,cost:{input:0,output:0,cacheRead:0,cacheWrite:0,total:0}},stopReason:"stop",timestamp:n}:{role:"user",content:re(s.content),timestamp:n});if(e){const s=r.filter(f=>f.role==="system").map(f=>f.content).join(`
-`).slice(0,16384),c=r.findIndex(f=>f.role==="user");if(s&&c>=0){const f=r[c];r[c]={...f,content:typeof f.content=="string"?`${s}
+var de=Object.defineProperty;var u=(t,e)=>de(t,"name",{value:e,configurable:!0});import{randomUUID as ce}from"node:crypto";import{streamSimple as fe}from"@earendil-works/pi-ai/compat";import{expandModelCandidates as Y,parseModelSpec as ue,probeRouteAvailability as me}from"../model-routes.js";import{readMergedLaunchConfig as ee}from"../config.js";import{openModelRegistry as pe}from"../runtime/model-registry.js";import{coolingUntilFor as te,isManagedProvider as E,markSubscriptionRateLimited as he,markSubscriptionSuccess as _e,readSubscriptionPool as ye,retryAfterMsFromHeaders as Z,retryAfterMsFromError as ge,refreshSubscriptionCredential as ve,subscriptionNeedsSuccessWrite as be}from"../subscription-state.js";import{classifyRateLimit as we,describeRateLimitScope as oe,rateLimitHeaders as xe}from"../rate-limit-scope.js";import{RunError as l}from"../runs/operations.js";import{recognizeProviderError as ne}from"../provider-error-recognition.js";import{isSafetyRefusal as ke}from"./refusal.js";import{assertUnderCap as je,recordModelUsage as qe}from"../usage/ledger.js";function Ae(t){return typeof t=="string"||t===null||Array.isArray(t)&&t.every(e=>e&&typeof e=="object"&&(e.type==="text"&&typeof e.text=="string"||e.type==="image_url"&&typeof e.image_url?.url=="string"&&/^data:image\/(png|jpeg|gif|webp);base64,[A-Za-z0-9+/]+={0,2}$/.test(e.image_url.url)))}u(Ae,"validContent");function re(t){return Array.isArray(t)?t.map(e=>{if(e.type==="text")return e;const[,n,r]=/^data:(image\/(?:png|jpeg|gif|webp));base64,(.*)$/.exec(e.image_url.url);return{type:"image",mimeType:n,data:r}}):t??""}u(re,"imageContent");function We(t){if(!t||typeof t!="object"||Array.isArray(t))throw new l(400,"invalid_request","body must be an object");const e=t;if(!Array.isArray(e.messages)||!e.messages.every(n=>n&&typeof n=="object"&&["system","user","assistant","tool"].includes(n.role)&&Ae(n.content)))throw new l(400,"invalid_request","messages must be chat messages","messages");if(e.messages.some(n=>["system","assistant"].includes(n.role)&&typeof n.content!="string"&&n.content!==null))throw new l(400,"invalid_request","system and assistant content must be text","messages");if(e.model!==void 0&&typeof e.model!="string")throw new l(400,"invalid_request","model must be a string","model");if(e.model_policy!==void 0&&!["exact","substitute"].includes(e.model_policy))throw new l(400,"invalid_request","invalid model_policy","model_policy");if(e.payer!==void 0&&!["user","app"].includes(e.payer))throw new l(400,"invalid_request","invalid payer","payer");if(e.reasoning_effort!==void 0&&!["minimal","low","medium","high","xhigh","max"].includes(e.reasoning_effort))throw new l(400,"invalid_request","invalid reasoning_effort","reasoning_effort");if(e.stream!==void 0&&typeof e.stream!="boolean")throw new l(400,"invalid_request","stream must be boolean","stream");if(e.tools!==void 0&&(!Array.isArray(e.tools)||!e.tools.every(n=>n?.type==="function"&&typeof n.function?.name=="string"&&n.function.name.length>0&&(n.function.description===void 0||typeof n.function.description=="string")&&(n.function.parameters===void 0||n.function.parameters!==null&&typeof n.function.parameters=="object"&&!Array.isArray(n.function.parameters)&&n.function.parameters.type==="object"))))throw new l(400,"invalid_request","invalid tools","tools");if(e.tool_choice!==void 0&&e.tool_choice!=="auto"&&e.tool_choice!=="none"&&e.tool_choice!=="required"&&(typeof e.tool_choice!="object"||e.tool_choice===null||e.tool_choice.type!=="function"||typeof e.tool_choice.function?.name!="string"))throw new l(400,"invalid_request","invalid tool_choice","tool_choice");if(e.messages.some(n=>n.role==="assistant"&&n.tool_calls!==void 0&&(!Array.isArray(n.tool_calls)||!n.tool_calls.every(r=>r&&typeof r.id=="string"&&typeof r.function?.name=="string"&&typeof r.function.arguments=="string"&&Me(r.function.arguments)))))throw new l(400,"invalid_request","invalid assistant tool_calls","messages");if(e.stop!==void 0&&!(typeof e.stop=="string"||Array.isArray(e.stop)&&e.stop.every(n=>typeof n=="string")))throw new l(400,"invalid_request","invalid stop","stop");if(e.seed!==void 0&&!Number.isSafeInteger(e.seed))throw new l(400,"invalid_request","invalid seed","seed");for(const[n,r]of[["temperature",e.temperature],["top_p",e.top_p],["max_tokens",e.max_tokens]])if(r!==void 0&&(typeof r!="number"||!Number.isFinite(r)||n==="temperature"&&(r<0||r>2)||n==="top_p"&&(r<0||r>1)||n==="max_tokens"&&(!Number.isSafeInteger(r)||r<1)))throw new l(400,"invalid_request",`invalid ${n}`,n);if(e.response_format!==void 0){const n=e.response_format;if(!n||typeof n!="object"||Array.isArray(n)||!["text","json_object","json_schema"].includes(n.type)||n.type==="json_schema"&&(typeof n.json_schema?.name!="string"||!n.json_schema.name||!n.json_schema.schema||typeof n.json_schema.schema!="object"||Array.isArray(n.json_schema.schema)))throw new l(400,"invalid_request","invalid response_format","response_format")}if(e.metadata!==void 0&&Buffer.byteLength(JSON.stringify(e.metadata)??"")>4096)throw new l(400,"invalid_request","metadata exceeds 4 KB","metadata");return e}u(We,"parseLlmRequest");function Me(t){try{const e=JSON.parse(t);return e!==null&&typeof e=="object"&&!Array.isArray(e)}catch{return!1}}u(Me,"isJsonObject");function Re(t){const e=Date.now(),n=t.messages.map(a=>a.role==="system"?{role:"system",content:a.content??"",timestamp:e}:a.role==="tool"?{role:"toolResult",toolCallId:a.tool_call_id??"",toolName:"",content:typeof a.content=="string"||a.content===null?[{type:"text",text:a.content??""}]:re(a.content),isError:!1,timestamp:e}:a.role==="assistant"?{role:"assistant",content:[...a.content?[{type:"text",text:a.content}]:[],...(a.tool_calls??[]).map(w=>({type:"toolCall",id:w.id,name:w.function.name,arguments:JSON.parse(w.function.arguments)}))],api:"openai-completions",provider:"openai",model:t.model??"",usage:{input:0,output:0,cacheRead:0,cacheWrite:0,totalTokens:0,cost:{input:0,output:0,cacheRead:0,cacheWrite:0,total:0}},stopReason:"stop",timestamp:e}:{role:"user",content:re(a.content),timestamp:e}),r=n.findIndex(a=>a.role!=="system"),y=n.slice(0,r<0?n.length:r).map(a=>a.content),m=y.join(`
 
-${f.content}`:[{type:"text",text:s},...f.content]}}s&&c<0&&r.unshift({role:"user",content:s,timestamp:n});for(let f=r.length-1;f>=0;f--)r[f]?.role==="system"&&r.splice(f,1)}const _=t.tools?.map(({function:s})=>({name:s.name,description:s.description??"",parameters:s.parameters??{type:"object",properties:{}}}));return{messages:r,..._?{tools:_}:{}}}p($e,"contextOf");function se(t,e,n,r){if(!t){const s=ee(n,r),c=Object.values({...s.modelRoutes??{},...s.legacyModelRoutes??{}}),f=c.length?[s.modelLadders.defaultProvider,...s.modelRouting?.familyOrder??[],...c.map(k=>k.family)].find(k=>c.some(I=>I.family===k)):s.modelLadders.defaultProvider??"anthropic";return{kind:"logical",family:f,strength:"strong",spec:`${f}/strong`}}const _=/^([^/]+)\/(ultra|strong|medium|light)$/.exec(t);return _?{kind:"logical",family:_[1],strength:_[2],spec:t}:{kind:e==="exact"?"exact":"concrete",spec:t}}p(se,"requestFor");function Re(t,e){const{input:n,output:r,cacheRead:_,cacheWrite:s}=t.cost;return n===0&&r===0&&_===0&&s===0?null:Number.isFinite(e)&&e>=0?e:null}p(Re,"usdEquivalent");function Ie(t,e){return t.compat?.allowedFallbackModels?.find(r=>r.provider===t.provider&&r.model===e)}p(Ie,"allowedFallback");async function He(t,e,n,r,_){if(t.payer==="app")throw new l(403,"payer_not_app","app-paid inference routes are not registered");je(_);const s=await pe(),c=se(t.model,t.model_policy,r?.cwd,r?.profileId),f=p(m=>m.routeId!=="concrete"&&m.routeId!=="exact","configured"),k=p(m=>{const a=s.find(m.providerId,m.modelId);return a!==void 0&&(!t.reasoning_effort||a.reasoning)&&(t.seed===void 0&&t.response_format===void 0||a.api==="openai-completions")&&(t.stop===void 0||["openai-completions","anthropic-messages"].includes(a.api))},"matchesFeatures"),I=Y(c.kind==="exact"?{kind:"concrete",spec:c.spec}:c,r?.cwd,r?.profileId).filter(f);let w;if(c.kind==="logical")w=t.model_policy==="exact"?I.filter(m=>m.family===c.family&&m.strength===c.strength).slice(0,1):I;else{const m=ue(c.spec),a=I.filter(u=>u.providerId===m?.providerId&&u.modelId===m.modelId&&(t.model_policy!=="exact"||u.thinkingLevel===m.thinkingLevel)&&k(u));if(t.model_policy==="exact")w=a.slice(0,1);else if(a.length)w=a;else{const u=ee(r?.cwd,r?.profileId),q=[...Object.values(u.modelRoutes??{}).flatMap(i=>Object.entries(i.models)),...[u.modelLadders.anthropic,u.modelLadders.openai].flatMap(i=>Object.entries(i))].find(([,i])=>i===c.spec)?.[0],L=se(void 0,"substitute",r?.cwd,r?.profileId);w=q&&L.kind==="logical"?Y({...L,strength:q,spec:c.spec},r?.cwd,r?.profileId).filter(i=>f(i)&&i.strength===q):[]}}if(w=w.filter(k),w.length===0)throw t.model_policy==="exact"?new l(400,"substitution_refused","no eligible model route is configured"):new l(404,"model_unavailable","no eligible model route is configured");const G=`chatcmpl-${ce()}`;let j=1/0,y,U;for(const m of w){if(!me(m,s).authenticated)continue;const a=s.find(m.providerId,m.modelId);if(!a)continue;const u=m.providerId,S=E(u)?_e(u).filter(i=>!i.authFailure):[],q=S.filter(i=>te(i,a.id)<=Date.now());if(S.length&&!q.length){const i=Math.min(...S.map(A=>te(A,a.id)));i<j&&(j=i,U=`${a.provider}/${a.id}`);continue}const L=S.length?q:[void 0];for(const i of L){const A=Date.now();let Q=i?.access;if(i&&i.expires<=Date.now()&&E(u)){const o=s.getProvider(u)?.auth.oauth;if(!o)throw new l(503,"upstream_model_unavailable",`no OAuth refresh for ${u}`,void 0,{served_model:`${a.provider}/${a.id}`});Q=(await ve(u,i.label,async(g,X)=>{const P=await o.refresh({type:"oauth",refresh:g,access:"",expires:0,...X?{clientId:X}:{}},e??new AbortController().signal),z=P.clientId;return{refresh:P.refresh,access:P.access,expires:P.expires,...typeof z=="string"&&z?{clientId:z}:{}}})).access}const ie=$e(t,!!i),M=i?void 0:await s.getApiKeyAndHeaders(a);if(M&&!M.ok)continue;const v=`${a.provider}/${a.id}`,$=t.model??v;let x=!1,V,b=3e4,R;const J=[],T=[],O=[];let d;const N=p((o,h=null)=>({id:G,object:"chat.completion.chunk",created:Math.floor(A/1e3),model:v,served_model:v,requested_model:$,substituted:$!==v&&!/^([^/]+)\/(ultra|strong|medium|light)$/.test($),choices:[{index:0,delta:o,finish_reason:h}]}),"chunk"),D=p(o=>{J.push(o),n?.(o)},"emit"),ae=fe(a,ie,{apiKey:Q??(M?.ok?M.apiKey:void 0),...M?.ok?{headers:M.headers}:{},signal:e,reasoning:t.reasoning_effort??m.thinkingLevel,...t.temperature!==void 0?{temperature:t.temperature}:{},...t.max_tokens!==void 0?{maxTokens:t.max_tokens}:{},...t.top_p!==void 0?{samplingParams:{top_p:t.top_p}}:{},...t.tool_choice==="none"?{toolChoice:"none"}:{},onPayload:p(o=>{if(!o||typeof o!="object")return;const h=t.tool_choice,g=h==="required"?a.api==="anthropic-messages"?{type:"any"}:"required":typeof h=="object"&&h!==null?a.api==="anthropic-messages"?{type:"tool",name:h.function.name}:h:void 0;return{...o,...g?{tool_choice:g}:{},...t.stop!==void 0?{[a.api==="anthropic-messages"?"stop_sequences":"stop"]:typeof t.stop=="string"?[t.stop]:t.stop}:{},...t.seed!==void 0?{seed:t.seed}:{},...t.response_format!==void 0?{response_format:t.response_format}:{}}},"onPayload"),fetch:p(async(o,h)=>{const g=await fetch(o,h);return V=g.status,g.status===429&&(x=!0,R=g.headers,b=Z(g.headers)??15e3),g},"fetch"),onResponse:p(o=>{o.status===429&&(x=!0,R=o.headers,b=Z(o.headers)??15e3)},"onResponse")});try{for await(const o of ae)o.type==="text_delta"&&(T.push(o.delta),D(N({content:o.delta}))),o.type==="thinking_delta"&&(O.push(o.delta),D(N({reasoning_content:o.delta}))),o.type==="done"&&(d=o.message),o.type==="error"&&(d=o.error)}catch(o){const h=ge(o);R===void 0&&o&&typeof o=="object"&&"headers"in o&&Object.keys(xe(o.headers)).length&&(R=o.headers);const g=ne({status:o&&typeof o=="object"&&"status"in o?o.status:void 0,message:o instanceof Error?o.message:String(o)});if(!x&&g.kind!=="rate-limit")throw J.length?new l(503,"upstream_model_unavailable",o instanceof Error?o.message:String(o),void 0,{served_model:v}):o;x=!0,b=h??b}if(!x&&d?.stopReason==="error"&&ne({message:d.errorMessage}).kind==="rate-limit"&&(x=!0,b=15e3),x){const o=we(R,a.id);if(Z(R)===void 0&&o.resetAt!==void 0&&(b=Math.max(b,o.resetAt-Date.now())),y={provider:u,model:a.id,...i?{account:i.label}:{},observation:o},console.error(`[llm] 429 from ${v}${i?` account "${i.label}"`:""}: scope=${oe(o.scope)} retry_after_ms=${b} headers=${JSON.stringify(o.headers)}`),i&&E(u)&&await he(u,i.label,b,A,Date.now(),o.scope),Date.now()+b<j&&(j=Date.now()+b,U=v),T.length||O.length)throw new l(503,"service_unavailable","vendor rate limit after output began",void 0,{served_model:v});continue}const F=ke(d);if(V===400&&t.tool_choice!==void 0&&t.tool_choice!=="auto"&&t.tool_choice!=="none"&&/\btool[_ ]choice\b/i.test(d?.errorMessage??"")&&/not supported|does not support|unsupported|cannot|must be auto/i.test(d?.errorMessage??""))throw new l(400,"invalid_request",d.errorMessage,"tool_choice",{served_model:v});if(!d||d.stopReason==="error"&&!F||d.stopReason==="aborted")throw new l(503,"upstream_model_unavailable",d?.errorMessage??"model request failed",void 0,{served_model:v});i&&E(u)&&be(i,a.id)&&await ye(u,i.label,A,a.id);const W=d.content.filter(o=>o.type==="toolCall").map(o=>({id:o.id,type:"function",function:{name:o.name,arguments:JSON.stringify(o.arguments)}})),H=F?"content_filter":d.stopReason==="toolUse"?"tool_calls":d.stopReason==="length"?"length":"stop",K=d.responseModel?Ie(a,d.responseModel):void 0,C=K?`${a.provider}/${K.model}`:v,B={prompt_tokens:d.usage.input,completion_tokens:d.usage.output,total_tokens:d.usage.totalTokens,cache_read_tokens:d.usage.cacheRead,cache_write_tokens:d.usage.cacheWrite,usd_equivalent:Re(K??a,d.usage.cost.total),served_model:C};_&&qe(_.grantee,"user",C,d.usage,B.usd_equivalent);const le={id:G,object:"chat.completion",created:Math.floor(A/1e3),model:C,served_model:C,requested_model:$,substituted:$!==C&&!/^([^/]+)\/(ultra|strong|medium|light)$/.test($),choices:[{index:0,message:{role:"assistant",content:T.join("")||null,...O.length?{reasoning_content:O.join("")}:{},...W.length?{tool_calls:W}:{},...F?{refusal:d.errorMessage??"Content filtered"}:{}},finish_reason:H}],usage:B,...t.metadata!==void 0?{metadata:t.metadata}:{}};for(const[o,h]of W.entries())D(N({tool_calls:[{index:o,...h}]}));return D({...N({},H),...F?{choices:[{index:0,delta:{},finish_reason:H,message:{refusal:d.errorMessage??"Content filtered"}}]}:{},usage:B}),{completion:le,chunks:J}}}throw j!==1/0?new l(429,"all_routes_exhausted","all eligible model routes are rate-limited",void 0,{retry_after_s:Math.max(1,Math.ceil((j-Date.now())/1e3)),served_model:y?`${y.provider}/${y.model}`:U,...y?{rate_limit:{provider:y.provider,model:y.model,...y.account?{account:y.account}:{},scope:oe(y.observation.scope),...y.observation.claim?{claim:y.observation.claim}:{},headers:y.observation.headers}}:{}},void 0,!0):t.model_policy==="exact"?new l(400,"substitution_refused","the requested model is unavailable"):new l(404,"model_unavailable","no usable model route is available")}p(He,"runLlm");export{We as parseLlmRequest,He as runLlm};
+`),p=t.tools?.map(({function:a})=>({name:a.name,description:a.description??"",parameters:a.parameters??{type:"object",properties:{}}}));return{...m?{systemPrompt:m}:{},messages:n.slice(y.length),...p?{tools:p}:{}}}u(Re,"contextOf");function se(t,e,n,r){if(!t){const m=ee(n,r),p=Object.values({...m.modelRoutes??{},...m.legacyModelRoutes??{}}),a=p.length?[m.modelLadders.defaultProvider,...m.modelRouting?.familyOrder??[],...p.map(w=>w.family)].find(w=>p.some(I=>I.family===w)):m.modelLadders.defaultProvider??"anthropic";return{kind:"logical",family:a,strength:"strong",spec:`${a}/strong`}}const y=/^([^/]+)\/(ultra|strong|medium|light)$/.exec(t);return y?{kind:"logical",family:y[1],strength:y[2],spec:t}:{kind:e==="exact"?"exact":"concrete",spec:t}}u(se,"requestFor");function $e(t,e){const{input:n,output:r,cacheRead:y,cacheWrite:m}=t.cost;return n===0&&r===0&&y===0&&m===0?null:Number.isFinite(e)&&e>=0?e:null}u($e,"usdEquivalent");function Ie(t,e){return t.compat?.allowedFallbackModels?.find(r=>r.provider===t.provider&&r.model===e)}u(Ie,"allowedFallback");async function He(t,e,n,r,y){if(t.payer==="app")throw new l(403,"payer_not_app","app-paid inference routes are not registered");je(y);const m=await pe(),p=se(t.model,t.model_policy,r?.cwd,r?.profileId),a=u(f=>f.routeId!=="concrete"&&f.routeId!=="exact","configured"),w=u(f=>{const s=m.find(f.providerId,f.modelId);return s!==void 0&&(!t.reasoning_effort||s.reasoning)&&(t.seed===void 0&&t.response_format===void 0||s.api==="openai-completions")&&(t.stop===void 0||["openai-completions","anthropic-messages"].includes(s.api))},"matchesFeatures"),I=Y(p.kind==="exact"?{kind:"concrete",spec:p.spec}:p,r?.cwd,r?.profileId).filter(a);let x;if(p.kind==="logical")x=t.model_policy==="exact"?I.filter(f=>f.family===p.family&&f.strength===p.strength).slice(0,1):I;else{const f=ue(p.spec),s=I.filter(c=>c.providerId===f?.providerId&&c.modelId===f.modelId&&(t.model_policy!=="exact"||c.thinkingLevel===f.thinkingLevel)&&w(c));if(t.model_policy==="exact")x=s.slice(0,1);else if(s.length)x=s;else{const c=ee(r?.cwd,r?.profileId),q=[...Object.values(c.modelRoutes??{}).flatMap(i=>Object.entries(i.models)),...[c.modelLadders.anthropic,c.modelLadders.openai].flatMap(i=>Object.entries(i))].find(([,i])=>i===p.spec)?.[0],L=se(void 0,"substitute",r?.cwd,r?.profileId);x=q&&L.kind==="logical"?Y({...L,strength:q,spec:p.spec},r?.cwd,r?.profileId).filter(i=>a(i)&&i.strength===q):[]}}if(x=x.filter(w),x.length===0)throw t.model_policy==="exact"?new l(400,"substitution_refused","no eligible model route is configured"):new l(404,"model_unavailable","no eligible model route is configured");const G=`chatcmpl-${ce()}`;let j=1/0,_,J;for(const f of x){if(!me(f,m).authenticated)continue;const s=m.find(f.providerId,f.modelId);if(!s)continue;const c=f.providerId,S=E(c)?ye(c).filter(i=>!i.authFailure):[],q=S.filter(i=>te(i,s.id)<=Date.now());if(S.length&&!q.length){const i=Math.min(...S.map(A=>te(A,s.id)));i<j&&(j=i,J=`${s.provider}/${s.id}`);continue}const L=S.length?q:[void 0];for(const i of L){const A=Date.now();let Q=i?.access;if(i&&i.expires<=Date.now()&&E(c)){const o=m.getProvider(c)?.auth.oauth;if(!o)throw new l(503,"upstream_model_unavailable",`no OAuth refresh for ${c}`,void 0,{served_model:`${s.provider}/${s.id}`});Q=(await ve(c,i.label,async(g,X)=>{const P=await o.refresh({type:"oauth",refresh:g,access:"",expires:0,...X?{clientId:X}:{}},e??new AbortController().signal),B=P.clientId;return{refresh:P.refresh,access:P.access,expires:P.expires,...typeof B=="string"&&B?{clientId:B}:{}}})).access}const ie=Re(t),M=i?void 0:await m.getApiKeyAndHeaders(s);if(M&&!M.ok)continue;const v=`${s.provider}/${s.id}`,R=t.model??v;let k=!1,V,b=3e4,$;const U=[],T=[],O=[];let d;const N=u((o,h=null)=>({id:G,object:"chat.completion.chunk",created:Math.floor(A/1e3),model:v,served_model:v,requested_model:R,substituted:R!==v&&!/^([^/]+)\/(ultra|strong|medium|light)$/.test(R),choices:[{index:0,delta:o,finish_reason:h}]}),"chunk"),D=u(o=>{U.push(o),n?.(o)},"emit"),ae=fe(s,ie,{apiKey:Q??(M?.ok?M.apiKey:void 0),...M?.ok?{headers:M.headers}:{},signal:e,reasoning:t.reasoning_effort??f.thinkingLevel,...t.temperature!==void 0?{temperature:t.temperature}:{},...t.max_tokens!==void 0?{maxTokens:t.max_tokens}:{},...t.top_p!==void 0?{samplingParams:{top_p:t.top_p}}:{},...t.tool_choice==="none"?{toolChoice:"none"}:{},onPayload:u(o=>{if(!o||typeof o!="object")return;const h=t.tool_choice,g=h==="required"?s.api==="anthropic-messages"?{type:"any"}:"required":typeof h=="object"&&h!==null?s.api==="anthropic-messages"?{type:"tool",name:h.function.name}:h:void 0;return{...o,...g?{tool_choice:g}:{},...t.stop!==void 0?{[s.api==="anthropic-messages"?"stop_sequences":"stop"]:typeof t.stop=="string"?[t.stop]:t.stop}:{},...t.seed!==void 0?{seed:t.seed}:{},...t.response_format!==void 0?{response_format:t.response_format}:{}}},"onPayload"),fetch:u(async(o,h)=>{const g=await fetch(o,h);return V=g.status,g.status===429&&(k=!0,$=g.headers,b=Z(g.headers)??15e3),g},"fetch"),onResponse:u(o=>{o.status===429&&(k=!0,$=o.headers,b=Z(o.headers)??15e3)},"onResponse")});try{for await(const o of ae)o.type==="text_delta"&&(T.push(o.delta),D(N({content:o.delta}))),o.type==="thinking_delta"&&(O.push(o.delta),D(N({reasoning_content:o.delta}))),o.type==="done"&&(d=o.message),o.type==="error"&&(d=o.error)}catch(o){const h=ge(o);$===void 0&&o&&typeof o=="object"&&"headers"in o&&Object.keys(xe(o.headers)).length&&($=o.headers);const g=ne({status:o&&typeof o=="object"&&"status"in o?o.status:void 0,message:o instanceof Error?o.message:String(o)});if(!k&&g.kind!=="rate-limit")throw U.length?new l(503,"upstream_model_unavailable",o instanceof Error?o.message:String(o),void 0,{served_model:v}):o;k=!0,b=h??b}if(!k&&d?.stopReason==="error"&&ne({message:d.errorMessage}).kind==="rate-limit"&&(k=!0,b=15e3),k){const o=we($,s.id);if(Z($)===void 0&&o.resetAt!==void 0&&(b=Math.max(b,o.resetAt-Date.now())),_={provider:c,model:s.id,...i?{account:i.label}:{},observation:o},console.error(`[llm] 429 from ${v}${i?` account "${i.label}"`:""}: scope=${oe(o.scope)} retry_after_ms=${b} headers=${JSON.stringify(o.headers)}`),i&&E(c)&&await he(c,i.label,b,A,Date.now(),o.scope),Date.now()+b<j&&(j=Date.now()+b,J=v),T.length||O.length)throw new l(503,"service_unavailable","vendor rate limit after output began",void 0,{served_model:v});continue}const F=ke(d);if(V===400&&t.tool_choice!==void 0&&t.tool_choice!=="auto"&&t.tool_choice!=="none"&&/\btool[_ ]choice\b/i.test(d?.errorMessage??"")&&/not supported|does not support|unsupported|cannot|must be auto/i.test(d?.errorMessage??""))throw new l(400,"invalid_request",d.errorMessage,"tool_choice",{served_model:v});if(!d||d.stopReason==="error"&&!F||d.stopReason==="aborted")throw new l(503,"upstream_model_unavailable",d?.errorMessage??"model request failed",void 0,{served_model:v});i&&E(c)&&be(i,s.id)&&await _e(c,i.label,A,s.id);const W=d.content.filter(o=>o.type==="toolCall").map(o=>({id:o.id,type:"function",function:{name:o.name,arguments:JSON.stringify(o.arguments)}})),H=F?"content_filter":d.stopReason==="toolUse"?"tool_calls":d.stopReason==="length"?"length":"stop",K=d.responseModel?Ie(s,d.responseModel):void 0,C=K?`${s.provider}/${K.model}`:v,z={prompt_tokens:d.usage.input,completion_tokens:d.usage.output,total_tokens:d.usage.totalTokens,cache_read_tokens:d.usage.cacheRead,cache_write_tokens:d.usage.cacheWrite,usd_equivalent:$e(K??s,d.usage.cost.total),served_model:C};y&&qe(y.grantee,"user",C,d.usage,z.usd_equivalent);const le={id:G,object:"chat.completion",created:Math.floor(A/1e3),model:C,served_model:C,requested_model:R,substituted:R!==C&&!/^([^/]+)\/(ultra|strong|medium|light)$/.test(R),choices:[{index:0,message:{role:"assistant",content:T.join("")||null,...O.length?{reasoning_content:O.join("")}:{},...W.length?{tool_calls:W}:{},...F?{refusal:d.errorMessage??"Content filtered"}:{}},finish_reason:H}],usage:z,...t.metadata!==void 0?{metadata:t.metadata}:{}};for(const[o,h]of W.entries())D(N({tool_calls:[{index:o,...h}]}));return D({...N({},H),...F?{choices:[{index:0,delta:{},finish_reason:H,message:{refusal:d.errorMessage??"Content filtered"}}]}:{},usage:z}),{completion:le,chunks:U}}}throw j!==1/0?new l(429,"all_routes_exhausted","all eligible model routes are rate-limited",void 0,{retry_after_s:Math.max(1,Math.ceil((j-Date.now())/1e3)),served_model:_?`${_.provider}/${_.model}`:J,..._?{rate_limit:{provider:_.provider,model:_.model,..._.account?{account:_.account}:{},scope:oe(_.observation.scope),..._.observation.claim?{claim:_.observation.claim}:{},headers:_.observation.headers}}:{}},void 0,!0):t.model_policy==="exact"?new l(400,"substitution_refused","the requested model is unavailable"):new l(404,"model_unavailable","no usable model route is available")}u(He,"runLlm");export{We as parseLlmRequest,He as runLlm};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.493",
+  "version": "0.3.494",
   "description": "crtr — agent runtime with memory, plugins, and marketplaces",
   "type": "module",
   "main": "dist/index.js",
@@ -138,7 +138,7 @@
   },
   "license": "UNLICENSED",
   "dependencies": {
-    "@north-light/identity": "0.3.493",
+    "@north-light/identity": "0.3.494",
     "@earendil-works/pi-agent-core": "1.1.0",
     "@earendil-works/pi-ai": "1.1.0",
     "@earendil-works/pi-coding-agent": "1.1.0",

package/packages/crouter-identity/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@north-light/identity",
-  "version": "0.3.493",
+  "version": "0.3.494",
   "description": "Northlight access-token verification and scope rules",
   "type": "module",
   "main": "./dist/index.js",

package/runtime.lock.json

@@ -1,12 +1,12 @@
 {
   "name": "@north-light/cli",
-  "version": "0.3.493",
+  "version": "0.3.494",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "@north-light/cli",
-      "version": "0.3.493",
+      "version": "0.3.494",
       "hasInstallScript": true,
       "license": "UNLICENSED",
       "workspaces": [
@@ -18,7 +18,7 @@
         "@earendil-works/pi-ai": "1.1.0",
         "@earendil-works/pi-coding-agent": "1.1.0",
         "@earendil-works/pi-tui": "1.1.0",
-        "@north-light/identity": "0.3.493",
+        "@north-light/identity": "0.3.494",
         "cron-parser": "^5.6.0",
         "esbuild": "^0.27.7",
         "jose": "^6.2.12",
@@ -12116,17 +12116,17 @@
     },
     "packages/crouter-api": {
       "name": "@north-light/api",
-      "version": "0.3.493",
+      "version": "0.3.494",
       "license": "UNLICENSED"
     },
     "packages/crouter-env-docker": {
       "name": "@north-light/env-docker",
-      "version": "0.3.493",
+      "version": "0.3.494",
       "license": "UNLICENSED"
     },
     "packages/crouter-identity": {
       "name": "@north-light/identity",
-      "version": "0.3.493",
+      "version": "0.3.494",
       "license": "UNLICENSED",
       "dependencies": {
         "jose": "^6.2.1"
@@ -12134,20 +12134,20 @@
     },
     "packages/crouter-plugin": {
       "name": "@north-light/plugin",
-      "version": "0.3.493",
+      "version": "0.3.494",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.493",
-        "@north-light/identity": "0.3.493"
+        "@north-light/api": "0.3.494",
+        "@north-light/identity": "0.3.494"
       }
     },
     "packages/crouter-sdk": {
       "name": "@north-light/sdk",
-      "version": "0.3.493",
+      "version": "0.3.494",
       "license": "UNLICENSED",
       "dependencies": {
-        "@north-light/api": "0.3.493",
-        "@north-light/identity": "0.3.493",
+        "@north-light/api": "0.3.494",
+        "@north-light/identity": "0.3.494",
         "jose": "^6.2.1"
       },
       "bin": {